Move recording setting from Developer menu to Transcribe app; add prominent indicator

This commit is contained in:
James Babcock committed 2026-11-11 15:55:52 -08:00
1 parent e158452ba4
commit 30de2c4dd7
24 files changed
+482 -153

No files matched your search

@@ -136,8 +136,6 @@ class FaceclawVoiceController(context: Context) {
override fun release() {}
}
}
override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) }
}
private val session = VoiceCaptureSession(host)
@@ -155,11 +153,6 @@ class FaceclawVoiceController(context: Context) {
session.setUsePhoneMic(usePhoneMic)
}
/** When true, the decoded mic PCM for each session is saved as a WAV. */
fun setSaveRecordings(saveRecordings: Boolean) {
session.setSaveRecordings(saveRecordings)
}
/**
* Which on-device model {@link #start}("onboard") should load: "whisper"
* selects the second on-device model (sherpa-onnx offline Whisper); any
@@ -233,29 +226,6 @@ class FaceclawVoiceController(context: Context) {
stop()
}
/** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */
private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) {
try {
val dir = File(appContext.getExternalFilesDir(null), "voice-recordings")
if (!dir.exists() && !dir.mkdirs()) {
Log.w(TAG, "could not create voice-recordings dir")
return
}
val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US)
.format(java.util.Date())
val file = File(dir, "voice-$stamp.wav")
java.io.FileOutputStream(file).use { fos ->
fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1))
fos.write(pcmBytes)
}
Log.i(TAG, "saved voice recording " + file.absolutePath
+ " samples=" + (pcmBytes.size / 2)
+ " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate))
} catch (t: Throwable) {
Log.w(TAG, "failed to save voice recording", t)
}
}
/**
* The on-device model directory for the given kind, populated by the
* download flow in asr-model.ts (releases before 0.5.0 copied the
+18 -1
View File
@@ -11,6 +11,7 @@ import { ensureLocationPermission, hasLocationPermission } from "../../../native
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
import { readPhoneBatteryState } from "../../../native/phone-battery";
import { voiceControlBridge } from "../../../native/voice-control";
import { weatherBridge } from "../../../native/weather";
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
import {
@@ -26,6 +27,7 @@ import {
watchBatteryVisibilitySetting,
type BatteryIndicatorVisibility,
} from "../../../ui/dashboard-settings";
import { RECORDING_ICON } from "../../../ui/listening-header";
import { type MenuItem } from "../../../ui/menu";
import { shell } from "../../../ui/shell/shell";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness";
@@ -149,13 +151,16 @@ function requestWeatherLocationAccess(): void {
* off, and the layout mirrored either way, in the card's submenu of
* Glanceboard > Settings. Battery style and per-device visibility follow
* Settings > Customization > Battery indicators, so the card agrees with
* the bar. Countdowns tick once a second only while one is running.
* the bar. Countdowns tick once a second only while one is running. While
* Transcribe or Translate is saving the mic's audio (their Save Recording),
* the recording circle sits under the batteries and weather.
*/
export class SystemCardWidget implements GlanceWidget {
private requestRender: (() => void) | null = null;
private minuteTimer: ReturnType<typeof setTimeout> | null = null;
private secondTimer: ReturnType<typeof setInterval> | null = null;
private unsubscribeNotifications: (() => void) | null = null;
private unsubscribeRecording: (() => void) | null = null;
private unsubscribeSettings: (() => void) | null = null;
private unsubscribeTimers: (() => void) | null = null;
private unsubscribeWeather: (() => void) | null = null;
@@ -164,6 +169,7 @@ export class SystemCardWidget implements GlanceWidget {
this.requestRender = requestRender;
this.scheduleMinuteTick();
this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.());
this.unsubscribeRecording = voiceControlBridge.onRecordingChange(() => this.requestRender?.());
this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.());
this.unsubscribeTimers = timerEngine.onChange(() => {
this.syncSecondTick();
@@ -182,6 +188,8 @@ export class SystemCardWidget implements GlanceWidget {
this.secondTimer = null;
this.unsubscribeNotifications?.();
this.unsubscribeNotifications = null;
this.unsubscribeRecording?.();
this.unsubscribeRecording = null;
this.unsubscribeSettings?.();
this.unsubscribeSettings = null;
this.unsubscribeTimers?.();
@@ -237,6 +245,8 @@ export class SystemCardWidget implements GlanceWidget {
if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems));
const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null;
if (weather) sideBlocks.push(weather);
// Last, so it comes and goes without moving the others.
if (voiceControlBridge.recording) sideBlocks.push(RECORDING_BLOCK);
const side: Rect[] = [];
let sideOffset = PAD;
for (const block of sideBlocks) {
@@ -382,6 +392,13 @@ function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock {
};
}
/** The recording circle: the mic's audio is being saved (see RECORDING_ICON). */
const RECORDING_BLOCK: SideBlock = {
width: RECORDING_ICON.width,
height: RECORDING_ICON.height,
draw: (image, left, top) => image.drawImage(RECORDING_ICON, left, top),
};
/** The current weather's condition icon and temperature, or null without a fresh reading. */
function weatherBlock(font: UiFont, nowMs: number): SideBlock | null {
const reading = weatherReading(weatherBridge.snapshot(), nowMs);
+28 -10
View File
@@ -1,4 +1,7 @@
import { imageFromAsciiArt } from "../../graphics/image";
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
import { voiceControlBridge } from "../../native/voice-control";
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { withRecordingIcon } from "../../ui/listening-header";
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
import {
createInProcessWindow,
@@ -13,8 +16,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
const TRAY_ICON_ID = "transcribe";
// 16x18 microphone glyph for the top-bar tray (also Translate's).
export const MIC_ICON = imageFromAsciiArt(
// 14x15 microphone glyph for the top-bar tray (also Translate's).
const MIC_ICON = imageFromAsciiArt(
[
" #### ",
" ###### ",
@@ -34,6 +37,12 @@ export const MIC_ICON = imageFromAsciiArt(
],
{ "#": 220 },
);
const MIC_RECORDING_ICON = withRecordingIcon(MIC_ICON);
/** The listening apps' tray icon: a microphone, with the recording circle while their audio is saved. */
export function micTrayIcon(recording: boolean): GrayImage {
return recording ? MIC_RECORDING_ICON : MIC_ICON;
}
export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */
@@ -59,18 +68,23 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
closeable: true,
actions: options.actions,
baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
menuItems: () => [
{
label: "Save",
onSelect: (ctx) => {
ctx.stack.pop();
layer.saveTranscript();
},
},
}],
// The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
],
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
unsubscribeRecording();
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
@@ -79,7 +93,11 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
app.requestRender();
});
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
return app;
}
+5 -2
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
@@ -52,7 +52,10 @@ export class TranscribeLayer implements Layer {
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
const bodyTop = paintListeningHeader(
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, voiceControlBridge.recording,
);
const step = lineStep(font) + 2;
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
+24 -8
View File
@@ -7,11 +7,12 @@ import {
translateProviderSetting,
translateShowOriginalSetting,
} from "./translate-settings";
import { MIC_ICON } from "../transcribe/transcribe-app";
import { micTrayIcon } from "../transcribe/transcribe-app";
import { transcriptTimestamp } from "../transcribe/transcribe";
import { voiceControlBridge } from "../../native/voice-control";
import { type ContinuousCaptureOptions } from "../../ui/layers";
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import {
createInProcessWindow,
YieldAtRootLayer,
@@ -44,6 +45,11 @@ export type TranslateAppOptions = InProcessAppOptions & {
* provider (Translate settings), which a shared capture switches to.
*/
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
// One timestamp per session names both the audio recording and a saved transcript.
const stamp = transcriptTimestamp();
// Whether the mic tray icon is up (capture was asked for).
let trayIconShown = false;
const syncTrayIcon = () => shell.setTrayIcon(TRAY_ICON_ID, trayIconShown ? micTrayIcon(voiceControlBridge.recording) : null);
// Checks the setup and (re)starts capture with the chosen provider and
// language: at launch, when either setting changes, and on a tap after a
// problem.
@@ -51,15 +57,18 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
const provider = resolveTranslateProvider();
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
layer.setProblem(problem);
if (!provider || problem) {
trayIconShown = Boolean(provider) && !problem;
if (provider && !problem) {
options.startContinuousVoiceCapture({
recordAs: `translate-${stamp}`,
transcription: { provider, language: resolveTranslateLanguage() },
});
} else {
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
return;
}
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
syncTrayIcon();
};
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
const layer = new TranslateLayer(stamp, startCapture);
const app = createInProcessWindow({
appId: "translate",
windowId: TRANSLATE_WINDOW_ID,
@@ -77,6 +86,8 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
layer.saveTranscript();
},
},
// The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
{
label: "Translate settings",
onSelect: (ctx) => {
@@ -94,6 +105,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
removeSurface: options.removeSurface,
onClosed: () => {
layer.onRemoved();
unsubscribeRecording();
options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed();
@@ -102,6 +114,10 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
// Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived.
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
syncTrayIcon();
app.requestRender();
});
startCapture();
return app;
}
+5 -3
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers";
@@ -77,9 +77,11 @@ export class TranslateLayer implements Layer {
const measure = (line: string) => font.measureText(line);
const maxWidth = width - 64;
const status = this.problem ? "Setup needed" : this.translationError || this.status;
const status = this.problem
? "Setup needed"
: voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.translationError || this.status;
const meter = this.listening && !this.problem ? this.meter : null;
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, voiceControlBridge.recording);
let lines: TranslationLine[];
if (this.problem) {
+16 -2
View File
@@ -77,7 +77,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import {
getInstalledEvenHubAppById,
@@ -487,6 +487,7 @@ class DashboardController {
this.applyAppLayoutsIfChanged();
this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged();
this.syncContinuousRecording();
// A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on
// the Java side).
@@ -2177,12 +2178,14 @@ class DashboardController {
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options);
this.syncContinuousRecording();
this.retargetContinuousCapture();
this.ensureContinuousCapture(true);
}
private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return;
this.syncContinuousRecording();
if (this.continuousClients.size > 0) {
// The window that needed its own provider may be the one that left.
this.retargetContinuousCapture();
@@ -2208,6 +2211,18 @@ class DashboardController {
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
}
/**
* Record to the first client's requested name while Save Recording is on
* (it is toggled from those windows' menus, so this also runs when a
* setting changes).
*/
private syncContinuousRecording(): void {
const name = saveRecordingSetting.get()
? Array.from(this.continuousClients.values(), (options) => options.recordAs).find(Boolean) ?? ""
: "";
voiceControlBridge.recordTo(name);
}
/** Move a running continuous capture onto the provider its windows now call for. */
private retargetContinuousCapture(): void {
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
@@ -2257,7 +2272,6 @@ class DashboardController {
whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
// "My voice only" (Microphones app): verify command utterances
// against the enrolled wearer voice-print; non-matching speakers'
// finals are suppressed by the bridge.
+8 -4
View File
@@ -53,7 +53,7 @@ import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveRecordingSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
@@ -418,6 +418,7 @@ export class IosPreviewController {
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
this.offSettings = onAnySettingChanged(() => {
this.syncLockSetting()
this.syncContinuousRecording()
this.pushBrightness()
this.relayout()
shell.foregroundWindow()?.requestRender()
@@ -1109,7 +1110,7 @@ export class IosPreviewController {
* Continuous capture on behalf of `client` (one id per window: Transcribe,
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
* Transcribe audio is on.
* Recording is on.
*/
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options)
@@ -1198,9 +1199,12 @@ export class IosPreviewController {
private syncContinuousClaim(): void {
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
}
/** Record to the first client's requested name while Save Transcribe audio is on. */
/**
* Record to the first client's requested name while Save Recording is on (it is
* toggled from those windows' menus, so this also runs when a setting changes).
*/
private syncContinuousRecording(): void {
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
const name = saveRecordingSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
iosVoiceInput.recordTo(name)
}
/**
+20 -5
View File
@@ -62,6 +62,9 @@ export class IosVoiceControlBridge {
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = ''
private recorder: WavRecorder | null = null
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false
private readonly recordingListeners = new Set<() => void>()
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
// rawListeners while any owner wants it, from the session setRawSession
// bound. It shares the one glasses mic with speech capture, which preempts
@@ -212,16 +215,28 @@ export class IosVoiceControlBridge {
recordTo(name: string): void {
if (name === this.recordingName) return
this.recorder?.close(); this.recorder = null
this.recordingName = name
this.setRecording(name, false)
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean { return this.recordingName !== '' && !this.recordingFailure }
/** Whether the recording asked for could not be saved (its file could not be created). */
get recordingFailed(): boolean { return this.recordingFailure }
/** `recording` or `recordingFailed` changed. */
onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => { this.recordingListeners.delete(listener) } }
private setRecording(name: string, failed: boolean): void {
const wasRecording = this.recording, hadFailed = this.recordingFailure
this.recordingName = name; this.recordingFailure = failed
if (wasRecording === this.recording && hadFailed === failed) return
for (const fn of [...this.recordingListeners]) fn()
}
private record(pcm: Uint8Array): void {
if (!this.recorder && this.recordingName) {
if (!this.recording) return
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName)
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
// One attempt per recording, not one per packet.
if (!this.recorder) this.recordingName = ''
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
}
this.recorder?.append(pcm)
this.recorder.append(pcm)
}
/** End continuous capture, transcribing the audio since the last pause. */
stopContinuousCapture(): Promise<void> {
+61 -3
View File
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause";
import { WavRecorder } from "./wav-recorder";
import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level";
@@ -60,7 +61,6 @@ export type PushToTalkOptions = {
whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string;
saveRecording: boolean;
/**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
* ("Hey Even") capture, which has no button release to end the utterance.
@@ -150,6 +150,14 @@ export class FaceclawVoiceControlBridge {
// the bridge swallow it.
private verificationActive = false;
private verificationRejected = false;
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = "";
private recorder: WavRecorder | null = null;
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false;
private readonly recordingListeners = new Set<() => void>();
onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener);
@@ -242,6 +250,56 @@ export class FaceclawVoiceControlBridge {
this.acquireCapture("continuous", options);
}
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* until called with another name or "".
*/
recordTo(name: string): void {
if (name === this.recordingName) return;
this.recorder?.close();
this.recorder = null;
this.setRecording(name, false);
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean {
return this.recordingName !== "" && !this.recordingFailure;
}
/** Whether the recording asked for could not be saved (its file could not be created). */
get recordingFailed(): boolean {
return this.recordingFailure;
}
/** `recording` or `recordingFailed` changed. */
onRecordingChange(listener: () => void): () => void {
this.recordingListeners.add(listener);
return () => this.recordingListeners.delete(listener);
}
private setRecording(name: string, failed: boolean): void {
const wasRecording = this.recording;
const hadFailed = this.recordingFailure;
this.recordingName = name;
this.recordingFailure = failed;
if (wasRecording === this.recording && hadFailed === failed) return;
for (const listener of Array.from(this.recordingListeners)) listener();
}
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
private record(pcm: any): void {
if (!this.recording) return;
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName);
if (!this.recorder) {
this.setRecording(this.recordingName, true);
return;
}
console.log(`Voice: recording audio to ${this.recorder.path}`);
}
this.recorder.append(pcm);
}
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
onRawPcm(listener: RawPcmListener): () => void {
this.rawPcmListeners.add(listener);
@@ -305,7 +363,6 @@ export class FaceclawVoiceControlBridge {
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
// preview-mode capture may have left on the shared controller.
this.controller?.setUsePhoneMic(false);
this.controller?.setSaveRecordings(false);
this.controller?.setEndpointing(false);
// Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing.
@@ -354,7 +411,6 @@ export class FaceclawVoiceControlBridge {
this.ensureController();
this.controller?.setCommunicator(options.communicator);
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
this.controller?.setSaveRecordings(options.saveRecording);
this.controller?.setEndpointing(Boolean(options.endpointing));
this.captureEndpointing = Boolean(options.endpointing);
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
@@ -554,6 +610,8 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes);
// Push-to-talk on a capture of its own is not recorded.
if (this.captureHolders.has("continuous")) this.record(pcm);
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener();
}
+42
View File
@@ -0,0 +1,42 @@
declare const android: any;
declare const java: any;
declare const com: any;
const SAMPLE_RATE = 16000;
/**
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
* native recorder keeps the header's sizes current, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private constructor(private recorder: any, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}.wav`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
const path = String(file.getAbsolutePath());
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
} catch (error) {
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
return null;
}
}
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
append(pcm: any): void {
this.recorder?.append(pcm);
}
close(): void {
this.recorder?.finish();
this.recorder = null;
}
}
+9 -13
View File
@@ -1030,20 +1030,16 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
});
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({
id: "save-transcribe-audio",
label: "Save Transcribe audio",
storageKey: "voice.saveTranscribeAudio",
defaultValue: true,
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.",
});
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
id: "save-voice-recordings",
label: "Save voice recordings",
storageKey: "developer.saveVoiceRecordings",
/**
* Toggled from the Transcribe and Translate apps' menus. One setting for
* both: they share the mic stream, so its audio is either being saved or not.
*/
export const saveRecordingSetting = new ConfigSettingBoolean({
id: "save-recording",
label: "Save Recording",
storageKey: "voice.saveRecording",
defaultValue: false,
description: "Keep a copy of captured voice audio on the phone, for debugging transcription problems.",
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
});
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
+1 -4
View File
@@ -69,7 +69,6 @@ import {
ringConnectionModeSetting,
glassesTypeSetting,
sonioxApiKeySetting,
saveTranscribeAudioSetting,
whisperServerApiKeySetting,
whisperServerLanguageSetting,
whisperServerModelSetting,
@@ -80,7 +79,6 @@ import {
lockScreenEnabledSetting,
menuAnimationSetting,
newNotificationDisplaySetting,
saveVoiceRecordingsSetting,
showBleBandwidthSetting,
suspendEvenHubWhenScreenOffSetting,
temperatureUnitSetting,
@@ -247,7 +245,6 @@ function settingsSections(): SettingsSection[] {
// Whether and how the phone opens its own BLE link to the R1 ring.
// Applied at connect time.
ringConnectionMenuItem(),
toggleSettingMenuItem(saveVoiceRecordingsSetting),
toggleSettingMenuItem(firmwareDebugFlagsSetting),
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
toggleSettingMenuItem(useMicControlSetting),
@@ -294,7 +291,7 @@ function settingsSections(): SettingsSection[] {
toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
enumSettingMenuItem(voiceProviderSetting), ...whisperServerMenuItems()] };
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
item.label !== screenTimeoutSetting.label) };
return section;
+1 -1
View File
@@ -12,7 +12,7 @@ export type TextSettingsEditToggle = {
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
recordAs?: string;
/**
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
+50 -4
View File
@@ -1,4 +1,4 @@
import { type GrayImage, type UiFont } from "../graphics/image";
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control";
import { lineStep } from "./metrics";
@@ -12,6 +12,10 @@ const SEGMENTS = 8;
const SEGMENT_WIDTH = 4;
const SEGMENT_GAP = 2;
const METER_GAP = 10;
/** Between the recording circle and the meter. */
const RECORDING_ICON_GAP = 4;
/** Between a tray icon and its recording circle: tight, so they read as one item in the bar. */
const TRAY_RECORDING_ICON_GAP = 2;
const LIT_GRAY = 200;
const UNLIT_GRAY = 45;
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
@@ -19,6 +23,38 @@ const TICK_MS = 200;
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
const DECAY = 0.5;
/** The listening views' status while Save Recording is on but its file could not be created. */
export const RECORDING_FAILED_NOTICE = "Recording failed (check file access).";
/**
* 9x9 filled circle: the mic's audio is being saved (Save Recording, in the
* Transcribe and Translate menus). Shown beside the mic level meter and the
* mic tray icon, and on Glanceboard's system card.
*/
export const RECORDING_ICON = imageFromAsciiArt(
[
" ### ",
" ####### ",
" ####### ",
"#########",
"#########",
"#########",
" ####### ",
" ####### ",
" ### ",
],
{ "#": 220 },
);
/** `icon` with the recording circle to its right: a tray icon while its app's audio is saved. */
export function withRecordingIcon(icon: GrayImage): GrayImage {
const circleX = icon.width + TRAY_RECORDING_ICON_GAP;
const image = new GrayImage(circleX + RECORDING_ICON.width, Math.max(icon.height, RECORDING_ICON.height), 0);
image.bitBlt(icon, 0, (image.height - icon.height) >> 1);
image.bitBlt(RECORDING_ICON, circleX, (image.height - RECORDING_ICON.height) >> 1);
return image;
}
/**
* A compact segmented mic-level meter for listening views. Levels arrive per
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
@@ -66,8 +102,9 @@ export class MicLevelMeter {
/**
* The one-line header of the listening views (Transcribe, Translate):
* "Title · status", the status truncated to fit, then the mic level meter at
* the line's end when one is given (while listening). Returns the y at which
* the view's body starts.
* the line's end when one is given (while listening), with the recording
* circle before it while the audio it shows is being saved. Returns the y at
* which the view's body starts.
*/
export function paintListeningHeader(
image: GrayImage,
@@ -75,13 +112,22 @@ export function paintListeningHeader(
title: string,
status: string,
meter: MicLevelMeter | null,
recording = false,
): number {
let textRight = image.width - HEADER_X;
if (meter) {
const meterX = textRight - meter.width;
// Roughly cap height, sitting on the text's baseline.
meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6)));
const baseline = HEADER_Y + font.ascent;
const meterHeight = Math.max(4, Math.round(font.ascent * 0.6));
meter.paint(image, meterX, baseline, meterHeight);
textRight = meterX - METER_GAP;
if (recording) {
const iconX = meterX - RECORDING_ICON_GAP - RECORDING_ICON.width;
// Centred on the meter's height.
image.drawImage(RECORDING_ICON, iconX, Math.round(baseline - (meterHeight + RECORDING_ICON.height) / 2));
textRight = iconX - METER_GAP;
}
}
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
if (status) {
@@ -40,6 +40,10 @@ constructor(
private var file: RandomAccessFile?
private var dataBytes = 0L
// The PCM bytes the header on disk accounts for. It is rewritten about once a second of
// audio, so a process death mid-recording still leaves a playable file.
private var headerDataBytes = 0L
init {
val parent = File(path).parentFile
if (parent != null && !parent.exists() && !parent.mkdirs())
@@ -58,11 +62,26 @@ constructor(
try {
current.write(pcm16le)
dataBytes += pcm16le.size
if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current)
} catch (e: IOException) {
Log.w("FaceclawWavRecorder", "append failed", e)
}
}
/** Rewrite the header to cover the PCM written so far, leaving the position at the end. */
private fun writeHeader(target: RandomAccessFile) {
val header =
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
target.seek(0)
target.write(header)
target.seek(header.size + dataBytes)
headerDataBytes = dataBytes
}
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
fun getPath(): String = path
@@ -72,16 +91,7 @@ constructor(
val current = file ?: return dataBytes
file = null
try {
current.use {
it.seek(0)
it.write(
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
)
}
current.use { writeHeader(it) }
} catch (e: IOException) {
Log.w("FaceclawWavRecorder", "finish failed", e)
}
@@ -35,14 +35,11 @@ interface VoiceCaptureHost {
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** Where a saved recording goes, or null when recordings cannot be stored. */
fun recordingSink(): RecordingSink?
}
/**
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the
@@ -158,11 +155,8 @@ class VoiceCaptureSession(
private var lastTranscriptDecodeAtMs: Long = 0
private var lastTranscript = ""
@Volatile
private var saveRecordings = false
@Volatile
private var endpointing = false
private val endpointDetector = VoiceEndpointDetector()
private var recordingPcm: ByteSink? = null
@Volatile
private var verifySpeakerModelPath: String? = null
@Volatile
@@ -199,10 +193,6 @@ class VoiceCaptureSession(
this.usePhoneMic = usePhoneMic
}
fun setSaveRecordings(saveRecordings: Boolean) {
this.saveRecordings = saveRecordings
}
fun setOnboardModelKind(kind: String?) {
this.onboardModelKind = parseModelKind(kind)
}
@@ -363,7 +353,6 @@ class VoiceCaptureSession(
}
endpointDetector.reset()
suppressor?.reset()
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0
@@ -404,7 +393,6 @@ class VoiceCaptureSession(
emitStatus("Voice control failed: " + error.message)
} finally {
stopG2Audio()
writeRecordingIfAny()
releaseRecognizer()
releaseDecoder()
lock.withLock {
@@ -418,25 +406,6 @@ class VoiceCaptureSession(
}
}
private fun appendRecording(pcm: ShortArray, count: Int) {
val out = recordingPcm ?: return
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
}
private fun writeRecordingIfAny() {
val out = recordingPcm
recordingPcm = null
if (out == null || out.size() == 0) {
return
}
val sink = host.recordingSink() ?: return
try {
sink.save(out.toByteArray(), SAMPLE_RATE)
} catch (t: Throwable) {
logWarn("failed to save voice recording (${t.message})")
}
}
private fun startG2Audio(): Boolean {
val source = host.g2AudioSource() ?: return false
resetAudioStats()
@@ -489,9 +458,6 @@ class VoiceCaptureSession(
if (suppressionEnabled) {
applySuppression(pcm, count)
}
if (recordingPcm != null) {
appendRecording(pcm, count)
}
val verifyBuffer = this.verifyBuffer
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
@@ -34,11 +34,6 @@ fun interface CallbackDispatcher {
fun post(action: () -> Unit)
}
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
fun interface RecordingSink {
fun save(pcm16le: ByteArray, sampleRate: Int)
}
/**
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
+10 -1
View File
@@ -368,6 +368,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
const calls = [], claims = [], providers = [];
let saveRecording = false;
// Models the bridge's continuous state: listening on a source, or draining after a stop.
const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
supportsContinuous: voice => voice.provider !== 'onboard',
@@ -385,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
setRawSession: session => { bridge.rawSession = session; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
saveTranscribeAudioSetting: { get: () => true } };
saveRecordingSetting: { get: () => saveRecording } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
@@ -404,7 +405,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
host.startContinuousVoiceCapture('calculator'); await settle();
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
// Save Recording (in the window's menu) starts and stops the recording while it is open.
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1');
saveRecording = false; host.syncContinuousRecording();
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1');
assert.deepEqual(calls, [['glasses', true]], 'without touching the capture');
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
+34 -1
View File
@@ -31,8 +31,13 @@ function harness() {
},
};
const recordings = [];
let failRecordings = false, recordingAttempts = 0;
class FakeRecorder {
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
static open(name) {
recordingAttempts++;
if (failRecordings) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(bytes) { this.pcm.push(...bytes); }
close() { this.closed = true; }
}
@@ -56,6 +61,7 @@ function harness() {
} };
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts,
analyzerSupported: value => { analyzerSupported = value; },
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
@@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
assert.equal(h.bridge.recording, false);
h.bridge.recordTo('transcribe-2026-10-17-120000');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]);
// A start that fails before any audio leaves no file behind.
h.native.error = 'No phone microphone is available.';
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
@@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
h.bridge.recordTo('');
assert.equal(h.recordings[0].closed, true);
assert.deepEqual([h.bridge.recording, changes], [false, 2]);
});
test('a recording whose file cannot be created is reported, and not retried per packet', async () => {
const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
h.failRecordings(true);
h.bridge.recordTo('transcribe-1');
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
h.native.pcmHandler(new Uint8Array([1, 2]));
h.native.pcmHandler(new Uint8Array([3, 4]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]);
// Asking for the same recording again changes nothing; another one (Save Recording
// switched off and on) is a fresh attempt.
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([5]));
assert.equal(h.recordingAttempts(), 1);
h.failRecordings(false);
h.bridge.recordTo('');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]);
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([6]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]);
assert.deepEqual(h.recordings[0].pcm, [6]);
});
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
@@ -21,4 +21,31 @@ class WavRecorderRepairTest {
file.delete()
}
}
@Test
fun keepsTheHeaderCurrentWhileRecording() {
val file = File.createTempFile("recording", ".wav")
try {
val recorder = FaceclawWavRecorder(file.path, 16000, 1)
// Under a second of audio: still the empty header from the start.
recorder.append(ByteArray(16000))
assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList())
// Past a second the header covers what is on disk, without waiting for finish(), and
// later audio still lands at the end.
recorder.append(ByteArray(16000) { 1 })
recorder.append(ByteArray(100) { 2 })
var bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList())
assertEquals(0.toByte(), bytes[44])
assertEquals(1.toByte(), bytes[44 + 16000])
assertEquals(2.toByte(), bytes.last())
assertEquals(32100L, recorder.finish())
bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList())
} finally {
file.delete()
}
}
}
@@ -63,7 +63,6 @@ class VoiceCaptureSessionTest {
override val dispatcher = CallbackDispatcher { it() }
var packetListener: FaceclawAudioPacketListener? = null
var g2Stops = 0
val recordings = mutableListOf<ByteArray>()
override fun onSessionStarting() {}
override fun isG2SessionReady(): Boolean = g2Ready
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
@@ -96,7 +95,6 @@ class VoiceCaptureSessionTest {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
override fun release() {}
}
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
}
private val logs = mutableListOf<String>()
@@ -196,7 +194,6 @@ class VoiceCaptureSessionTest {
val session = session(host, platform, "voice-test-onboard")
session.setListener(events)
session.setUsePhoneMic(true)
session.setSaveRecordings(true)
session.start("onboard", 1)
assertTrue(events.stoppedLatch.await(10000))
assertTrue(mic.closed)
@@ -211,8 +208,6 @@ class VoiceCaptureSessionTest {
// Every segment handed to the model is at most 8 s and was peak-normalized.
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
assertEquals(1, host.recordings.size)
assertEquals(170 * 800 * 2, host.recordings[0].size)
assertEquals(170, events.pcmChunks.size)
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
+44 -4
View File
@@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts');
function harness() {
const levelListeners = new Set();
let tick = null;
const { MicLevelMeter, paintListeningHeader } = loader({
const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({
setInterval(fn) { tick = fn; return 1; },
clearInterval() { tick = null; },
}, {
@@ -16,14 +16,15 @@ function harness() {
} },
})('app/ui/listening-header.ts');
const level = value => { for (const listener of levelListeners) listener(value); };
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
}
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
function fakeImage(width = 576) {
const texts = [], rects = [];
return { width, texts, rects,
const texts = [], rects = [], images = [];
return { width, texts, rects, images,
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
drawImage(source, x, y) { images.push({ source, x, y }); },
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
}
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
@@ -93,8 +94,47 @@ test('the header puts title and status on one line, truncated before the meter',
// Segments sit on the text baseline.
assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent));
assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved');
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null);
assert.equal(idle.rects.length, 0);
assert.equal(idle.texts[1].text, ' · Starting microphone...');
});
test('while the audio is being saved, the recording circle sits beside the meter', () => {
const h = harness();
const meter = new h.MicLevelMeter();
const image = fakeImage(300);
h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, true);
const meterX = Math.min(...image.rects.map(rect => rect.x));
assert.equal(meterX, 300 - 24 - meter.width, 'the meter stays where it was');
assert.equal(image.images.length, 1);
const [circle] = image.images;
assert.equal(circle.source, h.RECORDING_ICON);
assert.equal(circle.x + circle.source.width + 4, meterX);
// Centred on the meter, whose 7 rows sit on the text baseline.
const meterTop = Math.min(...image.rects.map(rect => rect.y));
assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2);
const status = image.texts[1].text;
assert.ok(24 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle');
// The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark.
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, true);
assert.equal(idle.images.length, 0);
});
test('a tray icon gains the recording circle to its right, vertically centred', () => {
const h = harness();
const { GrayImage } = loader()('app/graphics/image.ts');
const mic = new GrayImage(14, 15, 0);
mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200);
const icon = h.withRecordingIcon(mic);
assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]);
assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]);
assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them');
// The circle's widest row is the icon's middle row; its corners are clear.
for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220);
assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]);
});
+58 -2
View File
@@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
});
/** The Android voice bridge over a fake native controller (on-device recognition only). */
function androidBridge() {
function androidBridge(modules = {}) {
const controller = {
starts: 0,
setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; },
@@ -172,6 +172,7 @@ function androidBridge() {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
...modules,
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; },
@@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is
assert.equal(controller.active, true);
assert.equal(bridge.holdsCapture('ptt'), true);
});
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
const recordings = [];
let fail = false, attempts = 0;
class FakeRecorder {
static open(name) {
attempts++;
if (fail) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(pcm) { this.pcm.push(...pcm); }
close() { this.closed = true; }
}
const { bridge, controller } = androidBridge({
'./wav-recorder': { WavRecorder: FakeRecorder },
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
'../util/audio-level': { pcmLevel: () => 0 },
});
const options = { provider: 'onboard' };
let changes = 0; bridge.onRecordingChange(() => changes++);
bridge.recordTo('transcribe-1');
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]);
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
bridge.startPushToTalk(options);
controller.listener.onPcm([9, 9]);
assert.equal(attempts, 0);
bridge.startContinuousCapture(options);
controller.listener.onPcm([1, 2]);
controller.listener.onPcm([3, 4]);
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
bridge.handleSessionEnded();
bridge.resumeCapture(options);
controller.listener.onPcm([5]);
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
bridge.recordTo('');
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
controller.listener.onPcm([6]);
assert.equal(recordings[0].pcm.length, 5);
// One attempt per recording, not one per packet; another recording is a fresh attempt.
fail = true; attempts = 0;
bridge.recordTo('transcribe-2');
controller.listener.onPcm([7]); controller.listener.onPcm([8]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]);
bridge.recordTo('transcribe-2');
controller.listener.onPcm([9]);
assert.equal(attempts, 1);
fail = false;
bridge.recordTo('');
bridge.recordTo('transcribe-2');
controller.listener.onPcm([10]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
});