This commit is contained in:
James Babcock committed 2026-11-12 12:20:29 -08:00
commit c162c8e633
24 files changed
+486 -154

No files matched your search

@@ -136,8 +136,6 @@ class FaceclawVoiceController(context: Context) {
override fun release() {} override fun release() {}
} }
} }
override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) }
} }
private val session = VoiceCaptureSession(host) private val session = VoiceCaptureSession(host)
@@ -155,11 +153,6 @@ class FaceclawVoiceController(context: Context) {
session.setUsePhoneMic(usePhoneMic) session.setUsePhoneMic(usePhoneMic)
} }
/** When true, the decoded mic PCM for each session is saved as a WAV. */
fun setSaveRecordings(saveRecordings: Boolean) {
session.setSaveRecordings(saveRecordings)
}
/** /**
* Which on-device model {@link #start}("onboard") should load: "whisper" * Which on-device model {@link #start}("onboard") should load: "whisper"
* selects the second on-device model (sherpa-onnx offline Whisper); any * selects the second on-device model (sherpa-onnx offline Whisper); any
@@ -233,29 +226,6 @@ class FaceclawVoiceController(context: Context) {
stop() stop()
} }
/** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */
private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) {
try {
val dir = File(appContext.getExternalFilesDir(null), "voice-recordings")
if (!dir.exists() && !dir.mkdirs()) {
Log.w(TAG, "could not create voice-recordings dir")
return
}
val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US)
.format(java.util.Date())
val file = File(dir, "voice-$stamp.wav")
java.io.FileOutputStream(file).use { fos ->
fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1))
fos.write(pcmBytes)
}
Log.i(TAG, "saved voice recording " + file.absolutePath
+ " samples=" + (pcmBytes.size / 2)
+ " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate))
} catch (t: Throwable) {
Log.w(TAG, "failed to save voice recording", t)
}
}
/** /**
* The on-device model directory for the given kind, populated by the * The on-device model directory for the given kind, populated by the
* download flow in asr-model.ts (releases before 0.5.0 copied the * download flow in asr-model.ts (releases before 0.5.0 copied the
+18 -1
View File
@@ -11,6 +11,7 @@ import { ensureLocationPermission, hasLocationPermission } from "../../../native
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons"; import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types"; import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
import { readPhoneBatteryState } from "../../../native/phone-battery"; import { readPhoneBatteryState } from "../../../native/phone-battery";
import { voiceControlBridge } from "../../../native/voice-control";
import { weatherBridge } from "../../../native/weather"; import { weatherBridge } from "../../../native/weather";
import { formatClockDate, formatClockTime } from "../../../ui/clock-format"; import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
import { import {
@@ -26,6 +27,7 @@ import {
watchBatteryVisibilitySetting, watchBatteryVisibilitySetting,
type BatteryIndicatorVisibility, type BatteryIndicatorVisibility,
} from "../../../ui/dashboard-settings"; } from "../../../ui/dashboard-settings";
import { RECORDING_ICON } from "../../../ui/listening-header";
import { type MenuItem } from "../../../ui/menu"; import { type MenuItem } from "../../../ui/menu";
import { shell } from "../../../ui/shell/shell"; import { shell } from "../../../ui/shell/shell";
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness"; import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness";
@@ -149,13 +151,16 @@ function requestWeatherLocationAccess(): void {
* off, and the layout mirrored either way, in the card's submenu of * off, and the layout mirrored either way, in the card's submenu of
* Glanceboard > Settings. Battery style and per-device visibility follow * Glanceboard > Settings. Battery style and per-device visibility follow
* Settings > Customization > Battery indicators, so the card agrees with * Settings > Customization > Battery indicators, so the card agrees with
* the bar. Countdowns tick once a second only while one is running. * the bar. Countdowns tick once a second only while one is running. While
* Transcribe or Translate is saving the mic's audio (their Save Recording),
* the recording circle sits under the batteries and weather.
*/ */
export class SystemCardWidget implements GlanceWidget { export class SystemCardWidget implements GlanceWidget {
private requestRender: (() => void) | null = null; private requestRender: (() => void) | null = null;
private minuteTimer: ReturnType<typeof setTimeout> | null = null; private minuteTimer: ReturnType<typeof setTimeout> | null = null;
private secondTimer: ReturnType<typeof setInterval> | null = null; private secondTimer: ReturnType<typeof setInterval> | null = null;
private unsubscribeNotifications: (() => void) | null = null; private unsubscribeNotifications: (() => void) | null = null;
private unsubscribeRecording: (() => void) | null = null;
private unsubscribeSettings: (() => void) | null = null; private unsubscribeSettings: (() => void) | null = null;
private unsubscribeTimers: (() => void) | null = null; private unsubscribeTimers: (() => void) | null = null;
private unsubscribeWeather: (() => void) | null = null; private unsubscribeWeather: (() => void) | null = null;
@@ -164,6 +169,7 @@ export class SystemCardWidget implements GlanceWidget {
this.requestRender = requestRender; this.requestRender = requestRender;
this.scheduleMinuteTick(); this.scheduleMinuteTick();
this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.()); this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.());
this.unsubscribeRecording = voiceControlBridge.onRecordingChange(() => this.requestRender?.());
this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.()); this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.());
this.unsubscribeTimers = timerEngine.onChange(() => { this.unsubscribeTimers = timerEngine.onChange(() => {
this.syncSecondTick(); this.syncSecondTick();
@@ -182,6 +188,8 @@ export class SystemCardWidget implements GlanceWidget {
this.secondTimer = null; this.secondTimer = null;
this.unsubscribeNotifications?.(); this.unsubscribeNotifications?.();
this.unsubscribeNotifications = null; this.unsubscribeNotifications = null;
this.unsubscribeRecording?.();
this.unsubscribeRecording = null;
this.unsubscribeSettings?.(); this.unsubscribeSettings?.();
this.unsubscribeSettings = null; this.unsubscribeSettings = null;
this.unsubscribeTimers?.(); this.unsubscribeTimers?.();
@@ -237,6 +245,8 @@ export class SystemCardWidget implements GlanceWidget {
if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems)); if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems));
const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null; const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null;
if (weather) sideBlocks.push(weather); if (weather) sideBlocks.push(weather);
// Last, so it comes and goes without moving the others.
if (voiceControlBridge.recording) sideBlocks.push(RECORDING_BLOCK);
const side: Rect[] = []; const side: Rect[] = [];
let sideOffset = PAD; let sideOffset = PAD;
for (const block of sideBlocks) { for (const block of sideBlocks) {
@@ -382,6 +392,13 @@ function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock {
}; };
} }
/** The recording circle: the mic's audio is being saved (see RECORDING_ICON). */
const RECORDING_BLOCK: SideBlock = {
width: RECORDING_ICON.width,
height: RECORDING_ICON.height,
draw: (image, left, top) => image.drawImage(RECORDING_ICON, left, top),
};
/** The current weather's condition icon and temperature, or null without a fresh reading. */ /** The current weather's condition icon and temperature, or null without a fresh reading. */
function weatherBlock(font: UiFont, nowMs: number): SideBlock | null { function weatherBlock(font: UiFont, nowMs: number): SideBlock | null {
const reading = weatherReading(weatherBridge.snapshot(), nowMs); const reading = weatherReading(weatherBridge.snapshot(), nowMs);
+28 -10
View File
@@ -1,4 +1,7 @@
import { imageFromAsciiArt } from "../../graphics/image"; import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
import { voiceControlBridge } from "../../native/voice-control";
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { withRecordingIcon } from "../../ui/listening-header";
import { TranscribeLayer, transcriptTimestamp } from "./transcribe"; import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
import { import {
createInProcessWindow, createInProcessWindow,
@@ -13,8 +16,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
const TRAY_ICON_ID = "transcribe"; const TRAY_ICON_ID = "transcribe";
// 16x18 microphone glyph for the top-bar tray (also Translate's). // 14x15 microphone glyph for the top-bar tray (also Translate's).
export const MIC_ICON = imageFromAsciiArt( const MIC_ICON = imageFromAsciiArt(
[ [
" #### ", " #### ",
" ###### ", " ###### ",
@@ -34,6 +37,12 @@ export const MIC_ICON = imageFromAsciiArt(
], ],
{ "#": 220 }, { "#": 220 },
); );
const MIC_RECORDING_ICON = withRecordingIcon(MIC_ICON);
/** The listening apps' tray icon: a microphone, with the recording circle while their audio is saved. */
export function micTrayIcon(recording: boolean): GrayImage {
return recording ? MIC_RECORDING_ICON : MIC_ICON;
}
export type TranscribeAppOptions = InProcessAppOptions & { export type TranscribeAppOptions = InProcessAppOptions & {
/** Begin continuous mic capture (kept on while the window is open). */ /** Begin continuous mic capture (kept on while the window is open). */
@@ -59,18 +68,23 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
closeable: true, closeable: true,
actions: options.actions, actions: options.actions,
baseLayer: new YieldAtRootLayer(layer), baseLayer: new YieldAtRootLayer(layer),
menuItems: () => [{ menuItems: () => [
label: "Save", {
onSelect: (ctx) => { label: "Save",
ctx.stack.pop(); onSelect: (ctx) => {
layer.saveTranscript(); ctx.stack.pop();
layer.saveTranscript();
},
}, },
}], // The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
],
submitFrame: options.submitFrame, submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible, setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface, removeSurface: options.removeSurface,
onClosed: () => { onClosed: () => {
layer.onRemoved(); layer.onRemoved();
unsubscribeRecording();
options.stopContinuousVoiceCapture(); options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null); shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed(); options.onClosed();
@@ -79,7 +93,11 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
// Wire async state changes to this window's renderer, rather than the // Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived. // controller action from which the window-specific actions were derived.
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID)); layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
app.requestRender();
});
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
return app; return app;
} }
+6 -3
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image"; import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics"; import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access"; import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control"; import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers"; import { Layer, type LayerContext } from "../../ui/layers";
@@ -50,8 +50,11 @@ export class TranscribeLayer implements Layer {
const viewport = ctx.stack.getBaseSize(); const viewport = ctx.stack.getBaseSize();
const image = new GrayImage(viewport.width, viewport.height, 0); const image = new GrayImage(viewport.width, viewport.height, 0);
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null, const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
viewport.padding); const bodyTop = paintListeningHeader(
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, viewport.padding,
voiceControlBridge.recording,
);
const step = lineStep(font) + 2; const step = lineStep(font) + 2;
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step); const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail); const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
+24 -8
View File
@@ -7,11 +7,12 @@ import {
translateProviderSetting, translateProviderSetting,
translateShowOriginalSetting, translateShowOriginalSetting,
} from "./translate-settings"; } from "./translate-settings";
import { MIC_ICON } from "../transcribe/transcribe-app"; import { micTrayIcon } from "../transcribe/transcribe-app";
import { transcriptTimestamp } from "../transcribe/transcribe"; import { transcriptTimestamp } from "../transcribe/transcribe";
import { voiceControlBridge } from "../../native/voice-control";
import { type ContinuousCaptureOptions } from "../../ui/layers"; import { type ContinuousCaptureOptions } from "../../ui/layers";
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel"; import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings"; import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
import { import {
createInProcessWindow, createInProcessWindow,
YieldAtRootLayer, YieldAtRootLayer,
@@ -44,6 +45,11 @@ export type TranslateAppOptions = InProcessAppOptions & {
* provider (Translate settings), which a shared capture switches to. * provider (Translate settings), which a shared capture switches to.
*/ */
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow { export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
// One timestamp per session names both the audio recording and a saved transcript.
const stamp = transcriptTimestamp();
// Whether the mic tray icon is up (capture was asked for).
let trayIconShown = false;
const syncTrayIcon = () => shell.setTrayIcon(TRAY_ICON_ID, trayIconShown ? micTrayIcon(voiceControlBridge.recording) : null);
// Checks the setup and (re)starts capture with the chosen provider and // Checks the setup and (re)starts capture with the chosen provider and
// language: at launch, when either setting changes, and on a tap after a // language: at launch, when either setting changes, and on a tap after a
// problem. // problem.
@@ -51,15 +57,18 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
const provider = resolveTranslateProvider(); const provider = resolveTranslateProvider();
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null; const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
layer.setProblem(problem); layer.setProblem(problem);
if (!provider || problem) { trayIconShown = Boolean(provider) && !problem;
if (provider && !problem) {
options.startContinuousVoiceCapture({
recordAs: `translate-${stamp}`,
transcription: { provider, language: resolveTranslateLanguage() },
});
} else {
options.stopContinuousVoiceCapture(); options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null);
return;
} }
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } }); syncTrayIcon();
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
}; };
const layer = new TranslateLayer(transcriptTimestamp(), startCapture); const layer = new TranslateLayer(stamp, startCapture);
const app = createInProcessWindow({ const app = createInProcessWindow({
appId: "translate", appId: "translate",
windowId: TRANSLATE_WINDOW_ID, windowId: TRANSLATE_WINDOW_ID,
@@ -77,6 +86,8 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
layer.saveTranscript(); layer.saveTranscript();
}, },
}, },
// The controller starts and stops the recording as the setting changes.
toggleSettingMenuItem(saveRecordingSetting),
{ {
label: "Translate settings", label: "Translate settings",
onSelect: (ctx) => { onSelect: (ctx) => {
@@ -94,6 +105,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
removeSurface: options.removeSurface, removeSurface: options.removeSurface,
onClosed: () => { onClosed: () => {
layer.onRemoved(); layer.onRemoved();
unsubscribeRecording();
options.stopContinuousVoiceCapture(); options.stopContinuousVoiceCapture();
shell.setTrayIcon(TRAY_ICON_ID, null); shell.setTrayIcon(TRAY_ICON_ID, null);
options.onClosed(); options.onClosed();
@@ -102,6 +114,10 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
// Wire async state changes to this window's renderer, rather than the // Wire async state changes to this window's renderer, rather than the
// controller action from which the window-specific actions were derived. // controller action from which the window-specific actions were derived.
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID)); layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
syncTrayIcon();
app.requestRender();
});
startCapture(); startCapture();
return app; return app;
} }
+7 -3
View File
@@ -1,7 +1,7 @@
import { GrayImage } from "../../graphics/image"; import { GrayImage } from "../../graphics/image";
import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { lineStep } from "../../ui/metrics"; import { lineStep } from "../../ui/metrics";
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header"; import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
import { writeTextToDownloads } from "../../native/file-access"; import { writeTextToDownloads } from "../../native/file-access";
import { voiceControlBridge } from "../../native/voice-control"; import { voiceControlBridge } from "../../native/voice-control";
import { Layer, type LayerContext } from "../../ui/layers"; import { Layer, type LayerContext } from "../../ui/layers";
@@ -76,9 +76,13 @@ export class TranslateLayer implements Layer {
const image = new GrayImage(viewport.width, viewport.height, 0); const image = new GrayImage(viewport.width, viewport.height, 0);
const measure = (line: string) => font.measureText(line); const measure = (line: string) => font.measureText(line);
const status = this.problem ? "Setup needed" : this.translationError || this.status; const status = this.problem
? "Setup needed"
: voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.translationError || this.status;
const meter = this.listening && !this.problem ? this.meter : null; const meter = this.listening && !this.problem ? this.meter : null;
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, viewport.padding); const bodyTop = paintListeningHeader(
image, font, "Translate", this.saveNotice || status, meter, viewport.padding, voiceControlBridge.recording,
);
const step = lineStep(font) + 2; const step = lineStep(font) + 2;
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step); const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
const maxWidth = layout.textWidth; const maxWidth = layout.textWidth;
+16 -2
View File
@@ -78,7 +78,7 @@ import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarSt
import { sameEdges } from "../ui/viewport"; import { sameEdges } from "../ui/viewport";
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers"; import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
import { import {
getInstalledEvenHubAppById, getInstalledEvenHubAppById,
@@ -488,6 +488,7 @@ class DashboardController {
if (!resized && !appsResized) this.applyWindowEdgesIfChanged(); if (!resized && !appsResized) this.applyWindowEdgesIfChanged();
this.syncAssistantBridgeIfChanged(); this.syncAssistantBridgeIfChanged();
this.syncLockScreenSettingIfChanged(); this.syncLockScreenSettingIfChanged();
this.syncContinuousRecording();
// A Preview color or scaling change should show on the mirror at once, // A Preview color or scaling change should show on the mirror at once,
// not at the next composited frame (cheap: floor-limited, dedup'd on // not at the next composited frame (cheap: floor-limited, dedup'd on
// the Java side). // the Java side).
@@ -2194,12 +2195,14 @@ class DashboardController {
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options); this.continuousClients.set(client, options);
this.syncContinuousRecording();
this.retargetContinuousCapture(); this.retargetContinuousCapture();
this.ensureContinuousCapture(true); this.ensureContinuousCapture(true);
} }
private stopContinuousVoiceCapture(client: string): void { private stopContinuousVoiceCapture(client: string): void {
if (!this.continuousClients.delete(client)) return; if (!this.continuousClients.delete(client)) return;
this.syncContinuousRecording();
if (this.continuousClients.size > 0) { if (this.continuousClients.size > 0) {
// The window that needed its own provider may be the one that left. // The window that needed its own provider may be the one that left.
this.retargetContinuousCapture(); this.retargetContinuousCapture();
@@ -2225,6 +2228,18 @@ class DashboardController {
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim()); void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
} }
/**
* Record to the first client's requested name while Save Recording is on
* (it is toggled from those windows' menus, so this also runs when a
* setting changes).
*/
private syncContinuousRecording(): void {
const name = saveRecordingSetting.get()
? Array.from(this.continuousClients.values(), (options) => options.recordAs).find(Boolean) ?? ""
: "";
voiceControlBridge.recordTo(name);
}
/** Move a running continuous capture onto the provider its windows now call for. */ /** Move a running continuous capture onto the provider its windows now call for. */
private retargetContinuousCapture(): void { private retargetContinuousCapture(): void {
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator)); voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
@@ -2274,7 +2289,6 @@ class DashboardController {
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerUrl: whisperServerUrlSetting.get(),
whisperServerModel: whisperServerModelSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
whisperServerApiKey: whisperServerApiKeySetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get(),
saveRecording: saveVoiceRecordingsSetting.get(),
// "My voice only" (Microphones app): verify command utterances // "My voice only" (Microphones app): verify command utterances
// against the enrolled wearer voice-print; non-matching speakers' // against the enrolled wearer voice-print; non-matching speakers'
// finals are suppressed by the bridge. // finals are suppressed by the bridge.
+8 -4
View File
@@ -53,7 +53,7 @@ import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings' whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveRecordingSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures' import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
@@ -418,6 +418,7 @@ export class IosPreviewController {
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`) this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
this.offSettings = onAnySettingChanged(() => { this.offSettings = onAnySettingChanged(() => {
this.syncLockSetting() this.syncLockSetting()
this.syncContinuousRecording()
this.pushBrightness() this.pushBrightness()
this.relayout() this.relayout()
shell.foregroundWindow()?.requestRender() shell.foregroundWindow()?.requestRender()
@@ -1111,7 +1112,7 @@ export class IosPreviewController {
* Continuous capture on behalf of `client` (one id per window: Transcribe, * Continuous capture on behalf of `client` (one id per window: Transcribe,
* Translate, Teleprompter, Calculator); it runs while any client wants it, as * Translate, Teleprompter, Calculator); it runs while any client wants it, as
* on Android. `recordAs` names a WAV of the audio in Downloads when Save * on Android. `recordAs` names a WAV of the audio in Downloads when Save
* Transcribe audio is on. * Recording is on.
*/ */
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
this.continuousClients.set(client, options) this.continuousClients.set(client, options)
@@ -1200,9 +1201,12 @@ export class IosPreviewController {
private syncContinuousClaim(): void { private syncContinuousClaim(): void {
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
} }
/** Record to the first client's requested name while Save Transcribe audio is on. */ /**
* Record to the first client's requested name while Save Recording is on (it is
* toggled from those windows' menus, so this also runs when a setting changes).
*/
private syncContinuousRecording(): void { private syncContinuousRecording(): void {
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : '' const name = saveRecordingSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
iosVoiceInput.recordTo(name) iosVoiceInput.recordTo(name)
} }
/** /**
+20 -5
View File
@@ -62,6 +62,9 @@ export class IosVoiceControlBridge {
// and it is created by the first audio, so a failed start leaves no file. // and it is created by the first audio, so a failed start leaves no file.
private recordingName = '' private recordingName = ''
private recorder: WavRecorder | null = null private recorder: WavRecorder | null = null
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false
private readonly recordingListeners = new Set<() => void>()
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to // Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
// rawListeners while any owner wants it, from the session setRawSession // rawListeners while any owner wants it, from the session setRawSession
// bound. It shares the one glasses mic with speech capture, which preempts // bound. It shares the one glasses mic with speech capture, which preempts
@@ -212,16 +215,28 @@ export class IosVoiceControlBridge {
recordTo(name: string): void { recordTo(name: string): void {
if (name === this.recordingName) return if (name === this.recordingName) return
this.recorder?.close(); this.recorder = null this.recorder?.close(); this.recorder = null
this.recordingName = name this.setRecording(name, false)
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean { return this.recordingName !== '' && !this.recordingFailure }
/** Whether the recording asked for could not be saved (its file could not be created). */
get recordingFailed(): boolean { return this.recordingFailure }
/** `recording` or `recordingFailed` changed. */
onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => { this.recordingListeners.delete(listener) } }
private setRecording(name: string, failed: boolean): void {
const wasRecording = this.recording, hadFailed = this.recordingFailure
this.recordingName = name; this.recordingFailure = failed
if (wasRecording === this.recording && hadFailed === failed) return
for (const fn of [...this.recordingListeners]) fn()
} }
private record(pcm: Uint8Array): void { private record(pcm: Uint8Array): void {
if (!this.recorder && this.recordingName) { if (!this.recording) return
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName) this.recorder = WavRecorder.open(this.recordingName)
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
// One attempt per recording, not one per packet. if (!this.recorder) { this.setRecording(this.recordingName, true); return }
if (!this.recorder) this.recordingName = ''
} }
this.recorder?.append(pcm) this.recorder.append(pcm)
} }
/** End continuous capture, transcribing the audio since the last pause. */ /** End continuous capture, transcribing the audio since the last pause. */
stopContinuousCapture(): Promise<void> { stopContinuousCapture(): Promise<void> {
+61 -3
View File
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
import { SpeechPauseDetector } from "./speech-pause"; import { SpeechPauseDetector } from "./speech-pause";
import { WavRecorder } from "./wav-recorder";
import { toUint8Array } from "../util/array-util"; import { toUint8Array } from "../util/array-util";
import { pcmLevel } from "../util/audio-level"; import { pcmLevel } from "../util/audio-level";
@@ -60,7 +61,6 @@ export type PushToTalkOptions = {
whisperServerApiKey?: string; whisperServerApiKey?: string;
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */ /** Spoken language for cloud providers (see VoiceProviderSelection.language). */
language?: string; language?: string;
saveRecording: boolean;
/** /**
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
* ("Hey Even") capture, which has no button release to end the utterance. * ("Hey Even") capture, which has no button release to end the utterance.
@@ -150,6 +150,14 @@ export class FaceclawVoiceControlBridge {
// the bridge swallow it. // the bridge swallow it.
private verificationActive = false; private verificationActive = false;
private verificationRejected = false; private verificationRejected = false;
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
// set. The file outlives a capture, so a restarted one keeps appending to it,
// and it is created by the first audio, so a failed start leaves no file.
private recordingName = "";
private recorder: WavRecorder | null = null;
// The file could not be created: one attempt per recording, not one per packet.
private recordingFailure = false;
private readonly recordingListeners = new Set<() => void>();
onStatus(listener: (state: VoiceControlState) => void): () => void { onStatus(listener: (state: VoiceControlState) => void): () => void {
this.statusListeners.add(listener); this.statusListeners.add(listener);
@@ -242,6 +250,56 @@ export class FaceclawVoiceControlBridge {
this.acquireCapture("continuous", options); this.acquireCapture("continuous", options);
} }
/**
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
* until called with another name or "".
*/
recordTo(name: string): void {
if (name === this.recordingName) return;
this.recorder?.close();
this.recorder = null;
this.setRecording(name, false);
}
/** Whether continuous captures' audio is being saved (see recordTo). */
get recording(): boolean {
return this.recordingName !== "" && !this.recordingFailure;
}
/** Whether the recording asked for could not be saved (its file could not be created). */
get recordingFailed(): boolean {
return this.recordingFailure;
}
/** `recording` or `recordingFailed` changed. */
onRecordingChange(listener: () => void): () => void {
this.recordingListeners.add(listener);
return () => this.recordingListeners.delete(listener);
}
private setRecording(name: string, failed: boolean): void {
const wasRecording = this.recording;
const hadFailed = this.recordingFailure;
this.recordingName = name;
this.recordingFailure = failed;
if (wasRecording === this.recording && hadFailed === failed) return;
for (const listener of Array.from(this.recordingListeners)) listener();
}
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
private record(pcm: any): void {
if (!this.recording) return;
if (!this.recorder) {
this.recorder = WavRecorder.open(this.recordingName);
if (!this.recorder) {
this.setRecording(this.recordingName, true);
return;
}
console.log(`Voice: recording audio to ${this.recorder.path}`);
}
this.recorder.append(pcm);
}
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
onRawPcm(listener: RawPcmListener): () => void { onRawPcm(listener: RawPcmListener): () => void {
this.rawPcmListeners.add(listener); this.rawPcmListeners.add(listener);
@@ -305,7 +363,6 @@ export class FaceclawVoiceControlBridge {
// The raw tap is strictly the G2 stream; clear any phone-mic flag a // The raw tap is strictly the G2 stream; clear any phone-mic flag a
// preview-mode capture may have left on the shared controller. // preview-mode capture may have left on the shared controller.
this.controller?.setUsePhoneMic(false); this.controller?.setUsePhoneMic(false);
this.controller?.setSaveRecordings(false);
this.controller?.setEndpointing(false); this.controller?.setEndpointing(false);
// Raw means raw: the Microphones session (and EvenHub apps) get the // Raw means raw: the Microphones session (and EvenHub apps) get the
// unprocessed stream and run their own beam/suppression processing. // unprocessed stream and run their own beam/suppression processing.
@@ -354,7 +411,6 @@ export class FaceclawVoiceControlBridge {
this.ensureController(); this.ensureController();
this.controller?.setCommunicator(options.communicator); this.controller?.setCommunicator(options.communicator);
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
this.controller?.setSaveRecordings(options.saveRecording);
this.controller?.setEndpointing(Boolean(options.endpointing)); this.controller?.setEndpointing(Boolean(options.endpointing));
this.captureEndpointing = Boolean(options.endpointing); this.captureEndpointing = Boolean(options.endpointing);
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression)); this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
@@ -554,6 +610,8 @@ export class FaceclawVoiceControlBridge {
onPcm: (pcm: any) => { onPcm: (pcm: any) => {
const bytes = toUint8Array(pcm); const bytes = toUint8Array(pcm);
this.cloudClient?.acceptPcm(bytes); this.cloudClient?.acceptPcm(bytes);
// Push-to-talk on a capture of its own is not recorded.
if (this.captureHolders.has("continuous")) this.record(pcm);
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) { if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
for (const listener of this.speechPauseListeners) listener(); for (const listener of this.speechPauseListeners) listener();
} }
+42
View File
@@ -0,0 +1,42 @@
declare const android: any;
declare const java: any;
declare const com: any;
const SAMPLE_RATE = 16000;
/**
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
* native recorder keeps the header's sizes current, so a crash leaves a
* playable file of everything up to the last second.
*/
export class WavRecorder {
private constructor(private recorder: any, readonly path: string) {}
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
static open(name: string): WavRecorder | null {
try {
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
android.os.Environment.DIRECTORY_DOWNLOADS,
);
downloads.mkdirs();
let file = new java.io.File(downloads, `${name}.wav`);
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
const path = String(file.getAbsolutePath());
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
} catch (error) {
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
return null;
}
}
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
append(pcm: any): void {
this.recorder?.append(pcm);
}
close(): void {
this.recorder?.finish();
this.recorder = null;
}
}
+9 -13
View File
@@ -1045,20 +1045,16 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.", description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
}); });
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({ /**
id: "save-transcribe-audio", * Toggled from the Transcribe and Translate apps' menus. One setting for
label: "Save Transcribe audio", * both: they share the mic stream, so its audio is either being saved or not.
storageKey: "voice.saveTranscribeAudio", */
defaultValue: true, export const saveRecordingSetting = new ConfigSettingBoolean({
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.", id: "save-recording",
}); label: "Save Recording",
storageKey: "voice.saveRecording",
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
id: "save-voice-recordings",
label: "Save voice recordings",
storageKey: "developer.saveVoiceRecordings",
defaultValue: false, defaultValue: false,
description: "Keep a copy of captured voice audio on the phone, for debugging transcription problems.", description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
}); });
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
+1 -4
View File
@@ -70,7 +70,6 @@ import {
ringConnectionModeSetting, ringConnectionModeSetting,
glassesTypeSetting, glassesTypeSetting,
sonioxApiKeySetting, sonioxApiKeySetting,
saveTranscribeAudioSetting,
whisperServerApiKeySetting, whisperServerApiKeySetting,
whisperServerLanguageSetting, whisperServerLanguageSetting,
whisperServerModelSetting, whisperServerModelSetting,
@@ -81,7 +80,6 @@ import {
lockScreenEnabledSetting, lockScreenEnabledSetting,
menuAnimationSetting, menuAnimationSetting,
newNotificationDisplaySetting, newNotificationDisplaySetting,
saveVoiceRecordingsSetting,
showBleBandwidthSetting, showBleBandwidthSetting,
suspendEvenHubWhenScreenOffSetting, suspendEvenHubWhenScreenOffSetting,
temperatureUnitSetting, temperatureUnitSetting,
@@ -250,7 +248,6 @@ function settingsSections(): SettingsSection[] {
// Whether and how the phone opens its own BLE link to the R1 ring. // Whether and how the phone opens its own BLE link to the R1 ring.
// Applied at connect time. // Applied at connect time.
ringConnectionMenuItem(), ringConnectionMenuItem(),
toggleSettingMenuItem(saveVoiceRecordingsSetting),
toggleSettingMenuItem(firmwareDebugFlagsSetting), toggleSettingMenuItem(firmwareDebugFlagsSetting),
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting), toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
toggleSettingMenuItem(useMicControlSetting), toggleSettingMenuItem(useMicControlSetting),
@@ -297,7 +294,7 @@ function settingsSections(): SettingsSection[] {
toggleSettingMenuItem(showBleBandwidthSetting)] }; toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer. // No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; enumSettingMenuItem(voiceProviderSetting), ...whisperServerMenuItems()] };
if (section.label === "Display") return { ...section, items: section.items.filter(item => if (section.label === "Display") return { ...section, items: section.items.filter(item =>
item.label !== screenTimeoutSetting.label) }; item.label !== screenTimeoutSetting.label) };
return section; return section;
+1 -1
View File
@@ -13,7 +13,7 @@ export type TextSettingsEditToggle = {
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */ /** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
export type ContinuousCaptureOptions = { export type ContinuousCaptureOptions = {
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */ /** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
recordAs?: string; recordAs?: string;
/** /**
* Transcribe with this provider, expecting this spoken language (ISO 639-1, * Transcribe with this provider, expecting this spoken language (ISO 639-1,
+50 -4
View File
@@ -1,4 +1,4 @@
import { type GrayImage, type UiFont } from "../graphics/image"; import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
import { truncateText } from "../graphics/textwrap"; import { truncateText } from "../graphics/textwrap";
import { voiceControlBridge } from "../native/voice-control"; import { voiceControlBridge } from "../native/voice-control";
import { lineStep } from "./metrics"; import { lineStep } from "./metrics";
@@ -11,6 +11,10 @@ const SEGMENTS = 8;
const SEGMENT_WIDTH = 4; const SEGMENT_WIDTH = 4;
const SEGMENT_GAP = 2; const SEGMENT_GAP = 2;
const METER_GAP = 10; const METER_GAP = 10;
/** Between the recording circle and the meter. */
const RECORDING_ICON_GAP = 4;
/** Between a tray icon and its recording circle: tight, so they read as one item in the bar. */
const TRAY_RECORDING_ICON_GAP = 2;
const LIT_GRAY = 200; const LIT_GRAY = 200;
const UNLIT_GRAY = 45; const UNLIT_GRAY = 45;
/** How often the meter may ask for a repaint (each one is a frame over BLE). */ /** How often the meter may ask for a repaint (each one is a frame over BLE). */
@@ -18,6 +22,38 @@ const TICK_MS = 200;
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */ /** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
const DECAY = 0.5; const DECAY = 0.5;
/** The listening views' status while Save Recording is on but its file could not be created. */
export const RECORDING_FAILED_NOTICE = "Recording failed (check file access).";
/**
* 9x9 filled circle: the mic's audio is being saved (Save Recording, in the
* Transcribe and Translate menus). Shown beside the mic level meter and the
* mic tray icon, and on Glanceboard's system card.
*/
export const RECORDING_ICON = imageFromAsciiArt(
[
" ### ",
" ####### ",
" ####### ",
"#########",
"#########",
"#########",
" ####### ",
" ####### ",
" ### ",
],
{ "#": 220 },
);
/** `icon` with the recording circle to its right: a tray icon while its app's audio is saved. */
export function withRecordingIcon(icon: GrayImage): GrayImage {
const circleX = icon.width + TRAY_RECORDING_ICON_GAP;
const image = new GrayImage(circleX + RECORDING_ICON.width, Math.max(icon.height, RECORDING_ICON.height), 0);
image.bitBlt(icon, 0, (image.height - icon.height) >> 1);
image.bitBlt(RECORDING_ICON, circleX, (image.height - RECORDING_ICON.height) >> 1);
return image;
}
/** /**
* A compact segmented mic-level meter for listening views. Levels arrive per * A compact segmented mic-level meter for listening views. Levels arrive per
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms, * 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
@@ -65,8 +101,9 @@ export class MicLevelMeter {
/** /**
* The one-line header of the listening views (Transcribe, Translate): * The one-line header of the listening views (Transcribe, Translate):
* "Title · status", the status truncated to fit, then the mic level meter at * "Title · status", the status truncated to fit, then the mic level meter at
* the line's end when one is given (while listening). It spans the image * the line's end when one is given (while listening), with the recording
* inside `padding`, the padding suggested for the window's edges * circle before it while the audio it shows is being saved. It spans the
* image inside `padding`, the padding suggested for the window's edges
* (viewport.ts). Returns the y at which the view's body starts. * (viewport.ts). Returns the y at which the view's body starts.
*/ */
export function paintListeningHeader( export function paintListeningHeader(
@@ -76,14 +113,23 @@ export function paintListeningHeader(
status: string, status: string,
meter: MicLevelMeter | null, meter: MicLevelMeter | null,
padding: Edges<number>, padding: Edges<number>,
recording = false,
): number { ): number {
const x = padding.left, y = padding.top; const x = padding.left, y = padding.top;
let textRight = image.width - padding.right; let textRight = image.width - padding.right;
if (meter) { if (meter) {
const meterX = textRight - meter.width; const meterX = textRight - meter.width;
// Roughly cap height, sitting on the text's baseline. // Roughly cap height, sitting on the text's baseline.
meter.paint(image, meterX, y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6))); const baseline = y + font.ascent;
const meterHeight = Math.max(4, Math.round(font.ascent * 0.6));
meter.paint(image, meterX, baseline, meterHeight);
textRight = meterX - METER_GAP; textRight = meterX - METER_GAP;
if (recording) {
const iconX = meterX - RECORDING_ICON_GAP - RECORDING_ICON.width;
// Centred on the meter's height.
image.drawImage(RECORDING_ICON, iconX, Math.round(baseline - (meterHeight + RECORDING_ICON.height) / 2));
textRight = iconX - METER_GAP;
}
} }
image.drawText(font, x, y, title, TITLE_GRAY); image.drawText(font, x, y, title, TITLE_GRAY);
if (status) { if (status) {
@@ -40,6 +40,10 @@ constructor(
private var file: RandomAccessFile? private var file: RandomAccessFile?
private var dataBytes = 0L private var dataBytes = 0L
// The PCM bytes the header on disk accounts for. It is rewritten about once a second of
// audio, so a process death mid-recording still leaves a playable file.
private var headerDataBytes = 0L
init { init {
val parent = File(path).parentFile val parent = File(path).parentFile
if (parent != null && !parent.exists() && !parent.mkdirs()) if (parent != null && !parent.exists() && !parent.mkdirs())
@@ -58,11 +62,26 @@ constructor(
try { try {
current.write(pcm16le) current.write(pcm16le)
dataBytes += pcm16le.size dataBytes += pcm16le.size
if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current)
} catch (e: IOException) { } catch (e: IOException) {
Log.w("FaceclawWavRecorder", "append failed", e) Log.w("FaceclawWavRecorder", "append failed", e)
} }
} }
/** Rewrite the header to cover the PCM written so far, leaving the position at the end. */
private fun writeHeader(target: RandomAccessFile) {
val header =
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
target.seek(0)
target.write(header)
target.seek(header.size + dataBytes)
headerDataBytes = dataBytes
}
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L) @Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
fun getPath(): String = path fun getPath(): String = path
@@ -72,16 +91,7 @@ constructor(
val current = file ?: return dataBytes val current = file ?: return dataBytes
file = null file = null
try { try {
current.use { current.use { writeHeader(it) }
it.seek(0)
it.write(
BinaryEncoding.wavHeader(
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
sampleRate,
channels,
)
)
}
} catch (e: IOException) { } catch (e: IOException) {
Log.w("FaceclawWavRecorder", "finish failed", e) Log.w("FaceclawWavRecorder", "finish failed", e)
} }
@@ -35,14 +35,11 @@ interface VoiceCaptureHost {
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */ /** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** Where a saved recording goes, or null when recordings cannot be stored. */
fun recordingSink(): RecordingSink?
} }
/** /**
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late * Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker * accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with * verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
* REPLACE-semantics partials, and speaker verification at session end. The session owns its * REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the * worker thread (started through the shared [startThread]); listener callbacks go through the
@@ -158,11 +155,8 @@ class VoiceCaptureSession(
private var lastTranscriptDecodeAtMs: Long = 0 private var lastTranscriptDecodeAtMs: Long = 0
private var lastTranscript = "" private var lastTranscript = ""
@Volatile @Volatile
private var saveRecordings = false
@Volatile
private var endpointing = false private var endpointing = false
private val endpointDetector = VoiceEndpointDetector() private val endpointDetector = VoiceEndpointDetector()
private var recordingPcm: ByteSink? = null
@Volatile @Volatile
private var verifySpeakerModelPath: String? = null private var verifySpeakerModelPath: String? = null
@Volatile @Volatile
@@ -199,10 +193,6 @@ class VoiceCaptureSession(
this.usePhoneMic = usePhoneMic this.usePhoneMic = usePhoneMic
} }
fun setSaveRecordings(saveRecordings: Boolean) {
this.saveRecordings = saveRecordings
}
fun setOnboardModelKind(kind: String?) { fun setOnboardModelKind(kind: String?) {
this.onboardModelKind = parseModelKind(kind) this.onboardModelKind = parseModelKind(kind)
} }
@@ -363,7 +353,6 @@ class VoiceCaptureSession(
} }
endpointDetector.reset() endpointDetector.reset()
suppressor?.reset() suppressor?.reset()
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0 verifyCount = 0
@@ -404,7 +393,6 @@ class VoiceCaptureSession(
emitStatus("Voice control failed: " + error.message) emitStatus("Voice control failed: " + error.message)
} finally { } finally {
stopG2Audio() stopG2Audio()
writeRecordingIfAny()
releaseRecognizer() releaseRecognizer()
releaseDecoder() releaseDecoder()
lock.withLock { lock.withLock {
@@ -418,25 +406,6 @@ class VoiceCaptureSession(
} }
} }
private fun appendRecording(pcm: ShortArray, count: Int) {
val out = recordingPcm ?: return
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
}
private fun writeRecordingIfAny() {
val out = recordingPcm
recordingPcm = null
if (out == null || out.size() == 0) {
return
}
val sink = host.recordingSink() ?: return
try {
sink.save(out.toByteArray(), SAMPLE_RATE)
} catch (t: Throwable) {
logWarn("failed to save voice recording (${t.message})")
}
}
private fun startG2Audio(): Boolean { private fun startG2Audio(): Boolean {
val source = host.g2AudioSource() ?: return false val source = host.g2AudioSource() ?: return false
resetAudioStats() resetAudioStats()
@@ -489,9 +458,6 @@ class VoiceCaptureSession(
if (suppressionEnabled) { if (suppressionEnabled) {
applySuppression(pcm, count) applySuppression(pcm, count)
} }
if (recordingPcm != null) {
appendRecording(pcm, count)
}
val verifyBuffer = this.verifyBuffer val verifyBuffer = this.verifyBuffer
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) { if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount) val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
@@ -34,11 +34,6 @@ fun interface CallbackDispatcher {
fun post(action: () -> Unit) fun post(action: () -> Unit)
} }
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
fun interface RecordingSink {
fun save(pcm16le: ByteArray, sampleRate: Int)
}
/** /**
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared * A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus. * [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
+10 -1
View File
@@ -365,6 +365,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => { test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
const calls = [], claims = [], providers = []; const calls = [], claims = [], providers = [];
let saveRecording = false;
// Models the bridge's continuous state: listening on a source, or draining after a stop. // Models the bridge's continuous state: listening on a source, or draining after a stop.
const bridge = { source: null, draining: false, recording: '', rawSession: undefined, const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
supportsContinuous: voice => voice.provider !== 'onboard', supportsContinuous: voice => voice.provider !== 'onboard',
@@ -382,7 +383,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
setRawSession: session => { bridge.rawSession = session; } }; setRawSession: session => { bridge.rawSession = session; } };
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' }, const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' }, whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
saveTranscribeAudioSetting: { get: () => true } }; saveRecordingSetting: { get: () => saveRecording } };
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', { const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } } : id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
@@ -401,7 +402,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle(); host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
host.startContinuousVoiceCapture('calculator'); await settle(); host.startContinuousVoiceCapture('calculator'); await settle();
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture'); assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
// Save Recording (in the window's menu) starts and stops the recording while it is open.
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1'); assert.equal(bridge.recording, 'transcribe-1');
saveRecording = false; host.syncContinuousRecording();
assert.equal(bridge.recording, '');
saveRecording = true; host.syncContinuousRecording();
assert.equal(bridge.recording, 'transcribe-1');
assert.deepEqual(calls, [['glasses', true]], 'without touching the capture');
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']); assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
host.stopContinuousVoiceCapture('calculator'); host.stopContinuousVoiceCapture('calculator');
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening'); assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
+34 -1
View File
@@ -31,8 +31,13 @@ function harness() {
}, },
}; };
const recordings = []; const recordings = [];
let failRecordings = false, recordingAttempts = 0;
class FakeRecorder { class FakeRecorder {
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; } static open(name) {
recordingAttempts++;
if (failRecordings) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(bytes) { this.pcm.push(...bytes); } append(bytes) { this.pcm.push(...bytes); }
close() { this.closed = true; } close() { this.closed = true; }
} }
@@ -56,6 +61,7 @@ function harness() {
} }; } };
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' }; const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(), return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts,
analyzerSupported: value => { analyzerSupported = value; }, analyzerSupported: value => { analyzerSupported = value; },
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' }, onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests, microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
@@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => { test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
const h = harness(); const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
assert.equal(h.bridge.recording, false);
h.bridge.recordTo('transcribe-2026-10-17-120000'); h.bridge.recordTo('transcribe-2026-10-17-120000');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]);
// A start that fails before any audio leaves no file behind. // A start that fails before any audio leaves no file behind.
h.native.error = 'No phone microphone is available.'; h.native.error = 'No phone microphone is available.';
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true); await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
@@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]); assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
h.bridge.recordTo(''); h.bridge.recordTo('');
assert.equal(h.recordings[0].closed, true); assert.equal(h.recordings[0].closed, true);
assert.deepEqual([h.bridge.recording, changes], [false, 2]);
});
test('a recording whose file cannot be created is reported, and not retried per packet', async () => {
const h = harness();
let changes = 0; h.bridge.onRecordingChange(() => changes++);
h.failRecordings(true);
h.bridge.recordTo('transcribe-1');
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
h.native.pcmHandler(new Uint8Array([1, 2]));
h.native.pcmHandler(new Uint8Array([3, 4]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]);
// Asking for the same recording again changes nothing; another one (Save Recording
// switched off and on) is a fresh attempt.
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([5]));
assert.equal(h.recordingAttempts(), 1);
h.failRecordings(false);
h.bridge.recordTo('');
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]);
h.bridge.recordTo('transcribe-1');
h.native.pcmHandler(new Uint8Array([6]));
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]);
assert.deepEqual(h.recordings[0].pcm, [6]);
}); });
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => { test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
@@ -21,4 +21,31 @@ class WavRecorderRepairTest {
file.delete() file.delete()
} }
} }
@Test
fun keepsTheHeaderCurrentWhileRecording() {
val file = File.createTempFile("recording", ".wav")
try {
val recorder = FaceclawWavRecorder(file.path, 16000, 1)
// Under a second of audio: still the empty header from the start.
recorder.append(ByteArray(16000))
assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList())
// Past a second the header covers what is on disk, without waiting for finish(), and
// later audio still lands at the end.
recorder.append(ByteArray(16000) { 1 })
recorder.append(ByteArray(100) { 2 })
var bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList())
assertEquals(0.toByte(), bytes[44])
assertEquals(1.toByte(), bytes[44 + 16000])
assertEquals(2.toByte(), bytes.last())
assertEquals(32100L, recorder.finish())
bytes = file.readBytes()
assertEquals(44 + 32100, bytes.size)
assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList())
} finally {
file.delete()
}
}
} }
@@ -63,7 +63,6 @@ class VoiceCaptureSessionTest {
override val dispatcher = CallbackDispatcher { it() } override val dispatcher = CallbackDispatcher { it() }
var packetListener: FaceclawAudioPacketListener? = null var packetListener: FaceclawAudioPacketListener? = null
var g2Stops = 0 var g2Stops = 0
val recordings = mutableListOf<ByteArray>()
override fun onSessionStarting() {} override fun onSessionStarting() {}
override fun isG2SessionReady(): Boolean = g2Ready override fun isG2SessionReady(): Boolean = g2Ready
override fun isG2AudioCaptureActive(): Boolean = packetListener != null override fun isG2AudioCaptureActive(): Boolean = packetListener != null
@@ -96,7 +95,6 @@ class VoiceCaptureSessionTest {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
override fun release() {} override fun release() {}
} }
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
} }
private val logs = mutableListOf<String>() private val logs = mutableListOf<String>()
@@ -196,7 +194,6 @@ class VoiceCaptureSessionTest {
val session = session(host, platform, "voice-test-onboard") val session = session(host, platform, "voice-test-onboard")
session.setListener(events) session.setListener(events)
session.setUsePhoneMic(true) session.setUsePhoneMic(true)
session.setSaveRecordings(true)
session.start("onboard", 1) session.start("onboard", 1)
assertTrue(events.stoppedLatch.await(10000)) assertTrue(events.stoppedLatch.await(10000))
assertTrue(mic.closed) assertTrue(mic.closed)
@@ -211,8 +208,6 @@ class VoiceCaptureSessionTest {
// Every segment handed to the model is at most 8 s and was peak-normalized. // Every segment handed to the model is at most 8 s and was peak-normalized.
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES }) assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString()) assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
assertEquals(1, host.recordings.size)
assertEquals(170 * 800 * 2, host.recordings[0].size)
assertEquals(170, events.pcmChunks.size) assertEquals(170, events.pcmChunks.size)
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x")) assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y")) assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
+45 -4
View File
@@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts');
function harness() { function harness() {
const levelListeners = new Set(); const levelListeners = new Set();
let tick = null; let tick = null;
const { MicLevelMeter, paintListeningHeader } = loader({ const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({
setInterval(fn) { tick = fn; return 1; }, setInterval(fn) { tick = fn; return 1; },
clearInterval() { tick = null; }, clearInterval() { tick = null; },
}, { }, {
@@ -16,14 +16,15 @@ function harness() {
} }, } },
})('app/ui/listening-header.ts'); })('app/ui/listening-header.ts');
const level = value => { for (const listener of levelListeners) listener(value); }; const level = value => { for (const listener of levelListeners) listener(value); };
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners }; return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
} }
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */ /** Records draws; text is 6px per character, like a fixed-width bitmap face. */
function fakeImage(width = 576) { function fakeImage(width = 576) {
const texts = [], rects = []; const texts = [], rects = [], images = [];
return { width, texts, rects, return { width, texts, rects, images,
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); }, drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
drawImage(source, x, y) { images.push({ source, x, y }); },
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } }; fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
} }
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 }; const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
@@ -96,8 +97,48 @@ test('the header puts title and status on one line, truncated before the meter',
// Segments sit on the text baseline. // Segments sit on the text baseline.
assert.ok(image.rects.every(rect => rect.y + rect.h === 8 + font.ascent)); assert.ok(image.rects.every(rect => rect.y + rect.h === 8 + font.ascent));
assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved');
const idle = fakeImage(300); const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding); h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding);
assert.equal(idle.rects.length, 0); assert.equal(idle.rects.length, 0);
assert.equal(idle.texts[1].text, ' · Starting microphone...'); assert.equal(idle.texts[1].text, ' · Starting microphone...');
}); });
test('while the audio is being saved, the recording circle sits beside the meter', () => {
const h = harness();
const meter = new h.MicLevelMeter();
const image = fakeImage(300);
const padding = { top: 8, right: 0, bottom: 8, left: 8 };
h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, padding, true);
const meterX = Math.min(...image.rects.map(rect => rect.x));
assert.equal(meterX, 300 - meter.width, 'the meter stays where it was');
assert.equal(image.images.length, 1);
const [circle] = image.images;
assert.equal(circle.source, h.RECORDING_ICON);
assert.equal(circle.x + circle.source.width + 4, meterX);
// Centred on the meter, whose 7 rows sit on the text baseline.
const meterTop = Math.min(...image.rects.map(rect => rect.y));
assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2);
const status = image.texts[1].text;
assert.ok(8 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle');
// The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark.
const idle = fakeImage(300);
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, padding, true);
assert.equal(idle.images.length, 0);
});
test('a tray icon gains the recording circle to its right, vertically centred', () => {
const h = harness();
const { GrayImage } = loader()('app/graphics/image.ts');
const mic = new GrayImage(14, 15, 0);
mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200);
const icon = h.withRecordingIcon(mic);
assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]);
assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]);
assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them');
// The circle's widest row is the icon's middle row; its corners are clear.
for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220);
assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]);
});
+58 -2
View File
@@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
}); });
/** The Android voice bridge over a fake native controller (on-device recognition only). */ /** The Android voice bridge over a fake native controller (on-device recognition only). */
function androidBridge() { function androidBridge(modules = {}) {
const controller = { const controller = {
starts: 0, starts: 0,
setListener(listener) { this.listener = listener; }, setListener(listener) { this.listener = listener; },
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {}, setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
stop() { this.active = false; }, stop() { this.active = false; },
@@ -172,6 +172,7 @@ function androidBridge() {
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
'./speech-pause': { SpeechPauseDetector: class { reset() {} } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } },
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider }, './cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
...modules,
}, { global: { isAndroid: true }, com: { faceclaw: { app: { }, { global: { isAndroid: true }, com: { faceclaw: { app: {
FaceclawVoiceController: function () { return controller; }, FaceclawVoiceController: function () { return controller; },
FaceclawVoiceControllerListener: function (listener) { return listener; }, FaceclawVoiceControllerListener: function (listener) { return listener; },
@@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is
assert.equal(controller.active, true); assert.equal(controller.active, true);
assert.equal(bridge.holdsCapture('ptt'), true); assert.equal(bridge.holdsCapture('ptt'), true);
}); });
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
const recordings = [];
let fail = false, attempts = 0;
class FakeRecorder {
static open(name) {
attempts++;
if (fail) return null;
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
}
append(pcm) { this.pcm.push(...pcm); }
close() { this.closed = true; }
}
const { bridge, controller } = androidBridge({
'./wav-recorder': { WavRecorder: FakeRecorder },
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
'../util/audio-level': { pcmLevel: () => 0 },
});
const options = { provider: 'onboard' };
let changes = 0; bridge.onRecordingChange(() => changes++);
bridge.recordTo('transcribe-1');
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]);
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
bridge.startPushToTalk(options);
controller.listener.onPcm([9, 9]);
assert.equal(attempts, 0);
bridge.startContinuousCapture(options);
controller.listener.onPcm([1, 2]);
controller.listener.onPcm([3, 4]);
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
bridge.handleSessionEnded();
bridge.resumeCapture(options);
controller.listener.onPcm([5]);
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
bridge.recordTo('');
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
controller.listener.onPcm([6]);
assert.equal(recordings[0].pcm.length, 5);
// One attempt per recording, not one per packet; another recording is a fresh attempt.
fail = true; attempts = 0;
bridge.recordTo('transcribe-2');
controller.listener.onPcm([7]); controller.listener.onPcm([8]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]);
bridge.recordTo('transcribe-2');
controller.listener.onPcm([9]);
assert.equal(attempts, 1);
fail = false;
bridge.recordTo('');
bridge.recordTo('transcribe-2');
controller.listener.onPcm([10]);
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
});