mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Move recording setting from Developer menu to Transcribe app; add prominent indicator
This commit is contained in:
1 parent
e158452ba4
commit
30de2c4dd7
24 files changed
+482
-153
No files matched your search
@@ -136,8 +136,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
override fun release() {}
|
override fun release() {}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
override fun recordingSink(): RecordingSink = RecordingSink { pcmBytes, sampleRate -> saveRecording(pcmBytes, sampleRate) }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private val session = VoiceCaptureSession(host)
|
private val session = VoiceCaptureSession(host)
|
||||||
@@ -155,11 +153,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
session.setUsePhoneMic(usePhoneMic)
|
session.setUsePhoneMic(usePhoneMic)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** When true, the decoded mic PCM for each session is saved as a WAV. */
|
|
||||||
fun setSaveRecordings(saveRecordings: Boolean) {
|
|
||||||
session.setSaveRecordings(saveRecordings)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Which on-device model {@link #start}("onboard") should load: "whisper"
|
* Which on-device model {@link #start}("onboard") should load: "whisper"
|
||||||
* selects the second on-device model (sherpa-onnx offline Whisper); any
|
* selects the second on-device model (sherpa-onnx offline Whisper); any
|
||||||
@@ -233,29 +226,6 @@ class FaceclawVoiceController(context: Context) {
|
|||||||
stop()
|
stop()
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Save the session's decoded mic PCM as a 16 kHz mono 16-bit WAV. */
|
|
||||||
private fun saveRecording(pcmBytes: ByteArray, sampleRate: Int) {
|
|
||||||
try {
|
|
||||||
val dir = File(appContext.getExternalFilesDir(null), "voice-recordings")
|
|
||||||
if (!dir.exists() && !dir.mkdirs()) {
|
|
||||||
Log.w(TAG, "could not create voice-recordings dir")
|
|
||||||
return
|
|
||||||
}
|
|
||||||
val stamp = java.text.SimpleDateFormat("yyyyMMdd-HHmmss-SSS", java.util.Locale.US)
|
|
||||||
.format(java.util.Date())
|
|
||||||
val file = File(dir, "voice-$stamp.wav")
|
|
||||||
java.io.FileOutputStream(file).use { fos ->
|
|
||||||
fos.write(BinaryEncoding.wavHeader(pcmBytes.size, sampleRate, 1))
|
|
||||||
fos.write(pcmBytes)
|
|
||||||
}
|
|
||||||
Log.i(TAG, "saved voice recording " + file.absolutePath
|
|
||||||
+ " samples=" + (pcmBytes.size / 2)
|
|
||||||
+ " sec=" + String.format(java.util.Locale.US, "%.2f", pcmBytes.size / 2.0 / sampleRate))
|
|
||||||
} catch (t: Throwable) {
|
|
||||||
Log.w(TAG, "failed to save voice recording", t)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* The on-device model directory for the given kind, populated by the
|
* The on-device model directory for the given kind, populated by the
|
||||||
* download flow in asr-model.ts (releases before 0.5.0 copied the
|
* download flow in asr-model.ts (releases before 0.5.0 copied the
|
||||||
|
|||||||
@@ -11,6 +11,7 @@ import { ensureLocationPermission, hasLocationPermission } from "../../../native
|
|||||||
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||||
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
|
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
|
||||||
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
||||||
|
import { voiceControlBridge } from "../../../native/voice-control";
|
||||||
import { weatherBridge } from "../../../native/weather";
|
import { weatherBridge } from "../../../native/weather";
|
||||||
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
||||||
import {
|
import {
|
||||||
@@ -26,6 +27,7 @@ import {
|
|||||||
watchBatteryVisibilitySetting,
|
watchBatteryVisibilitySetting,
|
||||||
type BatteryIndicatorVisibility,
|
type BatteryIndicatorVisibility,
|
||||||
} from "../../../ui/dashboard-settings";
|
} from "../../../ui/dashboard-settings";
|
||||||
|
import { RECORDING_ICON } from "../../../ui/listening-header";
|
||||||
import { type MenuItem } from "../../../ui/menu";
|
import { type MenuItem } from "../../../ui/menu";
|
||||||
import { shell } from "../../../ui/shell/shell";
|
import { shell } from "../../../ui/shell/shell";
|
||||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness";
|
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness";
|
||||||
@@ -149,13 +151,16 @@ function requestWeatherLocationAccess(): void {
|
|||||||
* off, and the layout mirrored either way, in the card's submenu of
|
* off, and the layout mirrored either way, in the card's submenu of
|
||||||
* Glanceboard > Settings. Battery style and per-device visibility follow
|
* Glanceboard > Settings. Battery style and per-device visibility follow
|
||||||
* Settings > Customization > Battery indicators, so the card agrees with
|
* Settings > Customization > Battery indicators, so the card agrees with
|
||||||
* the bar. Countdowns tick once a second only while one is running.
|
* the bar. Countdowns tick once a second only while one is running. While
|
||||||
|
* Transcribe or Translate is saving the mic's audio (their Save Recording),
|
||||||
|
* the recording circle sits under the batteries and weather.
|
||||||
*/
|
*/
|
||||||
export class SystemCardWidget implements GlanceWidget {
|
export class SystemCardWidget implements GlanceWidget {
|
||||||
private requestRender: (() => void) | null = null;
|
private requestRender: (() => void) | null = null;
|
||||||
private minuteTimer: ReturnType<typeof setTimeout> | null = null;
|
private minuteTimer: ReturnType<typeof setTimeout> | null = null;
|
||||||
private secondTimer: ReturnType<typeof setInterval> | null = null;
|
private secondTimer: ReturnType<typeof setInterval> | null = null;
|
||||||
private unsubscribeNotifications: (() => void) | null = null;
|
private unsubscribeNotifications: (() => void) | null = null;
|
||||||
|
private unsubscribeRecording: (() => void) | null = null;
|
||||||
private unsubscribeSettings: (() => void) | null = null;
|
private unsubscribeSettings: (() => void) | null = null;
|
||||||
private unsubscribeTimers: (() => void) | null = null;
|
private unsubscribeTimers: (() => void) | null = null;
|
||||||
private unsubscribeWeather: (() => void) | null = null;
|
private unsubscribeWeather: (() => void) | null = null;
|
||||||
@@ -164,6 +169,7 @@ export class SystemCardWidget implements GlanceWidget {
|
|||||||
this.requestRender = requestRender;
|
this.requestRender = requestRender;
|
||||||
this.scheduleMinuteTick();
|
this.scheduleMinuteTick();
|
||||||
this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.());
|
this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.());
|
||||||
|
this.unsubscribeRecording = voiceControlBridge.onRecordingChange(() => this.requestRender?.());
|
||||||
this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.());
|
this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.());
|
||||||
this.unsubscribeTimers = timerEngine.onChange(() => {
|
this.unsubscribeTimers = timerEngine.onChange(() => {
|
||||||
this.syncSecondTick();
|
this.syncSecondTick();
|
||||||
@@ -182,6 +188,8 @@ export class SystemCardWidget implements GlanceWidget {
|
|||||||
this.secondTimer = null;
|
this.secondTimer = null;
|
||||||
this.unsubscribeNotifications?.();
|
this.unsubscribeNotifications?.();
|
||||||
this.unsubscribeNotifications = null;
|
this.unsubscribeNotifications = null;
|
||||||
|
this.unsubscribeRecording?.();
|
||||||
|
this.unsubscribeRecording = null;
|
||||||
this.unsubscribeSettings?.();
|
this.unsubscribeSettings?.();
|
||||||
this.unsubscribeSettings = null;
|
this.unsubscribeSettings = null;
|
||||||
this.unsubscribeTimers?.();
|
this.unsubscribeTimers?.();
|
||||||
@@ -237,6 +245,8 @@ export class SystemCardWidget implements GlanceWidget {
|
|||||||
if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems));
|
if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems));
|
||||||
const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null;
|
const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null;
|
||||||
if (weather) sideBlocks.push(weather);
|
if (weather) sideBlocks.push(weather);
|
||||||
|
// Last, so it comes and goes without moving the others.
|
||||||
|
if (voiceControlBridge.recording) sideBlocks.push(RECORDING_BLOCK);
|
||||||
const side: Rect[] = [];
|
const side: Rect[] = [];
|
||||||
let sideOffset = PAD;
|
let sideOffset = PAD;
|
||||||
for (const block of sideBlocks) {
|
for (const block of sideBlocks) {
|
||||||
@@ -382,6 +392,13 @@ function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** The recording circle: the mic's audio is being saved (see RECORDING_ICON). */
|
||||||
|
const RECORDING_BLOCK: SideBlock = {
|
||||||
|
width: RECORDING_ICON.width,
|
||||||
|
height: RECORDING_ICON.height,
|
||||||
|
draw: (image, left, top) => image.drawImage(RECORDING_ICON, left, top),
|
||||||
|
};
|
||||||
|
|
||||||
/** The current weather's condition icon and temperature, or null without a fresh reading. */
|
/** The current weather's condition icon and temperature, or null without a fresh reading. */
|
||||||
function weatherBlock(font: UiFont, nowMs: number): SideBlock | null {
|
function weatherBlock(font: UiFont, nowMs: number): SideBlock | null {
|
||||||
const reading = weatherReading(weatherBridge.snapshot(), nowMs);
|
const reading = weatherReading(weatherBridge.snapshot(), nowMs);
|
||||||
|
|||||||
@@ -1,4 +1,7 @@
|
|||||||
import { imageFromAsciiArt } from "../../graphics/image";
|
import { imageFromAsciiArt, type GrayImage } from "../../graphics/image";
|
||||||
|
import { voiceControlBridge } from "../../native/voice-control";
|
||||||
|
import { saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||||
|
import { withRecordingIcon } from "../../ui/listening-header";
|
||||||
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
import { TranscribeLayer, transcriptTimestamp } from "./transcribe";
|
||||||
import {
|
import {
|
||||||
createInProcessWindow,
|
createInProcessWindow,
|
||||||
@@ -13,8 +16,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe";
|
|||||||
|
|
||||||
const TRAY_ICON_ID = "transcribe";
|
const TRAY_ICON_ID = "transcribe";
|
||||||
|
|
||||||
// 16x18 microphone glyph for the top-bar tray (also Translate's).
|
// 14x15 microphone glyph for the top-bar tray (also Translate's).
|
||||||
export const MIC_ICON = imageFromAsciiArt(
|
const MIC_ICON = imageFromAsciiArt(
|
||||||
[
|
[
|
||||||
" #### ",
|
" #### ",
|
||||||
" ###### ",
|
" ###### ",
|
||||||
@@ -34,6 +37,12 @@ export const MIC_ICON = imageFromAsciiArt(
|
|||||||
],
|
],
|
||||||
{ "#": 220 },
|
{ "#": 220 },
|
||||||
);
|
);
|
||||||
|
const MIC_RECORDING_ICON = withRecordingIcon(MIC_ICON);
|
||||||
|
|
||||||
|
/** The listening apps' tray icon: a microphone, with the recording circle while their audio is saved. */
|
||||||
|
export function micTrayIcon(recording: boolean): GrayImage {
|
||||||
|
return recording ? MIC_RECORDING_ICON : MIC_ICON;
|
||||||
|
}
|
||||||
|
|
||||||
export type TranscribeAppOptions = InProcessAppOptions & {
|
export type TranscribeAppOptions = InProcessAppOptions & {
|
||||||
/** Begin continuous mic capture (kept on while the window is open). */
|
/** Begin continuous mic capture (kept on while the window is open). */
|
||||||
@@ -59,18 +68,23 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
|||||||
closeable: true,
|
closeable: true,
|
||||||
actions: options.actions,
|
actions: options.actions,
|
||||||
baseLayer: new YieldAtRootLayer(layer),
|
baseLayer: new YieldAtRootLayer(layer),
|
||||||
menuItems: () => [{
|
menuItems: () => [
|
||||||
label: "Save",
|
{
|
||||||
onSelect: (ctx) => {
|
label: "Save",
|
||||||
ctx.stack.pop();
|
onSelect: (ctx) => {
|
||||||
layer.saveTranscript();
|
ctx.stack.pop();
|
||||||
|
layer.saveTranscript();
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}],
|
// The controller starts and stops the recording as the setting changes.
|
||||||
|
toggleSettingMenuItem(saveRecordingSetting),
|
||||||
|
],
|
||||||
submitFrame: options.submitFrame,
|
submitFrame: options.submitFrame,
|
||||||
setSurfaceVisible: options.setSurfaceVisible,
|
setSurfaceVisible: options.setSurfaceVisible,
|
||||||
removeSurface: options.removeSurface,
|
removeSurface: options.removeSurface,
|
||||||
onClosed: () => {
|
onClosed: () => {
|
||||||
layer.onRemoved();
|
layer.onRemoved();
|
||||||
|
unsubscribeRecording();
|
||||||
options.stopContinuousVoiceCapture();
|
options.stopContinuousVoiceCapture();
|
||||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||||
options.onClosed();
|
options.onClosed();
|
||||||
@@ -79,7 +93,11 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
|||||||
// Wire async state changes to this window's renderer, rather than the
|
// Wire async state changes to this window's renderer, rather than the
|
||||||
// controller action from which the window-specific actions were derived.
|
// controller action from which the window-specific actions were derived.
|
||||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
||||||
|
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
|
||||||
|
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||||
|
app.requestRender();
|
||||||
|
});
|
||||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
shell.setTrayIcon(TRAY_ICON_ID, micTrayIcon(voiceControlBridge.recording));
|
||||||
return app;
|
return app;
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { GrayImage } from "../../graphics/image";
|
import { GrayImage } from "../../graphics/image";
|
||||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
import { lineStep } from "../../ui/metrics";
|
import { lineStep } from "../../ui/metrics";
|
||||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||||
import { writeTextToDownloads } from "../../native/file-access";
|
import { writeTextToDownloads } from "../../native/file-access";
|
||||||
import { voiceControlBridge } from "../../native/voice-control";
|
import { voiceControlBridge } from "../../native/voice-control";
|
||||||
import { Layer, type LayerContext } from "../../ui/layers";
|
import { Layer, type LayerContext } from "../../ui/layers";
|
||||||
@@ -52,7 +52,10 @@ export class TranscribeLayer implements Layer {
|
|||||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
||||||
|
|
||||||
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
|
const status = voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.status;
|
||||||
|
const bodyTop = paintListeningHeader(
|
||||||
|
image, font, "Transcribe", this.saveNotice || status, this.listening ? this.meter : null, voiceControlBridge.recording,
|
||||||
|
);
|
||||||
const step = lineStep(font) + 2;
|
const step = lineStep(font) + 2;
|
||||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||||
|
|||||||
@@ -7,11 +7,12 @@ import {
|
|||||||
translateProviderSetting,
|
translateProviderSetting,
|
||||||
translateShowOriginalSetting,
|
translateShowOriginalSetting,
|
||||||
} from "./translate-settings";
|
} from "./translate-settings";
|
||||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
import { micTrayIcon } from "../transcribe/transcribe-app";
|
||||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||||
|
import { voiceControlBridge } from "../../native/voice-control";
|
||||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||||
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
|
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
|
||||||
import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
import { enumSettingMenuItem, saveRecordingSetting, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||||
import {
|
import {
|
||||||
createInProcessWindow,
|
createInProcessWindow,
|
||||||
YieldAtRootLayer,
|
YieldAtRootLayer,
|
||||||
@@ -44,6 +45,11 @@ export type TranslateAppOptions = InProcessAppOptions & {
|
|||||||
* provider (Translate settings), which a shared capture switches to.
|
* provider (Translate settings), which a shared capture switches to.
|
||||||
*/
|
*/
|
||||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||||
|
// One timestamp per session names both the audio recording and a saved transcript.
|
||||||
|
const stamp = transcriptTimestamp();
|
||||||
|
// Whether the mic tray icon is up (capture was asked for).
|
||||||
|
let trayIconShown = false;
|
||||||
|
const syncTrayIcon = () => shell.setTrayIcon(TRAY_ICON_ID, trayIconShown ? micTrayIcon(voiceControlBridge.recording) : null);
|
||||||
// Checks the setup and (re)starts capture with the chosen provider and
|
// Checks the setup and (re)starts capture with the chosen provider and
|
||||||
// language: at launch, when either setting changes, and on a tap after a
|
// language: at launch, when either setting changes, and on a tap after a
|
||||||
// problem.
|
// problem.
|
||||||
@@ -51,15 +57,18 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
|||||||
const provider = resolveTranslateProvider();
|
const provider = resolveTranslateProvider();
|
||||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||||
layer.setProblem(problem);
|
layer.setProblem(problem);
|
||||||
if (!provider || problem) {
|
trayIconShown = Boolean(provider) && !problem;
|
||||||
|
if (provider && !problem) {
|
||||||
|
options.startContinuousVoiceCapture({
|
||||||
|
recordAs: `translate-${stamp}`,
|
||||||
|
transcription: { provider, language: resolveTranslateLanguage() },
|
||||||
|
});
|
||||||
|
} else {
|
||||||
options.stopContinuousVoiceCapture();
|
options.stopContinuousVoiceCapture();
|
||||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
|
||||||
return;
|
|
||||||
}
|
}
|
||||||
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
|
syncTrayIcon();
|
||||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
|
||||||
};
|
};
|
||||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
const layer = new TranslateLayer(stamp, startCapture);
|
||||||
const app = createInProcessWindow({
|
const app = createInProcessWindow({
|
||||||
appId: "translate",
|
appId: "translate",
|
||||||
windowId: TRANSLATE_WINDOW_ID,
|
windowId: TRANSLATE_WINDOW_ID,
|
||||||
@@ -77,6 +86,8 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
|||||||
layer.saveTranscript();
|
layer.saveTranscript();
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
// The controller starts and stops the recording as the setting changes.
|
||||||
|
toggleSettingMenuItem(saveRecordingSetting),
|
||||||
{
|
{
|
||||||
label: "Translate settings",
|
label: "Translate settings",
|
||||||
onSelect: (ctx) => {
|
onSelect: (ctx) => {
|
||||||
@@ -94,6 +105,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
|||||||
removeSurface: options.removeSurface,
|
removeSurface: options.removeSurface,
|
||||||
onClosed: () => {
|
onClosed: () => {
|
||||||
layer.onRemoved();
|
layer.onRemoved();
|
||||||
|
unsubscribeRecording();
|
||||||
options.stopContinuousVoiceCapture();
|
options.stopContinuousVoiceCapture();
|
||||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||||
options.onClosed();
|
options.onClosed();
|
||||||
@@ -102,6 +114,10 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
|||||||
// Wire async state changes to this window's renderer, rather than the
|
// Wire async state changes to this window's renderer, rather than the
|
||||||
// controller action from which the window-specific actions were derived.
|
// controller action from which the window-specific actions were derived.
|
||||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
||||||
|
const unsubscribeRecording = voiceControlBridge.onRecordingChange(() => {
|
||||||
|
syncTrayIcon();
|
||||||
|
app.requestRender();
|
||||||
|
});
|
||||||
startCapture();
|
startCapture();
|
||||||
return app;
|
return app;
|
||||||
}
|
}
|
||||||
@@ -1,7 +1,7 @@
|
|||||||
import { GrayImage } from "../../graphics/image";
|
import { GrayImage } from "../../graphics/image";
|
||||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||||
import { lineStep } from "../../ui/metrics";
|
import { lineStep } from "../../ui/metrics";
|
||||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
||||||
import { writeTextToDownloads } from "../../native/file-access";
|
import { writeTextToDownloads } from "../../native/file-access";
|
||||||
import { voiceControlBridge } from "../../native/voice-control";
|
import { voiceControlBridge } from "../../native/voice-control";
|
||||||
import { Layer, type LayerContext } from "../../ui/layers";
|
import { Layer, type LayerContext } from "../../ui/layers";
|
||||||
@@ -77,9 +77,11 @@ export class TranslateLayer implements Layer {
|
|||||||
const measure = (line: string) => font.measureText(line);
|
const measure = (line: string) => font.measureText(line);
|
||||||
const maxWidth = width - 64;
|
const maxWidth = width - 64;
|
||||||
|
|
||||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
const status = this.problem
|
||||||
|
? "Setup needed"
|
||||||
|
: voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.translationError || this.status;
|
||||||
const meter = this.listening && !this.problem ? this.meter : null;
|
const meter = this.listening && !this.problem ? this.meter : null;
|
||||||
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
|
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, voiceControlBridge.recording);
|
||||||
|
|
||||||
let lines: TranslationLine[];
|
let lines: TranslationLine[];
|
||||||
if (this.problem) {
|
if (this.problem) {
|
||||||
|
|||||||
@@ -77,7 +77,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
|||||||
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
import { appViewportRect, isOnSwitcherEdge, screenHeight, screenWidth, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||||
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
import { appSwitcherPositionSetting, glassesModelName, glassesTypeSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, effectiveRingConnectionMode, type RingConnectionMode, saveRecordingSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, shouldPopUpNewNotification, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||||
import {
|
import {
|
||||||
getInstalledEvenHubAppById,
|
getInstalledEvenHubAppById,
|
||||||
@@ -487,6 +487,7 @@ class DashboardController {
|
|||||||
this.applyAppLayoutsIfChanged();
|
this.applyAppLayoutsIfChanged();
|
||||||
this.syncAssistantBridgeIfChanged();
|
this.syncAssistantBridgeIfChanged();
|
||||||
this.syncLockScreenSettingIfChanged();
|
this.syncLockScreenSettingIfChanged();
|
||||||
|
this.syncContinuousRecording();
|
||||||
// A Preview color or scaling change should show on the mirror at once,
|
// A Preview color or scaling change should show on the mirror at once,
|
||||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||||
// the Java side).
|
// the Java side).
|
||||||
@@ -2177,12 +2178,14 @@ class DashboardController {
|
|||||||
|
|
||||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||||
this.continuousClients.set(client, options);
|
this.continuousClients.set(client, options);
|
||||||
|
this.syncContinuousRecording();
|
||||||
this.retargetContinuousCapture();
|
this.retargetContinuousCapture();
|
||||||
this.ensureContinuousCapture(true);
|
this.ensureContinuousCapture(true);
|
||||||
}
|
}
|
||||||
|
|
||||||
private stopContinuousVoiceCapture(client: string): void {
|
private stopContinuousVoiceCapture(client: string): void {
|
||||||
if (!this.continuousClients.delete(client)) return;
|
if (!this.continuousClients.delete(client)) return;
|
||||||
|
this.syncContinuousRecording();
|
||||||
if (this.continuousClients.size > 0) {
|
if (this.continuousClients.size > 0) {
|
||||||
// The window that needed its own provider may be the one that left.
|
// The window that needed its own provider may be the one that left.
|
||||||
this.retargetContinuousCapture();
|
this.retargetContinuousCapture();
|
||||||
@@ -2208,6 +2211,18 @@ class DashboardController {
|
|||||||
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
|
void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record to the first client's requested name while Save Recording is on
|
||||||
|
* (it is toggled from those windows' menus, so this also runs when a
|
||||||
|
* setting changes).
|
||||||
|
*/
|
||||||
|
private syncContinuousRecording(): void {
|
||||||
|
const name = saveRecordingSetting.get()
|
||||||
|
? Array.from(this.continuousClients.values(), (options) => options.recordAs).find(Boolean) ?? ""
|
||||||
|
: "";
|
||||||
|
voiceControlBridge.recordTo(name);
|
||||||
|
}
|
||||||
|
|
||||||
/** Move a running continuous capture onto the provider its windows now call for. */
|
/** Move a running continuous capture onto the provider its windows now call for. */
|
||||||
private retargetContinuousCapture(): void {
|
private retargetContinuousCapture(): void {
|
||||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||||
@@ -2257,7 +2272,6 @@ class DashboardController {
|
|||||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||||
whisperServerModel: whisperServerModelSetting.get(),
|
whisperServerModel: whisperServerModelSetting.get(),
|
||||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
|
||||||
// "My voice only" (Microphones app): verify command utterances
|
// "My voice only" (Microphones app): verify command utterances
|
||||||
// against the enrolled wearer voice-print; non-matching speakers'
|
// against the enrolled wearer voice-print; non-matching speakers'
|
||||||
// finals are suppressed by the bridge.
|
// finals are suppressed by the bridge.
|
||||||
|
|||||||
@@ -53,7 +53,7 @@ import { onAnySettingChanged,
|
|||||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
|
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveRecordingSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
|
||||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||||
@@ -418,6 +418,7 @@ export class IosPreviewController {
|
|||||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||||
this.offSettings = onAnySettingChanged(() => {
|
this.offSettings = onAnySettingChanged(() => {
|
||||||
this.syncLockSetting()
|
this.syncLockSetting()
|
||||||
|
this.syncContinuousRecording()
|
||||||
this.pushBrightness()
|
this.pushBrightness()
|
||||||
this.relayout()
|
this.relayout()
|
||||||
shell.foregroundWindow()?.requestRender()
|
shell.foregroundWindow()?.requestRender()
|
||||||
@@ -1109,7 +1110,7 @@ export class IosPreviewController {
|
|||||||
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
||||||
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
||||||
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
||||||
* Transcribe audio is on.
|
* Recording is on.
|
||||||
*/
|
*/
|
||||||
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
||||||
this.continuousClients.set(client, options)
|
this.continuousClients.set(client, options)
|
||||||
@@ -1198,9 +1199,12 @@ export class IosPreviewController {
|
|||||||
private syncContinuousClaim(): void {
|
private syncContinuousClaim(): void {
|
||||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||||
}
|
}
|
||||||
/** Record to the first client's requested name while Save Transcribe audio is on. */
|
/**
|
||||||
|
* Record to the first client's requested name while Save Recording is on (it is
|
||||||
|
* toggled from those windows' menus, so this also runs when a setting changes).
|
||||||
|
*/
|
||||||
private syncContinuousRecording(): void {
|
private syncContinuousRecording(): void {
|
||||||
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
|
const name = saveRecordingSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
|
||||||
iosVoiceInput.recordTo(name)
|
iosVoiceInput.recordTo(name)
|
||||||
}
|
}
|
||||||
/**
|
/**
|
||||||
|
|||||||
@@ -62,6 +62,9 @@ export class IosVoiceControlBridge {
|
|||||||
// and it is created by the first audio, so a failed start leaves no file.
|
// and it is created by the first audio, so a failed start leaves no file.
|
||||||
private recordingName = ''
|
private recordingName = ''
|
||||||
private recorder: WavRecorder | null = null
|
private recorder: WavRecorder | null = null
|
||||||
|
// The file could not be created: one attempt per recording, not one per packet.
|
||||||
|
private recordingFailure = false
|
||||||
|
private readonly recordingListeners = new Set<() => void>()
|
||||||
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
// Raw-PCM tap (EvenHub mic apps): decode-only glasses audio, broadcast to
|
||||||
// rawListeners while any owner wants it, from the session setRawSession
|
// rawListeners while any owner wants it, from the session setRawSession
|
||||||
// bound. It shares the one glasses mic with speech capture, which preempts
|
// bound. It shares the one glasses mic with speech capture, which preempts
|
||||||
@@ -212,16 +215,28 @@ export class IosVoiceControlBridge {
|
|||||||
recordTo(name: string): void {
|
recordTo(name: string): void {
|
||||||
if (name === this.recordingName) return
|
if (name === this.recordingName) return
|
||||||
this.recorder?.close(); this.recorder = null
|
this.recorder?.close(); this.recorder = null
|
||||||
this.recordingName = name
|
this.setRecording(name, false)
|
||||||
|
}
|
||||||
|
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||||
|
get recording(): boolean { return this.recordingName !== '' && !this.recordingFailure }
|
||||||
|
/** Whether the recording asked for could not be saved (its file could not be created). */
|
||||||
|
get recordingFailed(): boolean { return this.recordingFailure }
|
||||||
|
/** `recording` or `recordingFailed` changed. */
|
||||||
|
onRecordingChange(listener: () => void): () => void { this.recordingListeners.add(listener); return () => { this.recordingListeners.delete(listener) } }
|
||||||
|
private setRecording(name: string, failed: boolean): void {
|
||||||
|
const wasRecording = this.recording, hadFailed = this.recordingFailure
|
||||||
|
this.recordingName = name; this.recordingFailure = failed
|
||||||
|
if (wasRecording === this.recording && hadFailed === failed) return
|
||||||
|
for (const fn of [...this.recordingListeners]) fn()
|
||||||
}
|
}
|
||||||
private record(pcm: Uint8Array): void {
|
private record(pcm: Uint8Array): void {
|
||||||
if (!this.recorder && this.recordingName) {
|
if (!this.recording) return
|
||||||
|
if (!this.recorder) {
|
||||||
this.recorder = WavRecorder.open(this.recordingName)
|
this.recorder = WavRecorder.open(this.recordingName)
|
||||||
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording')
|
||||||
// One attempt per recording, not one per packet.
|
if (!this.recorder) { this.setRecording(this.recordingName, true); return }
|
||||||
if (!this.recorder) this.recordingName = ''
|
|
||||||
}
|
}
|
||||||
this.recorder?.append(pcm)
|
this.recorder.append(pcm)
|
||||||
}
|
}
|
||||||
/** End continuous capture, transcribing the audio since the last pause. */
|
/** End continuous capture, transcribing the audio since the last pause. */
|
||||||
stopContinuousCapture(): Promise<void> {
|
stopContinuousCapture(): Promise<void> {
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import { Utils } from "@nativescript/core";
|
|||||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||||
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider";
|
||||||
import { SpeechPauseDetector } from "./speech-pause";
|
import { SpeechPauseDetector } from "./speech-pause";
|
||||||
|
import { WavRecorder } from "./wav-recorder";
|
||||||
import { toUint8Array } from "../util/array-util";
|
import { toUint8Array } from "../util/array-util";
|
||||||
import { pcmLevel } from "../util/audio-level";
|
import { pcmLevel } from "../util/audio-level";
|
||||||
|
|
||||||
@@ -60,7 +61,6 @@ export type PushToTalkOptions = {
|
|||||||
whisperServerApiKey?: string;
|
whisperServerApiKey?: string;
|
||||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||||
language?: string;
|
language?: string;
|
||||||
saveRecording: boolean;
|
|
||||||
/**
|
/**
|
||||||
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
* Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free
|
||||||
* ("Hey Even") capture, which has no button release to end the utterance.
|
* ("Hey Even") capture, which has no button release to end the utterance.
|
||||||
@@ -150,6 +150,14 @@ export class FaceclawVoiceControlBridge {
|
|||||||
// the bridge swallow it.
|
// the bridge swallow it.
|
||||||
private verificationActive = false;
|
private verificationActive = false;
|
||||||
private verificationRejected = false;
|
private verificationRejected = false;
|
||||||
|
// Continuous captures' audio goes to Downloads/<recordingName>.wav while it is
|
||||||
|
// set. The file outlives a capture, so a restarted one keeps appending to it,
|
||||||
|
// and it is created by the first audio, so a failed start leaves no file.
|
||||||
|
private recordingName = "";
|
||||||
|
private recorder: WavRecorder | null = null;
|
||||||
|
// The file could not be created: one attempt per recording, not one per packet.
|
||||||
|
private recordingFailure = false;
|
||||||
|
private readonly recordingListeners = new Set<() => void>();
|
||||||
|
|
||||||
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
onStatus(listener: (state: VoiceControlState) => void): () => void {
|
||||||
this.statusListeners.add(listener);
|
this.statusListeners.add(listener);
|
||||||
@@ -242,6 +250,56 @@ export class FaceclawVoiceControlBridge {
|
|||||||
this.acquireCapture("continuous", options);
|
this.acquireCapture("continuous", options);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Record continuous captures' audio to Downloads/<name>.wav, across restarts,
|
||||||
|
* until called with another name or "".
|
||||||
|
*/
|
||||||
|
recordTo(name: string): void {
|
||||||
|
if (name === this.recordingName) return;
|
||||||
|
this.recorder?.close();
|
||||||
|
this.recorder = null;
|
||||||
|
this.setRecording(name, false);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether continuous captures' audio is being saved (see recordTo). */
|
||||||
|
get recording(): boolean {
|
||||||
|
return this.recordingName !== "" && !this.recordingFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Whether the recording asked for could not be saved (its file could not be created). */
|
||||||
|
get recordingFailed(): boolean {
|
||||||
|
return this.recordingFailure;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `recording` or `recordingFailed` changed. */
|
||||||
|
onRecordingChange(listener: () => void): () => void {
|
||||||
|
this.recordingListeners.add(listener);
|
||||||
|
return () => this.recordingListeners.delete(listener);
|
||||||
|
}
|
||||||
|
|
||||||
|
private setRecording(name: string, failed: boolean): void {
|
||||||
|
const wasRecording = this.recording;
|
||||||
|
const hadFailed = this.recordingFailure;
|
||||||
|
this.recordingName = name;
|
||||||
|
this.recordingFailure = failed;
|
||||||
|
if (wasRecording === this.recording && hadFailed === failed) return;
|
||||||
|
for (const listener of Array.from(this.recordingListeners)) listener();
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `pcm` is the controller's Java byte[], handed to the recorder as it came. */
|
||||||
|
private record(pcm: any): void {
|
||||||
|
if (!this.recording) return;
|
||||||
|
if (!this.recorder) {
|
||||||
|
this.recorder = WavRecorder.open(this.recordingName);
|
||||||
|
if (!this.recorder) {
|
||||||
|
this.setRecording(this.recordingName, true);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
console.log(`Voice: recording audio to ${this.recorder.path}`);
|
||||||
|
}
|
||||||
|
this.recorder.append(pcm);
|
||||||
|
}
|
||||||
|
|
||||||
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
/** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */
|
||||||
onRawPcm(listener: RawPcmListener): () => void {
|
onRawPcm(listener: RawPcmListener): () => void {
|
||||||
this.rawPcmListeners.add(listener);
|
this.rawPcmListeners.add(listener);
|
||||||
@@ -305,7 +363,6 @@ export class FaceclawVoiceControlBridge {
|
|||||||
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
|
// The raw tap is strictly the G2 stream; clear any phone-mic flag a
|
||||||
// preview-mode capture may have left on the shared controller.
|
// preview-mode capture may have left on the shared controller.
|
||||||
this.controller?.setUsePhoneMic(false);
|
this.controller?.setUsePhoneMic(false);
|
||||||
this.controller?.setSaveRecordings(false);
|
|
||||||
this.controller?.setEndpointing(false);
|
this.controller?.setEndpointing(false);
|
||||||
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
// Raw means raw: the Microphones session (and EvenHub apps) get the
|
||||||
// unprocessed stream and run their own beam/suppression processing.
|
// unprocessed stream and run their own beam/suppression processing.
|
||||||
@@ -354,7 +411,6 @@ export class FaceclawVoiceControlBridge {
|
|||||||
this.ensureController();
|
this.ensureController();
|
||||||
this.controller?.setCommunicator(options.communicator);
|
this.controller?.setCommunicator(options.communicator);
|
||||||
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic));
|
||||||
this.controller?.setSaveRecordings(options.saveRecording);
|
|
||||||
this.controller?.setEndpointing(Boolean(options.endpointing));
|
this.controller?.setEndpointing(Boolean(options.endpointing));
|
||||||
this.captureEndpointing = Boolean(options.endpointing);
|
this.captureEndpointing = Boolean(options.endpointing);
|
||||||
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
this.controller?.setNoiseSuppression(Boolean(options.noiseSuppression));
|
||||||
@@ -554,6 +610,8 @@ export class FaceclawVoiceControlBridge {
|
|||||||
onPcm: (pcm: any) => {
|
onPcm: (pcm: any) => {
|
||||||
const bytes = toUint8Array(pcm);
|
const bytes = toUint8Array(pcm);
|
||||||
this.cloudClient?.acceptPcm(bytes);
|
this.cloudClient?.acceptPcm(bytes);
|
||||||
|
// Push-to-talk on a capture of its own is not recorded.
|
||||||
|
if (this.captureHolders.has("continuous")) this.record(pcm);
|
||||||
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
if (!this.cloudClient && this.captureHolders.has("continuous") && this.speechPause.accept(bytes)) {
|
||||||
for (const listener of this.speechPauseListeners) listener();
|
for (const listener of this.speechPauseListeners) listener();
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,42 @@
|
|||||||
|
declare const android: any;
|
||||||
|
declare const java: any;
|
||||||
|
declare const com: any;
|
||||||
|
|
||||||
|
const SAMPLE_RATE = 16000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Appends 16 kHz mono PCM16 to a WAV file in the public Downloads directory
|
||||||
|
* while it is captured (Android; wav-recorder.ios.ts is the iOS one). The
|
||||||
|
* native recorder keeps the header's sizes current, so a crash leaves a
|
||||||
|
* playable file of everything up to the last second.
|
||||||
|
*/
|
||||||
|
export class WavRecorder {
|
||||||
|
private constructor(private recorder: any, readonly path: string) {}
|
||||||
|
|
||||||
|
/** A recorder for Downloads/<name>.wav (a -2, -3... suffix if taken), or null. */
|
||||||
|
static open(name: string): WavRecorder | null {
|
||||||
|
try {
|
||||||
|
const downloads = android.os.Environment.getExternalStoragePublicDirectory(
|
||||||
|
android.os.Environment.DIRECTORY_DOWNLOADS,
|
||||||
|
);
|
||||||
|
downloads.mkdirs();
|
||||||
|
let file = new java.io.File(downloads, `${name}.wav`);
|
||||||
|
for (let n = 2; file.exists(); n++) file = new java.io.File(downloads, `${name}-${n}.wav`);
|
||||||
|
const path = String(file.getAbsolutePath());
|
||||||
|
return new WavRecorder(new com.faceclaw.app.FaceclawWavRecorder(path, SAMPLE_RATE, 1), path);
|
||||||
|
} catch (error) {
|
||||||
|
console.warn(`WavRecorder: could not create ${name}.wav in Downloads: ${error}`);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/** `pcm` is the Java byte[] the voice controller delivered, passed back as it came. */
|
||||||
|
append(pcm: any): void {
|
||||||
|
this.recorder?.append(pcm);
|
||||||
|
}
|
||||||
|
|
||||||
|
close(): void {
|
||||||
|
this.recorder?.finish();
|
||||||
|
this.recorder = null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1030,20 +1030,16 @@ export const wakeWordActionSetting = new ConfigSettingEnum<WakeWordAction>({
|
|||||||
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
|
description: "What saying \"Hey Even\" does: start voice input, just turn the screen on, or nothing.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const saveTranscribeAudioSetting = new ConfigSettingBoolean({
|
/**
|
||||||
id: "save-transcribe-audio",
|
* Toggled from the Transcribe and Translate apps' menus. One setting for
|
||||||
label: "Save Transcribe audio",
|
* both: they share the mic stream, so its audio is either being saved or not.
|
||||||
storageKey: "voice.saveTranscribeAudio",
|
*/
|
||||||
defaultValue: true,
|
export const saveRecordingSetting = new ConfigSettingBoolean({
|
||||||
description: "Record each Transcribe session's audio as a WAV file in Downloads (about 2 MB per minute), named like its transcript. Recordings are kept until you delete them.",
|
id: "save-recording",
|
||||||
});
|
label: "Save Recording",
|
||||||
|
storageKey: "voice.saveRecording",
|
||||||
export const saveVoiceRecordingsSetting = new ConfigSettingBoolean({
|
|
||||||
id: "save-voice-recordings",
|
|
||||||
label: "Save voice recordings",
|
|
||||||
storageKey: "developer.saveVoiceRecordings",
|
|
||||||
defaultValue: false,
|
defaultValue: false,
|
||||||
description: "Keep a copy of captured voice audio on the phone, for debugging transcription problems.",
|
description: "Record the audio of Transcribe and Translate sessions as WAV files in Downloads (about 2 MB per minute), named after the app and the session's start time. Recordings are kept until you delete them.",
|
||||||
});
|
});
|
||||||
|
|
||||||
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||||
|
|||||||
@@ -69,7 +69,6 @@ import {
|
|||||||
ringConnectionModeSetting,
|
ringConnectionModeSetting,
|
||||||
glassesTypeSetting,
|
glassesTypeSetting,
|
||||||
sonioxApiKeySetting,
|
sonioxApiKeySetting,
|
||||||
saveTranscribeAudioSetting,
|
|
||||||
whisperServerApiKeySetting,
|
whisperServerApiKeySetting,
|
||||||
whisperServerLanguageSetting,
|
whisperServerLanguageSetting,
|
||||||
whisperServerModelSetting,
|
whisperServerModelSetting,
|
||||||
@@ -80,7 +79,6 @@ import {
|
|||||||
lockScreenEnabledSetting,
|
lockScreenEnabledSetting,
|
||||||
menuAnimationSetting,
|
menuAnimationSetting,
|
||||||
newNotificationDisplaySetting,
|
newNotificationDisplaySetting,
|
||||||
saveVoiceRecordingsSetting,
|
|
||||||
showBleBandwidthSetting,
|
showBleBandwidthSetting,
|
||||||
suspendEvenHubWhenScreenOffSetting,
|
suspendEvenHubWhenScreenOffSetting,
|
||||||
temperatureUnitSetting,
|
temperatureUnitSetting,
|
||||||
@@ -247,7 +245,6 @@ function settingsSections(): SettingsSection[] {
|
|||||||
// Whether and how the phone opens its own BLE link to the R1 ring.
|
// Whether and how the phone opens its own BLE link to the R1 ring.
|
||||||
// Applied at connect time.
|
// Applied at connect time.
|
||||||
ringConnectionMenuItem(),
|
ringConnectionMenuItem(),
|
||||||
toggleSettingMenuItem(saveVoiceRecordingsSetting),
|
|
||||||
toggleSettingMenuItem(firmwareDebugFlagsSetting),
|
toggleSettingMenuItem(firmwareDebugFlagsSetting),
|
||||||
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
|
toggleSettingMenuItem(suspendEvenHubWhenScreenOffSetting),
|
||||||
toggleSettingMenuItem(useMicControlSetting),
|
toggleSettingMenuItem(useMicControlSetting),
|
||||||
@@ -294,7 +291,7 @@ function settingsSections(): SettingsSection[] {
|
|||||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
enumSettingMenuItem(voiceProviderSetting), ...whisperServerMenuItems()] };
|
||||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||||
item.label !== screenTimeoutSetting.label) };
|
item.label !== screenTimeoutSetting.label) };
|
||||||
return section;
|
return section;
|
||||||
|
|||||||
+1
-1
@@ -12,7 +12,7 @@ export type TextSettingsEditToggle = {
|
|||||||
|
|
||||||
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
/** A continuous-capture client's needs (see LayerActions.startContinuousVoiceCapture). */
|
||||||
export type ContinuousCaptureOptions = {
|
export type ContinuousCaptureOptions = {
|
||||||
/** Names a WAV of the captured audio in Downloads (iOS, when Save Transcribe audio is on). */
|
/** Names a WAV of the captured audio in Downloads, written while Save Recording is on. */
|
||||||
recordAs?: string;
|
recordAs?: string;
|
||||||
/**
|
/**
|
||||||
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
* Transcribe with this provider, expecting this spoken language (ISO 639-1,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
import { type GrayImage, type UiFont } from "../graphics/image";
|
import { GrayImage, imageFromAsciiArt, type UiFont } from "../graphics/image";
|
||||||
import { truncateText } from "../graphics/textwrap";
|
import { truncateText } from "../graphics/textwrap";
|
||||||
import { voiceControlBridge } from "../native/voice-control";
|
import { voiceControlBridge } from "../native/voice-control";
|
||||||
import { lineStep } from "./metrics";
|
import { lineStep } from "./metrics";
|
||||||
@@ -12,6 +12,10 @@ const SEGMENTS = 8;
|
|||||||
const SEGMENT_WIDTH = 4;
|
const SEGMENT_WIDTH = 4;
|
||||||
const SEGMENT_GAP = 2;
|
const SEGMENT_GAP = 2;
|
||||||
const METER_GAP = 10;
|
const METER_GAP = 10;
|
||||||
|
/** Between the recording circle and the meter. */
|
||||||
|
const RECORDING_ICON_GAP = 4;
|
||||||
|
/** Between a tray icon and its recording circle: tight, so they read as one item in the bar. */
|
||||||
|
const TRAY_RECORDING_ICON_GAP = 2;
|
||||||
const LIT_GRAY = 200;
|
const LIT_GRAY = 200;
|
||||||
const UNLIT_GRAY = 45;
|
const UNLIT_GRAY = 45;
|
||||||
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
|
/** How often the meter may ask for a repaint (each one is a frame over BLE). */
|
||||||
@@ -19,6 +23,38 @@ const TICK_MS = 200;
|
|||||||
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
|
/** Per tick, the shown level falls to this fraction without louder audio (~1 s to empty). */
|
||||||
const DECAY = 0.5;
|
const DECAY = 0.5;
|
||||||
|
|
||||||
|
/** The listening views' status while Save Recording is on but its file could not be created. */
|
||||||
|
export const RECORDING_FAILED_NOTICE = "Recording failed (check file access).";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 9x9 filled circle: the mic's audio is being saved (Save Recording, in the
|
||||||
|
* Transcribe and Translate menus). Shown beside the mic level meter and the
|
||||||
|
* mic tray icon, and on Glanceboard's system card.
|
||||||
|
*/
|
||||||
|
export const RECORDING_ICON = imageFromAsciiArt(
|
||||||
|
[
|
||||||
|
" ### ",
|
||||||
|
" ####### ",
|
||||||
|
" ####### ",
|
||||||
|
"#########",
|
||||||
|
"#########",
|
||||||
|
"#########",
|
||||||
|
" ####### ",
|
||||||
|
" ####### ",
|
||||||
|
" ### ",
|
||||||
|
],
|
||||||
|
{ "#": 220 },
|
||||||
|
);
|
||||||
|
|
||||||
|
/** `icon` with the recording circle to its right: a tray icon while its app's audio is saved. */
|
||||||
|
export function withRecordingIcon(icon: GrayImage): GrayImage {
|
||||||
|
const circleX = icon.width + TRAY_RECORDING_ICON_GAP;
|
||||||
|
const image = new GrayImage(circleX + RECORDING_ICON.width, Math.max(icon.height, RECORDING_ICON.height), 0);
|
||||||
|
image.bitBlt(icon, 0, (image.height - icon.height) >> 1);
|
||||||
|
image.bitBlt(RECORDING_ICON, circleX, (image.height - RECORDING_ICON.height) >> 1);
|
||||||
|
return image;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A compact segmented mic-level meter for listening views. Levels arrive per
|
* A compact segmented mic-level meter for listening views. Levels arrive per
|
||||||
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
|
* 50 ms audio chunk; the meter holds their peak and, at most every 200 ms,
|
||||||
@@ -66,8 +102,9 @@ export class MicLevelMeter {
|
|||||||
/**
|
/**
|
||||||
* The one-line header of the listening views (Transcribe, Translate):
|
* The one-line header of the listening views (Transcribe, Translate):
|
||||||
* "Title · status", the status truncated to fit, then the mic level meter at
|
* "Title · status", the status truncated to fit, then the mic level meter at
|
||||||
* the line's end when one is given (while listening). Returns the y at which
|
* the line's end when one is given (while listening), with the recording
|
||||||
* the view's body starts.
|
* circle before it while the audio it shows is being saved. Returns the y at
|
||||||
|
* which the view's body starts.
|
||||||
*/
|
*/
|
||||||
export function paintListeningHeader(
|
export function paintListeningHeader(
|
||||||
image: GrayImage,
|
image: GrayImage,
|
||||||
@@ -75,13 +112,22 @@ export function paintListeningHeader(
|
|||||||
title: string,
|
title: string,
|
||||||
status: string,
|
status: string,
|
||||||
meter: MicLevelMeter | null,
|
meter: MicLevelMeter | null,
|
||||||
|
recording = false,
|
||||||
): number {
|
): number {
|
||||||
let textRight = image.width - HEADER_X;
|
let textRight = image.width - HEADER_X;
|
||||||
if (meter) {
|
if (meter) {
|
||||||
const meterX = textRight - meter.width;
|
const meterX = textRight - meter.width;
|
||||||
// Roughly cap height, sitting on the text's baseline.
|
// Roughly cap height, sitting on the text's baseline.
|
||||||
meter.paint(image, meterX, HEADER_Y + font.ascent, Math.max(4, Math.round(font.ascent * 0.6)));
|
const baseline = HEADER_Y + font.ascent;
|
||||||
|
const meterHeight = Math.max(4, Math.round(font.ascent * 0.6));
|
||||||
|
meter.paint(image, meterX, baseline, meterHeight);
|
||||||
textRight = meterX - METER_GAP;
|
textRight = meterX - METER_GAP;
|
||||||
|
if (recording) {
|
||||||
|
const iconX = meterX - RECORDING_ICON_GAP - RECORDING_ICON.width;
|
||||||
|
// Centred on the meter's height.
|
||||||
|
image.drawImage(RECORDING_ICON, iconX, Math.round(baseline - (meterHeight + RECORDING_ICON.height) / 2));
|
||||||
|
textRight = iconX - METER_GAP;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
|
image.drawText(font, HEADER_X, HEADER_Y, title, TITLE_GRAY);
|
||||||
if (status) {
|
if (status) {
|
||||||
|
|||||||
+20
-10
@@ -40,6 +40,10 @@ constructor(
|
|||||||
private var file: RandomAccessFile?
|
private var file: RandomAccessFile?
|
||||||
private var dataBytes = 0L
|
private var dataBytes = 0L
|
||||||
|
|
||||||
|
// The PCM bytes the header on disk accounts for. It is rewritten about once a second of
|
||||||
|
// audio, so a process death mid-recording still leaves a playable file.
|
||||||
|
private var headerDataBytes = 0L
|
||||||
|
|
||||||
init {
|
init {
|
||||||
val parent = File(path).parentFile
|
val parent = File(path).parentFile
|
||||||
if (parent != null && !parent.exists() && !parent.mkdirs())
|
if (parent != null && !parent.exists() && !parent.mkdirs())
|
||||||
@@ -58,11 +62,26 @@ constructor(
|
|||||||
try {
|
try {
|
||||||
current.write(pcm16le)
|
current.write(pcm16le)
|
||||||
dataBytes += pcm16le.size
|
dataBytes += pcm16le.size
|
||||||
|
if (dataBytes - headerDataBytes >= sampleRate * channels * 2L) writeHeader(current)
|
||||||
} catch (e: IOException) {
|
} catch (e: IOException) {
|
||||||
Log.w("FaceclawWavRecorder", "append failed", e)
|
Log.w("FaceclawWavRecorder", "append failed", e)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/** Rewrite the header to cover the PCM written so far, leaving the position at the end. */
|
||||||
|
private fun writeHeader(target: RandomAccessFile) {
|
||||||
|
val header =
|
||||||
|
BinaryEncoding.wavHeader(
|
||||||
|
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
|
||||||
|
sampleRate,
|
||||||
|
channels,
|
||||||
|
)
|
||||||
|
target.seek(0)
|
||||||
|
target.write(header)
|
||||||
|
target.seek(header.size + dataBytes)
|
||||||
|
headerDataBytes = dataBytes
|
||||||
|
}
|
||||||
|
|
||||||
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
|
@Synchronized fun getDurationMs(): Long = dataBytes * 1000L / (sampleRate * channels * 2L)
|
||||||
|
|
||||||
fun getPath(): String = path
|
fun getPath(): String = path
|
||||||
@@ -72,16 +91,7 @@ constructor(
|
|||||||
val current = file ?: return dataBytes
|
val current = file ?: return dataBytes
|
||||||
file = null
|
file = null
|
||||||
try {
|
try {
|
||||||
current.use {
|
current.use { writeHeader(it) }
|
||||||
it.seek(0)
|
|
||||||
it.write(
|
|
||||||
BinaryEncoding.wavHeader(
|
|
||||||
minOf(dataBytes, Int.MAX_VALUE.toLong()).toInt(),
|
|
||||||
sampleRate,
|
|
||||||
channels,
|
|
||||||
)
|
|
||||||
)
|
|
||||||
}
|
|
||||||
} catch (e: IOException) {
|
} catch (e: IOException) {
|
||||||
Log.w("FaceclawWavRecorder", "finish failed", e)
|
Log.w("FaceclawWavRecorder", "finish failed", e)
|
||||||
}
|
}
|
||||||
|
|||||||
+1
-35
@@ -35,14 +35,11 @@ interface VoiceCaptureHost {
|
|||||||
|
|
||||||
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
|
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
|
||||||
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
|
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
|
||||||
|
|
||||||
/** Where a saved recording goes, or null when recordings cannot be stored. */
|
|
||||||
fun recordingSink(): RecordingSink?
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
|
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
|
||||||
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker
|
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, speaker
|
||||||
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
|
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
|
||||||
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
|
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
|
||||||
* worker thread (started through the shared [startThread]); listener callbacks go through the
|
* worker thread (started through the shared [startThread]); listener callbacks go through the
|
||||||
@@ -158,11 +155,8 @@ class VoiceCaptureSession(
|
|||||||
private var lastTranscriptDecodeAtMs: Long = 0
|
private var lastTranscriptDecodeAtMs: Long = 0
|
||||||
private var lastTranscript = ""
|
private var lastTranscript = ""
|
||||||
@Volatile
|
@Volatile
|
||||||
private var saveRecordings = false
|
|
||||||
@Volatile
|
|
||||||
private var endpointing = false
|
private var endpointing = false
|
||||||
private val endpointDetector = VoiceEndpointDetector()
|
private val endpointDetector = VoiceEndpointDetector()
|
||||||
private var recordingPcm: ByteSink? = null
|
|
||||||
@Volatile
|
@Volatile
|
||||||
private var verifySpeakerModelPath: String? = null
|
private var verifySpeakerModelPath: String? = null
|
||||||
@Volatile
|
@Volatile
|
||||||
@@ -199,10 +193,6 @@ class VoiceCaptureSession(
|
|||||||
this.usePhoneMic = usePhoneMic
|
this.usePhoneMic = usePhoneMic
|
||||||
}
|
}
|
||||||
|
|
||||||
fun setSaveRecordings(saveRecordings: Boolean) {
|
|
||||||
this.saveRecordings = saveRecordings
|
|
||||||
}
|
|
||||||
|
|
||||||
fun setOnboardModelKind(kind: String?) {
|
fun setOnboardModelKind(kind: String?) {
|
||||||
this.onboardModelKind = parseModelKind(kind)
|
this.onboardModelKind = parseModelKind(kind)
|
||||||
}
|
}
|
||||||
@@ -363,7 +353,6 @@ class VoiceCaptureSession(
|
|||||||
}
|
}
|
||||||
endpointDetector.reset()
|
endpointDetector.reset()
|
||||||
suppressor?.reset()
|
suppressor?.reset()
|
||||||
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
|
|
||||||
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
|
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
|
||||||
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
|
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
|
||||||
verifyCount = 0
|
verifyCount = 0
|
||||||
@@ -404,7 +393,6 @@ class VoiceCaptureSession(
|
|||||||
emitStatus("Voice control failed: " + error.message)
|
emitStatus("Voice control failed: " + error.message)
|
||||||
} finally {
|
} finally {
|
||||||
stopG2Audio()
|
stopG2Audio()
|
||||||
writeRecordingIfAny()
|
|
||||||
releaseRecognizer()
|
releaseRecognizer()
|
||||||
releaseDecoder()
|
releaseDecoder()
|
||||||
lock.withLock {
|
lock.withLock {
|
||||||
@@ -418,25 +406,6 @@ class VoiceCaptureSession(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private fun appendRecording(pcm: ShortArray, count: Int) {
|
|
||||||
val out = recordingPcm ?: return
|
|
||||||
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun writeRecordingIfAny() {
|
|
||||||
val out = recordingPcm
|
|
||||||
recordingPcm = null
|
|
||||||
if (out == null || out.size() == 0) {
|
|
||||||
return
|
|
||||||
}
|
|
||||||
val sink = host.recordingSink() ?: return
|
|
||||||
try {
|
|
||||||
sink.save(out.toByteArray(), SAMPLE_RATE)
|
|
||||||
} catch (t: Throwable) {
|
|
||||||
logWarn("failed to save voice recording (${t.message})")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private fun startG2Audio(): Boolean {
|
private fun startG2Audio(): Boolean {
|
||||||
val source = host.g2AudioSource() ?: return false
|
val source = host.g2AudioSource() ?: return false
|
||||||
resetAudioStats()
|
resetAudioStats()
|
||||||
@@ -489,9 +458,6 @@ class VoiceCaptureSession(
|
|||||||
if (suppressionEnabled) {
|
if (suppressionEnabled) {
|
||||||
applySuppression(pcm, count)
|
applySuppression(pcm, count)
|
||||||
}
|
}
|
||||||
if (recordingPcm != null) {
|
|
||||||
appendRecording(pcm, count)
|
|
||||||
}
|
|
||||||
val verifyBuffer = this.verifyBuffer
|
val verifyBuffer = this.verifyBuffer
|
||||||
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
|
||||||
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
|
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
|
||||||
|
|||||||
@@ -34,11 +34,6 @@ fun interface CallbackDispatcher {
|
|||||||
fun post(action: () -> Unit)
|
fun post(action: () -> Unit)
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
|
|
||||||
fun interface RecordingSink {
|
|
||||||
fun save(pcm16le: ByteArray, sampleRate: Int)
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
|
* A glasses microphone packet decoder to 16 kHz mono PCM: the G2's LC3 (the shared
|
||||||
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
|
* [Lc3PacketFramer] over a platform LC3 codec) or the RayNeo iO's Opus.
|
||||||
|
|||||||
@@ -368,6 +368,7 @@ test('iOS preview voice uses the phone microphone, while connected capture stays
|
|||||||
|
|
||||||
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
|
test('iOS continuous capture is shared by windows and restarts after a disconnect, a lock or backgrounding', async () => {
|
||||||
const calls = [], claims = [], providers = [];
|
const calls = [], claims = [], providers = [];
|
||||||
|
let saveRecording = false;
|
||||||
// Models the bridge's continuous state: listening on a source, or draining after a stop.
|
// Models the bridge's continuous state: listening on a source, or draining after a stop.
|
||||||
const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
|
const bridge = { source: null, draining: false, recording: '', rawSession: undefined,
|
||||||
supportsContinuous: voice => voice.provider !== 'onboard',
|
supportsContinuous: voice => voice.provider !== 'onboard',
|
||||||
@@ -385,7 +386,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
|||||||
setRawSession: session => { bridge.rawSession = session; } };
|
setRawSession: session => { bridge.rawSession = session; } };
|
||||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||||
saveTranscribeAudioSetting: { get: () => true } };
|
saveRecordingSetting: { get: () => saveRecording } };
|
||||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||||
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
|
: id === '../ui/shell/voice-activity' ? { voiceActivity: { setActive: (active, holder) => claims.push([active, holder]) } }
|
||||||
@@ -404,7 +405,15 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
|||||||
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
|
host.startContinuousVoiceCapture('transcribe', { recordAs: 'transcribe-1' }); await settle();
|
||||||
host.startContinuousVoiceCapture('calculator'); await settle();
|
host.startContinuousVoiceCapture('calculator'); await settle();
|
||||||
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
|
assert.deepEqual(calls, [['glasses', true]], 'the second window shares the capture');
|
||||||
|
// Save Recording (in the window's menu) starts and stops the recording while it is open.
|
||||||
|
assert.equal(bridge.recording, '');
|
||||||
|
saveRecording = true; host.syncContinuousRecording();
|
||||||
assert.equal(bridge.recording, 'transcribe-1');
|
assert.equal(bridge.recording, 'transcribe-1');
|
||||||
|
saveRecording = false; host.syncContinuousRecording();
|
||||||
|
assert.equal(bridge.recording, '');
|
||||||
|
saveRecording = true; host.syncContinuousRecording();
|
||||||
|
assert.equal(bridge.recording, 'transcribe-1');
|
||||||
|
assert.deepEqual(calls, [['glasses', true]], 'without touching the capture');
|
||||||
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
|
assert.deepEqual(claims.at(-1), [true, 'continuous-capture']);
|
||||||
host.stopContinuousVoiceCapture('calculator');
|
host.stopContinuousVoiceCapture('calculator');
|
||||||
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
|
assert.deepEqual(calls, [['glasses', true]], 'closing one window keeps the other listening');
|
||||||
|
|||||||
@@ -31,8 +31,13 @@ function harness() {
|
|||||||
},
|
},
|
||||||
};
|
};
|
||||||
const recordings = [];
|
const recordings = [];
|
||||||
|
let failRecordings = false, recordingAttempts = 0;
|
||||||
class FakeRecorder {
|
class FakeRecorder {
|
||||||
static open(name) { const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r; }
|
static open(name) {
|
||||||
|
recordingAttempts++;
|
||||||
|
if (failRecordings) return null;
|
||||||
|
const r = new FakeRecorder(); r.name = name; r.path = `/Downloads/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||||
|
}
|
||||||
append(bytes) { this.pcm.push(...bytes); }
|
append(bytes) { this.pcm.push(...bytes); }
|
||||||
close() { this.closed = true; }
|
close() { this.closed = true; }
|
||||||
}
|
}
|
||||||
@@ -56,6 +61,7 @@ function harness() {
|
|||||||
} };
|
} };
|
||||||
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
const onboard = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: '', sonioxApiKey: '' };
|
||||||
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
|
return { bridge, native, session, audio, commands, text, timers, clouds, analyzers, recordings, release: () => release(),
|
||||||
|
failRecordings: value => { failRecordings = value; }, recordingAttempts: () => recordingAttempts,
|
||||||
analyzerSupported: value => { analyzerSupported = value; },
|
analyzerSupported: value => { analyzerSupported = value; },
|
||||||
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
onboard, cloud: { ...onboard, provider: 'soniox', sonioxApiKey: 'key' },
|
||||||
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
microphoneAuthorization: value => { microphoneAuthorization = value; }, microphoneRequests: () => microphoneRequests,
|
||||||
@@ -307,7 +313,10 @@ test('push-to-talk after Transcribe closed starts its own capture instead of joi
|
|||||||
|
|
||||||
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
|
test('continuous captures record to the requested WAV, created by their first audio and kept across restarts', async () => {
|
||||||
const h = harness();
|
const h = harness();
|
||||||
|
let changes = 0; h.bridge.onRecordingChange(() => changes++);
|
||||||
|
assert.equal(h.bridge.recording, false);
|
||||||
h.bridge.recordTo('transcribe-2026-10-17-120000');
|
h.bridge.recordTo('transcribe-2026-10-17-120000');
|
||||||
|
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 1]);
|
||||||
// A start that fails before any audio leaves no file behind.
|
// A start that fails before any audio leaves no file behind.
|
||||||
h.native.error = 'No phone microphone is available.';
|
h.native.error = 'No phone microphone is available.';
|
||||||
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
await h.bridge.startPhoneCapture(h.cloud, () => {}, false, true);
|
||||||
@@ -334,6 +343,30 @@ test('continuous captures record to the requested WAV, created by their first au
|
|||||||
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
assert.deepEqual(h.recordings[0].pcm, [1, 2, 3, 4, 5, 6, 7]);
|
||||||
h.bridge.recordTo('');
|
h.bridge.recordTo('');
|
||||||
assert.equal(h.recordings[0].closed, true);
|
assert.equal(h.recordings[0].closed, true);
|
||||||
|
assert.deepEqual([h.bridge.recording, changes], [false, 2]);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a recording whose file cannot be created is reported, and not retried per packet', async () => {
|
||||||
|
const h = harness();
|
||||||
|
let changes = 0; h.bridge.onRecordingChange(() => changes++);
|
||||||
|
h.failRecordings(true);
|
||||||
|
h.bridge.recordTo('transcribe-1');
|
||||||
|
const start = h.bridge.startGlassesCapture(h.session, h.cloud, () => {}, false, true); h.release(); await start;
|
||||||
|
h.native.pcmHandler(new Uint8Array([1, 2]));
|
||||||
|
h.native.pcmHandler(new Uint8Array([3, 4]));
|
||||||
|
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes, h.recordingAttempts()], [false, true, 2, 1]);
|
||||||
|
// Asking for the same recording again changes nothing; another one (Save Recording
|
||||||
|
// switched off and on) is a fresh attempt.
|
||||||
|
h.bridge.recordTo('transcribe-1');
|
||||||
|
h.native.pcmHandler(new Uint8Array([5]));
|
||||||
|
assert.equal(h.recordingAttempts(), 1);
|
||||||
|
h.failRecordings(false);
|
||||||
|
h.bridge.recordTo('');
|
||||||
|
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [false, false, 3]);
|
||||||
|
h.bridge.recordTo('transcribe-1');
|
||||||
|
h.native.pcmHandler(new Uint8Array([6]));
|
||||||
|
assert.deepEqual([h.bridge.recording, h.bridge.recordingFailed, changes], [true, false, 4]);
|
||||||
|
assert.deepEqual(h.recordings[0].pcm, [6]);
|
||||||
});
|
});
|
||||||
|
|
||||||
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
|
test('a glasses session ending drains continuous capture instead of discarding what it heard', async () => {
|
||||||
|
|||||||
@@ -21,4 +21,31 @@ class WavRecorderRepairTest {
|
|||||||
file.delete()
|
file.delete()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@Test
|
||||||
|
fun keepsTheHeaderCurrentWhileRecording() {
|
||||||
|
val file = File.createTempFile("recording", ".wav")
|
||||||
|
try {
|
||||||
|
val recorder = FaceclawWavRecorder(file.path, 16000, 1)
|
||||||
|
// Under a second of audio: still the empty header from the start.
|
||||||
|
recorder.append(ByteArray(16000))
|
||||||
|
assertEquals(BinaryEncoding.wavHeader(0, 16000, 1).toList(), file.readBytes().copyOf(44).toList())
|
||||||
|
// Past a second the header covers what is on disk, without waiting for finish(), and
|
||||||
|
// later audio still lands at the end.
|
||||||
|
recorder.append(ByteArray(16000) { 1 })
|
||||||
|
recorder.append(ByteArray(100) { 2 })
|
||||||
|
var bytes = file.readBytes()
|
||||||
|
assertEquals(44 + 32100, bytes.size)
|
||||||
|
assertEquals(BinaryEncoding.wavHeader(32000, 16000, 1).toList(), bytes.copyOf(44).toList())
|
||||||
|
assertEquals(0.toByte(), bytes[44])
|
||||||
|
assertEquals(1.toByte(), bytes[44 + 16000])
|
||||||
|
assertEquals(2.toByte(), bytes.last())
|
||||||
|
assertEquals(32100L, recorder.finish())
|
||||||
|
bytes = file.readBytes()
|
||||||
|
assertEquals(44 + 32100, bytes.size)
|
||||||
|
assertEquals(BinaryEncoding.wavHeader(32100, 16000, 1).toList(), bytes.copyOf(44).toList())
|
||||||
|
} finally {
|
||||||
|
file.delete()
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
@@ -63,7 +63,6 @@ class VoiceCaptureSessionTest {
|
|||||||
override val dispatcher = CallbackDispatcher { it() }
|
override val dispatcher = CallbackDispatcher { it() }
|
||||||
var packetListener: FaceclawAudioPacketListener? = null
|
var packetListener: FaceclawAudioPacketListener? = null
|
||||||
var g2Stops = 0
|
var g2Stops = 0
|
||||||
val recordings = mutableListOf<ByteArray>()
|
|
||||||
override fun onSessionStarting() {}
|
override fun onSessionStarting() {}
|
||||||
override fun isG2SessionReady(): Boolean = g2Ready
|
override fun isG2SessionReady(): Boolean = g2Ready
|
||||||
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
|
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
|
||||||
@@ -96,7 +95,6 @@ class VoiceCaptureSessionTest {
|
|||||||
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
|
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
|
||||||
override fun release() {}
|
override fun release() {}
|
||||||
}
|
}
|
||||||
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private val logs = mutableListOf<String>()
|
private val logs = mutableListOf<String>()
|
||||||
@@ -196,7 +194,6 @@ class VoiceCaptureSessionTest {
|
|||||||
val session = session(host, platform, "voice-test-onboard")
|
val session = session(host, platform, "voice-test-onboard")
|
||||||
session.setListener(events)
|
session.setListener(events)
|
||||||
session.setUsePhoneMic(true)
|
session.setUsePhoneMic(true)
|
||||||
session.setSaveRecordings(true)
|
|
||||||
session.start("onboard", 1)
|
session.start("onboard", 1)
|
||||||
assertTrue(events.stoppedLatch.await(10000))
|
assertTrue(events.stoppedLatch.await(10000))
|
||||||
assertTrue(mic.closed)
|
assertTrue(mic.closed)
|
||||||
@@ -211,8 +208,6 @@ class VoiceCaptureSessionTest {
|
|||||||
// Every segment handed to the model is at most 8 s and was peak-normalized.
|
// Every segment handed to the model is at most 8 s and was peak-normalized.
|
||||||
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
|
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
|
||||||
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
|
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
|
||||||
assertEquals(1, host.recordings.size)
|
|
||||||
assertEquals(170 * 800 * 2, host.recordings[0].size)
|
|
||||||
assertEquals(170, events.pcmChunks.size)
|
assertEquals(170, events.pcmChunks.size)
|
||||||
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
|
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
|
||||||
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
|
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
|
||||||
|
|||||||
@@ -7,7 +7,7 @@ const { pcmLevel } = loader()('app/util/audio-level.ts');
|
|||||||
function harness() {
|
function harness() {
|
||||||
const levelListeners = new Set();
|
const levelListeners = new Set();
|
||||||
let tick = null;
|
let tick = null;
|
||||||
const { MicLevelMeter, paintListeningHeader } = loader({
|
const { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon } = loader({
|
||||||
setInterval(fn) { tick = fn; return 1; },
|
setInterval(fn) { tick = fn; return 1; },
|
||||||
clearInterval() { tick = null; },
|
clearInterval() { tick = null; },
|
||||||
}, {
|
}, {
|
||||||
@@ -16,14 +16,15 @@ function harness() {
|
|||||||
} },
|
} },
|
||||||
})('app/ui/listening-header.ts');
|
})('app/ui/listening-header.ts');
|
||||||
const level = value => { for (const listener of levelListeners) listener(value); };
|
const level = value => { for (const listener of levelListeners) listener(value); };
|
||||||
return { MicLevelMeter, paintListeningHeader, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
|
return { MicLevelMeter, paintListeningHeader, RECORDING_ICON, withRecordingIcon, level, tick: () => tick?.(), ticking: () => tick !== null, levelListeners };
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
|
/** Records draws; text is 6px per character, like a fixed-width bitmap face. */
|
||||||
function fakeImage(width = 576) {
|
function fakeImage(width = 576) {
|
||||||
const texts = [], rects = [];
|
const texts = [], rects = [], images = [];
|
||||||
return { width, texts, rects,
|
return { width, texts, rects, images,
|
||||||
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
|
drawText(_font, x, y, text, gray) { texts.push({ x, y, text, gray }); },
|
||||||
|
drawImage(source, x, y) { images.push({ source, x, y }); },
|
||||||
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
|
fillRect(x, y, w, h, gray) { rects.push({ x, y, w, h, gray }); } };
|
||||||
}
|
}
|
||||||
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
|
const font = { ascent: 12, lineHeight: 16, measureText: text => text.length * 6 };
|
||||||
@@ -93,8 +94,47 @@ test('the header puts title and status on one line, truncated before the meter',
|
|||||||
// Segments sit on the text baseline.
|
// Segments sit on the text baseline.
|
||||||
assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent));
|
assert.ok(image.rects.every(rect => rect.y + rect.h === 20 + font.ascent));
|
||||||
|
|
||||||
|
assert.equal(image.images.length, 0, 'no recording circle unless the audio is being saved');
|
||||||
|
|
||||||
const idle = fakeImage(300);
|
const idle = fakeImage(300);
|
||||||
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null);
|
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null);
|
||||||
assert.equal(idle.rects.length, 0);
|
assert.equal(idle.rects.length, 0);
|
||||||
assert.equal(idle.texts[1].text, ' · Starting microphone...');
|
assert.equal(idle.texts[1].text, ' · Starting microphone...');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('while the audio is being saved, the recording circle sits beside the meter', () => {
|
||||||
|
const h = harness();
|
||||||
|
const meter = new h.MicLevelMeter();
|
||||||
|
const image = fakeImage(300);
|
||||||
|
h.paintListeningHeader(image, font, 'Translate', 'Listening (ElevenLabs)...', meter, true);
|
||||||
|
const meterX = Math.min(...image.rects.map(rect => rect.x));
|
||||||
|
assert.equal(meterX, 300 - 24 - meter.width, 'the meter stays where it was');
|
||||||
|
assert.equal(image.images.length, 1);
|
||||||
|
const [circle] = image.images;
|
||||||
|
assert.equal(circle.source, h.RECORDING_ICON);
|
||||||
|
assert.equal(circle.x + circle.source.width + 4, meterX);
|
||||||
|
// Centred on the meter, whose 7 rows sit on the text baseline.
|
||||||
|
const meterTop = Math.min(...image.rects.map(rect => rect.y));
|
||||||
|
assert.equal(circle.y + circle.source.height / 2, meterTop + 7 / 2);
|
||||||
|
const status = image.texts[1].text;
|
||||||
|
assert.ok(24 + 54 + status.length * 6 <= circle.x - 10, 'status stops short of the circle');
|
||||||
|
|
||||||
|
// The mic isn't live (starting, or waiting for the glasses): nothing is being saved to mark.
|
||||||
|
const idle = fakeImage(300);
|
||||||
|
h.paintListeningHeader(idle, font, 'Transcribe', 'Starting microphone...', null, true);
|
||||||
|
assert.equal(idle.images.length, 0);
|
||||||
|
});
|
||||||
|
|
||||||
|
test('a tray icon gains the recording circle to its right, vertically centred', () => {
|
||||||
|
const h = harness();
|
||||||
|
const { GrayImage } = loader()('app/graphics/image.ts');
|
||||||
|
const mic = new GrayImage(14, 15, 0);
|
||||||
|
mic.setPixel(0, 0, 200); mic.setPixel(13, 14, 200);
|
||||||
|
const icon = h.withRecordingIcon(mic);
|
||||||
|
assert.deepEqual([icon.width, icon.height], [14 + 2 + 9, 15]);
|
||||||
|
assert.deepEqual([icon.getPixel(0, 0), icon.getPixel(13, 14)], [200, 200]);
|
||||||
|
assert.deepEqual([icon.getPixel(14, 7), icon.getPixel(15, 7)], [0, 0], 'a gap separates them');
|
||||||
|
// The circle's widest row is the icon's middle row; its corners are clear.
|
||||||
|
for (let x = 16; x < 25; x++) assert.equal(icon.getPixel(x, 7), 220);
|
||||||
|
assert.deepEqual([icon.getPixel(16, 3), icon.getPixel(20, 3), icon.getPixel(20, 2), icon.getPixel(20, 12)], [0, 220, 0, 0]);
|
||||||
|
});
|
||||||
@@ -158,11 +158,11 @@ test('cloud fallback and a synchronous final still send exactly once', () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
/** The Android voice bridge over a fake native controller (on-device recognition only). */
|
/** The Android voice bridge over a fake native controller (on-device recognition only). */
|
||||||
function androidBridge() {
|
function androidBridge(modules = {}) {
|
||||||
const controller = {
|
const controller = {
|
||||||
starts: 0,
|
starts: 0,
|
||||||
setListener(listener) { this.listener = listener; },
|
setListener(listener) { this.listener = listener; },
|
||||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
setCommunicator() {}, setUsePhoneMic() {}, setEndpointing() {},
|
||||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||||
stop() { this.active = false; },
|
stop() { this.active = false; },
|
||||||
@@ -172,6 +172,7 @@ function androidBridge() {
|
|||||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||||
|
...modules,
|
||||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||||
FaceclawVoiceController: function () { return controller; },
|
FaceclawVoiceController: function () { return controller; },
|
||||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||||
@@ -215,3 +216,58 @@ test('a shared capture restarts for a holder that needs another provider, and is
|
|||||||
assert.equal(controller.active, true);
|
assert.equal(controller.active, true);
|
||||||
assert.equal(bridge.holdsCapture('ptt'), true);
|
assert.equal(bridge.holdsCapture('ptt'), true);
|
||||||
});
|
});
|
||||||
|
|
||||||
|
test('Android continuous captures record to the requested WAV, and report one that cannot be created', () => {
|
||||||
|
const recordings = [];
|
||||||
|
let fail = false, attempts = 0;
|
||||||
|
class FakeRecorder {
|
||||||
|
static open(name) {
|
||||||
|
attempts++;
|
||||||
|
if (fail) return null;
|
||||||
|
const r = new FakeRecorder(); r.name = name; r.path = `/Download/${name}.wav`; r.pcm = []; r.closed = false; recordings.push(r); return r;
|
||||||
|
}
|
||||||
|
append(pcm) { this.pcm.push(...pcm); }
|
||||||
|
close() { this.closed = true; }
|
||||||
|
}
|
||||||
|
const { bridge, controller } = androidBridge({
|
||||||
|
'./wav-recorder': { WavRecorder: FakeRecorder },
|
||||||
|
'./speech-pause': { SpeechPauseDetector: class { reset() {} accept() { return false; } } },
|
||||||
|
'../util/array-util': { toUint8Array: pcm => Uint8Array.from(pcm) },
|
||||||
|
'../util/audio-level': { pcmLevel: () => 0 },
|
||||||
|
});
|
||||||
|
const options = { provider: 'onboard' };
|
||||||
|
let changes = 0; bridge.onRecordingChange(() => changes++);
|
||||||
|
bridge.recordTo('transcribe-1');
|
||||||
|
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes], [true, false, 1]);
|
||||||
|
// Push-to-talk on a capture of its own is not recorded, and the file waits for audio to save.
|
||||||
|
bridge.startPushToTalk(options);
|
||||||
|
controller.listener.onPcm([9, 9]);
|
||||||
|
assert.equal(attempts, 0);
|
||||||
|
bridge.startContinuousCapture(options);
|
||||||
|
controller.listener.onPcm([1, 2]);
|
||||||
|
controller.listener.onPcm([3, 4]);
|
||||||
|
assert.deepEqual(recordings.map(r => [r.name, r.pcm, r.closed]), [['transcribe-1', [1, 2, 3, 4], false]]);
|
||||||
|
// The file outlives a capture: one restarted after the glasses reconnect keeps appending to it.
|
||||||
|
bridge.handleSessionEnded();
|
||||||
|
bridge.resumeCapture(options);
|
||||||
|
controller.listener.onPcm([5]);
|
||||||
|
assert.deepEqual(recordings.map(r => [r.pcm, r.closed]), [[[1, 2, 3, 4, 5], false]]);
|
||||||
|
bridge.recordTo('');
|
||||||
|
assert.deepEqual([recordings[0].closed, bridge.recording, changes], [true, false, 2]);
|
||||||
|
controller.listener.onPcm([6]);
|
||||||
|
assert.equal(recordings[0].pcm.length, 5);
|
||||||
|
|
||||||
|
// One attempt per recording, not one per packet; another recording is a fresh attempt.
|
||||||
|
fail = true; attempts = 0;
|
||||||
|
bridge.recordTo('transcribe-2');
|
||||||
|
controller.listener.onPcm([7]); controller.listener.onPcm([8]);
|
||||||
|
assert.deepEqual([bridge.recording, bridge.recordingFailed, changes, attempts], [false, true, 4, 1]);
|
||||||
|
bridge.recordTo('transcribe-2');
|
||||||
|
controller.listener.onPcm([9]);
|
||||||
|
assert.equal(attempts, 1);
|
||||||
|
fail = false;
|
||||||
|
bridge.recordTo('');
|
||||||
|
bridge.recordTo('transcribe-2');
|
||||||
|
controller.listener.onPcm([10]);
|
||||||
|
assert.deepEqual([bridge.recording, bridge.recordingFailed, recordings.at(-1).pcm], [true, false, [10]]);
|
||||||
|
});
|
||||||
Reference in new issue
Block a user