mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-20 13:03:14 +00:00
Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
cfda10d47f
|
||
|
|
55bd5f891c
|
No files matched your search
@@ -25,21 +25,18 @@ internal object AndroidSpeechEngines {
|
||||
private const val FEATURE_DIM = 80
|
||||
|
||||
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
|
||||
fun recognizerConfig(
|
||||
modelDir: File, kind: VoiceModelKind,
|
||||
whisper: WhisperModelFiles = WhisperModelFiles.forLanguage(null),
|
||||
): OfflineRecognizerConfig {
|
||||
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
|
||||
val modelConfig = OfflineModelConfig.builder()
|
||||
.setNumThreads(1)
|
||||
if (kind == VoiceModelKind.WHISPER) {
|
||||
modelConfig
|
||||
.setWhisper(OfflineWhisperModelConfig.builder()
|
||||
.setEncoder(File(modelDir, whisper.encoder).absolutePath)
|
||||
.setDecoder(File(modelDir, whisper.decoder).absolutePath)
|
||||
.setLanguage(whisper.language)
|
||||
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
|
||||
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
|
||||
.setLanguage("en")
|
||||
.setTask("transcribe")
|
||||
.build())
|
||||
.setTokens(File(modelDir, whisper.tokens).absolutePath)
|
||||
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
|
||||
} else {
|
||||
modelConfig
|
||||
.setMoonshine(OfflineMoonshineModelConfig.builder()
|
||||
|
||||
@@ -32,6 +32,17 @@ class FaceclawVoiceController(context: Context) {
|
||||
"decoder_model_merged.ort",
|
||||
"tokens.txt"
|
||||
)
|
||||
// Second on-device model: sherpa-onnx's offline Whisper backend (base.en,
|
||||
// int8-quantized -- see the model-choice note in asr-model.ts). Directory
|
||||
// shared with the TS-side download flow, same convention as ASR_MODEL_DIR.
|
||||
// Whisper re-encodes the whole buffer on every call, so the shared session
|
||||
// skips live partials for it; see VoiceCaptureSession.processRecognizer.
|
||||
private const val ASR_WHISPER_MODEL_DIR = "sherpa-onnx-whisper-base-en-int8"
|
||||
private val ASR_WHISPER_MODEL_FILES = arrayOf(
|
||||
"base.en-encoder.int8.onnx",
|
||||
"base.en-decoder.int8.onnx",
|
||||
"base.en-tokens.txt"
|
||||
)
|
||||
// 50 ms chunks match the G2 packet cadence the rest of the pipeline
|
||||
// (endpointing, transcript pacing) is tuned for.
|
||||
private const val PHONE_MIC_CHUNK_SAMPLES = SAMPLE_RATE / 20
|
||||
@@ -73,8 +84,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
@Volatile
|
||||
private var communicator: FaceclawBleCommunicator? = null
|
||||
private var whisperModel = WhisperModelFiles.forLanguage(null)
|
||||
private var frenchWhisper = false
|
||||
|
||||
private val host = object : VoiceCaptureHost {
|
||||
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
|
||||
@@ -113,7 +122,7 @@ class FaceclawVoiceController(context: Context) {
|
||||
|
||||
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber {
|
||||
val modelDir = findAsrModelDir(kind) ?: throw IllegalStateException("voice model missing")
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind, whisperModel))
|
||||
return AndroidSpeechEngines.SherpaTranscriber(AndroidSpeechEngines.recognizerConfig(modelDir, kind))
|
||||
}
|
||||
|
||||
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder {
|
||||
@@ -159,17 +168,6 @@ class FaceclawVoiceController(context: Context) {
|
||||
session.setOnboardModelKind(kind)
|
||||
}
|
||||
|
||||
/** Select the dedicated French fine-tune before setting the capture language. */
|
||||
fun setFrenchWhisper(enabled: Boolean) {
|
||||
frenchWhisper = enabled
|
||||
whisperModel = WhisperModelFiles.forSelection(whisperModel.language, enabled)
|
||||
}
|
||||
|
||||
/** Language for the next on-device Whisper capture; empty enables auto-detection. */
|
||||
fun setOnboardLanguage(language: String?) {
|
||||
whisperModel = WhisperModelFiles.forSelection(language, frenchWhisper)
|
||||
}
|
||||
|
||||
/**
|
||||
* When true, watch the decoded PCM and fire {@code onSpeechEnd} once the
|
||||
* speaker stops. Used by hands-free ("Hey Even") capture, which has no
|
||||
@@ -265,8 +263,8 @@ class FaceclawVoiceController(context: Context) {
|
||||
*/
|
||||
private fun findAsrModelDir(kind: VoiceModelKind): File? {
|
||||
val whisper = kind == VoiceModelKind.WHISPER
|
||||
val dirName = if (whisper) whisperModel.dirName else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) whisperModel.files else ASR_MODEL_FILES.toList()
|
||||
val dirName = if (whisper) ASR_WHISPER_MODEL_DIR else ASR_MODEL_DIR
|
||||
val fileNames = if (whisper) ASR_WHISPER_MODEL_FILES else ASR_MODEL_FILES
|
||||
val modelDir = File(appContext.filesDir, ASR_ROOT + File.separator + dirName)
|
||||
for (fileName in fileNames) {
|
||||
val file = File(modelDir, fileName)
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
* makes the content area exactly the 576x288 surface EvenHub apps expect.
|
||||
*/
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { EVENHUB_SCREEN_WIDTH } from "./compositor";
|
||||
import { EVENHUB_SCREEN_WIDTH, EVENHUB_SCREEN_HEIGHT } from "./compositor";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import {
|
||||
@@ -24,21 +24,26 @@ class EvenHubAppLayer implements Layer {
|
||||
* extension's source field should see that they came from the watch.
|
||||
*/
|
||||
readonly acceptsDirectional = true;
|
||||
private tallCanvas = false;
|
||||
|
||||
constructor(private readonly session: EvenHubSession) {}
|
||||
|
||||
setTallCanvas(tall: boolean): void {
|
||||
this.tallCanvas = tall;
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const size = ctx.stack.getBaseSize();
|
||||
// EvenHub apps lay out for the stock 576-wide surface. In the Full
|
||||
// Screen display mode the window is 640 wide; paint at stock width and
|
||||
// centre it rather than leaving all the slack on the right.
|
||||
if (size.width > EVENHUB_SCREEN_WIDTH) {
|
||||
const content = this.session.paint({ width: EVENHUB_SCREEN_WIDTH, height: size.height }, ctx.stack.isFocused());
|
||||
const image = new GrayImage(size.width, size.height, 0);
|
||||
content.composeInto(image, Math.floor((size.width - EVENHUB_SCREEN_WIDTH) / 2), 0);
|
||||
return image;
|
||||
}
|
||||
return this.session.paint(size, ctx.stack.isFocused());
|
||||
// Host full screen hides chrome; it does not enlarge the stock app canvas.
|
||||
// Only the app's explicit Faceclaw extended-layout request opts into height.
|
||||
const canvas = { width: EVENHUB_SCREEN_WIDTH, height: this.tallCanvas ? size.height : EVENHUB_SCREEN_HEIGHT };
|
||||
const content = this.session.paint(canvas, ctx.stack.isFocused());
|
||||
if (size.width === canvas.width && size.height === canvas.height) return content;
|
||||
const image = new GrayImage(size.width, size.height, 0);
|
||||
// Clip text to its logical canvas before placing it in the larger host window.
|
||||
image.bitBlt(content.withDrawsBaked(), Math.max(0, Math.floor((size.width - canvas.width) / 2)),
|
||||
Math.max(0, Math.floor((size.height - canvas.height) / 2)));
|
||||
return image;
|
||||
}
|
||||
|
||||
handleInput(event: InputEvent): void {
|
||||
@@ -60,6 +65,7 @@ export function createEvenHubWindow(
|
||||
requestShellRender: () => void,
|
||||
): InProcessWindow {
|
||||
const fallbackIcon = windowIcon("package", session.manifest.name.charAt(0).toUpperCase() || "E");
|
||||
const appLayer = new EvenHubAppLayer(session);
|
||||
const created = createInProcessWindow({
|
||||
appId,
|
||||
windowId,
|
||||
@@ -91,7 +97,7 @@ export function createEvenHubWindow(
|
||||
{ label: "Show phone UI", onSelect: (ctx: LayerContext) => { ctx.stack.pop(); onShowPhone(); } },
|
||||
],
|
||||
actions: options.actions,
|
||||
baseLayer: new EvenHubAppLayer(session),
|
||||
baseLayer: appLayer,
|
||||
submitFrame: options.submitFrame,
|
||||
setSurfaceVisible: options.setSurfaceVisible,
|
||||
removeSurface: options.removeSurface,
|
||||
@@ -109,7 +115,10 @@ export function createEvenHubWindow(
|
||||
pushOverlay: (layer) => created.stack.push(layer),
|
||||
// The extended layout uses the full 576x452 app area ("max"); stock apps
|
||||
// stay in the 576x288 band ("medium").
|
||||
setTallCanvas: (tall) => created.setHeightMode(tall ? "max" : "medium"),
|
||||
setTallCanvas: (tall) => {
|
||||
appLayer.setTallCanvas(tall);
|
||||
created.setHeightMode(tall ? "max" : "medium");
|
||||
},
|
||||
setAttention: (attention) => shell.setWindowAttention(windowId, attention),
|
||||
// Only while the app listens for text: otherwise the shell's Type Into App
|
||||
// destinations (voice dialog, phone keyboard, watch) would send into a void.
|
||||
|
||||
@@ -78,7 +78,7 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, openAiTranscriptionModelSetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -451,6 +451,8 @@ class DashboardController {
|
||||
openNotificationsWindow: () =>
|
||||
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
|
||||
onSwitcherBarHeightChanged: () => this.applySwitcherBarHeight(),
|
||||
onWindowLayoutChanged: () => this.applySwitcherBarHeight(),
|
||||
isLocked: () => this.glassesLocked,
|
||||
onScreenStateChanged: (on) => {
|
||||
// Any wake of the regular UI replaces a showing Glanceboard.
|
||||
if (on) this.glance.dismiss();
|
||||
@@ -483,7 +485,6 @@ class DashboardController {
|
||||
this.applyVerticalPositionIfChanged();
|
||||
this.applyAppLayoutsIfChanged();
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
this.retargetContinuousCapture();
|
||||
this.syncLockScreenSettingIfChanged();
|
||||
// A Preview color or scaling change should show on the mirror at once,
|
||||
// not at the next composited frame (cheap: floor-limited, dedup'd on
|
||||
@@ -2198,16 +2199,16 @@ class DashboardController {
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise use the shared transcription-language preference.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider,
|
||||
language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
|
||||
@@ -52,7 +52,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, openAiTranscriptionModelSetting, sonioxApiKeySetting, whisperServerApiKeySetting, getVoiceTranscriptionLanguage,
|
||||
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
||||
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings'
|
||||
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
@@ -221,6 +221,12 @@ export class IosPreviewController {
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
},
|
||||
onWindowLayoutChanged: () => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
},
|
||||
isLocked: () => this.glassesLocked,
|
||||
onScreenStateChanged: on => {
|
||||
if (on) this.glance.dismiss()
|
||||
// A wake unblanks only once the next shell render has staged an awake
|
||||
@@ -416,7 +422,6 @@ export class IosPreviewController {
|
||||
const phone = readPhoneBatteryState()
|
||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.syncContinuousCapture()
|
||||
this.syncLockSetting()
|
||||
this.pushBrightness()
|
||||
this.relayout()
|
||||
@@ -1199,13 +1204,14 @@ export class IosPreviewController {
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise use the shared language preference.
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : getVoiceTranscriptionLanguage(provider),
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), openAiTranscriptionModel: openAiTranscriptionModelSetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
|
||||
+2
-49
@@ -9,7 +9,7 @@ declare const java: any;
|
||||
* than bundled in the APK; FaceclawVoiceController reads the files from the
|
||||
* directory each model's `dirName` names.
|
||||
*
|
||||
* Downloadable models, same shape:
|
||||
* Two models, same shape:
|
||||
* - "moonshine": the original on-device option (three files). The model is
|
||||
* no longer bundled in the APK; it is fetched into the same filesDir
|
||||
* location that earlier releases copied the bundled files to, so
|
||||
@@ -29,20 +29,12 @@ declare const java: any;
|
||||
* used for Moonshine below. Every hash here was computed locally from a
|
||||
* freshly downloaded copy of the file, not copied from a doc page.
|
||||
*
|
||||
* - "whisper-base": multilingual base, int8-quantized. Used for non-English
|
||||
* speech and auto-detection; English-only downloads remain untouched.
|
||||
* SHA-256/size values come from the maintainer's LFS metadata, with the
|
||||
* vocabulary hash computed locally. URLs pin the source revision.
|
||||
*
|
||||
* - "whisper-small-fr": a dedicated French fine-tune, exported and quantized
|
||||
* with sherpa-onnx 1.13.0. Versioned release assets include its Apache license.
|
||||
*
|
||||
* Mirrors the on-phone assistant model flow in llama.ts, except each model
|
||||
* here is multiple files rather than one; they download sequentially through
|
||||
* FaceclawModelDownloader (resume + pinned sha256 per file).
|
||||
*/
|
||||
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base" | "whisper-small-fr";
|
||||
export type AsrModelId = "moonshine" | "whisper-base-en";
|
||||
|
||||
type AsrModelFile = {
|
||||
name: string;
|
||||
@@ -113,43 +105,6 @@ export const ASR_MODELS: Record<AsrModelId, AsrModelDef> = {
|
||||
],
|
||||
totalBytes: 160626066,
|
||||
},
|
||||
|
||||
"whisper-base": {
|
||||
label: "Whisper (multilingual, base)",
|
||||
dirName: "sherpa-onnx-whisper-base-int8",
|
||||
baseUrl: "https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/resolve/bb53ee204431c90d314c1cc08d28d23e5b7927cc/",
|
||||
files: [
|
||||
{
|
||||
name: "base-encoder.int8.onnx",
|
||||
sha256: "0b8fb1304b6109976038efff5ace81720e00386f3ff6b54ee8c75291ca0a1e11",
|
||||
sizeBytes: 29120534,
|
||||
},
|
||||
{
|
||||
name: "base-decoder.int8.onnx",
|
||||
sha256: "9759d217388a01b3a4c7c15533201067b48ae819c4daafc8624e64b9409dc02d",
|
||||
sizeBytes: 130672026,
|
||||
},
|
||||
{
|
||||
name: "base-tokens.txt",
|
||||
sha256: "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
||||
sizeBytes: 816730,
|
||||
},
|
||||
],
|
||||
totalBytes: 160609290,
|
||||
},
|
||||
"whisper-small-fr": {
|
||||
label: "Whisper (French, small)",
|
||||
dirName: "sherpa-onnx-whisper-small-fr-int8-v1",
|
||||
// Pinned export of bofenghuang/whisper-small-cv11-french; see docs/models/whisper-small-fr.
|
||||
baseUrl: "https://git.undefined-dev.fr/api/packages/Blendman974/generic/faceclaw-asr/whisper-small-fr-v1/",
|
||||
files: [
|
||||
{"name": "small-fr-encoder.int8.onnx", "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", "sizeBytes": 112413414},
|
||||
{"name": "small-fr-decoder.int8.onnx", "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", "sizeBytes": 262200764},
|
||||
{"name": "small-fr-tokens.txt", "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", "sizeBytes": 816730},
|
||||
{"name": "LICENSE.txt", "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "sizeBytes": 11358},
|
||||
],
|
||||
totalBytes: 375442266,
|
||||
},
|
||||
};
|
||||
|
||||
export type AsrModelState = {
|
||||
@@ -174,8 +129,6 @@ function freshRuntime(): ModelRuntime {
|
||||
const runtimes: Record<AsrModelId, ModelRuntime> = {
|
||||
moonshine: freshRuntime(),
|
||||
"whisper-base-en": freshRuntime(),
|
||||
"whisper-base": freshRuntime(),
|
||||
"whisper-small-fr": freshRuntime(),
|
||||
};
|
||||
|
||||
function modelDirPath(id: AsrModelId): string {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { CloudSttClient, CloudSttOptions } from "./cloud-stt";
|
||||
import { ElevenLabsSttClient } from "./elevenlabs-stt";
|
||||
import { DEFAULT_OPENAI_STT_MODEL, OpenAiRealtimeSttClient, type OpenAiSttModel } from "./openai-stt";
|
||||
import { OpenAiRealtimeSttClient } from "./openai-stt";
|
||||
import { ReconnectingSttClient } from "./reconnecting-stt";
|
||||
import { SonioxSttClient } from "./soniox-stt";
|
||||
import type { VoiceProviderKind } from "./voice-control";
|
||||
@@ -14,12 +14,11 @@ export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Also used by on-device Whisper.
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
@@ -53,7 +52,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
whisper: {
|
||||
name: "OpenAI",
|
||||
apiKey: (selection) => selection.openAiApiKey,
|
||||
create: (options, selection) => new OpenAiRealtimeSttClient({ ...options, model: selection.openAiTranscriptionModel }),
|
||||
create: (options) => new OpenAiRealtimeSttClient(options),
|
||||
},
|
||||
"whisper-server": {
|
||||
name: "Whisper server",
|
||||
@@ -77,11 +76,8 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language" | "openAiTranscriptionModel">): string {
|
||||
const fallback = selection.provider === "onboard-whisper-fr" ? "fr" : selection.provider === "onboard-whisper" ? "en" : "";
|
||||
const language = selection.provider === "onboard-whisper-fr" ? "fr" : (selection.language ?? fallback).trim().toLowerCase();
|
||||
const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : "";
|
||||
return `${selection.provider}:${language}${model}`;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
|
||||
@@ -3,7 +3,7 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
|
||||
/**
|
||||
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
||||
* on the Realtime API, with a configurable transcription model).
|
||||
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
||||
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
||||
*
|
||||
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
||||
@@ -16,11 +16,10 @@ import { openSocket, type SocketConnection } from "./socket";
|
||||
*/
|
||||
|
||||
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
||||
export const DEFAULT_OPENAI_STT_MODEL = "gpt-realtime-whisper";
|
||||
export type OpenAiSttModel = "gpt-realtime-whisper" | "gpt-4o-mini-transcribe" | "gpt-4o-transcribe";
|
||||
const MODEL_ID = "gpt-realtime-whisper";
|
||||
const TARGET_SAMPLE_RATE = 24000;
|
||||
|
||||
export type OpenAiSttOptions = CloudSttOptions & { model?: OpenAiSttModel };
|
||||
export type OpenAiSttOptions = CloudSttOptions;
|
||||
|
||||
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
private ws: SocketConnection | null = null;
|
||||
@@ -41,7 +40,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
this.open = true;
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "", this.options.model)));
|
||||
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
||||
for (const chunk of this.pendingChunks.splice(0)) {
|
||||
this.sendChunk(chunk);
|
||||
}
|
||||
@@ -166,9 +165,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
||||
return;
|
||||
}
|
||||
this.options.onError(message.error?.code === "model_not_found"
|
||||
? `OpenAI: ${this.options.model ?? DEFAULT_OPENAI_STT_MODEL} unavailable. Choose a model in Settings > Voice > OpenAI transcription model.`
|
||||
: `OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
@@ -204,7 +201,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient {
|
||||
|
||||
}
|
||||
|
||||
function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_STT_MODEL): object {
|
||||
function sessionConfig(language: string): object {
|
||||
return {
|
||||
type: "session.update",
|
||||
session: {
|
||||
@@ -212,7 +209,7 @@ function sessionConfig(language: string, model: OpenAiSttModel = DEFAULT_OPENAI_
|
||||
audio: {
|
||||
input: {
|
||||
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
||||
transcription: { model, ...(language ? { language } : {}) },
|
||||
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
||||
// Push-to-talk owns the utterance boundary; no server VAD.
|
||||
turn_detection: null,
|
||||
},
|
||||
|
||||
@@ -1,25 +0,0 @@
|
||||
/** Languages supported by both Whisper backends, in picker order. */
|
||||
export const TRANSCRIPTION_LANGUAGES = {
|
||||
en: "English", fr: "French", de: "German", es: "Spanish", it: "Italian",
|
||||
pt: "Portuguese", nl: "Dutch", ar: "Arabic", zh: "Chinese", cs: "Czech",
|
||||
da: "Danish", fi: "Finnish", el: "Greek", he: "Hebrew", hi: "Hindi",
|
||||
hu: "Hungarian", id: "Indonesian", ja: "Japanese", ko: "Korean",
|
||||
no: "Norwegian", pl: "Polish", ro: "Romanian", ru: "Russian", sk: "Slovak",
|
||||
sv: "Swedish", th: "Thai", tr: "Turkish", uk: "Ukrainian", vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPTION_LANGUAGES;
|
||||
|
||||
/** An empty language asks Whisper to detect it; default preserves each engine's old behavior. */
|
||||
export function resolveTranscriptionLanguage(
|
||||
choice: TranscriptionLanguage, provider: string, serverLanguage = "",
|
||||
): string {
|
||||
if (provider === "onboard-whisper-fr") return "fr";
|
||||
// Moonshine is English-only; Apple's on-device recognizer retains its own locale.
|
||||
if (provider === "onboard") return "";
|
||||
if (choice === "default") {
|
||||
if (provider === "onboard-whisper") return "en";
|
||||
return provider === "whisper-server" ? serverLanguage.trim().toLowerCase() : "";
|
||||
}
|
||||
return choice === "auto" ? "" : choice;
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { OpenAiSttModel } from "./openai-stt";
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt";
|
||||
@@ -25,14 +24,14 @@ export type VoiceControlState = {
|
||||
detail: string;
|
||||
};
|
||||
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (a configurable transcription model,
|
||||
// "whisper" (no "onboard-" prefix) is the OpenAI CLOUD provider (gpt-realtime-whisper,
|
||||
// see openai-stt.ts) -- an unfortunate pre-existing name collision with the model
|
||||
// family, not with "onboard-whisper" below, which is the on-device sherpa-onnx
|
||||
// Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than
|
||||
// renamed, since it's a persisted settings value on real installs already.
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP
|
||||
// (whisper-server-stt.ts).
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
export type VoiceTranscriptEvent = CloudSttTranscriptEvent & {
|
||||
/**
|
||||
@@ -55,12 +54,11 @@ export type PushToTalkOptions = {
|
||||
provider: VoiceProviderKind;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
openAiTranscriptionModel?: OpenAiSttModel;
|
||||
sonioxApiKey: string;
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerApiKey?: string;
|
||||
/** Spoken language for cloud providers and on-device Whisper; empty detects, absent keeps local English. */
|
||||
/** Spoken language for cloud providers (see VoiceProviderSelection.language). */
|
||||
language?: string;
|
||||
saveRecording: boolean;
|
||||
/**
|
||||
@@ -397,10 +395,7 @@ export class FaceclawVoiceControlBridge {
|
||||
this.started = true;
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
const frenchWhisper = options.provider === "onboard-whisper-fr";
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" || frenchWhisper ? "whisper" : "moonshine");
|
||||
this.controller?.setFrenchWhisper(frenchWhisper);
|
||||
this.controller?.setOnboardLanguage(frenchWhisper ? "fr" : options.language ?? "en");
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
|
||||
@@ -1,4 +1,3 @@
|
||||
import type { OpenAiSttModel } from "../native/openai-stt";
|
||||
import { normalizeNightscoutThreshold, type NightscoutThresholds } from "../apps/nightscout/nightscout-alerts";
|
||||
import { GESTURE_DOUBLE_CLICK, InputEvent } from "./gestures";
|
||||
import { DEFAULT_BRIGHTNESS_CURVE, normalizeBrightnessCurve, brightnessCurveError } from "../g2/brightness-curve";
|
||||
@@ -19,7 +18,6 @@ import {
|
||||
} from "~/assistant/models";
|
||||
import { isLocalModelReady } from "../native/local-llm";
|
||||
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
|
||||
import { TRANSCRIPTION_LANGUAGES, resolveTranscriptionLanguage, type TranscriptionLanguage } from "../native/transcription-language";
|
||||
import { normalizeOpenAiCompatibleUrl } from "../native/openai-chat";
|
||||
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
|
||||
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
|
||||
@@ -150,6 +148,39 @@ export class ConfigSettingBoolean<TId extends string = string> extends ConfigSet
|
||||
}
|
||||
}
|
||||
|
||||
const appFullscreenSettings = new Map<string, ConfigSettingBoolean>();
|
||||
|
||||
/** A user choice independent of an app's own temporary immersive mode. */
|
||||
export function appFullscreenSetting(appId: string): ConfigSettingBoolean {
|
||||
let setting = appFullscreenSettings.get(appId);
|
||||
if (!setting) {
|
||||
setting = new ConfigSettingBoolean({
|
||||
id: `app-fullscreen-${appId}`, label: "Full screen",
|
||||
storageKey: `display.fullscreen.${appId}`, defaultValue: false,
|
||||
reportInAnalytics: false,
|
||||
});
|
||||
appFullscreenSettings.set(appId, setting);
|
||||
}
|
||||
return setting;
|
||||
}
|
||||
|
||||
const appKeepAwakeSettings = new Map<string, ConfigSettingBoolean>();
|
||||
|
||||
/** Independent of full screen; applies only while this app is visible. */
|
||||
export function appKeepAwakeSetting(appId: string): ConfigSettingBoolean {
|
||||
let setting = appKeepAwakeSettings.get(appId);
|
||||
if (!setting) {
|
||||
setting = new ConfigSettingBoolean({
|
||||
id: `app-keep-awake-${appId}`, label: "Keep screen awake",
|
||||
description: "Prevent automatic sleep while this app is visible, in any display mode.",
|
||||
storageKey: `display.keepAwake.${appId}`, defaultValue: false,
|
||||
reportInAnalytics: false,
|
||||
});
|
||||
appKeepAwakeSettings.set(appId, setting);
|
||||
}
|
||||
return setting;
|
||||
}
|
||||
|
||||
type ConfigSettingEnumOptions<TValue extends string, TId extends string> = ConfigSettingOptions<TValue, TId> & {
|
||||
values: readonly TValue[];
|
||||
normalize?: (value: string | null | undefined) => TValue;
|
||||
@@ -884,14 +915,13 @@ export const ringConnectionModeSetting = new ConfigSettingEnum<RingConnectionMod
|
||||
// keeps its name (it's a persisted setting on real installs) but its label
|
||||
// below now says "OpenAI" to tell the two apart in the picker.
|
||||
// "whisper-server" is a self-hosted OpenAI-compatible Whisper (e.g. vLLM).
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
export type VoiceProvider = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server";
|
||||
|
||||
// On iOS "onboard" is Apple's on-device recognizer, and there is no
|
||||
// "onboard-whisper" (the sherpa-onnx models are Android-only).
|
||||
const voiceProviderLabels: Record<VoiceProvider, string> = {
|
||||
onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)",
|
||||
"onboard-whisper": "On-device (Whisper)",
|
||||
"onboard-whisper-fr": "On-device (Whisper French)",
|
||||
elevenlabs: "ElevenLabs",
|
||||
whisper: "OpenAI (Whisper)",
|
||||
soniox: "Soniox",
|
||||
@@ -905,7 +935,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
defaultValue: "onboard",
|
||||
values: global.isIOS
|
||||
? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"]
|
||||
: ["onboard", "onboard-whisper", "onboard-whisper-fr", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
: ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"],
|
||||
formatValue: (value) => voiceProviderLabels[value] ?? value,
|
||||
isDisabled: (value) => {
|
||||
if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0;
|
||||
@@ -916,7 +946,7 @@ export const voiceProviderSetting = new ConfigSettingEnum<VoiceProvider>({
|
||||
},
|
||||
description: global.isIOS
|
||||
? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone."
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The On-device options need their voice model downloaded (below) and never leave the phone. Whisper French uses the French-only small model and always transcribes in French.",
|
||||
: "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.",
|
||||
});
|
||||
|
||||
const wakeWordActionLabels: Record<WakeWordAction, string> = {
|
||||
@@ -1139,35 +1169,9 @@ export const whisperServerModelSetting = new ConfigSettingString({
|
||||
description: "Model id sent to the Whisper server. Leave empty to use the first model the server lists.",
|
||||
});
|
||||
|
||||
export const openAiTranscriptionModelSetting = new ConfigSettingEnum<OpenAiSttModel>({
|
||||
id: "openai-transcription-model",
|
||||
label: "OpenAI transcription model",
|
||||
storageKey: "voice.openAiTranscriptionModel",
|
||||
defaultValue: "gpt-realtime-whisper",
|
||||
values: ["gpt-realtime-whisper", "gpt-4o-mini-transcribe", "gpt-4o-transcribe"],
|
||||
description: "Cloud speech-to-text for Translate, Transcribe and dictation. Choose a model available to your OpenAI project. This does not change the Assistant translation model. The transcribe models return text after each committed audio segment.",
|
||||
});
|
||||
|
||||
export const voiceLanguageSetting = new ConfigSettingEnum<TranscriptionLanguage>({
|
||||
id: "voice-language",
|
||||
label: "Transcription language",
|
||||
storageKey: "voice.language",
|
||||
defaultValue: "default",
|
||||
values: ["default", "auto", ...(Object.keys(TRANSCRIPTION_LANGUAGES) as (keyof typeof TRANSCRIPTION_LANGUAGES)[])],
|
||||
formatValue: (value) => value === "default" ? "Provider default"
|
||||
: value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value],
|
||||
description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " +
|
||||
"Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " +
|
||||
"On-device Whisper needs the multilingual model for auto-detection or languages other than English. The dedicated Whisper French provider always uses French, without changing this preference. Moonshine remains English-only.",
|
||||
});
|
||||
|
||||
export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string {
|
||||
return resolveTranscriptionLanguage(voiceLanguageSetting.get(), provider, whisperServerLanguageSetting.get());
|
||||
}
|
||||
|
||||
export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
id: "whisper-server-language",
|
||||
label: "Whisper server language",
|
||||
label: "Whisper language",
|
||||
storageKey: "voice.whisperServerLanguage",
|
||||
defaultValue: "",
|
||||
editorTitle: "Language code, e.g. en or sk (empty: auto)",
|
||||
@@ -1175,7 +1179,7 @@ export const whisperServerLanguageSetting = new ConfigSettingString({
|
||||
formatValue: (value) => value || "Auto-detect",
|
||||
validate: (value) => (!value.trim() || /^[a-z]{2,3}$/i.test(value.trim()) ? null : "Use a language code like en, sk or cs."),
|
||||
description:
|
||||
"Spoken language for the Whisper server when Transcription language is Provider default. Leave empty to auto-detect.",
|
||||
"Spoken language for the Whisper server. Leave empty to auto-detect; a wrong fixed language makes Whisper translate into it instead of transcribing.",
|
||||
});
|
||||
|
||||
export const whisperServerApiKeySetting = new ConfigSettingString({
|
||||
|
||||
@@ -63,7 +63,6 @@ import {
|
||||
mapboxApiKeySetting,
|
||||
mirrorTouchSetting,
|
||||
openAiApiKeySetting,
|
||||
openAiTranscriptionModelSetting,
|
||||
previewColorSetting,
|
||||
previewScalingSetting,
|
||||
phoneRotationSetting,
|
||||
@@ -88,7 +87,6 @@ import {
|
||||
toggleSettingMenuItem,
|
||||
useMicControlSetting,
|
||||
voiceProviderSetting,
|
||||
voiceLanguageSetting,
|
||||
screenTimeoutSetting,
|
||||
wakeToLastAppSetting,
|
||||
wakeWordActionSetting,
|
||||
@@ -145,12 +143,9 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
enumSettingMenuItem(voiceLanguageSetting),
|
||||
...whisperServerMenuItems(),
|
||||
asrModelMenuItem("moonshine"),
|
||||
asrModelMenuItem("whisper-base-en"),
|
||||
asrModelMenuItem("whisper-base"),
|
||||
asrModelMenuItem("whisper-small-fr"),
|
||||
],
|
||||
},
|
||||
{
|
||||
@@ -289,7 +284,7 @@ function settingsSections(): SettingsSection[] {
|
||||
toggleSettingMenuItem(showBleBandwidthSetting)] };
|
||||
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
|
||||
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting), enumSettingMenuItem(voiceLanguageSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
|
||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||
item.label !== screenTimeoutSetting.label) };
|
||||
return section;
|
||||
@@ -532,7 +527,6 @@ function asrModelStatusText(id: AsrModelId): string {
|
||||
*/
|
||||
function whisperServerMenuItems(): MenuItem[] {
|
||||
return [
|
||||
enumSettingMenuItem(openAiTranscriptionModelSetting),
|
||||
textSettingMenuItem(whisperServerUrlSetting),
|
||||
textSettingMenuItem(whisperServerModelSetting),
|
||||
textSettingMenuItem(whisperServerLanguageSetting),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../../graphics/image";
|
||||
import {
|
||||
appSwitcherPositionSetting,
|
||||
appFullscreenSetting,
|
||||
type AppSwitcherPosition,
|
||||
displayLayoutSetting,
|
||||
type DisplayLayout,
|
||||
@@ -118,7 +119,7 @@ export function setAppImmersive(appId: string, immersive: boolean): boolean {
|
||||
}
|
||||
|
||||
export function isImmersive(appId?: string): boolean {
|
||||
return appId !== undefined && immersiveApps.has(appId);
|
||||
return appId !== undefined && (immersiveApps.has(appId) || appFullscreenSetting(appId).get());
|
||||
}
|
||||
|
||||
/** Whether an app's windows span the panel's whole width: Full Screen, or immersive. */
|
||||
|
||||
+66
-13
@@ -31,6 +31,9 @@ import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
appFullscreenSetting,
|
||||
appKeepAwakeSetting,
|
||||
toggleSettingMenuItem,
|
||||
assistantHermesKeySetting,
|
||||
assistantHermesUrlSetting,
|
||||
assistantBackendSetting,
|
||||
@@ -94,7 +97,8 @@ import type { AppSwitcherPosition } from "../dashboard-settings";
|
||||
* Input flow: every event enters via receiveInput. The shell consumes
|
||||
* everything while the sidebar or a shell overlay has focus and forwards the
|
||||
* rest to the focused window. Long-press opens the shell-owned system menu
|
||||
* (Focus app switcher, Voice input, Brightness, Close window, Debug) without reaching the
|
||||
* (Focus app switcher, Voice input, Brightness, Close current app, Close all apps,
|
||||
* Debug) without reaching the
|
||||
* app, so the shell keeps working when a window's handler hangs; a window
|
||||
* that claims long-press for a move of its own gets it forwarded instead, and
|
||||
* holding the press past the escape threshold still opens the system menu.
|
||||
@@ -109,7 +113,7 @@ export type ShellWindow = {
|
||||
title: string;
|
||||
/** Compositor surface this window renders to; configured at connect / launch. */
|
||||
surfaceId: string;
|
||||
/** Whether the system menu offers Close window (the launcher is pinned). */
|
||||
/** Whether the system menu can close this window (the launcher is pinned). */
|
||||
closeable: boolean;
|
||||
/**
|
||||
* True when tap-then-hold currently opens the window's own context menu
|
||||
@@ -222,6 +226,10 @@ export type ShellConfig = {
|
||||
* after the shell paint that found it.
|
||||
*/
|
||||
onSwitcherBarHeightChanged?: () => void;
|
||||
/** A per-app full-screen choice changed the viewport and compositor geometry. */
|
||||
onWindowLayoutChanged?: () => void;
|
||||
/** An app's keep-awake preference must not keep the lock screen awake. */
|
||||
isLocked?: () => boolean;
|
||||
/**
|
||||
* The Notifications app's window, opened in the background (neither
|
||||
* foregrounded nor focused) unless it is open already, for the switcher
|
||||
@@ -655,6 +663,12 @@ class Shell {
|
||||
if (window) this.closeWindow(window.windowId);
|
||||
}
|
||||
|
||||
/** Snapshot ids: close callbacks may remove themselves or other windows. */
|
||||
closeAllWindows(): void {
|
||||
const ids = this.windows.filter(window => window.closeable).map(window => window.windowId);
|
||||
for (const id of ids) this.closeWindow(id);
|
||||
}
|
||||
|
||||
getWindows(): readonly ShellWindow[] {
|
||||
return this.windows;
|
||||
}
|
||||
@@ -866,7 +880,9 @@ class Shell {
|
||||
// A foreground window that is playing video holds it off the same way.
|
||||
const foreground = this.foregroundWindow();
|
||||
if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() ||
|
||||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) {
|
||||
foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.() ||
|
||||
(foreground?.closeable && !this.config.isLocked?.() &&
|
||||
appKeepAwakeSetting(foreground.appId).get())) {
|
||||
this.lastInputAtMs = nowMs;
|
||||
return false;
|
||||
}
|
||||
@@ -1073,6 +1089,12 @@ class Shell {
|
||||
if (this.activeVoiceLayer || !this.stack.isAtBase()) {
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
// The switcher owns its menu even when the selected app uses a hold
|
||||
// for a game move or voice capture.
|
||||
if (this.focus === "sidebar") {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
const window = this.foregroundWindow();
|
||||
if (!window) {
|
||||
return { shell: true, window: false };
|
||||
@@ -1082,10 +1104,6 @@ class Shell {
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
if (!window.holdToTalk) this.startEscapeMenuTimer();
|
||||
if (this.focus === "sidebar") {
|
||||
this.focus = "window";
|
||||
window.onFocus?.(this.lastInput);
|
||||
}
|
||||
// The window owns frameId from here (render or explicit finish).
|
||||
await window.handleInput(event, frameId);
|
||||
return { shell: true, window: true };
|
||||
@@ -1096,6 +1114,10 @@ class Shell {
|
||||
// first), which answers with its own menu or asks for the system menu
|
||||
// when it has none.
|
||||
if (event.type === "short-then-long-press") {
|
||||
if (this.focus === "sidebar" && !this.activeVoiceLayer && this.stack.isAtBase()) {
|
||||
this.openEscapeMenu();
|
||||
return { shell: true, window: false };
|
||||
}
|
||||
const window = this.foregroundWindow();
|
||||
if (window?.holdToTalk) {
|
||||
this.openEscapeMenu();
|
||||
@@ -1913,7 +1935,7 @@ class Shell {
|
||||
|
||||
/**
|
||||
* The system/escape menu: the entries every window shares (Focus app
|
||||
* switcher, Voice input, Brightness, Close window) plus Debug. Shell-owned and
|
||||
* switcher, Voice input, Brightness, Close current app, Close all apps) plus Debug. Shell-owned and
|
||||
* shell-drawn (never the app's), so an unresponsive app can always be
|
||||
* closed. It opens for long-press (over the app's own menu too), after an
|
||||
* extended hold in a window that claims long-press, and on a window's
|
||||
@@ -1926,18 +1948,17 @@ class Shell {
|
||||
const items: MenuItem[] = [];
|
||||
if (foreground.closeable) {
|
||||
items.push({
|
||||
label: "Close window",
|
||||
label: "Close current app",
|
||||
onSelect: (ctx) => {
|
||||
// Pop the menu first (its onRemoved returns focus to the sidebar),
|
||||
// then close the window the menu was opened over.
|
||||
ctx.stack.pop();
|
||||
this.closeForegroundWindow();
|
||||
this.closeWindow(foreground.windowId);
|
||||
},
|
||||
});
|
||||
}
|
||||
// Close window sits first but the menu opens on Focus app switcher, so a
|
||||
// reflexive tap never closes the window.
|
||||
const initialSelection = items.length;
|
||||
// Prefer closing the current app; pinned windows fall back to the switcher.
|
||||
const initialSelection = 0;
|
||||
items.push(
|
||||
{
|
||||
// Defocus the app (hand focus to the sidebar) without closing it: the
|
||||
@@ -1959,6 +1980,38 @@ class Shell {
|
||||
},
|
||||
}]),
|
||||
);
|
||||
items.push({
|
||||
label: "Close all apps",
|
||||
disabled: () => !this.windows.some(window => window.closeable),
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
this.closeAllWindows();
|
||||
},
|
||||
});
|
||||
if (foreground.closeable) {
|
||||
const fullscreen = appFullscreenSetting(foreground.appId);
|
||||
items.push({
|
||||
label: fullscreen.get() ? "Exit full screen" : "Enter full screen",
|
||||
description: "Use the whole display for this app.",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
if (!this.windows.some(window => window.windowId === foreground.windowId)) return;
|
||||
fullscreen.set(!fullscreen.get());
|
||||
this.noteUserActivity();
|
||||
this.config.onWindowLayoutChanged?.();
|
||||
this.focusWindow(foreground.windowId);
|
||||
this.config.requestShellRender();
|
||||
},
|
||||
});
|
||||
items.push(toggleSettingMenuItem(appKeepAwakeSetting(foreground.appId), {
|
||||
onChange: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
this.noteUserActivity();
|
||||
this.focusWindow(foreground.windowId);
|
||||
this.config.requestShellRender();
|
||||
},
|
||||
}));
|
||||
}
|
||||
if (brightnessSetting.get() !== "auto") {
|
||||
items.push({
|
||||
label: "Brightness",
|
||||
|
||||
@@ -9,7 +9,7 @@ import type { WorkerAppReply } from "./shell/worker-window";
|
||||
* The window context menu, opened by tap-then-hold: an app's own menu,
|
||||
* holding its app-specific actions. The shell's system menu (long-press)
|
||||
* carries the entries every window shares — Focus app switcher, Voice input,
|
||||
* Close window — so an app with nothing of its own to offer has no menu of
|
||||
* Close current app, Close all apps — so an app with nothing of its own to offer has no menu of
|
||||
* its own: tap-then-hold then opens the system menu in its place, so both
|
||||
* gestures land on the same menu. The system menu is also the safety net:
|
||||
* shell-owned, so an unresponsive app can always be closed (a window that
|
||||
|
||||
@@ -1,202 +0,0 @@
|
||||
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -1,71 +0,0 @@
|
||||
# On-device Whisper French
|
||||
|
||||
Android Settings → Voice now offers `On-device (Whisper French)` and a separate
|
||||
`On-device model: Whisper (French, small)` download (~375 MB decimal). Download
|
||||
first, then choose the provider. Recognition stays on the phone; no API key is
|
||||
required. This is speech-to-text in French, not translation.
|
||||
|
||||
This provider always requests `language=fr`, `task=transcribe`, without writing
|
||||
the shared language preference. Switching back to existing Moonshine, English
|
||||
Whisper or multilingual Whisper restores their previous behavior. Downloads,
|
||||
cancellation/resume and deletion reuse the existing model manager and leave
|
||||
other model directories untouched. Missing French files do not silently select
|
||||
multilingual base. iOS and the Microphones app's separate Moonshine caption
|
||||
engine are outside this addition.
|
||||
|
||||
## Provenance and distribution
|
||||
|
||||
- Source: https://huggingface.co/bofenghuang/whisper-small-cv11-french
|
||||
- Pinned revision: `83d7df2928fb2218721c543dcc0be29ef8db2f96`.
|
||||
- The model author's declared license is Apache-2.0; `LICENSE.txt` is distributed
|
||||
with the downloadable model. The original Whisper implementation is MIT;
|
||||
sherpa-onnx's exporter is Apache-2.0.
|
||||
- Exporter: sherpa-onnx v1.13.0 `scripts/whisper/export-onnx.py`.
|
||||
- Export: strict tensor-name conversion from HF Whisper to OpenAI Whisper,
|
||||
then upstream sherpa tensor-cache export, opset 17, dynamic int8 MatMul.
|
||||
- Assets: https://git.undefined-dev.fr/Blendman974/faceclaw/releases/tag/models-whisper-small-fr-v1
|
||||
- Model files use the public Gitea generic package registry (`faceclaw-asr`,
|
||||
version `whisper-small-fr-v1`), because the encoder and decoder exceed the
|
||||
instance’s 100 MB attachment limit. No infrastructure limit is changed.
|
||||
- The JSON manifest records file lengths and SHA-256 hashes. Both the source
|
||||
checkpoint and exporter are checksum-verified before execution. The existing
|
||||
Android downloader checks each downloaded asset against its pinned hash.
|
||||
- Only encoder, decoder, vocabulary and license are downloaded to the phone;
|
||||
the original checkpoint and unquantized intermediate models are not shipped.
|
||||
|
||||
## Reproducing the export
|
||||
|
||||
Use an isolated Python environment with the versions used for this release:
|
||||
Python 3.14, torch 2.9.0+cpu (CPU wheel index), openai-whisper 20250625,
|
||||
onnx 1.23.2, onnxruntime 1.31.1 and sherpa-onnx 1.13.0. These are development
|
||||
requirements only; no Python or additional runtime is added to the app.
|
||||
On NixOS expose the host C++ runtime and zlib through LD_LIBRARY_PATH.
|
||||
|
||||
```sh
|
||||
python tools/export-whisper-small-fr.py --output /tmp/whisper-small-fr
|
||||
python tools/test-whisper-small-fr.py \
|
||||
--model-dir /tmp/whisper-small-fr --wav /path/to/french-test.wav
|
||||
```
|
||||
|
||||
The exporter downloads the pinned source (~484 MB), writes unquantized and
|
||||
quantized intermediate models, and produces `manifest.json`. Artifact changes
|
||||
require a new versioned release and explicit catalog hash changes; never replace
|
||||
v1 assets in place. These scripts do not upload or publish files automatically.
|
||||
|
||||
## Validation
|
||||
|
||||
The exported int8 files were actually decoded on CPU with sherpa-onnx 1.13.0,
|
||||
the same version used by Android. A public 4.97-second French sample from
|
||||
`csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3/test_wavs/fr.wav` produced:
|
||||
|
||||
> Ne vous demandez pas ce que votre pays peut faire pour vous, demandez-vous plutôt ce que vous pouvez faire pour lui.
|
||||
|
||||
This is a smoke test, not a recognition-quality benchmark or a measurement of
|
||||
Pixel latency. Application tests cover provider persistence, French selection
|
||||
without rewriting other languages, switching back, capture sharing, interrupted
|
||||
download/resume, file hashes and independent deletion. Kotlin tests cover the
|
||||
native file selection and preservation of existing filenames/defaults.
|
||||
|
||||
Hardware checks remain pending: real French dictation through phone and glasses
|
||||
microphones, latency and RAM on the Pixel, airplane-mode use after downloading,
|
||||
reconnection and long utterances. No firmware changes are required.
|
||||
@@ -1,29 +0,0 @@
|
||||
{
|
||||
"source": "bofenghuang/whisper-small-cv11-french",
|
||||
"revision": "83d7df2928fb2218721c543dcc0be29ef8db2f96",
|
||||
"sourceSha256": "855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb",
|
||||
"exporterSha256": "5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45",
|
||||
"torchVersion": "2.9.0+cpu",
|
||||
"files": [
|
||||
{
|
||||
"name": "small-fr-encoder.int8.onnx",
|
||||
"sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d",
|
||||
"sizeBytes": 112413414
|
||||
},
|
||||
{
|
||||
"name": "small-fr-decoder.int8.onnx",
|
||||
"sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0",
|
||||
"sizeBytes": 262200764
|
||||
},
|
||||
{
|
||||
"name": "small-fr-tokens.txt",
|
||||
"sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126",
|
||||
"sizeBytes": 816730
|
||||
},
|
||||
{
|
||||
"name": "LICENSE.txt",
|
||||
"sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30",
|
||||
"sizeBytes": 11358
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
# Per-app full screen and keep awake
|
||||
|
||||
Long-press opens the system menu. Closeable apps offer Enter full screen / Exit
|
||||
full screen. The choice persists per app through the existing settings store,
|
||||
independently of global display layout and app-controlled immersive mode.
|
||||
Full screen fills the 640x480 panel, hides chrome while the app has focus, and
|
||||
resizes existing in-process and worker windows without restarting the app.
|
||||
The switcher remains accessible through the system menu.
|
||||
|
||||
Stock EvenHub apps retain their logical 576x288 canvas in every host display
|
||||
mode, including full screen. Content is centered at native pixel scale and
|
||||
clipped to that canvas; extra space does not change wrapping or layout.
|
||||
Only an app's explicit Faceclaw extended-layout request enables a taller canvas.
|
||||
Regression tests compare the complete stock raster against its centered
|
||||
full-screen placement, including text crossing the logical canvas boundary.
|
||||
|
||||
Keep screen awake is a separate On/Off switch immediately after the full-screen
|
||||
action in the system menu. It defaults off and persists per app independently
|
||||
of full screen. Existing full-screen preferences remain unchanged and do not
|
||||
implicitly enable keep awake.
|
||||
|
||||
Automatic idle sleep is suspended only while an app with Keep screen awake
|
||||
enabled is in the foreground, in any display mode. A background app does not
|
||||
hold the display awake. Full screen alone does not suppress idle sleep.
|
||||
The lock screen and explicit sleep still work. The inactivity countdown starts
|
||||
fresh after disabling keep awake or leaving a kept-awake app. Existing automatic
|
||||
holds for voice capture, assistant turns and video playback remain in effect.
|
||||
|
||||
Software regressions cover geometry with every switcher position, preserving
|
||||
other display settings and persistence after reload, captured menu targets,
|
||||
exit, independent keep-awake persistence, foreground/background idle behavior,
|
||||
lock timeout and manual sleep.
|
||||
|
||||
On glasses, verify entry/exit from both in-process apps and worker apps,
|
||||
reflow, state retention, idle timeout, app switching, lock/off-head behavior,
|
||||
and reconnection. These hardware checks remain pending. No firmware change.
|
||||
@@ -0,0 +1,34 @@
|
||||
# App switcher close menu
|
||||
|
||||
Feature branch: `feat/app-controls`, based on `main` so it can be
|
||||
reviewed independently. The combined build lives on `build/all-features`.
|
||||
|
||||
With the app switcher focused, long-press or tap-then-hold opens the existing
|
||||
shell-owned System menu. “Close current app” closes the selected window when it
|
||||
is closeable. It is absent for the pinned launcher (the large application grid).
|
||||
“Close all apps” closes every closeable window, including background windows and
|
||||
multiple windows belonging to one app. Pinned windows remain available.
|
||||
|
||||
Both actions reuse the shell's normal close/cleanup path. The menu closes before
|
||||
the windows do, and focus returns to the switcher. It selects “Close current app” by default when available, otherwise “Focus app switcher”. With no closeable windows, “Close all apps” is disabled.
|
||||
No new preferences or persistence format are introduced.
|
||||
|
||||
Software: `tests/app-switcher-menu.test.cjs` covers both gestures, apps claiming
|
||||
holds/hold-to-talk, launcher exclusion, captured close targets, cancellation,
|
||||
default selection, multiple windows, reentrant removal, callback errors and
|
||||
screen-off behavior. Existing in-app gesture routing remains unchanged.
|
||||
|
||||
Manual validation on glasses remains pending:
|
||||
|
||||
1. Open several apps, focus the app switcher, and open the menu with both gestures.
|
||||
2. Close the selected app; verify its surface disappears and other apps remain.
|
||||
3. Select the large application grid; verify “Close current app” is absent.
|
||||
4. Use “Close all apps”; verify the launcher remains usable and apps can reopen.
|
||||
5. Repeat for left/right/bottom/popup switcher positions and with a running game,
|
||||
a hold-to-talk app and an app exposing multiple windows.
|
||||
6. Check cancel/back closes nothing; confirming the initial selection closes the current app when available. Check both
|
||||
menu gestures still work normally when focus is inside an app.
|
||||
|
||||
Closing windows retains each app's existing lifecycle semantics; it does not
|
||||
uninstall apps or reset their preferences. Existing app-specific shutdown behavior
|
||||
(such as a remote terminal connection) is the same as closing its window normally.
|
||||
@@ -1,68 +0,0 @@
|
||||
# Whisper transcription languages
|
||||
|
||||
Settings > Voice > Transcription language selects French, English, 27 other
|
||||
languages, or auto-detection. It applies to ordinary dictation and continuous
|
||||
transcription using OpenAI, on-device Whisper, and the other cloud providers.
|
||||
Translate retains its own spoken-language preference. Moonshine remains
|
||||
English-only; Apple's on-device speech recognizer retains its own locale.
|
||||
|
||||
The initial **Provider default** preserves existing installations: English for
|
||||
on-device Whisper, auto-detection for OpenAI/other cloud providers, and the saved
|
||||
Whisper server language for self-hosted Whisper. Selecting another shared
|
||||
language overrides the server language without deleting it. Returning to Provider
|
||||
default restores it. Settings persist through the existing settings store.
|
||||
|
||||
For French or auto-detection with **On-device (Whisper)**, download
|
||||
**On-device model: Whisper (multilingual, base)** in Settings > Voice (~161 MB).
|
||||
English uses the existing English-only base model; neither model is deleted or
|
||||
downloaded automatically. A missing model uses the existing “Voice model not
|
||||
downloaded” status. Language changes restart active continuous transcription;
|
||||
other captures use the new language when started.
|
||||
|
||||
## Software validation
|
||||
|
||||
- JavaScript regression tests exercise stored settings/reload, compatible
|
||||
defaults, French and auto-detection, Translate overrides in Android/iOS capture
|
||||
options, the OpenAI WebSocket configuration, and local bridge restart/sharing.
|
||||
- Kotlin tests exercise English/French/other/automatic model selection and file
|
||||
names. Android release compilation checks the actual sherpa-onnx bridge.
|
||||
- New model downloads use a pinned Hugging Face revision, per-file SHA-256 and
|
||||
the existing resumable downloader. Encoder/decoder hashes and sizes come from
|
||||
repository LFS metadata; the vocabulary hash was computed from its contents.
|
||||
|
||||
## Device validation still required
|
||||
|
||||
With both phone microphone preview and glasses microphone, check French dictation
|
||||
(including accents), auto-detection, English rollback, continuous capture while
|
||||
changing language, cancellation/resumption of model download, and persistence
|
||||
after restarting the app. Compare OpenAI and offline Whisper on the same phrases.
|
||||
No paid API call, microphone recording, or firmware update is needed for the
|
||||
software checks above; recognition accuracy and latency require device testing.
|
||||
|
||||
## References
|
||||
|
||||
- [OpenAI Realtime transcription: language configuration](https://developers.openai.com/api/docs/guides/realtime-transcription)
|
||||
- [sherpa-onnx Whisper configuration and automatic language detection](https://github.com/k2-fsa/sherpa-onnx/blob/master/sherpa-onnx/csrc/offline-whisper-model-config.cc)
|
||||
- [Multilingual base model, pinned revision](https://huggingface.co/csukuangfj/sherpa-onnx-whisper-base/tree/bb53ee204431c90d314c1cc08d28d23e5b7927cc)
|
||||
|
||||
## OpenAI transcription model selection
|
||||
|
||||
Settings > Voice > OpenAI transcription model selects gpt-realtime-whisper,
|
||||
gpt-4o-mini-transcribe or gpt-4o-transcribe. For a project whose model list lacks
|
||||
gpt-realtime-whisper but includes gpt-4o-mini-transcribe, select the latter.
|
||||
The unset default retains gpt-realtime-whisper for existing installations; no
|
||||
provider, language or Assistant model preference is overwritten. This setting
|
||||
applies to Translate's speech recognition as well as Transcribe and dictation.
|
||||
The two transcribe alternatives produce transcript deltas after a committed
|
||||
audio segment; existing pause segmentation and push-to-talk release supply the
|
||||
commits. Choosing a model retargets active continuous capture.
|
||||
|
||||
Model access errors name the requested model before the long project error and
|
||||
point to the setting. Factory/session-wire tests cover model, language, manual
|
||||
commit and final transcript behavior; settings tests cover persistence and the
|
||||
Android/iOS option plumbing. No real API inference was performed; actual project
|
||||
access, recognition and latency remain to be checked on a device. The model list
|
||||
provided by the operator establishes account visibility, not a successful audio
|
||||
request.
|
||||
|
||||
Reference: [OpenAI transcription session API](https://developers.openai.com/api/reference/resources/realtime/subresources/transcription_sessions/methods/create).
|
||||
-22
@@ -1,22 +0,0 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
/** Existing English/multilingual files stay usable; the dedicated French provider uses its own download. */
|
||||
data class WhisperModelFiles(val language: String, val frenchOnly: Boolean = false) {
|
||||
val englishOnly: Boolean get() = !frenchOnly && language == "en"
|
||||
val dirName: String get() = if (frenchOnly) "sherpa-onnx-whisper-small-fr-int8-v1" else if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8"
|
||||
private val prefix: String get() = if (frenchOnly) "small-fr" else if (englishOnly) "base.en" else "base"
|
||||
val encoder: String get() = "$prefix-encoder.int8.onnx"
|
||||
val decoder: String get() = "$prefix-decoder.int8.onnx"
|
||||
val tokens: String get() = "$prefix-tokens.txt"
|
||||
val files: List<String> get() = listOf(encoder, decoder, tokens)
|
||||
|
||||
companion object {
|
||||
/** A dedicated French fine-tune never silently substitutes the generic base model. */
|
||||
fun forSelection(language: String?, frenchOnly: Boolean): WhisperModelFiles =
|
||||
if (frenchOnly) WhisperModelFiles("fr", true) else forLanguage(language)
|
||||
|
||||
/** Null preserves the old English default; an empty code enables automatic detection. */
|
||||
fun forLanguage(language: String?): WhisperModelFiles =
|
||||
WhisperModelFiles(language?.trim()?.lowercase() ?: "en")
|
||||
}
|
||||
}
|
||||
@@ -267,6 +267,9 @@ function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {})
|
||||
}
|
||||
const voiceDialogs = [];
|
||||
const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' },
|
||||
appFullscreenSetting: () => ({ get: () => false }),
|
||||
appKeepAwakeSetting: () => ({ label: 'Keep screen awake', get: () => false }),
|
||||
toggleSettingMenuItem: setting => ({ label: setting.label }),
|
||||
assistantSkipConfirmationSetting: { get: () => skipConfirmation } };
|
||||
const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', {
|
||||
'../../graphics/shell-scene': { encodeShellScene: () => new Uint8Array([0, 0]) },
|
||||
@@ -370,6 +373,7 @@ test('ordinary apps retain long-press system menu and games retain the escape ti
|
||||
assert.deepEqual(env.input, []);
|
||||
env.shell.stack.clearToBase();
|
||||
env.app.window.claimsLongPress = () => true;
|
||||
env.shell.focusWindow('ai-chat');
|
||||
await env.shell.receiveInput(env.event('long-press'));
|
||||
assert.notEqual(env.shell.escapeMenuTimer, null);
|
||||
await env.shell.receiveInput(env.event('long-press-release'));
|
||||
|
||||
@@ -15,6 +15,8 @@ function geometry() {
|
||||
appSwitcherPositionSetting: 'left', uiDepthSetting: '0', statusBarPositionSetting: 'top',
|
||||
statusBarVisibilitySetting: 'always', windowBorderSetting: true,
|
||||
}).map(([name, value]) => [name, { value, get() { return this.value; } }]));
|
||||
const fullscreen = new Map();
|
||||
settings.appFullscreenSetting = id => ({ get: () => fullscreen.get(id) ?? false, set: value => fullscreen.set(id, value) });
|
||||
const context = { exports: {}, require: (name) => {
|
||||
if (name === '../../graphics/image') return { G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 };
|
||||
assert.equal(name, '../dashboard-settings');
|
||||
@@ -27,6 +29,25 @@ function geometry() {
|
||||
|
||||
const LAYOUTS = ['band-top', 'band-high', 'band-middle', 'band-low', 'band-bottom', 'tall', 'fullscreen'];
|
||||
|
||||
test('user full screen spans the panel with any switcher and survives app immersive cleanup', () => {
|
||||
const g = geometry();
|
||||
const previous = g.rect('min', 'test');
|
||||
for (const position of ['left', 'right', 'bottom', 'popup']) {
|
||||
g.settings.appSwitcherPositionSetting.value = position;
|
||||
g.settings.appFullscreenSetting('test').set(true);
|
||||
assert.deepEqual(g.rect('min', 'test'), { x: 0, y: 0, width: 640, height: 480 });
|
||||
assert.equal(g.sidebarStripVisible('window', 'test'), false);
|
||||
assert.equal(g.sidebarStripVisible('sidebar', 'test'), true);
|
||||
g.setAppImmersive('test', true);
|
||||
g.setAppImmersive('test', false);
|
||||
assert.equal(g.isImmersive('test'), true);
|
||||
assert.equal(g.isImmersive('other'), false);
|
||||
}
|
||||
g.settings.appSwitcherPositionSetting.value = 'left';
|
||||
g.settings.appFullscreenSetting('test').set(false);
|
||||
assert.deepEqual(g.rect('min', 'test'), previous);
|
||||
});
|
||||
|
||||
test('defaults preserve normal windows, EvenHub height and tall terminal sessions', () => {
|
||||
const g = geometry();
|
||||
assert.deepEqual(g.rect('min', 'navigate'), { x: 64, y: 124, width: 576, height: 260 });
|
||||
|
||||
@@ -0,0 +1,227 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function fixture(options = {}) {
|
||||
const source = ts.createSourceFile('shell.ts', fs.readFileSync(path.join(__dirname, '../app/ui/shell/shell.ts'), 'utf8'), ts.ScriptTarget.Latest, true);
|
||||
const shellClass = source.statements.find(node => ts.isClassDeclaration(node) && node.name.text === 'Shell');
|
||||
const names = new Set(['routeInput', 'openEscapeMenu', 'closeAllWindows', 'closeWindow', 'removeWindow',
|
||||
'foregroundWindow', 'noteWindowVisible', 'mostRecentWindowIndex', 'yieldFocusToSidebar', 'applyScreenTimeout']);
|
||||
const methods = shellClass.members.filter(node => names.has(node.name?.getText(source)));
|
||||
assert.equal(methods.length, names.size);
|
||||
const overlay = source.statements.find(node => ts.isClassDeclaration(node) && node.name.text === 'ShellOverlayMenuLayer');
|
||||
class MenuLayer {
|
||||
constructor(title, items) { this.title = title; this.items = items; }
|
||||
selectItem(index) { this.selectedIndex = index; }
|
||||
}
|
||||
const fullscreen = new Map();
|
||||
const keepAwake = new Map();
|
||||
const context = {
|
||||
exports: {}, MenuLayer, console: { warn: () => {} },
|
||||
appViewportRect: () => ({ x: 0, y: 0, width: 576 }), CONTEXT_MENU_DIM: 0.5,
|
||||
brightnessSetting: { get: () => 'auto' }, gestureHints: () => 'App menu',
|
||||
GESTURE_SHORT_THEN_LONG_PRESS: 'tap-hold', makeInputEvent: event => event,
|
||||
isWatchInput: event => event.source === 'watch', isDirectionalInput: () => false,
|
||||
appFullscreenSetting: id => ({ get: () => fullscreen.get(id) ?? false, set: value => fullscreen.set(id, value) }),
|
||||
appKeepAwakeSetting: id => ({ label: 'Keep screen awake', get: () => keepAwake.get(id) ?? false, set: value => keepAwake.set(id, value) }),
|
||||
toggleSettingMenuItem: (setting, options) => ({ label: setting.label, onSelect(ctx) {
|
||||
setting.set(!setting.get()); options.onChange(ctx);
|
||||
} }),
|
||||
};
|
||||
vm.runInNewContext(ts.transpileModule(`${overlay.getText(source)}\nexport class Shell { ${methods.map(node => node.getText(source)).join('\n')} }`, {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, context);
|
||||
const s = new context.exports.Shell();
|
||||
const closed = [], delivered = [], warnings = [], renders = [];
|
||||
context.console.warn = message => warnings.push(message);
|
||||
const window = (id, closeable = true, extra = {}) => ({
|
||||
appId: id, windowId: id, closeable,
|
||||
handleInput: async event => delivered.push([id, event.type]),
|
||||
requestRender: () => renders.push(id),
|
||||
close: () => closed.push(id),
|
||||
...extra,
|
||||
});
|
||||
const launcher = window('launcher', false);
|
||||
const app = window('current', true, options.app);
|
||||
const background = window('background');
|
||||
const layers = [];
|
||||
s.stack = {
|
||||
isAtBase: () => !layers.length,
|
||||
push: layer => layers.push(layer),
|
||||
pop: () => layers.pop()?.onRemoved?.(),
|
||||
popIfTop: predicate => { if (!layers.length || !predicate(layers.at(-1))) return false; s.stack.pop(); return true; },
|
||||
handleInput: async event => { if (event.type === 'double-click') s.stack.pop(); },
|
||||
};
|
||||
Object.assign(s, {
|
||||
windows: [launcher, app, background], selectedIndex: 1, focus: 'sidebar', screenOn: true,
|
||||
mruWindowIds: ['current', 'background', 'launcher'], attention: new Map(),
|
||||
config: { voiceInputEnabled: false, requestShellRender() {}, onWindowsChanged() {} },
|
||||
cancelEscapeMenuTimer() {}, startEscapeMenuTimer() { s.escapeStarted = true; },
|
||||
lastInput: null,
|
||||
lastInputAtMs: 0,
|
||||
noteUserActivity() { this.lastInputAtMs = 100; },
|
||||
focusWindow(id) { this.selectedIndex = this.windows.findIndex(w => w.windowId === id); this.focus = 'window'; },
|
||||
sleep() { this.screenOn = false; },
|
||||
});
|
||||
const input = type => s.routeInput({ type, source: 'ring', timestampMs: 100 }, 1);
|
||||
const menu = () => layers.at(-1);
|
||||
const choose = label => {
|
||||
const item = menu().items.find(item => item.label === label);
|
||||
assert.ok(item, label);
|
||||
item.onSelect({ stack: s.stack });
|
||||
};
|
||||
return { s, input, menu, choose, closed, delivered, warnings, renders, launcher, app, background, window, fullscreen, keepAwake };
|
||||
}
|
||||
|
||||
test('full screen targets the captured app without inhibiting idle sleep', async () => {
|
||||
const h = fixture();
|
||||
h.s.config.getScreenTimeoutMs = () => 1000;
|
||||
let relayouts = 0;
|
||||
h.s.config.onWindowLayoutChanged = () => relayouts++;
|
||||
await h.input('long-press');
|
||||
h.s.selectedIndex = 2;
|
||||
h.choose('Enter full screen');
|
||||
assert.equal(h.fullscreen.get('current'), true);
|
||||
assert.equal(h.s.foregroundWindow().windowId, 'current');
|
||||
assert.equal(h.s.focus, 'window');
|
||||
assert.equal(relayouts, 1);
|
||||
assert.equal(h.s.applyScreenTimeout(5000), true);
|
||||
h.s.screenOn = true;
|
||||
h.s.focusWindow('current');
|
||||
await h.input('long-press');
|
||||
h.choose('Exit full screen');
|
||||
assert.equal(h.fullscreen.get('current'), false);
|
||||
assert.equal(relayouts, 2);
|
||||
assert.equal(h.s.applyScreenTimeout(7000), true);
|
||||
});
|
||||
|
||||
test('keep awake is independent of full screen and applies only to the visible app', async () => {
|
||||
const h = fixture();
|
||||
h.s.config.getScreenTimeoutMs = () => 1000;
|
||||
await h.input('long-press');
|
||||
const labels = h.menu().items.map(item => item.label);
|
||||
assert.equal(labels.indexOf('Keep screen awake'), labels.indexOf('Enter full screen') + 1);
|
||||
h.s.selectedIndex = 2;
|
||||
h.choose('Keep screen awake');
|
||||
assert.equal(h.keepAwake.get('current'), true);
|
||||
assert.equal(h.fullscreen.get('current'), undefined);
|
||||
assert.equal(h.s.foregroundWindow().windowId, 'current');
|
||||
assert.equal(h.s.applyScreenTimeout(5000), false);
|
||||
h.s.focusWindow('background');
|
||||
assert.equal(h.s.applyScreenTimeout(5500), false);
|
||||
assert.equal(h.s.applyScreenTimeout(6000), true);
|
||||
h.s.screenOn = true;
|
||||
h.s.focusWindow('current');
|
||||
await h.input('long-press'); h.choose('Enter full screen');
|
||||
assert.equal(h.s.applyScreenTimeout(7000), false);
|
||||
await h.input('long-press'); h.choose('Exit full screen');
|
||||
assert.equal(h.keepAwake.get('current'), true);
|
||||
assert.equal(h.s.applyScreenTimeout(8000), false);
|
||||
await h.input('long-press'); h.choose('Keep screen awake');
|
||||
assert.equal(h.keepAwake.get('current'), false);
|
||||
assert.equal(h.s.applyScreenTimeout(9000), true);
|
||||
});
|
||||
|
||||
test('keep awake does not suppress lock timeout or manual sleep, and options are absent for the launcher', async () => {
|
||||
const h = fixture();
|
||||
h.fullscreen.set('current', true);
|
||||
h.keepAwake.set('current', true);
|
||||
h.s.config.getScreenTimeoutMs = () => 1000;
|
||||
h.s.config.isLocked = () => true;
|
||||
assert.equal(h.s.applyScreenTimeout(5000), true);
|
||||
h.s.screenOn = true;
|
||||
h.s.config.isLocked = () => false;
|
||||
h.s.sleep();
|
||||
assert.equal(h.s.screenOn, false);
|
||||
h.s.screenOn = true;
|
||||
h.s.selectedIndex = 0;
|
||||
await h.input('long-press');
|
||||
assert.ok(!h.menu().items.some(item => /full screen/.test(item.label)));
|
||||
assert.ok(!h.menu().items.some(item => item.label === 'Keep screen awake'));
|
||||
});
|
||||
|
||||
test('switcher opens its menu on hold or tap-hold even for an app claiming holds or hold-to-talk', async () => {
|
||||
for (const type of ['long-press', 'short-then-long-press']) {
|
||||
for (const app of [{}, { claimsLongPress: () => true }, { holdToTalk: true }]) {
|
||||
const h = fixture({ app });
|
||||
await h.input(type);
|
||||
assert.ok(h.menu());
|
||||
assert.ok(h.menu().items.some(item => item.label === 'Close current app'));
|
||||
assert.ok(h.menu().items.some(item => item.label === 'Close all apps'));
|
||||
assert.equal(h.menu().items[h.menu().selectedIndex].label, 'Close current app');
|
||||
assert.deepEqual(h.delivered, [['current', 'system-menu-opened']]);
|
||||
assert.equal(h.s.escapeStarted, undefined);
|
||||
assert.equal(h.s.focus, 'sidebar');
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('close current app closes the captured window and preserves the launcher and background windows', async () => {
|
||||
const h = fixture();
|
||||
await h.input('long-press');
|
||||
h.s.selectedIndex = 2; // An asynchronous focus change must not redirect the close action.
|
||||
h.menu().items[h.menu().selectedIndex].onSelect({ stack: h.s.stack });
|
||||
assert.deepEqual(h.closed, ['current']);
|
||||
assert.deepEqual(h.s.windows.map(w => w.windowId), ['launcher', 'background']);
|
||||
assert.equal(h.s.focus, 'sidebar');
|
||||
assert.equal(h.menu(), undefined);
|
||||
});
|
||||
|
||||
test('launcher offers close all but never close current; close all leaves only the launcher', async () => {
|
||||
const h = fixture();
|
||||
h.s.selectedIndex = 0;
|
||||
await h.input('short-then-long-press');
|
||||
assert.ok(!h.menu().items.some(item => item.label === 'Close current app'));
|
||||
assert.equal(h.menu().items[h.menu().selectedIndex].label, 'Focus app switcher');
|
||||
h.choose('Close all apps');
|
||||
assert.deepEqual(h.closed, ['current', 'background']);
|
||||
assert.deepEqual(h.s.windows.map(w => w.windowId), ['launcher']);
|
||||
assert.equal(h.s.selectedIndex, 0);
|
||||
assert.equal(h.s.focus, 'sidebar');
|
||||
await h.input('long-press');
|
||||
assert.equal(h.menu().items.find(item => item.label === 'Close all apps').disabled(), true);
|
||||
});
|
||||
|
||||
test('close all uses normal cleanup despite reentrant removal, multiple windows per app and throwing callbacks', () => {
|
||||
const h = fixture();
|
||||
h.s.windows.push(h.window('another', true, { appId: 'current' }), h.window('pinned', false));
|
||||
h.app.close = () => { h.closed.push('current'); h.s.removeWindow('current'); };
|
||||
h.background.close = () => { h.closed.push('background'); throw new Error('failed cleanup'); };
|
||||
h.s.closeAllWindows();
|
||||
assert.deepEqual(h.closed, ['current', 'background', 'another']);
|
||||
assert.deepEqual(h.s.windows.map(w => w.windowId), ['launcher', 'pinned']);
|
||||
assert.equal(h.warnings.length, 1);
|
||||
assert.equal(h.s.foregroundWindow().windowId, 'launcher');
|
||||
});
|
||||
|
||||
test('focusing switcher and cancelling do not close apps; an open overlay never receives a second menu', async () => {
|
||||
const h = fixture();
|
||||
await h.input('long-press');
|
||||
h.choose('Focus app switcher');
|
||||
assert.deepEqual(h.closed, []);
|
||||
await h.input('long-press');
|
||||
const menu = h.menu();
|
||||
await h.input('long-press');
|
||||
assert.equal(h.menu(), menu);
|
||||
await h.input('double-click');
|
||||
assert.equal(h.menu(), undefined);
|
||||
assert.deepEqual(h.closed, []);
|
||||
});
|
||||
|
||||
test('in-app hold actions and app menus keep their existing routing; asleep switcher stays asleep', async () => {
|
||||
for (const [app, type] of [[{ claimsLongPress: () => true }, 'long-press'], [{ holdToTalk: true }, 'long-press'], [{}, 'short-then-long-press']]) {
|
||||
const h = fixture({ app });
|
||||
h.s.focus = 'window';
|
||||
await h.input(type);
|
||||
assert.deepEqual(h.delivered, [['current', type]]);
|
||||
assert.equal(h.menu(), undefined);
|
||||
}
|
||||
const h = fixture();
|
||||
h.s.screenOn = false;
|
||||
await h.input('long-press');
|
||||
assert.equal(h.menu(), undefined);
|
||||
assert.deepEqual(h.closed, []);
|
||||
});
|
||||
@@ -54,9 +54,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') {
|
||||
'../../native/asr-model': {
|
||||
ASR_MODELS: {
|
||||
moonshine: { label: 'Moonshine', sizeBytes: 1 },
|
||||
'whisper-small-fr': { label: 'Whisper French Small', sizeBytes: 1 },
|
||||
'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 },
|
||||
'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 },
|
||||
},
|
||||
asrModelState: () => ({}), cancelAsrModelDownload() {},
|
||||
deleteAsrModel() {}, startAsrModelDownload() {}, onAsrModelStateChanged: () => () => {},
|
||||
|
||||
@@ -26,7 +26,6 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
|
||||
@@ -39,6 +39,9 @@ const menu = load('app/ui/menu.ts', {
|
||||
function fixture(initial = '50', viewport = { x: 64, y: 124, width: 576, height: 260 }) {
|
||||
let value = initial, writes = 0, closed = 0;
|
||||
const settings = {
|
||||
appFullscreenSetting: () => ({ get: () => false }),
|
||||
appKeepAwakeSetting: () => ({ label: 'Keep screen awake', get: () => false }),
|
||||
toggleSettingMenuItem: setting => ({ label: setting.label }),
|
||||
brightnessSetting: { get: () => value, set: (next) => { value = next; writes++; } },
|
||||
brightnessSettingToLevel: (value) => value === 'auto' ? null : Number(value),
|
||||
};
|
||||
@@ -120,13 +123,16 @@ test('system menu hides brightness in Auto and opens its picker between Voice in
|
||||
shell.openSystemMenu('test');
|
||||
let opened = shell.stack.layers.at(-1);
|
||||
assert.deepEqual(Array.from(opened.items, (item) => item.label),
|
||||
['Close window', 'Focus app switcher', 'Voice input', 'Debug']);
|
||||
['Close current app', 'Focus app switcher', 'Voice input', 'Close all apps', 'Enter full screen', 'Keep screen awake', 'Debug']);
|
||||
shell.stack.clearToBase();
|
||||
f.settings.brightnessSetting.set('40');
|
||||
shell.openSystemMenu('test');
|
||||
opened = shell.stack.layers.at(-1);
|
||||
assert.deepEqual(Array.from(opened.items, (item) => item.label),
|
||||
['Close window', 'Focus app switcher', 'Voice input', 'Brightness', 'Debug']);
|
||||
['Close current app', 'Focus app switcher', 'Voice input', 'Close all apps', 'Enter full screen', 'Keep screen awake', 'Brightness', 'Debug']);
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'scroll-down' }));
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'scroll-down' }));
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'scroll-down' }));
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'scroll-down' }));
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'scroll-down' }));
|
||||
await shell.stack.handleInput(gestures.makeInputEvent({ type: 'click', source: 'ring' }));
|
||||
|
||||
@@ -207,10 +207,6 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
// A running capture is shared only on the same provider and language.
|
||||
assert.equal(transcriptionKey({ provider: 'soniox', language: ' JA ' }), 'soniox:ja');
|
||||
assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' }));
|
||||
assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' }));
|
||||
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' }));
|
||||
assert.equal(transcriptionKey({ provider: 'onboard-whisper-fr', language: 'en' }), 'onboard-whisper-fr:fr');
|
||||
assert.notEqual(transcriptionKey({ provider: 'onboard-whisper-fr' }), transcriptionKey({ provider: 'onboard-whisper', language: 'fr' }));
|
||||
const statuses = [];
|
||||
const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} };
|
||||
const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' };
|
||||
@@ -229,13 +225,13 @@ test('provider selection falls back to on-device without a key and connects the
|
||||
});
|
||||
|
||||
test('a spoken-language hint reaches each provider on the wire, and is left out to auto-detect', () => {
|
||||
for (const language of ['fr', 'ja', '']) {
|
||||
for (const language of ['ja', '']) {
|
||||
const elevenlabs = harness('elevenlabs', true, { language });
|
||||
assert.equal(new URL(elevenlabs.sockets[0].url).searchParams.get('language_code'), language || null);
|
||||
|
||||
const soniox = harness('soniox', true, { language });
|
||||
soniox.open();
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? [language] : undefined);
|
||||
assert.deepEqual(JSON.parse(soniox.sockets[0].sent[0]).language_hints, language ? ['ja'] : undefined);
|
||||
|
||||
const openai = harness('openai', true, { language });
|
||||
openai.open();
|
||||
@@ -244,43 +240,3 @@ test('a spoken-language hint reaches each provider on the wire, and is left out
|
||||
assert.equal(transcription.language, language || undefined);
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI selected transcription models preserve French hints, manual commits and final transcripts', () => {
|
||||
for (const model of ['gpt-realtime-whisper', 'gpt-4o-mini-transcribe', 'gpt-4o-transcribe']) {
|
||||
const h = harness('openai', true, { model, language: 'fr' });
|
||||
h.open();
|
||||
const input = JSON.parse(h.sockets[0].sent[0]).session.audio.input;
|
||||
assert.equal(input.transcription.model, model);
|
||||
assert.equal(input.transcription.language, 'fr');
|
||||
assert.equal(input.turn_detection, null);
|
||||
h.client.acceptPcm(pcm(1200, 150));
|
||||
h.client.acceptPcm(pcm(0, 1500));
|
||||
assert.ok(h.sockets[0].sent.some(value => JSON.parse(value).type === 'input_audio_buffer.commit'));
|
||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'one' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'one', delta: 'Bonjour' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'one', transcript: 'Bonjour à tous.' });
|
||||
assert.equal(h.events.at(-1).text, 'Bonjour à tous.');
|
||||
assert.equal(h.events.at(-1).isFinal, true);
|
||||
h.message({ type: 'error', error: { code: 'model_not_found', message: 'Project private-id does not have access' } });
|
||||
assert.ok(h.errors[0].includes(model));
|
||||
assert.ok(h.errors[0].includes('OpenAI transcription model'));
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI model selection reaches the factory and separates shared captures only for OpenAI', () => {
|
||||
const created = [];
|
||||
const load = loader({ setTimeout, clearTimeout }, {
|
||||
'./openai-stt': {
|
||||
DEFAULT_OPENAI_STT_MODEL: 'gpt-realtime-whisper',
|
||||
OpenAiRealtimeSttClient: class { constructor(options) { created.push(options); } start() {} stop() {} },
|
||||
},
|
||||
});
|
||||
const { createCloudSttClient, transcriptionKey } = load('app/native/cloud-stt-provider.ts');
|
||||
const selection = { provider: 'whisper', language: 'fr', openAiApiKey: 'test', openAiTranscriptionModel: 'gpt-4o-mini-transcribe' };
|
||||
createCloudSttClient(selection, { onStatus() {}, onError() {}, onTranscript() {} }, () => false).start();
|
||||
assert.equal(created[0].model, selection.openAiTranscriptionModel);
|
||||
assert.equal(created[0].language, 'fr');
|
||||
assert.notEqual(transcriptionKey(selection), transcriptionKey({ ...selection, openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ provider: 'whisper' }), transcriptionKey({ provider: 'whisper', openAiTranscriptionModel: 'gpt-realtime-whisper' }));
|
||||
assert.equal(transcriptionKey({ ...selection, provider: 'soniox' }), transcriptionKey({ ...selection, provider: 'soniox', openAiTranscriptionModel: 'gpt-4o-transcribe' }));
|
||||
});
|
||||
@@ -12,7 +12,6 @@ function settingsFixture(store = new Map()) {
|
||||
};
|
||||
if (id.includes('brightness-curve')) return require('../.test-build/app/g2/brightness-curve.js');
|
||||
if (id.includes('animation-speed')) return require('../.test-build/app/ui/animation-speed.js');
|
||||
if (id.includes('transcription-language')) return require('../.test-build/app/native/transcription-language.js');
|
||||
return { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] };
|
||||
}};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/ui/dashboard-settings.ts', 'utf8'), {
|
||||
@@ -65,3 +64,32 @@ test('Full Screen is offered only with a bottom or popup app switcher', () => {
|
||||
assert.deepEqual(Array.from(api.DISPLAY_LAYOUT_VALUES, (value) => api.displayLayoutLabel(value)),
|
||||
['Band · Top', 'Band · High', 'Band · Middle', 'Band · Low', 'Band · Bottom', 'Tall', 'Full Screen']);
|
||||
});
|
||||
|
||||
test('per-app full screen defaults off, persists across restart and preserves display preferences', () => {
|
||||
const store = new Map([['display.layout', 'band-low'], ['terminal.displayLayout', 'tall']]);
|
||||
const prefs = settingsFixture(store);
|
||||
assert.equal(prefs.appFullscreenSetting('terminal').get(), false);
|
||||
prefs.appFullscreenSetting('terminal').set(true);
|
||||
const restarted = settingsFixture(store);
|
||||
assert.equal(restarted.appFullscreenSetting('terminal').get(), true);
|
||||
assert.equal(restarted.appFullscreenSetting('other').get(), false);
|
||||
restarted.appFullscreenSetting('terminal').set(false);
|
||||
assert.equal(restarted.displayLayoutSetting.get(), 'band-low');
|
||||
assert.equal(store.get('terminal.displayLayout'), 'tall');
|
||||
});
|
||||
|
||||
test('keep awake persists per app independently of full screen, including existing full-screen choices', () => {
|
||||
const store = new Map([['display.fullscreen.terminal', true]]);
|
||||
let prefs = settingsFixture(store);
|
||||
assert.equal(prefs.appFullscreenSetting('terminal').get(), true);
|
||||
assert.equal(prefs.appKeepAwakeSetting('terminal').get(), false);
|
||||
prefs.appKeepAwakeSetting('terminal').set(true);
|
||||
prefs = settingsFixture(store);
|
||||
assert.equal(prefs.appKeepAwakeSetting('terminal').get(), true);
|
||||
assert.equal(prefs.appKeepAwakeSetting('other').get(), false);
|
||||
prefs.appFullscreenSetting('terminal').set(false);
|
||||
assert.equal(prefs.appKeepAwakeSetting('terminal').get(), true);
|
||||
prefs.appKeepAwakeSetting('terminal').set(false);
|
||||
prefs = settingsFixture(store);
|
||||
assert.equal(prefs.appKeepAwakeSetting('terminal').get(), false);
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const { GrayImage } = require('../.test-build/app/graphics/image.js');
|
||||
|
||||
function fixture() {
|
||||
let options, hooks;
|
||||
const paints = [], heights = [];
|
||||
const session = {
|
||||
manifest: { name: 'Test', packageId: 'test.app' },
|
||||
renderWindowIcon() {}, contextMenuItems: () => [], keepsScreenOn: () => false,
|
||||
attachWindow(value) { hooks = value; },
|
||||
paint(size, focused) {
|
||||
paints.push({ ...size, focused });
|
||||
const image = new GrayImage(size.width, size.height);
|
||||
image.setPixel(0, 0, 255);
|
||||
image.setPixel(size.width - 1, size.height - 1, 120);
|
||||
// Deliberately overflow a deferred text run: stock clips it at 576x288.
|
||||
image.drawFwText({ bakeFwTextRun(target, run, dx, dy) {
|
||||
target.fillRect(run.x + dx, run.y + dy, 20, 20, run.value);
|
||||
} }, 570, 280, 200, [{}]);
|
||||
return image;
|
||||
},
|
||||
};
|
||||
const load = loader({}, {
|
||||
'../../graphics/image': { GrayImage },
|
||||
'./compositor': { EVENHUB_SCREEN_WIDTH: 576, EVENHUB_SCREEN_HEIGHT: 288 },
|
||||
'../../ui/gestures': {}, '../../ui/shell/shell': { shell: {} }, './session': {},
|
||||
'../../ui/shell/chrome-layer': { windowIcon: () => () => null, makeImageWindowIcon: () => () => null },
|
||||
'./installed-apps': {},
|
||||
'../../ui/shell/in-process-window': { createInProcessWindow(value) {
|
||||
options = value;
|
||||
return { window: {}, stack: {}, requestRender() {}, setHeightMode(mode) { heights.push(mode); } };
|
||||
} },
|
||||
});
|
||||
load('app/apps/evenhub/evenhub-window.ts').createEvenHubWindow('test', 'test', session, {}, () => {}, () => {});
|
||||
return { paints, heights, hooks: () => hooks,
|
||||
paint(width, height, focused = true) {
|
||||
return options.baseLayer.paint({ stack: { getBaseSize: () => ({ width, height }), isFocused: () => focused } });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test('stock EvenHub canvas remains 576x288 through full-screen entry, resize and exit', () => {
|
||||
const f = fixture();
|
||||
for (const [width, height] of [[576, 288], [640, 480], [576, 452], [640, 480], [576, 288]]) {
|
||||
const image = f.paint(width, height);
|
||||
assert.deepEqual(f.paints.at(-1), { width: 576, height: 288, focused: true });
|
||||
assert.equal(image.width, width);
|
||||
assert.equal(image.height, height);
|
||||
}
|
||||
});
|
||||
|
||||
test('full screen centers stock pixels at native scale and clips overflowing text to the stock canvas', () => {
|
||||
const f = fixture();
|
||||
const stock = f.paint(576, 288).withDrawsBaked();
|
||||
const full = f.paint(640, 480).withDrawsBaked();
|
||||
for (let y = 0; y < 480; y++) {
|
||||
for (let x = 0; x < 640; x++) {
|
||||
const expected = x >= 32 && x < 608 && y >= 96 && y < 384 ? stock.getPixel(x - 32, y - 96) : 0;
|
||||
assert.equal(full.getPixel(x, y), expected);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test('only an explicit extended-layout request opts into the taller canvas', () => {
|
||||
const f = fixture();
|
||||
f.paint(640, 480, false);
|
||||
assert.deepEqual(f.paints.at(-1), { width: 576, height: 288, focused: false });
|
||||
f.hooks().setTallCanvas(true);
|
||||
assert.deepEqual(f.heights, ['max']);
|
||||
f.paint(576, 452);
|
||||
assert.equal(f.paints.at(-1).height, 452);
|
||||
f.paint(640, 480);
|
||||
assert.equal(f.paints.at(-1).width, 576);
|
||||
assert.equal(f.paints.at(-1).height, 480);
|
||||
f.hooks().setTallCanvas(false);
|
||||
f.paint(640, 480);
|
||||
assert.deepEqual(f.heights, ['max', 'medium']);
|
||||
assert.equal(f.paints.at(-1).height, 288);
|
||||
});
|
||||
@@ -6,8 +6,7 @@ const ts = require('typescript');
|
||||
const path = require('node:path');
|
||||
// The provider settings ios-preview-controller hands the voice bridge.
|
||||
const voiceSettings = { voiceProviderSetting: { get: () => 'onboard' }, elevenLabsApiKeySetting: { get: () => '' },
|
||||
getVoiceTranscriptionLanguage: () => '',
|
||||
openAiTranscriptionModelSetting: { get: () => 'gpt-realtime-whisper' }, openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
openAiApiKeySetting: { get: () => '' }, sonioxApiKeySetting: { get: () => '' },
|
||||
whisperServerUrlSetting: { get: () => '' }, whisperServerModelSetting: { get: () => '' },
|
||||
whisperServerLanguageSetting: { get: () => '' }, whisperServerApiKeySetting: { get: () => '' } };
|
||||
function load(file, context) {
|
||||
@@ -169,7 +168,6 @@ test('settings-driven repaint runs after font cache invalidation, regardless of
|
||||
let cachedFont = 'Light';
|
||||
const api = load('app/ui/dashboard-settings.ts', { global: { isIOS: true },
|
||||
require: id => id.includes('settings-store') ? { onSettingsStoreChanged: fn => listeners.push(fn) }
|
||||
: id.endsWith('/transcription-language') ? load('app/native/transcription-language.ts', {})
|
||||
: { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] },
|
||||
setTimeout: fn => tasks.push(fn),
|
||||
});
|
||||
@@ -386,7 +384,7 @@ test('iOS continuous capture is shared by windows and restarts after a disconnec
|
||||
recordTo: name => { bridge.recording = name; },
|
||||
setRawSession: session => { bridge.rawSession = session; } };
|
||||
const settings = { ...voiceSettings, voiceProviderSetting: { get: () => 'whisper-server' },
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, getVoiceTranscriptionLanguage: () => 'en',
|
||||
whisperServerUrlSetting: { get: () => 'http://whisper.local:8000' }, whisperServerLanguageSetting: { get: () => 'en' },
|
||||
saveTranscribeAudioSetting: { get: () => true } };
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', {
|
||||
require: id => id === '../native/ios-voice-input' ? { iosVoiceInput: bridge } : id === '../ui/dashboard-settings' ? settings
|
||||
|
||||
@@ -1,46 +0,0 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.test.Test
|
||||
import kotlin.test.assertEquals
|
||||
import kotlin.test.assertFalse
|
||||
import kotlin.test.assertTrue
|
||||
|
||||
class WhisperModelFilesTest {
|
||||
@Test fun existingEnglishFilesRemainUsable() {
|
||||
for (language in listOf(null, "en", " EN ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertTrue(model.englishOnly)
|
||||
assertEquals("en", model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-en-int8", model.dirName)
|
||||
assertEquals(listOf("base.en-encoder.int8.onnx", "base.en-decoder.int8.onnx", "base.en-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun frenchAndOtherLanguagesUseMultilingualFiles() {
|
||||
for (language in listOf("fr", "de", "ja", " FR ")) {
|
||||
val model = WhisperModelFiles.forLanguage(language)
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals(language.trim().lowercase(), model.language)
|
||||
assertEquals("sherpa-onnx-whisper-base-int8", model.dirName)
|
||||
assertEquals(listOf("base-encoder.int8.onnx", "base-decoder.int8.onnx", "base-tokens.txt"), model.files)
|
||||
}
|
||||
}
|
||||
|
||||
@Test fun autoDetectionUsesMultilingualFilesWithAnEmptyLanguage() {
|
||||
val model = WhisperModelFiles.forLanguage("")
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals("", model.language)
|
||||
assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files)
|
||||
}
|
||||
@Test fun frenchFineTuneUsesDistinctFilesRegardlessOfTheGenericLanguage() {
|
||||
for (language in listOf(null, "en", "de", "", "fr")) {
|
||||
val model = WhisperModelFiles.forSelection(language, true)
|
||||
assertTrue(model.frenchOnly)
|
||||
assertFalse(model.englishOnly)
|
||||
assertEquals("fr", model.language)
|
||||
assertEquals("sherpa-onnx-whisper-small-fr-int8-v1", model.dirName)
|
||||
assertEquals(listOf("small-fr-encoder.int8.onnx", "small-fr-decoder.int8.onnx", "small-fr-tokens.txt"), model.files)
|
||||
assertEquals(WhisperModelFiles.forLanguage(language), WhisperModelFiles.forSelection(language, false))
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,6 @@
|
||||
"../app/ui/menu-core.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/native/java-direct-buffer.ts",
|
||||
"../app/native/transcription-language.ts",
|
||||
"../app/native/notification-types.ts",
|
||||
"../app/g2/firmware-compat.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
|
||||
@@ -163,9 +163,7 @@ function androidBridge() {
|
||||
starts: 0,
|
||||
setListener(listener) { this.listener = listener; },
|
||||
setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {},
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {},
|
||||
setFrenchWhisper(enabled) { this.frenchWhisper = enabled; },
|
||||
setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; },
|
||||
setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, setOnboardModelKind() {},
|
||||
start(_mode, id) { this.captureId = id; this.active = true; this.starts++; },
|
||||
stop() { this.active = false; },
|
||||
isCapturing() { return this.active; },
|
||||
@@ -173,8 +171,7 @@ function androidBridge() {
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null,
|
||||
transcriptionKey: options => `${options.provider}:${options.provider === 'onboard-whisper-fr' ? 'fr' : options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` },
|
||||
'./cloud-stt-provider': { createCloudSttClient: () => null, transcriptionKey: options => options.provider },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: function () { return controller; },
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
@@ -182,26 +179,6 @@ function androidBridge() {
|
||||
return { bridge: new FaceclawVoiceControlBridge(), controller };
|
||||
}
|
||||
|
||||
test('local Whisper receives French and restarts for auto or English, sharing an unchanged language', () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
bridge.startContinuousCapture({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.modelKind, 'whisper');
|
||||
assert.equal(controller.language, 'fr');
|
||||
bridge.startPushToTalk({ provider: 'onboard-whisper', language: 'fr' });
|
||||
assert.equal(controller.starts, 1);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: '' });
|
||||
assert.equal(controller.language, '');
|
||||
assert.equal(controller.starts, 2);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
||||
// An omitted language preserves the native bridge's original English behavior.
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper', language: 'en' });
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 3);
|
||||
bridge.stop();
|
||||
});
|
||||
|
||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
const options = { provider: 'onboard-whisper' };
|
||||
@@ -238,16 +215,3 @@ test('a shared capture restarts for a holder that needs another provider, and is
|
||||
assert.equal(controller.active, true);
|
||||
assert.equal(bridge.holdsCapture('ptt'), true);
|
||||
});
|
||||
|
||||
test('dedicated French Whisper selects its own files and switches back without retaining the override', () => {
|
||||
const { bridge, controller } = androidBridge();
|
||||
bridge.startContinuousCapture({ provider: 'onboard-whisper-fr', language: 'en' });
|
||||
assert.equal(controller.modelKind, 'whisper');
|
||||
assert.equal(controller.frenchWhisper, true);
|
||||
assert.equal(controller.language, 'fr');
|
||||
bridge.retargetContinuousCapture({ provider: 'onboard-whisper' });
|
||||
assert.equal(controller.frenchWhisper, false);
|
||||
assert.equal(controller.language, 'en');
|
||||
assert.equal(controller.starts, 2);
|
||||
bridge.stop();
|
||||
});
|
||||
@@ -1,210 +0,0 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, requireModule = () => ({}), globals = {}) {
|
||||
const exports = {};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, { exports, require: requireModule, console, global: { isAndroid: true }, ...globals });
|
||||
return exports;
|
||||
}
|
||||
const languages = load('app/native/transcription-language.ts');
|
||||
function settings(values = new Map()) {
|
||||
return load('app/ui/dashboard-settings.ts', name => name.endsWith('/transcription-language') ? languages
|
||||
: name.includes('settings-store') ? {
|
||||
getStringSetting: (key, fallback) => values.get(key) ?? fallback,
|
||||
setStringSetting: (key, value) => values.set(key, value),
|
||||
getBooleanSetting: (_key, fallback) => fallback,
|
||||
onSettingsStoreChanged() {},
|
||||
} : { Layer: class {}, ASSISTANT_MODEL_CHOICES: [] });
|
||||
}
|
||||
|
||||
test('language defaults preserve local English, cloud detection and an existing server preference', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
const s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.deepEqual([...values], [['voice.whisperServerLanguage', 'sk']]);
|
||||
});
|
||||
|
||||
test('French and auto-detection persist across restart without overwriting server exceptions', () => {
|
||||
const values = new Map([['voice.whisperServerLanguage', 'sk']]);
|
||||
let s = settings(values);
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.voiceLanguageSetting.displayValue(), 'French');
|
||||
for (const provider of ['whisper', 'onboard-whisper', 'whisper-server', 'elevenlabs', 'soniox']) {
|
||||
assert.equal(s.getVoiceTranscriptionLanguage(provider), 'fr');
|
||||
}
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard'), '');
|
||||
s.voiceLanguageSetting.set('auto');
|
||||
s = settings(values);
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), '');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
s.voiceLanguageSetting.set('default');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper-server'), 'sk');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
});
|
||||
|
||||
test('corrupt language preferences revert to the compatible provider defaults', () => {
|
||||
const s = settings(new Map([['voice.language', 'invalid']]));
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'en');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('whisper'), '');
|
||||
});
|
||||
|
||||
test('Android and iOS capture options use the shared language and preserve Translate overrides', () => {
|
||||
const s = settings();
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s.voiceProviderSetting.set('whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
for (const [file, name, method] of [
|
||||
['app/g2/dashboard-controller.ts', 'DashboardController', 'voiceCaptureOptions'],
|
||||
['app/g2/ios-preview-controller.ts', 'IosPreviewController', 'voiceProvider'],
|
||||
]) {
|
||||
const host = { continuousClients: new Map() };
|
||||
host.transcriptionChoice = () => [...host.continuousClients.values()].find(x => x.transcription)?.transcription;
|
||||
// Run each real capture-options method with only its native dependencies stubbed.
|
||||
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
|
||||
const tree = ts.createSourceFile(file, source, ts.ScriptTarget.Latest, true);
|
||||
const declaration = tree.statements.find(n => ts.isClassDeclaration(n) && n.name.text === name);
|
||||
const member = declaration.members.find(n => n.name?.getText(tree) === method);
|
||||
const capture = loadBody(member.body.getText(tree), s, host);
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
host.continuousClients.set('translate', { transcription: { provider: 'soniox', language: 'ja' } });
|
||||
assert.equal(capture().provider, 'soniox');
|
||||
assert.equal(capture().language, 'ja');
|
||||
host.continuousClients.clear();
|
||||
assert.equal(capture().language, 'fr');
|
||||
assert.equal(capture().openAiTranscriptionModel, 'gpt-4o-mini-transcribe');
|
||||
}
|
||||
});
|
||||
|
||||
function loadBody(body, s, host) {
|
||||
const globals = { ...s, wearerVerificationOptions: () => null, micSession: { captureProcessingOptions: () => ({}) } };
|
||||
return vm.runInNewContext(`(function(communicator) ${body})`, globals).bind(host, null);
|
||||
}
|
||||
|
||||
test('multilingual downloads and deletion leave an existing English model intact', () => {
|
||||
const files = new Map(), downloads = [];
|
||||
const asr = load('app/native/asr-model.ts', () => ({
|
||||
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
||||
}), {
|
||||
java: { io: { File: class {
|
||||
constructor(name) { this.name = name; }
|
||||
exists() { return files.has(this.name); }
|
||||
length() { return files.get(this.name) ?? 0; }
|
||||
delete() { return files.delete(this.name); }
|
||||
} } },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
||||
FaceclawModelDownloader: class {
|
||||
constructor(url, destination, hash, size, listener) {
|
||||
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
||||
}
|
||||
start() {}
|
||||
cancel() {}
|
||||
},
|
||||
} } },
|
||||
});
|
||||
const english = asr.ASR_MODELS['whisper-base-en'];
|
||||
for (const file of english.files) files.set(`/files/faceclaw-voice-asr/${english.dirName}/${file.name}`, file.sizeBytes);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
asr.startAsrModelDownload('whisper-base');
|
||||
assert.equal(asr.asrModelState('whisper-base').status, 'downloading');
|
||||
for (let index = 0; index < 3; index++) {
|
||||
const download = downloads[index];
|
||||
assert.match(download.url, /sherpa-onnx-whisper-base\/resolve\/[a-f0-9]{40}\/base-/);
|
||||
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
||||
files.set(download.destination, download.size);
|
||||
download.listener.onDone();
|
||||
}
|
||||
assert.equal(downloads.length, 3);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), true);
|
||||
asr.deleteAsrModel('whisper-base');
|
||||
assert.equal(asr.isAsrModelReady('whisper-base'), false);
|
||||
assert.equal(asr.isAsrModelReady('whisper-base-en'), true);
|
||||
});
|
||||
|
||||
test('OpenAI transcription model preserves legacy default and persists independently of language', () => {
|
||||
const values = new Map();
|
||||
let s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
s.openAiTranscriptionModelSetting.set('gpt-4o-mini-transcribe');
|
||||
s.voiceLanguageSetting.set('fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.openAiTranscriptionModelSetting.get(), 'gpt-4o-mini-transcribe');
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'fr');
|
||||
assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper');
|
||||
});
|
||||
|
||||
test('French provider persists independently and never overwrites shared language preferences', () => {
|
||||
const values = new Map([['voice.language', 'ja'], ['voice.provider', 'onboard-whisper']]);
|
||||
let s = settings(values);
|
||||
s.voiceProviderSetting.set('onboard-whisper-fr');
|
||||
s = settings(values);
|
||||
assert.equal(s.voiceProviderSetting.get(), 'onboard-whisper-fr');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper-fr'), 'fr');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja');
|
||||
assert.equal(s.voiceLanguageSetting.get(), 'ja');
|
||||
s.voiceProviderSetting.set('onboard-whisper');
|
||||
assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja');
|
||||
});
|
||||
|
||||
test('French model download resumes, verifies every file including its license, and preserves other models', () => {
|
||||
const files = new Map(), downloads = [];
|
||||
const asr = load('app/native/asr-model.ts', () => ({
|
||||
Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } },
|
||||
}), {
|
||||
java: { io: { File: class {
|
||||
constructor(name) { this.name = name; }
|
||||
exists() { return files.has(this.name); }
|
||||
length() { return files.get(this.name) ?? 0; }
|
||||
delete() { return files.delete(this.name); }
|
||||
} } },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawModelDownloaderListener: function(listener) { return listener; },
|
||||
FaceclawModelDownloader: class {
|
||||
constructor(url, destination, hash, size, listener) {
|
||||
Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this);
|
||||
}
|
||||
start() {}
|
||||
cancel() {}
|
||||
},
|
||||
} } },
|
||||
});
|
||||
for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) {
|
||||
const model = asr.ASR_MODELS[id];
|
||||
for (const file of model.files) files.set(`/files/faceclaw-voice-asr/${model.dirName}/${file.name}`, file.sizeBytes);
|
||||
}
|
||||
const model = asr.ASR_MODELS['whisper-small-fr'];
|
||||
const manifest = JSON.parse(fs.readFileSync(path.join(__dirname, '../docs/models/whisper-small-fr/manifest.json')));
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(model.files)), manifest.files);
|
||||
assert.equal(model.totalBytes, model.files.reduce((total, file) => total + file.sizeBytes, 0));
|
||||
assert.equal(asr.isAsrModelReady('whisper-small-fr'), false);
|
||||
asr.startAsrModelDownload('whisper-small-fr');
|
||||
files.set(downloads[0].destination, downloads[0].size);
|
||||
downloads[0].listener.onDone();
|
||||
downloads[1].listener.onError('network unavailable');
|
||||
assert.equal(asr.asrModelState('whisper-small-fr').status, 'absent');
|
||||
asr.startAsrModelDownload('whisper-small-fr');
|
||||
assert.equal(downloads[2].destination, downloads[1].destination, 'resume the missing decoder');
|
||||
for (let index = 2; index < 5; index++) {
|
||||
const download = downloads[index];
|
||||
assert.match(download.url, /api\/packages\/Blendman974\/generic\/faceclaw-asr\/whisper-small-fr-v1\//);
|
||||
assert.match(download.hash, /^[a-f0-9]{64}$/);
|
||||
files.set(download.destination, download.size);
|
||||
download.listener.onDone();
|
||||
}
|
||||
assert.equal(asr.isAsrModelReady('whisper-small-fr'), true);
|
||||
asr.deleteAsrModel('whisper-small-fr');
|
||||
assert.equal(asr.isAsrModelReady('whisper-small-fr'), false);
|
||||
for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) assert.equal(asr.isAsrModelReady(id), true);
|
||||
});
|
||||
@@ -1,124 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export the pinned French fine-tune to sherpa-onnx 1.13.0, using CPU only.
|
||||
|
||||
See docs/models/whisper-small-fr/README.md for the isolated export environment.
|
||||
This script never uploads assets or touches installed/downloaded phone models.
|
||||
"""
|
||||
import argparse
|
||||
import hashlib
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import shutil
|
||||
from types import SimpleNamespace
|
||||
import urllib.request
|
||||
|
||||
SOURCE_REVISION = '83d7df2928fb2218721c543dcc0be29ef8db2f96'
|
||||
SOURCE_BASE = f'https://huggingface.co/bofenghuang/whisper-small-cv11-french/resolve/{SOURCE_REVISION}/'
|
||||
WEIGHTS_SHA256 = '855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb'
|
||||
EXPORTER_URL = 'https://raw.githubusercontent.com/k2-fsa/sherpa-onnx/v1.13.0/scripts/whisper/export-onnx.py'
|
||||
EXPORTER_SHA256 = '5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45'
|
||||
MODEL_NAME = 'small-fr'
|
||||
|
||||
|
||||
def digest(path):
|
||||
with path.open('rb') as stream:
|
||||
return hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
|
||||
|
||||
def download(url, path, expected=None):
|
||||
if not path.exists():
|
||||
temp = path.with_suffix(path.suffix + '.part')
|
||||
urllib.request.urlretrieve(url, temp)
|
||||
temp.replace(path)
|
||||
if expected and digest(path) != expected:
|
||||
raise ValueError(f'Checksum mismatch: {path.name}')
|
||||
|
||||
|
||||
def openai_key(key):
|
||||
"""HF Whisper and OpenAI Whisper have identical tensors, differently named."""
|
||||
if not key.startswith('model.'):
|
||||
raise ValueError(f'Unexpected weight: {key}')
|
||||
key = key.removeprefix('model.')
|
||||
replacements = [
|
||||
('embed_positions.weight', 'positional_embedding'),
|
||||
('embed_tokens', 'token_embedding'), ('layers.', 'blocks.'),
|
||||
('self_attn_layer_norm', 'attn_ln'), ('encoder_attn_layer_norm', 'cross_attn_ln'),
|
||||
('final_layer_norm', 'mlp_ln'), ('self_attn', 'attn'), ('encoder_attn', 'cross_attn'),
|
||||
('k_proj', 'key'), ('q_proj', 'query'), ('v_proj', 'value'), ('out_proj', 'out'),
|
||||
('fc1', 'mlp.0'), ('fc2', 'mlp.2'),
|
||||
('encoder.layer_norm', 'encoder.ln_post'), ('decoder.layer_norm', 'decoder.ln'),
|
||||
]
|
||||
for old, new in replacements:
|
||||
key = key.replace(old, new)
|
||||
return key
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--output', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
output = args.output.resolve()
|
||||
output.mkdir(parents=True, exist_ok=True)
|
||||
download(SOURCE_BASE + 'pytorch_model.bin', output / 'pytorch_model.bin', WEIGHTS_SHA256)
|
||||
download(SOURCE_BASE + 'config.json', output / 'config.json')
|
||||
download(EXPORTER_URL, output / 'sherpa-export.py', EXPORTER_SHA256)
|
||||
|
||||
import torch
|
||||
import whisper
|
||||
from whisper.model import ModelDimensions, disable_sdpa
|
||||
torch.set_num_threads(1)
|
||||
torch.set_num_interop_threads(1)
|
||||
torch.manual_seed(0)
|
||||
config = json.loads((output / 'config.json').read_text())
|
||||
dimensions = ModelDimensions(
|
||||
n_mels=config['num_mel_bins'], n_audio_ctx=config['max_source_positions'],
|
||||
n_audio_state=config['d_model'], n_audio_head=config['encoder_attention_heads'],
|
||||
n_audio_layer=config['encoder_layers'], n_vocab=config['vocab_size'],
|
||||
n_text_ctx=config['max_target_positions'], n_text_state=config['d_model'],
|
||||
n_text_head=config['decoder_attention_heads'], n_text_layer=config['decoder_layers'],
|
||||
)
|
||||
model = whisper.Whisper(dimensions)
|
||||
weights = torch.load(output / 'pytorch_model.bin', map_location='cpu', weights_only=True)
|
||||
converted = {openai_key(key): value for key, value in weights.items()}
|
||||
model.load_state_dict(converted, strict=True)
|
||||
del weights, converted
|
||||
model.eval()
|
||||
|
||||
spec = importlib.util.spec_from_file_location('sherpa_export', output / 'sherpa-export.py')
|
||||
exporter = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(exporter)
|
||||
exporter.get_args = lambda: SimpleNamespace(model=MODEL_NAME)
|
||||
exporter.load_model = lambda _name: model
|
||||
original_export = torch.onnx.export
|
||||
|
||||
def legacy_export(*values, **options):
|
||||
options['dynamo'] = False # sherpa 1.13.0 uses TorchScript's exporter.
|
||||
return original_export(*values, **options)
|
||||
|
||||
torch.onnx.export = legacy_export
|
||||
old_cwd = Path.cwd()
|
||||
try:
|
||||
os.chdir(output)
|
||||
with disable_sdpa():
|
||||
exporter.main()
|
||||
finally:
|
||||
torch.onnx.export = original_export
|
||||
os.chdir(old_cwd)
|
||||
|
||||
license_path = Path(__file__).resolve().parents[1] / 'docs/models/whisper-small-fr/LICENSE.txt'
|
||||
shutil.copyfile(license_path, output / 'LICENSE.txt')
|
||||
names = [f'{MODEL_NAME}-encoder.int8.onnx', f'{MODEL_NAME}-decoder.int8.onnx', f'{MODEL_NAME}-tokens.txt', 'LICENSE.txt']
|
||||
manifest = {
|
||||
'source': 'bofenghuang/whisper-small-cv11-french', 'revision': SOURCE_REVISION,
|
||||
'sourceSha256': WEIGHTS_SHA256, 'exporterSha256': EXPORTER_SHA256,
|
||||
'torchVersion': torch.__version__,
|
||||
'files': [{'name': name, 'sha256': digest(output / name), 'sizeBytes': (output / name).stat().st_size} for name in names],
|
||||
}
|
||||
(output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
print(json.dumps(manifest, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -1,52 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Verify released hashes and decode a public/test WAV with sherpa-onnx CPU."""
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import wave
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--model-dir', type=Path, required=True)
|
||||
parser.add_argument('--wav', type=Path, required=True)
|
||||
parser.add_argument('--expected', help='Optional expected transcription substring')
|
||||
args = parser.parse_args()
|
||||
model_dir = args.model_dir
|
||||
manifest = json.loads((model_dir / 'manifest.json').read_text())
|
||||
for entry in manifest['files']:
|
||||
path = model_dir / entry['name']
|
||||
with path.open('rb') as stream:
|
||||
actual = hashlib.file_digest(stream, 'sha256').hexdigest()
|
||||
if actual != entry['sha256'] or path.stat().st_size != entry['sizeBytes']:
|
||||
raise ValueError(f'Model file integrity check failed: {path.name}')
|
||||
|
||||
import numpy as np
|
||||
import sherpa_onnx
|
||||
with wave.open(str(args.wav)) as audio:
|
||||
if audio.getnchannels() != 1 or audio.getsampwidth() != 2:
|
||||
raise ValueError('Provide a mono PCM16 WAV')
|
||||
sample_rate = audio.getframerate()
|
||||
samples = np.frombuffer(audio.readframes(audio.getnframes()), dtype='<i2').astype(np.float32) / 32768
|
||||
recognizer = sherpa_onnx.OfflineRecognizer.from_whisper(
|
||||
encoder=str(model_dir / 'small-fr-encoder.int8.onnx'),
|
||||
decoder=str(model_dir / 'small-fr-decoder.int8.onnx'),
|
||||
tokens=str(model_dir / 'small-fr-tokens.txt'), num_threads=1,
|
||||
language='fr', task='transcribe',
|
||||
)
|
||||
stream = recognizer.create_stream()
|
||||
stream.accept_waveform(sample_rate, samples)
|
||||
start = time.monotonic()
|
||||
recognizer.decode_stream(stream)
|
||||
text = stream.result.text.strip()
|
||||
if not text or (args.expected and args.expected.casefold() not in text.casefold()):
|
||||
raise ValueError(f'Unexpected transcription: {text!r}')
|
||||
print(json.dumps({'sherpaVersion': sherpa_onnx.__version__, 'text': text,
|
||||
'audioSeconds': len(samples) / sample_rate,
|
||||
'decodeSeconds': time.monotonic() - start}, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in new issue
Block a user