diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt index 8b18600e..a35c8daa 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.kt @@ -74,6 +74,7 @@ class FaceclawVoiceController(context: Context) { @Volatile private var communicator: FaceclawBleCommunicator? = null private var whisperModel = WhisperModelFiles.forLanguage(null) + private var frenchWhisper = false private val host = object : VoiceCaptureHost { override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } } @@ -158,9 +159,15 @@ class FaceclawVoiceController(context: Context) { session.setOnboardModelKind(kind) } + /** Select the dedicated French fine-tune before setting the capture language. */ + fun setFrenchWhisper(enabled: Boolean) { + frenchWhisper = enabled + whisperModel = WhisperModelFiles.forSelection(whisperModel.language, enabled) + } + /** Language for the next on-device Whisper capture; empty enables auto-detection. */ fun setOnboardLanguage(language: String?) { - whisperModel = WhisperModelFiles.forLanguage(language) + whisperModel = WhisperModelFiles.forSelection(language, frenchWhisper) } /** diff --git a/app/native/asr-model.ts b/app/native/asr-model.ts index c21675fc..9599d0a1 100644 --- a/app/native/asr-model.ts +++ b/app/native/asr-model.ts @@ -34,12 +34,15 @@ declare const java: any; * SHA-256/size values come from the maintainer's LFS metadata, with the * vocabulary hash computed locally. URLs pin the source revision. * + * - "whisper-small-fr": a dedicated French fine-tune, exported and quantized + * with sherpa-onnx 1.13.0. Versioned release assets include its Apache license. + * * Mirrors the on-phone assistant model flow in llama.ts, except each model * here is multiple files rather than one; they download sequentially through * FaceclawModelDownloader (resume + pinned sha256 per file). */ -export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base"; +export type AsrModelId = "moonshine" | "whisper-base-en" | "whisper-base" | "whisper-small-fr"; type AsrModelFile = { name: string; @@ -134,6 +137,19 @@ export const ASR_MODELS: Record = { ], totalBytes: 160609290, }, + "whisper-small-fr": { + label: "Whisper (French, small)", + dirName: "sherpa-onnx-whisper-small-fr-int8-v1", + // Pinned export of bofenghuang/whisper-small-cv11-french; see docs/models/whisper-small-fr. + baseUrl: "https://git.undefined-dev.fr/api/packages/Blendman974/generic/faceclaw-asr/whisper-small-fr-v1/", + files: [ + {"name": "small-fr-encoder.int8.onnx", "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", "sizeBytes": 112413414}, + {"name": "small-fr-decoder.int8.onnx", "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", "sizeBytes": 262200764}, + {"name": "small-fr-tokens.txt", "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", "sizeBytes": 816730}, + {"name": "LICENSE.txt", "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", "sizeBytes": 11358}, + ], + totalBytes: 375442266, + }, }; export type AsrModelState = { @@ -159,6 +175,7 @@ const runtimes: Record = { moonshine: freshRuntime(), "whisper-base-en": freshRuntime(), "whisper-base": freshRuntime(), + "whisper-small-fr": freshRuntime(), }; function modelDirPath(id: AsrModelId): string { diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts index 4dad2d70..5309a2e2 100644 --- a/app/native/cloud-stt-provider.ts +++ b/app/native/cloud-stt-provider.ts @@ -78,9 +78,10 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection * to expect. A running capture is shared only by holders asking for the same. */ export function transcriptionKey(selection: Pick): string { - const fallback = selection.provider === "onboard-whisper" ? "en" : ""; + const fallback = selection.provider === "onboard-whisper-fr" ? "fr" : selection.provider === "onboard-whisper" ? "en" : ""; + const language = selection.provider === "onboard-whisper-fr" ? "fr" : (selection.language ?? fallback).trim().toLowerCase(); const model = selection.provider === "whisper" ? `:${selection.openAiTranscriptionModel ?? DEFAULT_OPENAI_STT_MODEL}` : ""; - return `${selection.provider}:${(selection.language ?? fallback).trim().toLowerCase()}${model}`; + return `${selection.provider}:${language}${model}`; } /** Whether createCloudSttClient would return a cloud client for this selection. */ diff --git a/app/native/transcription-language.ts b/app/native/transcription-language.ts index 1b8ae0a7..3aa1e6c5 100644 --- a/app/native/transcription-language.ts +++ b/app/native/transcription-language.ts @@ -14,6 +14,7 @@ export type TranscriptionLanguage = "default" | "auto" | keyof typeof TRANSCRIPT export function resolveTranscriptionLanguage( choice: TranscriptionLanguage, provider: string, serverLanguage = "", ): string { + if (provider === "onboard-whisper-fr") return "fr"; // Moonshine is English-only; Apple's on-device recognizer retains its own locale. if (provider === "onboard") return ""; if (choice === "default") { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index 319a8673..22864a78 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -32,7 +32,7 @@ export type VoiceControlState = { // renamed, since it's a persisted settings value on real installs already. // "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP // (whisper-server-stt.ts). -export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "onboard-whisper-fr" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -397,8 +397,10 @@ export class FaceclawVoiceControlBridge { this.started = true; // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). - this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); - this.controller?.setOnboardLanguage(options.language ?? "en"); + const frenchWhisper = options.provider === "onboard-whisper-fr"; + this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" || frenchWhisper ? "whisper" : "moonshine"); + this.controller?.setFrenchWhisper(frenchWhisper); + this.controller?.setOnboardLanguage(frenchWhisper ? "fr" : options.language ?? "en"); const captureId = this.nextNativeCaptureId++; this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); this.controller?.start("onboard", captureId); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 427edf9c..8743142c 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -884,13 +884,14 @@ export const ringConnectionModeSetting = new ConfigSettingEnum = { onboard: global.isIOS ? "On-device (Apple)" : "On-device (Moonshine)", "onboard-whisper": "On-device (Whisper)", + "onboard-whisper-fr": "On-device (Whisper French)", elevenlabs: "ElevenLabs", whisper: "OpenAI (Whisper)", soniox: "Soniox", @@ -904,7 +905,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ defaultValue: "onboard", values: global.isIOS ? ["onboard", "elevenlabs", "whisper", "soniox", "whisper-server"] - : ["onboard", "onboard-whisper", "elevenlabs", "whisper", "soniox", "whisper-server"], + : ["onboard", "onboard-whisper", "onboard-whisper-fr", "elevenlabs", "whisper", "soniox", "whisper-server"], formatValue: (value) => voiceProviderLabels[value] ?? value, isDisabled: (value) => { if (value === "elevenlabs") return elevenLabsApiKeySetting.get().trim().length === 0; @@ -915,7 +916,7 @@ export const voiceProviderSetting = new ConfigSettingEnum({ }, description: global.isIOS ? "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key (Settings > API Keys), with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). On-device uses your iPhone's speech language and never leaves the phone." - : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The two On-device options need their voice model downloaded (below) and never leave the phone.", + : "Speech-to-text engine for voice input. ElevenLabs, OpenAI, and Soniox are cloud services that need an API key, with significantly better accuracy than on-device transcription. Whisper server is your own OpenAI-compatible Whisper (set its URL below). The On-device options need their voice model downloaded (below) and never leave the phone. Whisper French uses the French-only small model and always transcribes in French.", }); const wakeWordActionLabels: Record = { @@ -1157,7 +1158,7 @@ export const voiceLanguageSetting = new ConfigSettingEnum : value === "auto" ? "Auto-detect" : TRANSCRIPTION_LANGUAGES[value], description: "Spoken language for OpenAI, on-device Whisper and other cloud transcription providers. " + "Provider default preserves English for on-device Whisper and auto-detection for cloud providers, with the server's own language below. " + - "On-device Whisper needs the multilingual model for auto-detection or languages other than English. Moonshine remains English-only.", + "On-device Whisper needs the multilingual model for auto-detection or languages other than English. The dedicated Whisper French provider always uses French, without changing this preference. Moonshine remains English-only.", }); export function getVoiceTranscriptionLanguage(provider: VoiceProvider): string { diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 95ae991f..c889a4b4 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -150,6 +150,7 @@ function settingsSections(): SettingsSection[] { asrModelMenuItem("moonshine"), asrModelMenuItem("whisper-base-en"), asrModelMenuItem("whisper-base"), + asrModelMenuItem("whisper-small-fr"), ], }, { diff --git a/docs/models/whisper-small-fr/LICENSE.txt b/docs/models/whisper-small-fr/LICENSE.txt new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/docs/models/whisper-small-fr/LICENSE.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/docs/models/whisper-small-fr/README.md b/docs/models/whisper-small-fr/README.md new file mode 100644 index 00000000..40f31064 --- /dev/null +++ b/docs/models/whisper-small-fr/README.md @@ -0,0 +1,71 @@ +# On-device Whisper French + +Android Settings → Voice now offers `On-device (Whisper French)` and a separate +`On-device model: Whisper (French, small)` download (~375 MB decimal). Download +first, then choose the provider. Recognition stays on the phone; no API key is +required. This is speech-to-text in French, not translation. + +This provider always requests `language=fr`, `task=transcribe`, without writing +the shared language preference. Switching back to existing Moonshine, English +Whisper or multilingual Whisper restores their previous behavior. Downloads, +cancellation/resume and deletion reuse the existing model manager and leave +other model directories untouched. Missing French files do not silently select +multilingual base. iOS and the Microphones app's separate Moonshine caption +engine are outside this addition. + +## Provenance and distribution + +- Source: https://huggingface.co/bofenghuang/whisper-small-cv11-french +- Pinned revision: `83d7df2928fb2218721c543dcc0be29ef8db2f96`. +- The model author's declared license is Apache-2.0; `LICENSE.txt` is distributed + with the downloadable model. The original Whisper implementation is MIT; + sherpa-onnx's exporter is Apache-2.0. +- Exporter: sherpa-onnx v1.13.0 `scripts/whisper/export-onnx.py`. +- Export: strict tensor-name conversion from HF Whisper to OpenAI Whisper, + then upstream sherpa tensor-cache export, opset 17, dynamic int8 MatMul. +- Assets: https://git.undefined-dev.fr/Blendman974/faceclaw/releases/tag/models-whisper-small-fr-v1 +- Model files use the public Gitea generic package registry (`faceclaw-asr`, + version `whisper-small-fr-v1`), because the encoder and decoder exceed the + instance’s 100 MB attachment limit. No infrastructure limit is changed. +- The JSON manifest records file lengths and SHA-256 hashes. Both the source + checkpoint and exporter are checksum-verified before execution. The existing + Android downloader checks each downloaded asset against its pinned hash. +- Only encoder, decoder, vocabulary and license are downloaded to the phone; + the original checkpoint and unquantized intermediate models are not shipped. + +## Reproducing the export + +Use an isolated Python environment with the versions used for this release: +Python 3.14, torch 2.9.0+cpu (CPU wheel index), openai-whisper 20250625, +onnx 1.23.2, onnxruntime 1.31.1 and sherpa-onnx 1.13.0. These are development +requirements only; no Python or additional runtime is added to the app. +On NixOS expose the host C++ runtime and zlib through LD_LIBRARY_PATH. + +```sh +python tools/export-whisper-small-fr.py --output /tmp/whisper-small-fr +python tools/test-whisper-small-fr.py \ + --model-dir /tmp/whisper-small-fr --wav /path/to/french-test.wav +``` + +The exporter downloads the pinned source (~484 MB), writes unquantized and +quantized intermediate models, and produces `manifest.json`. Artifact changes +require a new versioned release and explicit catalog hash changes; never replace +v1 assets in place. These scripts do not upload or publish files automatically. + +## Validation + +The exported int8 files were actually decoded on CPU with sherpa-onnx 1.13.0, +the same version used by Android. A public 4.97-second French sample from +`csukuangfj/sherpa-onnx-nemo-parakeet-tdt-0.6b-v3/test_wavs/fr.wav` produced: + +> Ne vous demandez pas ce que votre pays peut faire pour vous, demandez-vous plutôt ce que vous pouvez faire pour lui. + +This is a smoke test, not a recognition-quality benchmark or a measurement of +Pixel latency. Application tests cover provider persistence, French selection +without rewriting other languages, switching back, capture sharing, interrupted +download/resume, file hashes and independent deletion. Kotlin tests cover the +native file selection and preservation of existing filenames/defaults. + +Hardware checks remain pending: real French dictation through phone and glasses +microphones, latency and RAM on the Pixel, airplane-mode use after downloading, +reconnection and long utterances. No firmware changes are required. diff --git a/docs/models/whisper-small-fr/manifest.json b/docs/models/whisper-small-fr/manifest.json new file mode 100644 index 00000000..8f3cf6ee --- /dev/null +++ b/docs/models/whisper-small-fr/manifest.json @@ -0,0 +1,29 @@ +{ + "source": "bofenghuang/whisper-small-cv11-french", + "revision": "83d7df2928fb2218721c543dcc0be29ef8db2f96", + "sourceSha256": "855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb", + "exporterSha256": "5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45", + "torchVersion": "2.9.0+cpu", + "files": [ + { + "name": "small-fr-encoder.int8.onnx", + "sha256": "ec34fc42182f37727ad982d254b18a6001f5dcccebc5a36060031995f1782d5d", + "sizeBytes": 112413414 + }, + { + "name": "small-fr-decoder.int8.onnx", + "sha256": "b554e35dc8c593b550a82cd01ebb00c1347e8c2b089b780b4c4fe16450cd7dc0", + "sizeBytes": 262200764 + }, + { + "name": "small-fr-tokens.txt", + "sha256": "b34b360dbb493e781e479794586d661700670d65564001f23024971d1f2fa126", + "sizeBytes": 816730 + }, + { + "name": "LICENSE.txt", + "sha256": "cfc7749b96f63bd31c3c42b5c471bf756814053e847c10f3eb003417bc523d30", + "sizeBytes": 11358 + } + ] +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt index 9ae3817f..99afe8b4 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/audio/WhisperModelFiles.kt @@ -1,16 +1,20 @@ package com.faceclaw.app -/** The English-only files remain usable after upgrading; other languages need multilingual base. */ -data class WhisperModelFiles(val language: String) { - val englishOnly: Boolean get() = language == "en" - val dirName: String get() = if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8" - private val prefix: String get() = if (englishOnly) "base.en" else "base" +/** Existing English/multilingual files stay usable; the dedicated French provider uses its own download. */ +data class WhisperModelFiles(val language: String, val frenchOnly: Boolean = false) { + val englishOnly: Boolean get() = !frenchOnly && language == "en" + val dirName: String get() = if (frenchOnly) "sherpa-onnx-whisper-small-fr-int8-v1" else if (englishOnly) "sherpa-onnx-whisper-base-en-int8" else "sherpa-onnx-whisper-base-int8" + private val prefix: String get() = if (frenchOnly) "small-fr" else if (englishOnly) "base.en" else "base" val encoder: String get() = "$prefix-encoder.int8.onnx" val decoder: String get() = "$prefix-decoder.int8.onnx" val tokens: String get() = "$prefix-tokens.txt" val files: List get() = listOf(encoder, decoder, tokens) companion object { + /** A dedicated French fine-tune never silently substitutes the generic base model. */ + fun forSelection(language: String?, frenchOnly: Boolean): WhisperModelFiles = + if (frenchOnly) WhisperModelFiles("fr", true) else forLanguage(language) + /** Null preserves the old English default; an empty code enables automatic detection. */ fun forLanguage(language: String?): WhisperModelFiles = WhisperModelFiles(language?.trim()?.lowercase() ?: "en") diff --git a/tests/assistant-settings-visibility.test.cjs b/tests/assistant-settings-visibility.test.cjs index 75a98697..4173342f 100644 --- a/tests/assistant-settings-visibility.test.cjs +++ b/tests/assistant-settings-visibility.test.cjs @@ -54,6 +54,7 @@ function settingsEnv(initialUrl = '', initialModel = 'openai-compatible') { '../../native/asr-model': { ASR_MODELS: { moonshine: { label: 'Moonshine', sizeBytes: 1 }, + 'whisper-small-fr': { label: 'Whisper French Small', sizeBytes: 1 }, 'whisper-base-en': { label: 'Whisper Base English', sizeBytes: 1 }, 'whisper-base': { label: 'Whisper Base Multilingual', sizeBytes: 1 }, }, diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs index 82c7890b..dbaa71f5 100644 --- a/tests/cloud-stt.test.cjs +++ b/tests/cloud-stt.test.cjs @@ -209,6 +209,8 @@ test('provider selection falls back to on-device without a key and connects the assert.notEqual(transcriptionKey({ provider: 'soniox' }), transcriptionKey({ provider: 'soniox', language: 'ja' })); assert.equal(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: 'en' })); assert.notEqual(transcriptionKey({ provider: 'onboard-whisper' }), transcriptionKey({ provider: 'onboard-whisper', language: '' })); + assert.equal(transcriptionKey({ provider: 'onboard-whisper-fr', language: 'en' }), 'onboard-whisper-fr:fr'); + assert.notEqual(transcriptionKey({ provider: 'onboard-whisper-fr' }), transcriptionKey({ provider: 'onboard-whisper', language: 'fr' })); const statuses = []; const options = { onTranscript() {}, onStatus: status => statuses.push(status), onError() {} }; const selection = { provider: 'onboard', elevenLabsApiKey: '', openAiApiKey: ' ', sonioxApiKey: 'soniox-key' }; diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt index 3cc37184..06a81ad6 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/WhisperModelFilesTest.kt @@ -32,4 +32,15 @@ class WhisperModelFilesTest { assertEquals("", model.language) assertEquals(WhisperModelFiles.forLanguage("fr").files, model.files) } + @Test fun frenchFineTuneUsesDistinctFilesRegardlessOfTheGenericLanguage() { + for (language in listOf(null, "en", "de", "", "fr")) { + val model = WhisperModelFiles.forSelection(language, true) + assertTrue(model.frenchOnly) + assertFalse(model.englishOnly) + assertEquals("fr", model.language) + assertEquals("sherpa-onnx-whisper-small-fr-int8-v1", model.dirName) + assertEquals(listOf("small-fr-encoder.int8.onnx", "small-fr-decoder.int8.onnx", "small-fr-tokens.txt"), model.files) + assertEquals(WhisperModelFiles.forLanguage(language), WhisperModelFiles.forSelection(language, false)) + } + } } diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs index 565151e0..1cafe7b3 100644 --- a/tests/voice-finalization.test.cjs +++ b/tests/voice-finalization.test.cjs @@ -164,6 +164,7 @@ function androidBridge() { setListener(listener) { this.listener = listener; }, setCommunicator() {}, setUsePhoneMic() {}, setSaveRecordings() {}, setEndpointing() {}, setNoiseSuppression() {}, setBeamFilter() {}, clearSpeakerVerification() {}, + setFrenchWhisper(enabled) { this.frenchWhisper = enabled; }, setOnboardModelKind(kind) { this.modelKind = kind; }, setOnboardLanguage(language) { this.language = language; }, start(_mode, id) { this.captureId = id; this.active = true; this.starts++; }, stop() { this.active = false; }, @@ -173,7 +174,7 @@ function androidBridge() { '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, './speech-pause': { SpeechPauseDetector: class { reset() {} } }, './cloud-stt-provider': { createCloudSttClient: () => null, - transcriptionKey: options => `${options.provider}:${options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` }, + transcriptionKey: options => `${options.provider}:${options.provider === 'onboard-whisper-fr' ? 'fr' : options.language ?? (options.provider === 'onboard-whisper' ? 'en' : '')}` }, }, { global: { isAndroid: true }, com: { faceclaw: { app: { FaceclawVoiceController: function () { return controller; }, FaceclawVoiceControllerListener: function (listener) { return listener; }, @@ -237,3 +238,16 @@ test('a shared capture restarts for a holder that needs another provider, and is assert.equal(controller.active, true); assert.equal(bridge.holdsCapture('ptt'), true); }); + +test('dedicated French Whisper selects its own files and switches back without retaining the override', () => { + const { bridge, controller } = androidBridge(); + bridge.startContinuousCapture({ provider: 'onboard-whisper-fr', language: 'en' }); + assert.equal(controller.modelKind, 'whisper'); + assert.equal(controller.frenchWhisper, true); + assert.equal(controller.language, 'fr'); + bridge.retargetContinuousCapture({ provider: 'onboard-whisper' }); + assert.equal(controller.frenchWhisper, false); + assert.equal(controller.language, 'en'); + assert.equal(controller.starts, 2); + bridge.stop(); +}); diff --git a/tests/whisper-language.test.cjs b/tests/whisper-language.test.cjs index 95435a47..77689e56 100644 --- a/tests/whisper-language.test.cjs +++ b/tests/whisper-language.test.cjs @@ -144,3 +144,67 @@ test('OpenAI transcription model preserves legacy default and persists independe assert.equal(s.voiceLanguageSetting.get(), 'fr'); assert.equal(settings(new Map([['voice.openAiTranscriptionModel', 'invalid']])).openAiTranscriptionModelSetting.get(), 'gpt-realtime-whisper'); }); + +test('French provider persists independently and never overwrites shared language preferences', () => { + const values = new Map([['voice.language', 'ja'], ['voice.provider', 'onboard-whisper']]); + let s = settings(values); + s.voiceProviderSetting.set('onboard-whisper-fr'); + s = settings(values); + assert.equal(s.voiceProviderSetting.get(), 'onboard-whisper-fr'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper-fr'), 'fr'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja'); + assert.equal(s.voiceLanguageSetting.get(), 'ja'); + s.voiceProviderSetting.set('onboard-whisper'); + assert.equal(s.getVoiceTranscriptionLanguage('onboard-whisper'), 'ja'); +}); + +test('French model download resumes, verifies every file including its license, and preserves other models', () => { + const files = new Map(), downloads = []; + const asr = load('app/native/asr-model.ts', () => ({ + Utils: { android: { getApplicationContext: () => ({ getFilesDir: () => ({ getAbsolutePath: () => '/files' }) }) } }, + }), { + java: { io: { File: class { + constructor(name) { this.name = name; } + exists() { return files.has(this.name); } + length() { return files.get(this.name) ?? 0; } + delete() { return files.delete(this.name); } + } } }, + com: { faceclaw: { app: { + FaceclawModelDownloaderListener: function(listener) { return listener; }, + FaceclawModelDownloader: class { + constructor(url, destination, hash, size, listener) { + Object.assign(this, { url, destination, hash, size, listener }); downloads.push(this); + } + start() {} + cancel() {} + }, + } } }, + }); + for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) { + const model = asr.ASR_MODELS[id]; + for (const file of model.files) files.set(`/files/faceclaw-voice-asr/${model.dirName}/${file.name}`, file.sizeBytes); + } + const model = asr.ASR_MODELS['whisper-small-fr']; + const manifest = JSON.parse(fs.readFileSync(path.join(__dirname, '../docs/models/whisper-small-fr/manifest.json'))); + assert.deepEqual(JSON.parse(JSON.stringify(model.files)), manifest.files); + assert.equal(model.totalBytes, model.files.reduce((total, file) => total + file.sizeBytes, 0)); + assert.equal(asr.isAsrModelReady('whisper-small-fr'), false); + asr.startAsrModelDownload('whisper-small-fr'); + files.set(downloads[0].destination, downloads[0].size); + downloads[0].listener.onDone(); + downloads[1].listener.onError('network unavailable'); + assert.equal(asr.asrModelState('whisper-small-fr').status, 'absent'); + asr.startAsrModelDownload('whisper-small-fr'); + assert.equal(downloads[2].destination, downloads[1].destination, 'resume the missing decoder'); + for (let index = 2; index < 5; index++) { + const download = downloads[index]; + assert.match(download.url, /api\/packages\/Blendman974\/generic\/faceclaw-asr\/whisper-small-fr-v1\//); + assert.match(download.hash, /^[a-f0-9]{64}$/); + files.set(download.destination, download.size); + download.listener.onDone(); + } + assert.equal(asr.isAsrModelReady('whisper-small-fr'), true); + asr.deleteAsrModel('whisper-small-fr'); + assert.equal(asr.isAsrModelReady('whisper-small-fr'), false); + for (const id of ['moonshine', 'whisper-base-en', 'whisper-base']) assert.equal(asr.isAsrModelReady(id), true); +}); diff --git a/tools/export-whisper-small-fr.py b/tools/export-whisper-small-fr.py new file mode 100644 index 00000000..f7f56a8d --- /dev/null +++ b/tools/export-whisper-small-fr.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Export the pinned French fine-tune to sherpa-onnx 1.13.0, using CPU only. + +See docs/models/whisper-small-fr/README.md for the isolated export environment. +This script never uploads assets or touches installed/downloaded phone models. +""" +import argparse +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +from types import SimpleNamespace +import urllib.request + +SOURCE_REVISION = '83d7df2928fb2218721c543dcc0be29ef8db2f96' +SOURCE_BASE = f'https://huggingface.co/bofenghuang/whisper-small-cv11-french/resolve/{SOURCE_REVISION}/' +WEIGHTS_SHA256 = '855a879fa3a71e75f29ed59fcfce287cf21bcdaab23530cd6f8c4ad192677afb' +EXPORTER_URL = 'https://raw.githubusercontent.com/k2-fsa/sherpa-onnx/v1.13.0/scripts/whisper/export-onnx.py' +EXPORTER_SHA256 = '5e0ddabfac8330ee9aecaba4c1b4286d1cd9f88df4cd30e54706b011bf976e45' +MODEL_NAME = 'small-fr' + + +def digest(path): + with path.open('rb') as stream: + return hashlib.file_digest(stream, 'sha256').hexdigest() + + +def download(url, path, expected=None): + if not path.exists(): + temp = path.with_suffix(path.suffix + '.part') + urllib.request.urlretrieve(url, temp) + temp.replace(path) + if expected and digest(path) != expected: + raise ValueError(f'Checksum mismatch: {path.name}') + + +def openai_key(key): + """HF Whisper and OpenAI Whisper have identical tensors, differently named.""" + if not key.startswith('model.'): + raise ValueError(f'Unexpected weight: {key}') + key = key.removeprefix('model.') + replacements = [ + ('embed_positions.weight', 'positional_embedding'), + ('embed_tokens', 'token_embedding'), ('layers.', 'blocks.'), + ('self_attn_layer_norm', 'attn_ln'), ('encoder_attn_layer_norm', 'cross_attn_ln'), + ('final_layer_norm', 'mlp_ln'), ('self_attn', 'attn'), ('encoder_attn', 'cross_attn'), + ('k_proj', 'key'), ('q_proj', 'query'), ('v_proj', 'value'), ('out_proj', 'out'), + ('fc1', 'mlp.0'), ('fc2', 'mlp.2'), + ('encoder.layer_norm', 'encoder.ln_post'), ('decoder.layer_norm', 'decoder.ln'), + ] + for old, new in replacements: + key = key.replace(old, new) + return key + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + output = args.output.resolve() + output.mkdir(parents=True, exist_ok=True) + download(SOURCE_BASE + 'pytorch_model.bin', output / 'pytorch_model.bin', WEIGHTS_SHA256) + download(SOURCE_BASE + 'config.json', output / 'config.json') + download(EXPORTER_URL, output / 'sherpa-export.py', EXPORTER_SHA256) + + import torch + import whisper + from whisper.model import ModelDimensions, disable_sdpa + torch.set_num_threads(1) + torch.set_num_interop_threads(1) + torch.manual_seed(0) + config = json.loads((output / 'config.json').read_text()) + dimensions = ModelDimensions( + n_mels=config['num_mel_bins'], n_audio_ctx=config['max_source_positions'], + n_audio_state=config['d_model'], n_audio_head=config['encoder_attention_heads'], + n_audio_layer=config['encoder_layers'], n_vocab=config['vocab_size'], + n_text_ctx=config['max_target_positions'], n_text_state=config['d_model'], + n_text_head=config['decoder_attention_heads'], n_text_layer=config['decoder_layers'], + ) + model = whisper.Whisper(dimensions) + weights = torch.load(output / 'pytorch_model.bin', map_location='cpu', weights_only=True) + converted = {openai_key(key): value for key, value in weights.items()} + model.load_state_dict(converted, strict=True) + del weights, converted + model.eval() + + spec = importlib.util.spec_from_file_location('sherpa_export', output / 'sherpa-export.py') + exporter = importlib.util.module_from_spec(spec) + spec.loader.exec_module(exporter) + exporter.get_args = lambda: SimpleNamespace(model=MODEL_NAME) + exporter.load_model = lambda _name: model + original_export = torch.onnx.export + + def legacy_export(*values, **options): + options['dynamo'] = False # sherpa 1.13.0 uses TorchScript's exporter. + return original_export(*values, **options) + + torch.onnx.export = legacy_export + old_cwd = Path.cwd() + try: + os.chdir(output) + with disable_sdpa(): + exporter.main() + finally: + torch.onnx.export = original_export + os.chdir(old_cwd) + + license_path = Path(__file__).resolve().parents[1] / 'docs/models/whisper-small-fr/LICENSE.txt' + shutil.copyfile(license_path, output / 'LICENSE.txt') + names = [f'{MODEL_NAME}-encoder.int8.onnx', f'{MODEL_NAME}-decoder.int8.onnx', f'{MODEL_NAME}-tokens.txt', 'LICENSE.txt'] + manifest = { + 'source': 'bofenghuang/whisper-small-cv11-french', 'revision': SOURCE_REVISION, + 'sourceSha256': WEIGHTS_SHA256, 'exporterSha256': EXPORTER_SHA256, + 'torchVersion': torch.__version__, + 'files': [{'name': name, 'sha256': digest(output / name), 'sizeBytes': (output / name).stat().st_size} for name in names], + } + (output / 'manifest.json').write_text(json.dumps(manifest, indent=2) + '\n') + print(json.dumps(manifest, indent=2)) + + +if __name__ == '__main__': + main() diff --git a/tools/test-whisper-small-fr.py b/tools/test-whisper-small-fr.py new file mode 100644 index 00000000..b1705570 --- /dev/null +++ b/tools/test-whisper-small-fr.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python3 +"""Verify released hashes and decode a public/test WAV with sherpa-onnx CPU.""" +import argparse +import hashlib +import json +from pathlib import Path +import time +import wave + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--model-dir', type=Path, required=True) + parser.add_argument('--wav', type=Path, required=True) + parser.add_argument('--expected', help='Optional expected transcription substring') + args = parser.parse_args() + model_dir = args.model_dir + manifest = json.loads((model_dir / 'manifest.json').read_text()) + for entry in manifest['files']: + path = model_dir / entry['name'] + with path.open('rb') as stream: + actual = hashlib.file_digest(stream, 'sha256').hexdigest() + if actual != entry['sha256'] or path.stat().st_size != entry['sizeBytes']: + raise ValueError(f'Model file integrity check failed: {path.name}') + + import numpy as np + import sherpa_onnx + with wave.open(str(args.wav)) as audio: + if audio.getnchannels() != 1 or audio.getsampwidth() != 2: + raise ValueError('Provide a mono PCM16 WAV') + sample_rate = audio.getframerate() + samples = np.frombuffer(audio.readframes(audio.getnframes()), dtype='