From bcc30545ada964a57b9fc386ecf4c5683bc692ac Mon Sep 17 00:00:00 2001 From: James Babcock Date: Fri, 1 May 2026 15:00:53 -0700 Subject: [PATCH] Add Transcribe screen --- .gitignore | 2 + App_Resources/Android/app.gradle | 24 ++- .../faceclaw/app/FaceclawVoiceController.java | 180 ++++++++++++++++-- .../app/FaceclawVoiceControllerListener.java | 2 + .../com/k2fsa/sherpa/onnx/EndpointConfig.java | 66 +++++++ .../com/k2fsa/sherpa/onnx/EndpointRule.java | 54 ++++++ .../sherpa/onnx/HomophoneReplacerConfig.java | 54 ++++++ .../onnx/OnlineCtcFstDecoderConfig.java | 42 ++++ .../com/k2fsa/sherpa/onnx/OnlineLMConfig.java | 42 ++++ .../k2fsa/sherpa/onnx/OnlineRecognizer.java | 66 +++++++ .../sherpa/onnx/OnlineRecognizerConfig.java | 146 ++++++++++++++ .../sherpa/onnx/OnlineRecognizerResult.java | 31 +++ app/g2/dashboard-controller.ts | 49 ++++- app/native/voice-control.ts | 35 +++- app/ui/dashboard.ts | 128 +++++++++++++ app/ui/layers.ts | 3 + 16 files changed, 904 insertions(+), 20 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java create mode 100644 App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java diff --git a/.gitignore b/.gitignore index 7b398a03..828e580f 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,8 @@ yarn-error.log* tmp/ typings/ .gradle-cursor +App_Resources/Android/src/main/jniLibs/**/*.so +App_Resources/Android/src/main/assets/faceclaw-voice-asr/ # Visual Studio Code .vscode/* diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 5c47d87c..457e23c4 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -7,8 +7,10 @@ dependencies { def faceclawSherpaOnnxVersion = "1.12.34" def faceclawLibLc3Version = "1.1.3" +def faceclawAsrModelName = "sherpa-onnx-streaming-zipformer-en-2023-06-26" def faceclawAbi = "arm64-v8a" def faceclawJniLibsDir = file("$projectDir/src/main/jniLibs/$faceclawAbi") +def faceclawAssetsDir = file("$projectDir/src/main/assets/faceclaw-voice-asr") def faceclawNativeBuildDir = file("$buildDir/faceclaw-native") def faceclawDownload = { String url, File destination -> @@ -56,11 +58,16 @@ tasks.register("prepareFaceclawNativeLibs") { outputs.files( file("$faceclawJniLibsDir/libonnxruntime.so"), file("$faceclawJniLibsDir/libsherpa-onnx-jni.so"), - file("$faceclawJniLibsDir/libfaceclaw_lc3.so") + file("$faceclawJniLibsDir/libfaceclaw_lc3.so"), + file("$faceclawAssetsDir/$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx"), + file("$faceclawAssetsDir/$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx"), + file("$faceclawAssetsDir/$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx"), + file("$faceclawAssetsDir/$faceclawAsrModelName/tokens.txt") ) doLast { faceclawJniLibsDir.mkdirs() + faceclawAssetsDir.mkdirs() def downloadsDir = file("$faceclawNativeBuildDir/downloads") def sherpaArchive = file("$downloadsDir/sherpa-onnx-v${faceclawSherpaOnnxVersion}-android.tar.bz2") @@ -77,6 +84,21 @@ tasks.register("prepareFaceclawNativeLibs") { into faceclawJniLibsDir } + def asrArchive = file("$downloadsDir/${faceclawAsrModelName}.tar.bz2") + faceclawDownload( + "https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/${faceclawAsrModelName}.tar.bz2", + asrArchive + ) + copy { + from tarTree(resources.bzip2(asrArchive)) + include "$faceclawAsrModelName/encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx" + include "$faceclawAsrModelName/decoder-epoch-99-avg-1-chunk-16-left-128.onnx" + include "$faceclawAsrModelName/joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx" + include "$faceclawAsrModelName/tokens.txt" + includeEmptyDirs = false + into faceclawAssetsDir + } + def lc3Archive = file("$downloadsDir/liblc3-v${faceclawLibLc3Version}.tar.gz") faceclawDownload( "https://github.com/google/liblc3/archive/refs/tags/v${faceclawLibLc3Version}.tar.gz", diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index a7a0194b..f6e86adf 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -12,6 +12,9 @@ import com.k2fsa.sherpa.onnx.KeywordSpotter; import com.k2fsa.sherpa.onnx.KeywordSpotterConfig; import com.k2fsa.sherpa.onnx.KeywordSpotterResult; import com.k2fsa.sherpa.onnx.OnlineModelConfig; +import com.k2fsa.sherpa.onnx.OnlineRecognizer; +import com.k2fsa.sherpa.onnx.OnlineRecognizerConfig; +import com.k2fsa.sherpa.onnx.OnlineRecognizerResult; import com.k2fsa.sherpa.onnx.OnlineStream; import com.k2fsa.sherpa.onnx.OnlineTransducerModelConfig; @@ -31,6 +34,8 @@ public class FaceclawVoiceController { private static final int STATS_INTERVAL_MS = 5_000; private static final String ASSET_ROOT = "faceclaw-voice"; private static final String MODEL_DIR = "sherpa-onnx-kws-zipformer-gigaspeech-3.3M-2024-01-01"; + private static final String ASR_ASSET_ROOT = "faceclaw-voice-asr"; + private static final String ASR_MODEL_DIR = "sherpa-onnx-streaming-zipformer-en-2023-06-26"; private static final String[] MODEL_FILES = { "encoder-epoch-12-avg-2-chunk-16-left-64.onnx", "decoder-epoch-12-avg-2-chunk-16-left-64.onnx", @@ -38,6 +43,17 @@ public class FaceclawVoiceController { "tokens.txt", "screen-on-keywords.txt" }; + private static final String[] ASR_MODEL_FILES = { + "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx", + "decoder-epoch-99-avg-1-chunk-16-left-128.onnx", + "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx", + "tokens.txt" + }; + + private enum VoiceInputMode { + WAKEWORD, + FULL + } private final Context appContext; private final Handler mainHandler = new Handler(Looper.getMainLooper()); @@ -48,9 +64,12 @@ public class FaceclawVoiceController { private volatile FaceclawBleCommunicator communicator; private Thread workerThread; private volatile boolean started; + private VoiceInputMode mode = VoiceInputMode.WAKEWORD; private KeywordSpotter keywordSpotter; + private OnlineRecognizer recognizer; private OnlineStream stream; private FaceclawLc3Decoder lc3Decoder; + private String lastTranscript = ""; private long queuedPackets; private long queueDroppedPackets; private long decodedSamples; @@ -73,6 +92,10 @@ public class FaceclawVoiceController { } public void start() { + start("wakeword"); + } + + public void start(String requestedMode) { synchronized (lock) { if (started) { emitStatus("Voice control is already listening."); @@ -82,6 +105,7 @@ public class FaceclawVoiceController { emitStatus("Voice control needs an active G2 connection."); return; } + mode = parseMode(requestedMode); started = true; workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); workerThread.start(); @@ -103,6 +127,13 @@ public class FaceclawVoiceController { } if (threadToJoin != null) { threadToJoin.interrupt(); + if (Thread.currentThread() != threadToJoin) { + try { + threadToJoin.join(1500); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } } } @@ -110,19 +141,35 @@ public class FaceclawVoiceController { stop(); } + private VoiceInputMode parseMode(String requestedMode) { + return "full".equals(requestedMode) ? VoiceInputMode.FULL : VoiceInputMode.WAKEWORD; + } + private void runLoop() { try { - emitStatus("Voice control loading wake-word model..."); - File modelDir = installModelFiles(); - keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); - stream = keywordSpotter.createStream(); + VoiceInputMode currentMode = mode; + emitStatus(currentMode == VoiceInputMode.FULL + ? "Voice control loading transcription model..." + : "Voice control loading wake-word model..."); + if (currentMode == VoiceInputMode.FULL) { + File modelDir = installAsrModelFiles(); + recognizer = new OnlineRecognizer(buildRecognizerConfig(modelDir)); + stream = recognizer.createStream(); + lastTranscript = ""; + } else { + File modelDir = installModelFiles(); + keywordSpotter = new KeywordSpotter(buildConfig(modelDir)); + stream = keywordSpotter.createStream(); + } lc3Decoder = new FaceclawLc3Decoder(); if (!startG2Audio()) { emitStatus("Voice control could not start G2 microphone input."); return; } - emitStatus("Voice control listening for \"screen on\" from the G2 mic."); + emitStatus(currentMode == VoiceInputMode.FULL + ? "Voice control transcribing from the G2 mic." + : "Voice control listening for \"screen on\" from the G2 mic."); processG2Audio(); } catch (Throwable error) { Log.e(TAG, "Voice control failed", error); @@ -152,6 +199,7 @@ public class FaceclawVoiceController { .build()) .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) .setModelType("zipformer2") + .setModelingUnit("") .setNumThreads(1) .build()) .setKeywordsFile(new File(modelDir, "screen-on-keywords.txt").getAbsolutePath()) @@ -160,6 +208,26 @@ public class FaceclawVoiceController { .build(); } + private OnlineRecognizerConfig buildRecognizerConfig(File modelDir) { + return OnlineRecognizerConfig.builder() + .setFeatureConfig(FeatureConfig.builder() + .setSampleRate(SAMPLE_RATE) + .setFeatureDim(FEATURE_DIM) + .build()) + .setOnlineModelConfig(OnlineModelConfig.builder() + .setTransducer(OnlineTransducerModelConfig.builder() + .setEncoder(new File(modelDir, "encoder-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath()) + .setDecoder(new File(modelDir, "decoder-epoch-99-avg-1-chunk-16-left-128.onnx").getAbsolutePath()) + .setJoiner(new File(modelDir, "joiner-epoch-99-avg-1-chunk-16-left-128.int8.onnx").getAbsolutePath()) + .build()) + .setTokens(new File(modelDir, "tokens.txt").getAbsolutePath()) + .setModelType("zipformer2") + .setNumThreads(1) + .build()) + .setEnableEndpoint(true) + .build(); + } + private File installModelFiles() throws IOException { File modelDir = new File(appContext.getFilesDir(), ASSET_ROOT + File.separator + MODEL_DIR); if (!modelDir.exists() && !modelDir.mkdirs()) { @@ -176,6 +244,22 @@ public class FaceclawVoiceController { return modelDir; } + private File installAsrModelFiles() throws IOException { + File modelDir = new File(appContext.getFilesDir(), ASR_ASSET_ROOT + File.separator + ASR_MODEL_DIR); + if (!modelDir.exists() && !modelDir.mkdirs()) { + throw new IOException("Could not create " + modelDir.getAbsolutePath()); + } + AssetManager assets = appContext.getAssets(); + for (String fileName : ASR_MODEL_FILES) { + copyAssetIfNeeded( + assets, + ASR_ASSET_ROOT + "/" + ASR_MODEL_DIR + "/" + fileName, + new File(modelDir, fileName) + ); + } + return modelDir; + } + private void copyAssetIfNeeded(AssetManager assets, String assetPath, File destination) throws IOException { if (destination.exists() && destination.length() > 0) { return; @@ -206,9 +290,8 @@ public class FaceclawVoiceController { short[] pcm = new short[FaceclawLc3Decoder.SAMPLES_PER_PACKET]; while (started && !Thread.currentThread().isInterrupted()) { OnlineStream currentStream = stream; - KeywordSpotter currentSpotter = keywordSpotter; FaceclawLc3Decoder currentDecoder = lc3Decoder; - if (currentStream == null || currentSpotter == null || currentDecoder == null) { + if (currentStream == null || currentDecoder == null) { return; } @@ -229,19 +312,73 @@ public class FaceclawVoiceController { } currentStream.acceptWaveform(samples, SAMPLE_RATE); - while (currentSpotter.isReady(currentStream)) { - currentSpotter.decode(currentStream); - KeywordSpotterResult result = currentSpotter.getResult(currentStream); - String keyword = result == null ? "" : result.getKeyword(); - if (keyword != null && keyword.trim().length() > 0) { - currentSpotter.reset(currentStream); - emitWakeWord(keyword); - } + if (mode == VoiceInputMode.FULL) { + processRecognizer(currentStream); + } else { + processKeywordSpotter(currentStream); } maybeEmitAudioStats(false); } } + private void processKeywordSpotter(OnlineStream currentStream) { + KeywordSpotter currentSpotter = keywordSpotter; + if (currentSpotter == null) { + return; + } + while (currentSpotter.isReady(currentStream)) { + currentSpotter.decode(currentStream); + KeywordSpotterResult result = currentSpotter.getResult(currentStream); + String keyword = result == null ? "" : result.getKeyword(); + if (keyword != null && keyword.trim().length() > 0) { + currentSpotter.reset(currentStream); + emitWakeWord(keyword); + } + } + } + + private void processRecognizer(OnlineStream currentStream) { + OnlineRecognizer currentRecognizer = recognizer; + if (currentRecognizer == null) { + return; + } + while (currentRecognizer.isReady(currentStream)) { + currentRecognizer.decode(currentStream); + } + emitRecognizerResult(currentRecognizer, currentStream, false); + if (currentRecognizer.isEndpoint(currentStream)) { + emitRecognizerResult(currentRecognizer, currentStream, true); + currentRecognizer.reset(currentStream); + lastTranscript = ""; + } + } + + private void emitRecognizerResult(OnlineRecognizer currentRecognizer, OnlineStream currentStream, boolean isFinal) { + OnlineRecognizerResult result = currentRecognizer.getResult(currentStream); + String text = result == null ? "" : result.getText(); + if (text == null) { + text = ""; + } + if (text.equals(lastTranscript)) { + if (isFinal && lastTranscript.length() > 0) { + emitTranscript("", true); + lastTranscript = ""; + } + return; + } + String delta = text; + if (lastTranscript.length() > 0 && text.startsWith(lastTranscript)) { + delta = text.substring(lastTranscript.length()); + } + if (delta.trim().length() > 0) { + emitTranscript(delta, false); + } + lastTranscript = isFinal ? "" : text; + if (isFinal) { + emitTranscript("", true); + } + } + private void stopG2Audio() { FaceclawBleCommunicator currentCommunicator = communicator; if (currentCommunicator != null) { @@ -259,6 +396,10 @@ public class FaceclawVoiceController { keywordSpotter.release(); keywordSpotter = null; } + if (recognizer != null) { + recognizer.release(); + recognizer = null; + } } private void releaseLc3() { @@ -339,6 +480,7 @@ public class FaceclawVoiceController { + " duplicate=" + duplicate + " late=" + latePackets + " maxGapMs=" + maxInterPacketMs + + "\n" + " queueDrop=" + queueDroppedPackets + " decodeErrors=" + decodeErrors + " wrongArm=" + wrongArmPackets @@ -363,6 +505,14 @@ public class FaceclawVoiceController { mainHandler.post(() -> currentListener.onWakeWord(keyword)); } + private void emitTranscript(String text, boolean isFinal) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener == null) { + return; + } + mainHandler.post(() -> currentListener.onTranscript(text, isFinal)); + } + private static final class AudioPacket { final byte[] data; final String arm; diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index 60b306b5..a41ee229 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -4,4 +4,6 @@ public interface FaceclawVoiceControllerListener { void onStatus(String status); void onWakeWord(String keyword); + + void onTranscript(String text, boolean isFinal); } diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java new file mode 100644 index 00000000..76a768a8 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointConfig.java @@ -0,0 +1,66 @@ +package com.k2fsa.sherpa.onnx; + +public class EndpointConfig { + private final EndpointRule rule1; + private final EndpointRule rule2; + private final EndpointRule rule3; + + private EndpointConfig(Builder builder) { + this.rule1 = builder.rule1; + this.rule2 = builder.rule2; + this.rule3 = builder.rule3; + } + + public static Builder builder() { + return new Builder(); + } + + public EndpointRule getRule1() { + return rule1; + } + + public EndpointRule getRule2() { + return rule2; + } + + public EndpointRule getRule3() { + return rule3; + } + + public static class Builder { + private EndpointRule rule1 = EndpointRule.builder() + .setMustContainNonSilence(false) + .setMinTrailingSilence(2.4f) + .setMinUtteranceLength(0.0f) + .build(); + private EndpointRule rule2 = EndpointRule.builder() + .setMustContainNonSilence(true) + .setMinTrailingSilence(1.2f) + .setMinUtteranceLength(0.0f) + .build(); + private EndpointRule rule3 = EndpointRule.builder() + .setMustContainNonSilence(false) + .setMinTrailingSilence(0.0f) + .setMinUtteranceLength(20.0f) + .build(); + + public Builder setRule1(EndpointRule rule1) { + this.rule1 = rule1; + return this; + } + + public Builder setRule2(EndpointRule rule2) { + this.rule2 = rule2; + return this; + } + + public Builder setRule3(EndpointRule rule3) { + this.rule3 = rule3; + return this; + } + + public EndpointConfig build() { + return new EndpointConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java new file mode 100644 index 00000000..b71bdf0b --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/EndpointRule.java @@ -0,0 +1,54 @@ +package com.k2fsa.sherpa.onnx; + +public class EndpointRule { + private final boolean mustContainNonSilence; + private final float minTrailingSilence; + private final float minUtteranceLength; + + private EndpointRule(Builder builder) { + this.mustContainNonSilence = builder.mustContainNonSilence; + this.minTrailingSilence = builder.minTrailingSilence; + this.minUtteranceLength = builder.minUtteranceLength; + } + + public static Builder builder() { + return new Builder(); + } + + public boolean getMustContainNonSilence() { + return mustContainNonSilence; + } + + public float getMinTrailingSilence() { + return minTrailingSilence; + } + + public float getMinUtteranceLength() { + return minUtteranceLength; + } + + public static class Builder { + private boolean mustContainNonSilence = false; + private float minTrailingSilence = 0.0f; + private float minUtteranceLength = 0.0f; + + public Builder setMustContainNonSilence(boolean mustContainNonSilence) { + this.mustContainNonSilence = mustContainNonSilence; + return this; + } + + public Builder setMinTrailingSilence(float minTrailingSilence) { + this.minTrailingSilence = minTrailingSilence; + return this; + } + + public Builder setMinUtteranceLength(float minUtteranceLength) { + this.minUtteranceLength = minUtteranceLength; + return this; + } + + public EndpointRule build() { + return new EndpointRule(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.java new file mode 100644 index 00000000..9e549383 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/HomophoneReplacerConfig.java @@ -0,0 +1,54 @@ +package com.k2fsa.sherpa.onnx; + +public class HomophoneReplacerConfig { + private final String dictDir; + private final String lexicon; + private final String ruleFsts; + + private HomophoneReplacerConfig(Builder builder) { + this.dictDir = builder.dictDir; + this.lexicon = builder.lexicon; + this.ruleFsts = builder.ruleFsts; + } + + public static Builder builder() { + return new Builder(); + } + + public String getDictDir() { + return dictDir; + } + + public String getLexicon() { + return lexicon; + } + + public String getRuleFsts() { + return ruleFsts; + } + + public static class Builder { + private String dictDir = ""; + private String lexicon = ""; + private String ruleFsts = ""; + + public Builder setDictDir(String dictDir) { + this.dictDir = dictDir; + return this; + } + + public Builder setLexicon(String lexicon) { + this.lexicon = lexicon; + return this; + } + + public Builder setRuleFsts(String ruleFsts) { + this.ruleFsts = ruleFsts; + return this; + } + + public HomophoneReplacerConfig build() { + return new HomophoneReplacerConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java new file mode 100644 index 00000000..ef8518e6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineCtcFstDecoderConfig.java @@ -0,0 +1,42 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineCtcFstDecoderConfig { + private final String graph; + private final int maxActive; + + private OnlineCtcFstDecoderConfig(Builder builder) { + this.graph = builder.graph; + this.maxActive = builder.maxActive; + } + + public static Builder builder() { + return new Builder(); + } + + public String getGraph() { + return graph; + } + + public int getMaxActive() { + return maxActive; + } + + public static class Builder { + private String graph = ""; + private int maxActive = 3000; + + public Builder setGraph(String graph) { + this.graph = graph; + return this; + } + + public Builder setMaxActive(int maxActive) { + this.maxActive = maxActive; + return this; + } + + public OnlineCtcFstDecoderConfig build() { + return new OnlineCtcFstDecoderConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java new file mode 100644 index 00000000..b7dad082 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineLMConfig.java @@ -0,0 +1,42 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineLMConfig { + private final String model; + private final float scale; + + private OnlineLMConfig(Builder builder) { + this.model = builder.model; + this.scale = builder.scale; + } + + public static Builder builder() { + return new Builder(); + } + + public String getModel() { + return model; + } + + public float getScale() { + return scale; + } + + public static class Builder { + private String model = ""; + private float scale = 0.5f; + + public Builder setModel(String model) { + this.model = model; + return this; + } + + public Builder setScale(float scale) { + this.scale = scale; + return this; + } + + public OnlineLMConfig build() { + return new OnlineLMConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java new file mode 100644 index 00000000..496a0fe9 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizer.java @@ -0,0 +1,66 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineRecognizer { + private long ptr; + + public OnlineRecognizer(OnlineRecognizerConfig config) { + System.loadLibrary("onnxruntime"); + System.loadLibrary("sherpa-onnx-jni"); + ptr = newFromFile(config); + if (ptr == 0) { + throw new IllegalArgumentException("Invalid OnlineRecognizerConfig"); + } + } + + public OnlineStream createStream() { + return createStream(""); + } + + public OnlineStream createStream(String hotwords) { + long streamPtr = createStream(ptr, hotwords == null ? "" : hotwords); + return new OnlineStream(streamPtr); + } + + public boolean isReady(OnlineStream stream) { + return stream != null && isReady(ptr, stream.getPtr()); + } + + public void decode(OnlineStream stream) { + if (stream != null) { + decode(ptr, stream.getPtr()); + } + } + + public void reset(OnlineStream stream) { + if (stream != null) { + reset(ptr, stream.getPtr()); + } + } + + public boolean isEndpoint(OnlineStream stream) { + return stream != null && isEndpoint(ptr, stream.getPtr()); + } + + public OnlineRecognizerResult getResult(OnlineStream stream) { + if (stream == null) { + return new OnlineRecognizerResult("", new String[0], new float[0], new float[0]); + } + return getResult(ptr, stream.getPtr()); + } + + public void release() { + if (ptr != 0) { + delete(ptr); + ptr = 0; + } + } + + private native long newFromFile(OnlineRecognizerConfig config); + private native void delete(long ptr); + private native long createStream(long ptr, String hotwords); + private native void reset(long ptr, long streamPtr); + private native void decode(long ptr, long streamPtr); + private native boolean isEndpoint(long ptr, long streamPtr); + private native boolean isReady(long ptr, long streamPtr); + private native OnlineRecognizerResult getResult(long ptr, long streamPtr); +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java new file mode 100644 index 00000000..234a3f63 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerConfig.java @@ -0,0 +1,146 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineRecognizerConfig { + private final FeatureConfig featConfig; + private final OnlineModelConfig modelConfig; + private final OnlineLMConfig lmConfig; + private final OnlineCtcFstDecoderConfig ctcFstDecoderConfig; + private final HomophoneReplacerConfig hr; + private final EndpointConfig endpointConfig; + private final boolean enableEndpoint; + private final String decodingMethod; + private final int maxActivePaths; + private final String hotwordsFile; + private final float hotwordsScore; + private final String ruleFsts; + private final String ruleFars; + private final float blankPenalty; + + private OnlineRecognizerConfig(Builder builder) { + this.featConfig = builder.featConfig; + this.modelConfig = builder.modelConfig; + this.lmConfig = builder.lmConfig; + this.ctcFstDecoderConfig = builder.ctcFstDecoderConfig; + this.hr = builder.hr; + this.endpointConfig = builder.endpointConfig; + this.enableEndpoint = builder.enableEndpoint; + this.decodingMethod = builder.decodingMethod; + this.maxActivePaths = builder.maxActivePaths; + this.hotwordsFile = builder.hotwordsFile; + this.hotwordsScore = builder.hotwordsScore; + this.ruleFsts = builder.ruleFsts; + this.ruleFars = builder.ruleFars; + this.blankPenalty = builder.blankPenalty; + } + + public static Builder builder() { + return new Builder(); + } + + public FeatureConfig getFeatConfig() { + return featConfig; + } + + public OnlineModelConfig getModelConfig() { + return modelConfig; + } + + public OnlineLMConfig getLmConfig() { + return lmConfig; + } + + public OnlineCtcFstDecoderConfig getCtcFstDecoderConfig() { + return ctcFstDecoderConfig; + } + + public HomophoneReplacerConfig getHr() { + return hr; + } + + public EndpointConfig getEndpointConfig() { + return endpointConfig; + } + + public boolean getEnableEndpoint() { + return enableEndpoint; + } + + public String getDecodingMethod() { + return decodingMethod; + } + + public int getMaxActivePaths() { + return maxActivePaths; + } + + public String getHotwordsFile() { + return hotwordsFile; + } + + public float getHotwordsScore() { + return hotwordsScore; + } + + public String getRuleFsts() { + return ruleFsts; + } + + public String getRuleFars() { + return ruleFars; + } + + public float getBlankPenalty() { + return blankPenalty; + } + + public static class Builder { + private FeatureConfig featConfig = FeatureConfig.builder().build(); + private OnlineModelConfig modelConfig = OnlineModelConfig.builder().build(); + private OnlineLMConfig lmConfig = OnlineLMConfig.builder().build(); + private OnlineCtcFstDecoderConfig ctcFstDecoderConfig = OnlineCtcFstDecoderConfig.builder().build(); + private HomophoneReplacerConfig hr = HomophoneReplacerConfig.builder().build(); + private EndpointConfig endpointConfig = EndpointConfig.builder().build(); + private boolean enableEndpoint = true; + private String decodingMethod = "greedy_search"; + private int maxActivePaths = 4; + private String hotwordsFile = ""; + private float hotwordsScore = 1.5f; + private String ruleFsts = ""; + private String ruleFars = ""; + private float blankPenalty = 0.0f; + + public Builder setFeatureConfig(FeatureConfig featConfig) { + this.featConfig = featConfig; + return this; + } + + public Builder setOnlineModelConfig(OnlineModelConfig modelConfig) { + this.modelConfig = modelConfig; + return this; + } + + public Builder setEndpointConfig(EndpointConfig endpointConfig) { + this.endpointConfig = endpointConfig; + return this; + } + + public Builder setEnableEndpoint(boolean enableEndpoint) { + this.enableEndpoint = enableEndpoint; + return this; + } + + public Builder setDecodingMethod(String decodingMethod) { + this.decodingMethod = decodingMethod; + return this; + } + + public Builder setMaxActivePaths(int maxActivePaths) { + this.maxActivePaths = maxActivePaths; + return this; + } + + public OnlineRecognizerConfig build() { + return new OnlineRecognizerConfig(this); + } + } +} diff --git a/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java new file mode 100644 index 00000000..5b765675 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/k2fsa/sherpa/onnx/OnlineRecognizerResult.java @@ -0,0 +1,31 @@ +package com.k2fsa.sherpa.onnx; + +public class OnlineRecognizerResult { + private final String text; + private final String[] tokens; + private final float[] timestamps; + private final float[] ysProbs; + + public OnlineRecognizerResult(String text, String[] tokens, float[] timestamps, float[] ysProbs) { + this.text = text == null ? "" : text; + this.tokens = tokens == null ? new String[0] : tokens; + this.timestamps = timestamps == null ? new float[0] : timestamps; + this.ysProbs = ysProbs == null ? new float[0] : ysProbs; + } + + public String getText() { + return text; + } + + public String[] getTokens() { + return tokens; + } + + public float[] getTimestamps() { + return timestamps; + } + + public float[] getYsProbs() { + return ysProbs; + } +} diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 7122082a..49112ad2 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -52,6 +52,7 @@ const CONTAINER_NAME = "dashboard"; const DASHBOARD_INTERVAL_MS = 60_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; const STOPWATCH_RENDER_INTERVAL_MS = 100; +const TRANSCRIBE_RENDER_INTERVAL_MS = 250; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000; const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000; @@ -120,6 +121,7 @@ class DashboardController { private dashboardTimer: ReturnType | null = null; private screenTimeoutTimer: ReturnType | null = null; private stopwatchRenderTimer: ReturnType | null = null; + private transcribeRenderTimer: ReturnType | null = null; private offState: (() => void) | null = null; private offLog: (() => void) | null = null; private offRing: (() => void) | null = null; @@ -149,6 +151,9 @@ class DashboardController { endTextSettingEdit: () => this.endTextSettingEdit(), setVoiceControlEnabled: (enabled) => this.setVoiceControlEnabled(enabled), setStopwatchRenderActive: (active) => this.setStopwatchRenderActive(active), + setTranscribeRenderActive: (active) => this.setTranscribeRenderActive(active), + startDedicatedVoiceInput: (mode) => this.startDedicatedVoiceInput(mode), + stopDedicatedVoiceInput: () => this.stopDedicatedVoiceInput(), }); } @@ -393,6 +398,10 @@ class DashboardController { this.offVoiceStatus = null; this.offVoiceWakeWord?.(); this.offVoiceWakeWord = null; + if (this.transcribeRenderTimer) { + clearInterval(this.transcribeRenderTimer); + this.transcribeRenderTimer = null; + } await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); @@ -452,6 +461,10 @@ class DashboardController { await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); + if (this.transcribeRenderTimer) { + clearInterval(this.transcribeRenderTimer); + this.transcribeRenderTimer = null; + } await communicator?.close().catch(() => {}); } finally { stopForegroundNotification(); @@ -673,7 +686,21 @@ class DashboardController { if (!this.communicator) return; if (!isDashboardVoiceControlEnabled()) return; this.updateConnectedForegroundNotification(); - voiceControlBridge.start(this.communicator.getNativeCommunicator()); + voiceControlBridge.start(this.communicator.getNativeCommunicator(), "wakeword"); + } + + private async startDedicatedVoiceInput(mode: "wakeword" | "full"): Promise { + if (!this.communicator) { + throw new Error("Voice input needs an active G2 connection."); + } + await ensureVoicePermissions(); + this.updateConnectedForegroundNotification(); + voiceControlBridge.start(this.communicator.getNativeCommunicator(), mode); + } + + private stopDedicatedVoiceInput(): void { + voiceControlBridge.stop(); + this.startVoiceControlIfEnabled(); } private setStopwatchRenderActive(active: boolean): void { @@ -692,6 +719,22 @@ class DashboardController { }, STOPWATCH_RENDER_INTERVAL_MS); } + private setTranscribeRenderActive(active: boolean): void { + if (!active || this.phase !== "connected" || !this.communicator) { + if (this.transcribeRenderTimer) { + clearInterval(this.transcribeRenderTimer); + this.transcribeRenderTimer = null; + } + return; + } + if (this.transcribeRenderTimer) return; + this.transcribeRenderTimer = setInterval(() => { + void this.requestRender("interval").catch((error) => { + this.appendLog(`transcribe render failed: ${this.formatError(error)}`); + }); + }, TRANSCRIBE_RENDER_INTERVAL_MS); + } + private clearDashboardTimer(): void { if (this.dashboardTimer) { clearInterval(this.dashboardTimer); @@ -705,6 +748,10 @@ class DashboardController { clearInterval(this.stopwatchRenderTimer); this.stopwatchRenderTimer = null; } + if (this.transcribeRenderTimer) { + clearInterval(this.transcribeRenderTimer); + this.transcribeRenderTimer = null; + } } private setPhase(phase: ConnectionPhase): void { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index db5e3a62..abf077d6 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -6,13 +6,21 @@ export type VoiceControlState = { status: string; }; +export type VoiceInputMode = "off" | "wakeword" | "full"; +export type VoiceTranscriptEvent = { + text: string; + isFinal: boolean; +}; + export class FaceclawVoiceControlBridge { private readonly statusListeners = new Set<(state: VoiceControlState) => void>(); private readonly wakeWordListeners = new Set<(keyword: string) => void>(); + private readonly transcriptListeners = new Set<(event: VoiceTranscriptEvent) => void>(); private controller: any | null = null; private listenerProxy: any | null = null; private status = "Voice control stopped."; private started = false; + private mode: VoiceInputMode = "off"; onStatus(listener: (state: VoiceControlState) => void): () => void { this.statusListeners.add(listener); @@ -25,19 +33,34 @@ export class FaceclawVoiceControlBridge { return () => this.wakeWordListeners.delete(listener); } - start(communicator?: any): void { - if (this.started || !global.isAndroid) return; + onTranscript(listener: (event: VoiceTranscriptEvent) => void): () => void { + this.transcriptListeners.add(listener); + return () => this.transcriptListeners.delete(listener); + } + + start(communicator?: any, mode: VoiceInputMode = "wakeword"): void { + if (mode === "off") { + this.stop(); + return; + } + if (!global.isAndroid) return; this.ensureController(); if (communicator) { this.controller?.setCommunicator(communicator); } + if (this.started && this.mode !== mode) { + this.controller?.stop(); + this.started = false; + } this.started = true; - this.controller?.start(); + this.mode = mode; + this.controller?.start(mode); } stop(): void { if (!this.started || !global.isAndroid) return; this.started = false; + this.mode = "off"; this.controller?.stop(); this.setStatus("Voice control stopped."); } @@ -58,6 +81,12 @@ export class FaceclawVoiceControlBridge { listener(String(keyword)); } }, + onTranscript: (text: string, isFinal: boolean) => { + const event = { text: String(text), isFinal: Boolean(isFinal) }; + for (const listener of this.transcriptListeners) { + listener(event); + } + }, }); this.controller.setListener(this.listenerProxy); } diff --git a/app/ui/dashboard.ts b/app/ui/dashboard.ts index df9a651f..2f764a7e 100644 --- a/app/ui/dashboard.ts +++ b/app/ui/dashboard.ts @@ -8,6 +8,7 @@ import { nightscoutBridge } from "../native/nightscout"; import { readActiveNotificationIcons } from "../native/notification-icons"; import { readPhoneBatteryState } from "../native/phone-battery"; import { readSystemStatusIcons } from "../native/system-status-icons"; +import { voiceControlBridge, type VoiceTranscriptEvent } from "../native/voice-control"; import { EventSourceType, OsEventTypeList } from "../g2/events"; import { getDashboardPlugin, @@ -85,6 +86,9 @@ const dashboardActions: LayerActions = { endTextSettingEdit: () => {}, setVoiceControlEnabled: () => {}, setStopwatchRenderActive: () => {}, + setTranscribeRenderActive: () => {}, + startDedicatedVoiceInput: () => {}, + stopDedicatedVoiceInput: () => {}, }; const dashboardFont = loadEmbeddedTerminus12(); const dashboardSystemFont = loadEmbeddedTerminus16(); @@ -487,6 +491,14 @@ function createAppsMenuLayer(): MenuLayer { void ctx.actions.setStopwatchRenderActive(true); }, }, + { + label: "Transcribe", + onSelect: (ctx) => { + const layer = new TranscribeLayer(); + ctx.stack.push(layer); + layer.start(ctx); + }, + }, ], TOP_LEFT_MENU_LAYOUT, ); @@ -539,6 +551,122 @@ class StopwatchLayer implements Layer { } } +class TranscribeLayer implements Layer { + private mode: "off" | "wakeword" | "full" = "full"; + private status = "Starting transcription..."; + private transcript = ""; + private unsubscribeTranscript: (() => void) | null = null; + private unsubscribeStatus: (() => void) | null = null; + + start(ctx: LayerContext): void { + this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); + this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + this.status = state.status; + }); + void ctx.actions.setTranscribeRenderActive(true); + void Promise.resolve(ctx.actions.startDedicatedVoiceInput("full")).catch((error) => { + this.status = `Could not start transcription: ${error instanceof Error ? error.message : String(error)}`; + }); + } + + paint(ctx: LayerContext): GrayImage { + const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0); + const modeLabel = this.mode === "full" ? "Full" : this.mode === "wakeword" ? "Wakeword" : "Off"; + const text = this.transcript || "Listening..."; + const wrapped = wrapTranscribeText(ctx.font, text, G2_LENS_WIDTH - 64); + + image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52); + image.drawText(ctx.font, 24, 24, "Transcribe", 200); + image.drawText(ctx.font, G2_LENS_WIDTH - 108, 24, modeLabel, this.mode === "full" ? 220 : 130); + //image.drawText(ctx.font, 24, 46, truncateTranscribeLine(this.status, 74), 110); + image.drawText(ctx.font, 24, 46, this.status, 110); + + const firstLine = Math.max(0, wrapped.length - 10); + for (let index = firstLine; index < wrapped.length; index++) { + const y = 72 + (index - firstLine) * 16; + image.drawText(ctx.font, 32, y, wrapped[index]!, 230); + } + + image.drawText(ctx.font, 24, 252, "Click: mode / Double-click: back", 110); + return image; + } + + handleInput(event: DashboardInputEvent, ctx: LayerContext): void { + if (event.type === "double-click") { + this.stop(ctx); + ctx.stack.pop(); + return; + } + if (event.type === "click") { + this.cycleMode(ctx); + } + } + + private cycleMode(ctx: LayerContext): void { + if (this.mode === "full") { + this.mode = "wakeword"; + void ctx.actions.startDedicatedVoiceInput("wakeword"); + } else if (this.mode === "wakeword") { + this.mode = "off"; + void ctx.actions.stopDedicatedVoiceInput(); + } else { + this.mode = "full"; + void ctx.actions.startDedicatedVoiceInput("full"); + } + } + + private stop(ctx: LayerContext): void { + this.unsubscribeTranscript?.(); + this.unsubscribeTranscript = null; + this.unsubscribeStatus?.(); + this.unsubscribeStatus = null; + void ctx.actions.setTranscribeRenderActive(false); + void ctx.actions.stopDedicatedVoiceInput(); + } + + private onTranscript(event: VoiceTranscriptEvent): void { + if (event.isFinal) { + if (this.transcript && !this.transcript.endsWith("\n")) { + this.transcript = `${this.transcript}\n`; + } + return; + } + this.transcript = appendTranscriptChunk(this.transcript, event.text); + } +} + +function appendTranscriptChunk(transcript: string, chunk: string): string { + const trimmed = chunk.replace(/\s+/g, " ").trim(); + if (!trimmed) return transcript; + if (!transcript || transcript.endsWith("\n") || /^[.,!?;:]/.test(trimmed)) { + return `${transcript}${trimmed}`; + } + return `${transcript} ${trimmed}`; +} + +function wrapTranscribeText(font: typeof dashboardFont, text: string, maxWidth: number): string[] { + const words = text.split(/\s+/).filter(Boolean); + const lines: string[] = []; + let line = ""; + for (const word of words) { + const candidate = line ? `${line} ${word}` : word; + if (line && font.measureText(candidate) > maxWidth) { + lines.push(line); + line = word; + } else { + line = candidate; + } + } + if (line) { + lines.push(line); + } + return lines.length ? lines : [""]; +} + +function truncateTranscribeLine(text: string, maxChars: number): string { + return text.length <= maxChars ? text : `${text.slice(0, maxChars - 3)}...`; +} + function formatStopwatchElapsed(elapsedMs: number): string { const totalTenths = Math.floor(elapsedMs / 100); const tenths = totalTenths % 10; diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 97efad2e..226f872e 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -17,6 +17,9 @@ export type LayerActions = { endTextSettingEdit: () => Promise | void; setVoiceControlEnabled: (enabled: boolean) => Promise | void; setStopwatchRenderActive: (active: boolean) => Promise | void; + setTranscribeRenderActive: (active: boolean) => Promise | void; + startDedicatedVoiceInput: (mode: "wakeword" | "full") => Promise | void; + stopDedicatedVoiceInput: () => Promise | void; }; export type PaintBelow = () => GrayImage;