diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java index d6b8171d..37a53d58 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java @@ -266,20 +266,26 @@ public class FaceclawVoiceController { } public void start(String requestedMode) { + start(requestedMode, 0); + } + + public void start(String requestedMode, int captureId) { synchronized (lock) { if (started) { emitStatus("Voice control is already listening."); + emitStopped(captureId); return; } if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) { emitStatus("Voice control needs an active G2 connection."); + emitStopped(captureId); return; } mode = parseMode(requestedMode); activePhoneMic = usePhoneMic; started = true; audioStarted = false; - workerThread = new Thread(this::runLoop, "FaceclawVoiceController"); + workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController"); workerThread.start(); } } @@ -350,7 +356,7 @@ public class FaceclawVoiceController { return VoiceInputMode.ONBOARD; } - private void runLoop() { + private void runLoop(int captureId) { try { deleteLegacyKwsFiles(); VoiceInputMode currentMode = mode; @@ -432,6 +438,7 @@ public class FaceclawVoiceController { audioStarted = false; workerThread = null; } + emitStopped(captureId); } } @@ -1014,6 +1021,13 @@ public class FaceclawVoiceController { mainHandler.post(currentListener::onSpeechEnd); } + private void emitStopped(int captureId) { + FaceclawVoiceControllerListener currentListener = listener; + if (currentListener != null) { + mainHandler.post(() -> currentListener.onStopped(captureId)); + } + } + /** * Decides when a hands-free utterance is over, so "Hey Even" capture can * stop without a button release. diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java index c60ec655..95c5826f 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceControllerListener.java @@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener { */ void onTranscript(String text, boolean isFinal); + /** Capture has finished, after the final transcript or a startup/recognition error. */ + void onStopped(int captureId); + /** * Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM. * Emitted in every mode so raw-PCM consumers (levels, recording, captions) diff --git a/app/apps/ai-chat/voice-draft.ts b/app/apps/ai-chat/voice-draft.ts index 0cb6854c..f91093fc 100644 --- a/app/apps/ai-chat/voice-draft.ts +++ b/app/apps/ai-chat/voice-draft.ts @@ -89,11 +89,22 @@ export class VoiceDraft { } private stopAndFinalize(): void { - // Install fallback first: stop may synchronously deliver the final. - this.timer = setTimeout(() => this.finish(), 1500); + const generation = this.generation; + const afterStop = () => { + if (generation !== this.generation || !this.isFinishing()) return; + this.timer = setTimeout(() => this.finish(), 1500); + }; if (this.started) { this.started = false; - void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error))); + // Native recognition can take longer than the synchronous stop join. + // Its final callback may finish the draft before this promise resolves. + const stopped = this.actions.stopVoiceCapture(); + if (!stopped) { afterStop(); return; } + void stopped.then(afterStop, (error) => { + if (generation === this.generation) this.cancel(String(error)); + }); + } else { + afterStop(); } } diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 84fb9548..a70a7c55 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1912,9 +1912,9 @@ class DashboardController { return this.beginVoiceCapture("ptt", endpointing); } - private stopVoiceCapture(): void { + private stopVoiceCapture(): Promise | void { ++this.pttCaptureGeneration; - voiceControlBridge.stopPushToTalk(); + return voiceControlBridge.stopPushToTalk(); } private startContinuousVoiceCapture(): void { diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index acfd9f31..f3acadab 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -108,6 +108,9 @@ export class FaceclawVoiceControlBridge { private listening = false; private detail = ""; private started = false; + private nativeCaptureFinished: Promise = Promise.resolve(); + private nextNativeCaptureId = 1; + private readonly nativeCaptureResolvers = new Map void>(); // The mic is a single shared stream; these are the reasons it is running. // The first holder starts capture (choosing the provider); the mic stops // when the last one releases. Transcript events broadcast to every @@ -189,8 +192,13 @@ export class FaceclawVoiceControlBridge { } /** End push-to-talk: for cloud, commit for a final result if it was the last holder. */ - stopPushToTalk(): void { + stopPushToTalk(): Promise | void { + // Shared continuous capture keeps running; cloud providers finalize via + // their own callbacks. Only a stopped native recognizer must be awaited. + const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1 + ? this.nativeCaptureFinished : undefined; this.releaseCapture("ptt", true); + return finished; } /** Begin continuous capture (Transcribe): the mic stays on until released. */ @@ -322,7 +330,9 @@ export class FaceclawVoiceControlBridge { // Which on-device model to load; a no-op setter for every provider except // "onboard-whisper" (FaceclawVoiceController defaults to Moonshine). this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine"); - this.controller?.start("onboard"); + const captureId = this.nextNativeCaptureId++; + this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); }); + this.controller?.start("onboard", captureId); } /** @@ -491,6 +501,10 @@ export class FaceclawVoiceControlBridge { onTranscript: (text: string, isFinal: boolean) => { this.emitTranscript(String(text), Boolean(isFinal)); }, + onStopped: (captureId: number) => { + this.nativeCaptureResolvers.get(captureId)?.(); + this.nativeCaptureResolvers.delete(captureId); + }, onPcm: (pcm: any) => { const bytes = toUint8Array(pcm); this.cloudClient?.acceptPcm(bytes); diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 50ad0473..32315136 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -28,7 +28,7 @@ export type LayerActions = { * (hands-free); otherwise it runs until stopVoiceCapture. */ startVoiceCapture: (endpointing?: boolean) => Promise | void; - /** Stop push-to-talk; for a cloud provider this also commits for a final result. */ + /** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */ stopVoiceCapture: () => Promise | void; /** Start continuous capture (Transcribe); shares the mic with push-to-talk. */ startContinuousVoiceCapture: () => Promise | void; diff --git a/app/ui/shell/voice-input.ts b/app/ui/shell/voice-input.ts index 41498570..bb594c49 100644 --- a/app/ui/shell/voice-input.ts +++ b/app/ui/shell/voice-input.ts @@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings"; import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures"; import { Layer, type LayerActions, type LayerContext } from "../layers"; import { paintInputDialog } from "./input-dialog"; -// After the mic stops, the provider's final transcript can trail in (cloud -// commit round-trip); wait this long for it before refining with what we have. +// After native recognition finishes, a cloud final can still trail in. const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200; /** @@ -80,6 +79,10 @@ export class VoiceInputLayer implements Layer { private capturing = false; /** Continuation stopped; waiting for the trailing final transcript. */ private followupFinalizeTimer: ReturnType | null = null; + private pendingFollowup = false; + private stoppingCapture = false; + private stopGeneration = 0; + private acceptingTranscript = true; private refineHandle: AnthropicStreamHandle | null = null; private menuIndex = 0; /** Auto-send (wakeword skip-confirmation) is waiting to fire. */ @@ -155,21 +158,48 @@ export class VoiceInputLayer implements Layer { } this.capturing = false; this.phase = "menu"; - void this.actions.stopVoiceCapture(); if (this.autoSend && this.sendTargets.length) { // Skip-confirmation (wakeword): send to the default target as soon as the // transcript finalizes, or after a short wait for the trailing final. this.pendingAutoSend = true; this.status = "Sending..."; - this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS); } else if (this.status.endsWith("...")) { // A progress status ("Starting microphone...", "Listening...") gives // way to the menu prompt; an error (ending in ".") stays visible. this.status = "Send, continue, or discard?"; } + this.stopCapture(() => { + if (this.pendingAutoSend) { + this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS); + } + }); this.actions.requestRender(); } + /** Native stop can outlast its synchronous join while Whisper is decoding. */ + private stopCapture(onStopped: () => void): void { + const generation = ++this.stopGeneration; + this.stoppingCapture = true; + const afterStop = () => { + if (generation !== this.stopGeneration) return; + this.stoppingCapture = false; + onStopped(); + this.actions.requestRender(); + }; + const stopped = this.actions.stopVoiceCapture(); + if (!stopped) { + afterStop(); + return; + } + void stopped.then(afterStop, (error) => { + if (generation !== this.stopGeneration) return; + this.stoppingCapture = false; + this.pendingAutoSend = false; + this.acceptingTranscript = false; + this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error)); + }); + } + /** Fire the queued skip-confirmation send (or fall back to the menu). */ private performAutoSend(): void { if (this.autoSendTimer !== null) { @@ -193,7 +223,7 @@ export class VoiceInputLayer implements Layer { /** The menu rows: one per send target, then Continue, then Discard. */ private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> { const text = this.displayText().trim(); - const hasText = text.length > 0; + const hasText = text.length > 0 && !this.stoppingCapture; const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0; const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = []; for (const target of this.sendTargets) { @@ -201,6 +231,7 @@ export class VoiceInputLayer implements Layer { label: target.label, dim: !hasText, onSelect: () => { + if (this.stoppingCapture) return; this.dismiss(); if (hasText) target.onSend(text); }, @@ -208,9 +239,9 @@ export class VoiceInputLayer implements Layer { } rows.push({ label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)", - dim: !hasLlmKey, + dim: !hasLlmKey || this.stoppingCapture, onSelect: () => { - if (hasLlmKey) this.startContinuation(); + if (hasLlmKey && !this.stoppingCapture) this.startContinuation(); }, }); rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() }); @@ -222,7 +253,7 @@ export class VoiceInputLayer implements Layer { const inMenu = this.phase === "menu"; paintInputDialog(image, { title: this.capturing && this.listening ? "Voice ●" : "Voice", - status: this.status, + status: this.stoppingCapture ? "Finishing transcription..." : this.status, text: this.displayText() || this.placeholderText(), rows: inMenu ? this.menuRows() : [], selectedRow: this.menuIndex, @@ -284,6 +315,7 @@ export class VoiceInputLayer implements Layer { /** Continue selected: keep the message aside and record a follow-up. */ private startContinuation(): void { + this.acceptingTranscript = true; this.baseText = this.displayText().trim(); this.finalizedText = ""; this.liveText = ""; @@ -298,15 +330,18 @@ export class VoiceInputLayer implements Layer { private endContinuationCapture(): void { if (!this.capturing) return; this.capturing = false; - void this.actions.stopVoiceCapture(); - this.status = "Refining..."; + this.pendingFollowup = true; + this.status = "Finishing transcription..."; + this.stopCapture(() => { + if (this.pendingFollowup) { + this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); + } + }); this.actions.requestRender(); - // The provider's committed transcript arrives shortly after stop; refine - // when it does, or after a timeout with whatever partials we have. - this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS); } private beginRefine(): void { + this.pendingFollowup = false; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; @@ -343,9 +378,10 @@ export class VoiceInputLayer implements Layer { /** Abort a continuation (mic or LLM stage) and restore the prior message. */ private cancelContinuation(status: string): void { + this.acceptingTranscript = false; if (this.capturing) { this.capturing = false; - void this.actions.stopVoiceCapture(); + this.stopCapture(() => {}); } this.refineHandle?.cancel(); this.refineHandle = null; @@ -353,6 +389,7 @@ export class VoiceInputLayer implements Layer { } private backToMenu(text: string, status: string): void { + this.pendingFollowup = false; if (this.followupFinalizeTimer !== null) { clearTimeout(this.followupFinalizeTimer); this.followupFinalizeTimer = null; @@ -366,6 +403,9 @@ export class VoiceInputLayer implements Layer { } onRemoved(): void { + ++this.stopGeneration; + this.acceptingTranscript = false; + this.pendingFollowup = false; this.unsubscribeTranscript?.(); this.unsubscribeTranscript = null; this.unsubscribeStatus?.(); @@ -428,14 +468,14 @@ export class VoiceInputLayer implements Layer { private onTranscript(event: VoiceTranscriptEvent): void { // The refine stream owns the text buffers once it starts; a transcript // that trails in after that point is stale. - if (this.phase === "refining") return; + if (!this.acceptingTranscript || this.phase === "refining") return; if (event.isFinal) { const finalText = event.text.trim() || this.liveText.trim(); if (finalText) { this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText; } this.liveText = ""; - if (this.phase === "continuing" && this.followupFinalizeTimer !== null) { + if (this.phase === "continuing" && this.pendingFollowup) { // The follow-up finalized; no need to keep waiting. this.beginRefine(); return; diff --git a/tests/ai-chat.test.cjs b/tests/ai-chat.test.cjs index 9423f06c..e4ded14a 100644 --- a/tests/ai-chat.test.cjs +++ b/tests/ai-chat.test.cjs @@ -125,7 +125,7 @@ test('synchronous provider errors do not leave a permanently active turn', () => assert.equal(session.status, 'failed'); }); -function draftEnv({ prepare = async () => true, start = async () => {}, initialStatus = 'Ready' } = {}) { +function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) { const transcripts = new Set(), statuses = new Set(), timers = new Map(); const sent = []; let starts = 0, stops = 0, mic = false, timerId = 0; @@ -139,7 +139,7 @@ function draftEnv({ prepare = async () => true, start = async () => {}, initialS setTimeout(cb) { timers.set(++timerId, cb); return timerId; }, clearTimeout(id) { timers.delete(id); }, }); - const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; } }, prepare, () => {}, (text) => sent.push(text)); + const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text)); return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic, transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); }, status(status) { for (const cb of [...statuses]) cb({ status }); }, @@ -174,6 +174,34 @@ test('release before permission resolves never starts a microphone', async () => assert.deepEqual(env.sent, []); }); +test('chat waits past its fallback deadline for native recognition', async () => { + let complete; + const stopped = new Promise(resolve => { complete = resolve; }); + const env = draftEnv({ stop: () => stopped }); + await env.draft.start(); + env.transcript('first segment'); + env.draft.release(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.draft.phase, 'finishing'); + env.transcript('first segment and the rest', true); + complete(); await stopped; env.timeout(); + assert.deepEqual(env.sent, ['first segment and the rest']); +}); + +test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => { + let complete; + const stopped = new Promise(resolve => { complete = resolve; }); + const env = draftEnv({ stop: () => stopped }); + await env.draft.start(); + env.draft.release(); env.draft.cancel(); + await env.draft.start(); + env.transcript('new speech'); + complete(); await stopped; env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.draft.phase, 'listening'); + env.draft.cancel(); +}); + test('release during async mic startup stops capture after it starts', async () => { let started; const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) }); diff --git a/tests/fixtures/TextureCacheProtocolTest.java b/tests/fixtures/TextureCacheProtocolTest.java new file mode 100644 index 00000000..77ecac96 --- /dev/null +++ b/tests/fixtures/TextureCacheProtocolTest.java @@ -0,0 +1,58 @@ +package com.faceclaw.app; + +import java.lang.reflect.*; +import java.util.*; + +public class TextureCacheProtocolTest { + static int u16(byte[] b, int p) { return (b[p]&255) | (b[p+1]&255)<<8; } + static int u32(byte[] b, int p) { return u16(b,p) | u16(b,p+2)<<16; } + public static void main(String[] args) throws Exception { + TextureCacheState cache = new TextureCacheState(); + byte[] pixels = new byte[255*255]; + for (int i=0;i=0; + ImageAtlas.Entry tiny = new ImageAtlas.Entry(1,1,new byte[]{15}); + int imageOffset = cache.ensureImage(100,tiny); + assert imageOffset>65535; + GlyphAtlas.Glyph glyph = new GlyphAtlas.Glyph(1,1,0,1,0,null,new byte[]{15}); + int glyphOffset = cache.ensureGlyph(42,'A',glyph); + int table = cache.fontTableOffset(42); + assert table>65535 && glyphOffset==table+96*4; + byte[] memory = new byte[262144]; + for (byte[] upload: cache.drainUploadPayloads(3600)) { + assert upload[0]==18; + int pos=1; + while (pos selected=Class.forName("com.faceclaw.app.TexturePlanner$Selected"); + Constructor ctor=selected.getDeclaredConstructors()[0]; ctor.setAccessible(true); + Object image=ctor.newInstance(SurfaceCompositor.ScreenDraw.image(100,321,123),null,tiny,321,15); + Method encode=TexturePlanner.class.getDeclaredMethod("encodeImageDraw",TextureCacheState.class,selected); + encode.setAccessible(true); + byte[] draw=(byte[])encode.invoke(null,cache,image); + assert draw.length==10 && draw[0]==19 && u32(draw,1)==imageOffset; + assert u16(draw,5)==321 && u16(draw,7)==123 && draw[9]==31; + Object letter=ctor.newInstance(SurfaceCompositor.ScreenDraw.glyph(42,'A',321,123,255),glyph,null,321,15); + Method runs=TexturePlanner.class.getDeclaredMethod("buildRuns",TextureCacheState.class,List.class,List.class); + runs.setAccessible(true); List out=new ArrayList<>(); + runs.invoke(null,cache,Collections.singletonList(letter),out); + byte[] run=out.get(0); + assert run.length==12 && run[0]==20 && u32(run,1)==table; + assert u16(run,5)==321 && u16(run,7)==123 && run[9]==31 && run[10]==1 && run[11]=='A'; + int id=200; + while (cache.ensureImage(id++,big)>=0) {} + int before=cache.usedBytes(); + assert cache.ensureImage(id,big)==-1 && cache.usedBytes()==before; + cache.reset(); assert cache.usedBytes()==0 && !cache.hasPendingUploads(); + assert cache.ensureImage(id,tiny)==0; + } +} diff --git a/tests/texture-cache-protocol.test.cjs b/tests/texture-cache-protocol.test.cjs new file mode 100644 index 00000000..dc397251 --- /dev/null +++ b/tests/texture-cache-protocol.test.cjs @@ -0,0 +1,23 @@ +const test = require('node:test'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const { execFileSync } = require('node:child_process'); + +test('256 KiB cache uploads and planner draws preserve wide image and glyph offsets', () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-texture-java-')); + const javaBin = (name) => process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', name) : name; + const root = path.join(__dirname, '../App_Resources/Android/src/main/java/com/faceclaw/app'); + try { + const clock = path.join(directory, 'SystemClock.java'); + const log = path.join(directory, 'Log.java'); + fs.writeFileSync(clock, 'package android.os; public class SystemClock { public static long elapsedRealtime() { return 0; } }'); + fs.writeFileSync(log, 'package android.util; public class Log { public static int i(String tag, String msg) { return 0; } }'); + const sources = ['GlyphAtlas.java', 'ImageAtlas.java', 'FwGlyphAtlas.java', 'SurfaceCompositor.java', + 'util/BmpUtil.java', 'util/CollectionUtils.java', 'g2protocol/BleProtocol.java', + 'g2protocol/BleImageOptimizer.java', 'g2protocol/TextureCacheState.java', 'g2protocol/TexturePlanner.java']; + execFileSync(javaBin('javac'), ['-d', directory, ...sources.map(file => path.join(root, file)), + clock, log, path.join(__dirname, 'fixtures/TextureCacheProtocolTest.java')], { stdio: 'pipe' }); + execFileSync(javaBin('java'), ['-ea', '-cp', directory, 'com.faceclaw.app.TextureCacheProtocolTest'], { stdio: 'pipe' }); + } finally { fs.rmSync(directory, { recursive: true, force: true }); } +}); diff --git a/tests/voice-finalization.test.cjs b/tests/voice-finalization.test.cjs new file mode 100644 index 00000000..2077d488 --- /dev/null +++ b/tests/voice-finalization.test.cjs @@ -0,0 +1,166 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function load(file, deps, globals = {}) { + const exports = {}; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, { exports, console, ...globals, require: name => deps[name] ?? {} }); + return exports; +} + +function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) { + const listeners = new Set(), timers = new Map(), sent = [], refinements = []; + let complete, timerId = 0, starts = 0, frame; + let completion = new Promise(resolve => { complete = resolve; }); + const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', { + '../../native/voice-control': { voiceControlBridge: { + onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); }, + onStatus() { return () => {}; }, + } }, + '../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } }, + '../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } }, + '../gestures': { gestureHints: () => '' }, + './input-dialog': { paintInputDialog(_image, options) { frame = options; } }, + }, { + setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; }, + clearTimeout(id) { timers.delete(id); }, + }); + const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); }; + const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(), + actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() { + if (synchronousFinal) transcript(synchronousFinal, true); + return deferred ? completion : undefined; + } }, + sendTargets: [{ id: 'send', label: 'Send', onSend: text => sent.push(text) }], + }); + layer.startCapture(); + return { layer, sent, refinements, transcript, starts: () => starts, + input: type => layer.handleInput({ type }, {}), + frame() { layer.paint({}, () => ({})); return frame; }, + timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } }, + async complete() { complete(); await completion; await Promise.resolve(); }, + nextCapture() { completion = new Promise(resolve => { complete = resolve; }); }, + timers: () => timers.size, + }; +} + +for (const partial of ['', 'Set a timer']) { + test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => { + const env = dialog({ autoSend: true }); + if (partial) env.transcript(partial); + env.layer.endCapture(); + env.timeout(); // well beyond the old fallback deadline + assert.deepEqual(env.sent, []); + assert.equal(env.timers(), 0); + env.transcript('Set a timer for five minutes', true); + await env.complete(); env.timeout(); + assert.deepEqual(env.sent, ['Set a timer for five minutes']); + assert.equal(env.timers(), 0); + }); +} + +async function continuation() { + const env = dialog(); + env.layer.endCapture(); + env.transcript('Set a timer for five minutes', true); + await env.complete(); + env.nextCapture(); + env.input('scroll-down'); env.input('click'); // Continue + env.input('click'); // stop follow-up + return env; +} + +test('Continue refines the complete late result instead of appending an editing instruction', async () => { + const env = await continuation(); + env.timeout(); + assert.equal(env.refinements.length, 0); + env.transcript('Change five to ten', true); + await env.complete(); env.timeout(); + assert.equal(env.refinements.length, 1); + assert.equal(env.refinements[0].original, 'Set a timer for five minutes'); + assert.equal(env.refinements[0].followup, 'Change five to ten'); +}); + +test('Send and Continue cannot consume partial text while native recognition is pending', async () => { + const env = dialog(); + env.transcript('Set a timer'); env.layer.endCapture(); + assert.equal(env.frame().status, 'Finishing transcription...'); + env.input('click'); env.input('scroll-down'); env.input('click'); + assert.deepEqual(env.sent, []); + assert.equal(env.starts(), 1); + env.transcript('Set a timer for five minutes', true); await env.complete(); + env.input('scroll-up'); env.input('click'); + assert.deepEqual(env.sent, ['Set a timer for five minutes']); +}); + +test('cancelling a pending continuation ignores its late transcript', async () => { + const env = await continuation(); + env.input('double-click'); + env.transcript('Change five to ten', true); await env.complete(); env.timeout(); + assert.equal(env.refinements.length, 0); + assert.equal(env.frame().text, 'Set a timer for five minutes'); +}); + +test('closing while native recognition is pending never schedules a fallback or sends', async () => { + const env = dialog({ autoSend: true }); + env.layer.endCapture(); env.layer.onRemoved(); + env.transcript('late', true); await env.complete(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.timers(), 0); +}); + +test('failed or empty native captures return to the menu without sending', async () => { + const env = dialog({ autoSend: true }); + env.layer.endCapture(); await env.complete(); env.timeout(); + assert.deepEqual(env.sent, []); + assert.equal(env.frame().status, 'Send, continue, or discard?'); +}); + +test('cloud fallback and a synchronous final still send exactly once', () => { + for (const synchronousFinal of [undefined, 'complete']) { + const env = dialog({ autoSend: true, deferred: false, synchronousFinal }); + env.transcript('partial'); env.layer.endCapture(); env.timeout(); + assert.deepEqual(env.sent, [synchronousFinal ?? 'partial']); + } +}); + +test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => { + let controller; + class Controller { + constructor() { controller = this; } + setListener(listener) { this.listener = listener; } + setCommunicator() {} setUsePhoneMic() {} setSaveRecordings() {} setEndpointing() {} + setNoiseSuppression() {} setBeamFilter() {} clearSpeakerVerification() {} setOnboardModelKind() {} + start(_mode, id) { this.captureId = id; this.active = true; } + stop() { this.active = false; } + isCapturing() { return this.active; } + } + const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', { + '@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } }, + './speech-pause': { SpeechPauseDetector: class { reset() {} } }, + }, { global: { isAndroid: true }, com: { faceclaw: { app: { + FaceclawVoiceController: Controller, + FaceclawVoiceControllerListener: function (listener) { return listener; }, + } } } }); + const bridge = new FaceclawVoiceControlBridge(); + const options = { provider: 'onboard-whisper' }; + const events = []; + bridge.onTranscript(event => events.push(event.text)); + bridge.startPushToTalk(options); + const stopped = bridge.stopPushToTalk().then(() => events.push('stopped')); + controller.listener.onStopped(0); // an older raw/cloud capture's callback + await Promise.resolve(); assert.deepEqual(events, []); + controller.listener.onTranscript('complete', true); + assert.deepEqual(events, ['complete']); + controller.listener.onStopped(controller.captureId); await stopped; + assert.deepEqual(events, ['complete', 'stopped']); + bridge.startContinuousCapture(options); bridge.startPushToTalk(options); + assert.equal(bridge.stopPushToTalk(), undefined); + assert.equal(controller.active, true); + bridge.stopContinuousCapture(); controller.listener.onStopped(controller.captureId); +});