mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Wait for final results from Whisper before completing
<details> <summary>Prompts Used</summary> User: This checkout corresponds to https://github.com/jimrandomh/faceclaw/pull/27. Review this branch. GPT-6 Astra: > Found one issue: > > - **[P2] Wait for Whisper’s final result before completing dictation** — [FaceclawVoiceController.java:729](/Users/jbabcock/repositories/g2/faceclaw/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVoiceController.java:729). Skipping partial decodes leaves short utterances without text until release, but the dialog still finalizes after its existing timeout. If decoding exceeds roughly 2.7 seconds (native stop’s 1.5-second wait plus the dialog’s 1.2-second timeout), auto-send can abandon a short command or send an incomplete longer one. Continue can append the late editing instruction literally instead of refining the message. I reproduced these outcomes with delayed transcript events. These flows should await recognition completion. > > Validation: TypeScript and a clean Android build passed. All 21 test failures also occur on `main`. Actual Whisper latency was not tested on hardware. <hr> User: Thanks, let's fix that issue. GPT-6 Astra: > Fixed. Dictation now waits for native recognition to finish before fallback timers run. Auto-send, Continue, and AI Chat preserve delayed Whisper results; cancellation ignores late transcripts. > > TypeScript and Android builds pass. All 11 new regression tests pass. The full suite still has the same 21 pre-existing failures. </details>
This commit is contained in:
1 parent
91890a25ac
commit
25c398d60f
11 files changed
+385
-28
No files matched your search
@@ -266,20 +266,26 @@ public class FaceclawVoiceController {
|
||||
}
|
||||
|
||||
public void start(String requestedMode) {
|
||||
start(requestedMode, 0);
|
||||
}
|
||||
|
||||
public void start(String requestedMode, int captureId) {
|
||||
synchronized (lock) {
|
||||
if (started) {
|
||||
emitStatus("Voice control is already listening.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
if (!usePhoneMic && (communicator == null || !communicator.isSessionReady())) {
|
||||
emitStatus("Voice control needs an active G2 connection.");
|
||||
emitStopped(captureId);
|
||||
return;
|
||||
}
|
||||
mode = parseMode(requestedMode);
|
||||
activePhoneMic = usePhoneMic;
|
||||
started = true;
|
||||
audioStarted = false;
|
||||
workerThread = new Thread(this::runLoop, "FaceclawVoiceController");
|
||||
workerThread = new Thread(() -> runLoop(captureId), "FaceclawVoiceController");
|
||||
workerThread.start();
|
||||
}
|
||||
}
|
||||
@@ -350,7 +356,7 @@ public class FaceclawVoiceController {
|
||||
return VoiceInputMode.ONBOARD;
|
||||
}
|
||||
|
||||
private void runLoop() {
|
||||
private void runLoop(int captureId) {
|
||||
try {
|
||||
deleteLegacyKwsFiles();
|
||||
VoiceInputMode currentMode = mode;
|
||||
@@ -432,6 +438,7 @@ public class FaceclawVoiceController {
|
||||
audioStarted = false;
|
||||
workerThread = null;
|
||||
}
|
||||
emitStopped(captureId);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1014,6 +1021,13 @@ public class FaceclawVoiceController {
|
||||
mainHandler.post(currentListener::onSpeechEnd);
|
||||
}
|
||||
|
||||
private void emitStopped(int captureId) {
|
||||
FaceclawVoiceControllerListener currentListener = listener;
|
||||
if (currentListener != null) {
|
||||
mainHandler.post(() -> currentListener.onStopped(captureId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Decides when a hands-free utterance is over, so "Hey Even" capture can
|
||||
* stop without a button release.
|
||||
|
||||
+3
@@ -10,6 +10,9 @@ public interface FaceclawVoiceControllerListener {
|
||||
*/
|
||||
void onTranscript(String text, boolean isFinal);
|
||||
|
||||
/** Capture has finished, after the final transcript or a startup/recognition error. */
|
||||
void onStopped(int captureId);
|
||||
|
||||
/**
|
||||
* Decoded microphone audio: 16 kHz mono signed 16-bit little-endian PCM.
|
||||
* Emitted in every mode so raw-PCM consumers (levels, recording, captions)
|
||||
|
||||
@@ -89,11 +89,22 @@ export class VoiceDraft {
|
||||
}
|
||||
|
||||
private stopAndFinalize(): void {
|
||||
// Install fallback first: stop may synchronously deliver the final.
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
const generation = this.generation;
|
||||
const afterStop = () => {
|
||||
if (generation !== this.generation || !this.isFinishing()) return;
|
||||
this.timer = setTimeout(() => this.finish(), 1500);
|
||||
};
|
||||
if (this.started) {
|
||||
this.started = false;
|
||||
void Promise.resolve(this.actions.stopVoiceCapture()).catch((error) => this.cancel(String(error)));
|
||||
// Native recognition can take longer than the synchronous stop join.
|
||||
// Its final callback may finish the draft before this promise resolves.
|
||||
const stopped = this.actions.stopVoiceCapture();
|
||||
if (!stopped) { afterStop(); return; }
|
||||
void stopped.then(afterStop, (error) => {
|
||||
if (generation === this.generation) this.cancel(String(error));
|
||||
});
|
||||
} else {
|
||||
afterStop();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1912,9 +1912,9 @@ class DashboardController {
|
||||
return this.beginVoiceCapture("ptt", endpointing);
|
||||
}
|
||||
|
||||
private stopVoiceCapture(): void {
|
||||
private stopVoiceCapture(): Promise<void> | void {
|
||||
++this.pttCaptureGeneration;
|
||||
voiceControlBridge.stopPushToTalk();
|
||||
return voiceControlBridge.stopPushToTalk();
|
||||
}
|
||||
|
||||
private startContinuousVoiceCapture(): void {
|
||||
|
||||
@@ -108,6 +108,9 @@ export class FaceclawVoiceControlBridge {
|
||||
private listening = false;
|
||||
private detail = "";
|
||||
private started = false;
|
||||
private nativeCaptureFinished: Promise<void> = Promise.resolve();
|
||||
private nextNativeCaptureId = 1;
|
||||
private readonly nativeCaptureResolvers = new Map<number, () => void>();
|
||||
// The mic is a single shared stream; these are the reasons it is running.
|
||||
// The first holder starts capture (choosing the provider); the mic stops
|
||||
// when the last one releases. Transcript events broadcast to every
|
||||
@@ -189,8 +192,13 @@ export class FaceclawVoiceControlBridge {
|
||||
}
|
||||
|
||||
/** End push-to-talk: for cloud, commit for a final result if it was the last holder. */
|
||||
stopPushToTalk(): void {
|
||||
stopPushToTalk(): Promise<void> | void {
|
||||
// Shared continuous capture keeps running; cloud providers finalize via
|
||||
// their own callbacks. Only a stopped native recognizer must be awaited.
|
||||
const finished = !this.cloudClient && this.captureHolders.has("ptt") && this.captureHolders.size === 1
|
||||
? this.nativeCaptureFinished : undefined;
|
||||
this.releaseCapture("ptt", true);
|
||||
return finished;
|
||||
}
|
||||
|
||||
/** Begin continuous capture (Transcribe): the mic stays on until released. */
|
||||
@@ -322,7 +330,9 @@ export class FaceclawVoiceControlBridge {
|
||||
// Which on-device model to load; a no-op setter for every provider except
|
||||
// "onboard-whisper" (FaceclawVoiceController defaults to Moonshine).
|
||||
this.controller?.setOnboardModelKind(options.provider === "onboard-whisper" ? "whisper" : "moonshine");
|
||||
this.controller?.start("onboard");
|
||||
const captureId = this.nextNativeCaptureId++;
|
||||
this.nativeCaptureFinished = new Promise((resolve) => { this.nativeCaptureResolvers.set(captureId, resolve); });
|
||||
this.controller?.start("onboard", captureId);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -491,6 +501,10 @@ export class FaceclawVoiceControlBridge {
|
||||
onTranscript: (text: string, isFinal: boolean) => {
|
||||
this.emitTranscript(String(text), Boolean(isFinal));
|
||||
},
|
||||
onStopped: (captureId: number) => {
|
||||
this.nativeCaptureResolvers.get(captureId)?.();
|
||||
this.nativeCaptureResolvers.delete(captureId);
|
||||
},
|
||||
onPcm: (pcm: any) => {
|
||||
const bytes = toUint8Array(pcm);
|
||||
this.cloudClient?.acceptPcm(bytes);
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@ export type LayerActions = {
|
||||
* (hands-free); otherwise it runs until stopVoiceCapture.
|
||||
*/
|
||||
startVoiceCapture: (endpointing?: boolean) => Promise<void> | void;
|
||||
/** Stop push-to-talk; for a cloud provider this also commits for a final result. */
|
||||
/** Stop push-to-talk; awaits native recognition, or commits a cloud provider for a final result. */
|
||||
stopVoiceCapture: () => Promise<void> | void;
|
||||
/** Start continuous capture (Transcribe); shares the mic with push-to-talk. */
|
||||
startContinuousVoiceCapture: () => Promise<void> | void;
|
||||
|
||||
+56
-16
@@ -5,8 +5,7 @@ import { anthropicApiKeySetting } from "../dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, gestureHints, type InputEvent } from "../gestures";
|
||||
import { Layer, type LayerActions, type LayerContext } from "../layers";
|
||||
import { paintInputDialog } from "./input-dialog";
|
||||
// After the mic stops, the provider's final transcript can trail in (cloud
|
||||
// commit round-trip); wait this long for it before refining with what we have.
|
||||
// After native recognition finishes, a cloud final can still trail in.
|
||||
const FOLLOWUP_FINALIZE_TIMEOUT_MS = 1200;
|
||||
|
||||
/**
|
||||
@@ -80,6 +79,10 @@ export class VoiceInputLayer implements Layer {
|
||||
private capturing = false;
|
||||
/** Continuation stopped; waiting for the trailing final transcript. */
|
||||
private followupFinalizeTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private pendingFollowup = false;
|
||||
private stoppingCapture = false;
|
||||
private stopGeneration = 0;
|
||||
private acceptingTranscript = true;
|
||||
private refineHandle: AnthropicStreamHandle | null = null;
|
||||
private menuIndex = 0;
|
||||
/** Auto-send (wakeword skip-confirmation) is waiting to fire. */
|
||||
@@ -155,21 +158,48 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
this.capturing = false;
|
||||
this.phase = "menu";
|
||||
void this.actions.stopVoiceCapture();
|
||||
if (this.autoSend && this.sendTargets.length) {
|
||||
// Skip-confirmation (wakeword): send to the default target as soon as the
|
||||
// transcript finalizes, or after a short wait for the trailing final.
|
||||
this.pendingAutoSend = true;
|
||||
this.status = "Sending...";
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
} else if (this.status.endsWith("...")) {
|
||||
// A progress status ("Starting microphone...", "Listening...") gives
|
||||
// way to the menu prompt; an error (ending in ".") stays visible.
|
||||
this.status = "Send, continue, or discard?";
|
||||
}
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingAutoSend) {
|
||||
this.autoSendTimer = setTimeout(() => this.performAutoSend(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
}
|
||||
|
||||
/** Native stop can outlast its synchronous join while Whisper is decoding. */
|
||||
private stopCapture(onStopped: () => void): void {
|
||||
const generation = ++this.stopGeneration;
|
||||
this.stoppingCapture = true;
|
||||
const afterStop = () => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
onStopped();
|
||||
this.actions.requestRender();
|
||||
};
|
||||
const stopped = this.actions.stopVoiceCapture();
|
||||
if (!stopped) {
|
||||
afterStop();
|
||||
return;
|
||||
}
|
||||
void stopped.then(afterStop, (error) => {
|
||||
if (generation !== this.stopGeneration) return;
|
||||
this.stoppingCapture = false;
|
||||
this.pendingAutoSend = false;
|
||||
this.acceptingTranscript = false;
|
||||
this.backToMenu(this.phase === "continuing" ? this.baseText : this.displayText(), String(error));
|
||||
});
|
||||
}
|
||||
|
||||
/** Fire the queued skip-confirmation send (or fall back to the menu). */
|
||||
private performAutoSend(): void {
|
||||
if (this.autoSendTimer !== null) {
|
||||
@@ -193,7 +223,7 @@ export class VoiceInputLayer implements Layer {
|
||||
/** The menu rows: one per send target, then Continue, then Discard. */
|
||||
private menuRows(): Array<{ label: string; dim: boolean; onSelect: () => void }> {
|
||||
const text = this.displayText().trim();
|
||||
const hasText = text.length > 0;
|
||||
const hasText = text.length > 0 && !this.stoppingCapture;
|
||||
const hasLlmKey = anthropicApiKeySetting.get().trim().length > 0;
|
||||
const rows: Array<{ label: string; dim: boolean; onSelect: () => void }> = [];
|
||||
for (const target of this.sendTargets) {
|
||||
@@ -201,6 +231,7 @@ export class VoiceInputLayer implements Layer {
|
||||
label: target.label,
|
||||
dim: !hasText,
|
||||
onSelect: () => {
|
||||
if (this.stoppingCapture) return;
|
||||
this.dismiss();
|
||||
if (hasText) target.onSend(text);
|
||||
},
|
||||
@@ -208,9 +239,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
rows.push({
|
||||
label: hasLlmKey ? "Continue" : "Continue (Needs LLM API key)",
|
||||
dim: !hasLlmKey,
|
||||
dim: !hasLlmKey || this.stoppingCapture,
|
||||
onSelect: () => {
|
||||
if (hasLlmKey) this.startContinuation();
|
||||
if (hasLlmKey && !this.stoppingCapture) this.startContinuation();
|
||||
},
|
||||
});
|
||||
rows.push({ label: "Discard", dim: false, onSelect: () => this.dismiss() });
|
||||
@@ -222,7 +253,7 @@ export class VoiceInputLayer implements Layer {
|
||||
const inMenu = this.phase === "menu";
|
||||
paintInputDialog(image, {
|
||||
title: this.capturing && this.listening ? "Voice ●" : "Voice",
|
||||
status: this.status,
|
||||
status: this.stoppingCapture ? "Finishing transcription..." : this.status,
|
||||
text: this.displayText() || this.placeholderText(),
|
||||
rows: inMenu ? this.menuRows() : [],
|
||||
selectedRow: this.menuIndex,
|
||||
@@ -284,6 +315,7 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Continue selected: keep the message aside and record a follow-up. */
|
||||
private startContinuation(): void {
|
||||
this.acceptingTranscript = true;
|
||||
this.baseText = this.displayText().trim();
|
||||
this.finalizedText = "";
|
||||
this.liveText = "";
|
||||
@@ -298,15 +330,18 @@ export class VoiceInputLayer implements Layer {
|
||||
private endContinuationCapture(): void {
|
||||
if (!this.capturing) return;
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.status = "Refining...";
|
||||
this.pendingFollowup = true;
|
||||
this.status = "Finishing transcription...";
|
||||
this.stopCapture(() => {
|
||||
if (this.pendingFollowup) {
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
});
|
||||
this.actions.requestRender();
|
||||
// The provider's committed transcript arrives shortly after stop; refine
|
||||
// when it does, or after a timeout with whatever partials we have.
|
||||
this.followupFinalizeTimer = setTimeout(() => this.beginRefine(), FOLLOWUP_FINALIZE_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private beginRefine(): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -343,9 +378,10 @@ export class VoiceInputLayer implements Layer {
|
||||
|
||||
/** Abort a continuation (mic or LLM stage) and restore the prior message. */
|
||||
private cancelContinuation(status: string): void {
|
||||
this.acceptingTranscript = false;
|
||||
if (this.capturing) {
|
||||
this.capturing = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.stopCapture(() => {});
|
||||
}
|
||||
this.refineHandle?.cancel();
|
||||
this.refineHandle = null;
|
||||
@@ -353,6 +389,7 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
private backToMenu(text: string, status: string): void {
|
||||
this.pendingFollowup = false;
|
||||
if (this.followupFinalizeTimer !== null) {
|
||||
clearTimeout(this.followupFinalizeTimer);
|
||||
this.followupFinalizeTimer = null;
|
||||
@@ -366,6 +403,9 @@ export class VoiceInputLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
++this.stopGeneration;
|
||||
this.acceptingTranscript = false;
|
||||
this.pendingFollowup = false;
|
||||
this.unsubscribeTranscript?.();
|
||||
this.unsubscribeTranscript = null;
|
||||
this.unsubscribeStatus?.();
|
||||
@@ -428,14 +468,14 @@ export class VoiceInputLayer implements Layer {
|
||||
private onTranscript(event: VoiceTranscriptEvent): void {
|
||||
// The refine stream owns the text buffers once it starts; a transcript
|
||||
// that trails in after that point is stale.
|
||||
if (this.phase === "refining") return;
|
||||
if (!this.acceptingTranscript || this.phase === "refining") return;
|
||||
if (event.isFinal) {
|
||||
const finalText = event.text.trim() || this.liveText.trim();
|
||||
if (finalText) {
|
||||
this.finalizedText = this.finalizedText ? `${this.finalizedText} ${finalText}` : finalText;
|
||||
}
|
||||
this.liveText = "";
|
||||
if (this.phase === "continuing" && this.followupFinalizeTimer !== null) {
|
||||
if (this.phase === "continuing" && this.pendingFollowup) {
|
||||
// The follow-up finalized; no need to keep waiting.
|
||||
this.beginRefine();
|
||||
return;
|
||||
|
||||
+30
-2
@@ -125,7 +125,7 @@ test('synchronous provider errors do not leave a permanently active turn', () =>
|
||||
assert.equal(session.status, 'failed');
|
||||
});
|
||||
|
||||
function draftEnv({ prepare = async () => true, start = async () => {}, initialStatus = 'Ready' } = {}) {
|
||||
function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) {
|
||||
const transcripts = new Set(), statuses = new Set(), timers = new Map();
|
||||
const sent = [];
|
||||
let starts = 0, stops = 0, mic = false, timerId = 0;
|
||||
@@ -139,7 +139,7 @@ function draftEnv({ prepare = async () => true, start = async () => {}, initialS
|
||||
setTimeout(cb) { timers.set(++timerId, cb); return timerId; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
});
|
||||
const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; } }, prepare, () => {}, (text) => sent.push(text));
|
||||
const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text));
|
||||
return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic,
|
||||
transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); },
|
||||
status(status) { for (const cb of [...statuses]) cb({ status }); },
|
||||
@@ -174,6 +174,34 @@ test('release before permission resolves never starts a microphone', async () =>
|
||||
assert.deepEqual(env.sent, []);
|
||||
});
|
||||
|
||||
test('chat waits past its fallback deadline for native recognition', async () => {
|
||||
let complete;
|
||||
const stopped = new Promise(resolve => { complete = resolve; });
|
||||
const env = draftEnv({ stop: () => stopped });
|
||||
await env.draft.start();
|
||||
env.transcript('first segment');
|
||||
env.draft.release(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.draft.phase, 'finishing');
|
||||
env.transcript('first segment and the rest', true);
|
||||
complete(); await stopped; env.timeout();
|
||||
assert.deepEqual(env.sent, ['first segment and the rest']);
|
||||
});
|
||||
|
||||
test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => {
|
||||
let complete;
|
||||
const stopped = new Promise(resolve => { complete = resolve; });
|
||||
const env = draftEnv({ stop: () => stopped });
|
||||
await env.draft.start();
|
||||
env.draft.release(); env.draft.cancel();
|
||||
await env.draft.start();
|
||||
env.transcript('new speech');
|
||||
complete(); await stopped; env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.draft.phase, 'listening');
|
||||
env.draft.cancel();
|
||||
});
|
||||
|
||||
test('release during async mic startup stops capture after it starts', async () => {
|
||||
let started;
|
||||
const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) });
|
||||
|
||||
+58
@@ -0,0 +1,58 @@
|
||||
package com.faceclaw.app;
|
||||
|
||||
import java.lang.reflect.*;
|
||||
import java.util.*;
|
||||
|
||||
public class TextureCacheProtocolTest {
|
||||
static int u16(byte[] b, int p) { return (b[p]&255) | (b[p+1]&255)<<8; }
|
||||
static int u32(byte[] b, int p) { return u16(b,p) | u16(b,p+2)<<16; }
|
||||
public static void main(String[] args) throws Exception {
|
||||
TextureCacheState cache = new TextureCacheState();
|
||||
byte[] pixels = new byte[255*255];
|
||||
for (int i=0;i<pixels.length;i++) pixels[i]=(byte)(i%15+1);
|
||||
ImageAtlas.Entry big = new ImageAtlas.Entry(255,255,pixels);
|
||||
for (int i=0;i<3;i++) assert cache.ensureImage(i,big)>=0;
|
||||
ImageAtlas.Entry tiny = new ImageAtlas.Entry(1,1,new byte[]{15});
|
||||
int imageOffset = cache.ensureImage(100,tiny);
|
||||
assert imageOffset>65535;
|
||||
GlyphAtlas.Glyph glyph = new GlyphAtlas.Glyph(1,1,0,1,0,null,new byte[]{15});
|
||||
int glyphOffset = cache.ensureGlyph(42,'A',glyph);
|
||||
int table = cache.fontTableOffset(42);
|
||||
assert table>65535 && glyphOffset==table+96*4;
|
||||
byte[] memory = new byte[262144];
|
||||
for (byte[] upload: cache.drainUploadPayloads(3600)) {
|
||||
assert upload[0]==18;
|
||||
int pos=1;
|
||||
while (pos<upload.length) {
|
||||
int offset=u32(upload,pos), n=u16(upload,pos+4);
|
||||
System.arraycopy(upload,pos+6,memory,offset,n); pos+=6+n;
|
||||
}
|
||||
assert pos==upload.length;
|
||||
}
|
||||
assert memory[imageOffset]==1 && memory[imageOffset+2]==0x1f;
|
||||
assert u32(memory,table+('A'-32)*4)==glyphOffset;
|
||||
assert !cache.hasPendingUploads();
|
||||
// Exercise the production planner encoders with offsets above 64 KiB.
|
||||
Class<?> selected=Class.forName("com.faceclaw.app.TexturePlanner$Selected");
|
||||
Constructor<?> ctor=selected.getDeclaredConstructors()[0]; ctor.setAccessible(true);
|
||||
Object image=ctor.newInstance(SurfaceCompositor.ScreenDraw.image(100,321,123),null,tiny,321,15);
|
||||
Method encode=TexturePlanner.class.getDeclaredMethod("encodeImageDraw",TextureCacheState.class,selected);
|
||||
encode.setAccessible(true);
|
||||
byte[] draw=(byte[])encode.invoke(null,cache,image);
|
||||
assert draw.length==10 && draw[0]==19 && u32(draw,1)==imageOffset;
|
||||
assert u16(draw,5)==321 && u16(draw,7)==123 && draw[9]==31;
|
||||
Object letter=ctor.newInstance(SurfaceCompositor.ScreenDraw.glyph(42,'A',321,123,255),glyph,null,321,15);
|
||||
Method runs=TexturePlanner.class.getDeclaredMethod("buildRuns",TextureCacheState.class,List.class,List.class);
|
||||
runs.setAccessible(true); List<byte[]> out=new ArrayList<>();
|
||||
runs.invoke(null,cache,Collections.singletonList(letter),out);
|
||||
byte[] run=out.get(0);
|
||||
assert run.length==12 && run[0]==20 && u32(run,1)==table;
|
||||
assert u16(run,5)==321 && u16(run,7)==123 && run[9]==31 && run[10]==1 && run[11]=='A';
|
||||
int id=200;
|
||||
while (cache.ensureImage(id++,big)>=0) {}
|
||||
int before=cache.usedBytes();
|
||||
assert cache.ensureImage(id,big)==-1 && cache.usedBytes()==before;
|
||||
cache.reset(); assert cache.usedBytes()==0 && !cache.hasPendingUploads();
|
||||
assert cache.ensureImage(id,tiny)==0;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
const test = require('node:test');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
const { execFileSync } = require('node:child_process');
|
||||
|
||||
test('256 KiB cache uploads and planner draws preserve wide image and glyph offsets', () => {
|
||||
const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-texture-java-'));
|
||||
const javaBin = (name) => process.env.JAVA_HOME ? path.join(process.env.JAVA_HOME, 'bin', name) : name;
|
||||
const root = path.join(__dirname, '../App_Resources/Android/src/main/java/com/faceclaw/app');
|
||||
try {
|
||||
const clock = path.join(directory, 'SystemClock.java');
|
||||
const log = path.join(directory, 'Log.java');
|
||||
fs.writeFileSync(clock, 'package android.os; public class SystemClock { public static long elapsedRealtime() { return 0; } }');
|
||||
fs.writeFileSync(log, 'package android.util; public class Log { public static int i(String tag, String msg) { return 0; } }');
|
||||
const sources = ['GlyphAtlas.java', 'ImageAtlas.java', 'FwGlyphAtlas.java', 'SurfaceCompositor.java',
|
||||
'util/BmpUtil.java', 'util/CollectionUtils.java', 'g2protocol/BleProtocol.java',
|
||||
'g2protocol/BleImageOptimizer.java', 'g2protocol/TextureCacheState.java', 'g2protocol/TexturePlanner.java'];
|
||||
execFileSync(javaBin('javac'), ['-d', directory, ...sources.map(file => path.join(root, file)),
|
||||
clock, log, path.join(__dirname, 'fixtures/TextureCacheProtocolTest.java')], { stdio: 'pipe' });
|
||||
execFileSync(javaBin('java'), ['-ea', '-cp', directory, 'com.faceclaw.app.TextureCacheProtocolTest'], { stdio: 'pipe' });
|
||||
} finally { fs.rmSync(directory, { recursive: true, force: true }); }
|
||||
});
|
||||
@@ -0,0 +1,166 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, deps, globals = {}) {
|
||||
const exports = {};
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, { exports, console, ...globals, require: name => deps[name] ?? {} });
|
||||
return exports;
|
||||
}
|
||||
|
||||
function dialog({ autoSend = false, deferred = true, synchronousFinal } = {}) {
|
||||
const listeners = new Set(), timers = new Map(), sent = [], refinements = [];
|
||||
let complete, timerId = 0, starts = 0, frame;
|
||||
let completion = new Promise(resolve => { complete = resolve; });
|
||||
const { VoiceInputLayer } = load('app/ui/shell/voice-input.ts', {
|
||||
'../../native/voice-control': { voiceControlBridge: {
|
||||
onTranscript(cb) { listeners.add(cb); return () => listeners.delete(cb); },
|
||||
onStatus() { return () => {}; },
|
||||
} },
|
||||
'../../native/anthropic': { refineDictation(options) { refinements.push(options); return { cancel() {} }; } },
|
||||
'../dashboard-settings': { anthropicApiKeySetting: { get: () => 'key' } },
|
||||
'../gestures': { gestureHints: () => '' },
|
||||
'./input-dialog': { paintInputDialog(_image, options) { frame = options; } },
|
||||
}, {
|
||||
setTimeout(cb) { const id = ++timerId; timers.set(id, cb); return id; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
});
|
||||
const transcript = (text, isFinal = false) => { for (const cb of [...listeners]) cb({ text, isFinal }); };
|
||||
const layer = new VoiceInputLayer({ autoSend, onClosed() {}, dismiss: () => layer.onRemoved(),
|
||||
actions: { requestRender() {}, startVoiceCapture() { starts++; }, stopVoiceCapture() {
|
||||
if (synchronousFinal) transcript(synchronousFinal, true);
|
||||
return deferred ? completion : undefined;
|
||||
} },
|
||||
sendTargets: [{ id: 'send', label: 'Send', onSend: text => sent.push(text) }],
|
||||
});
|
||||
layer.startCapture();
|
||||
return { layer, sent, refinements, transcript, starts: () => starts,
|
||||
input: type => layer.handleInput({ type }, {}),
|
||||
frame() { layer.paint({}, () => ({})); return frame; },
|
||||
timeout() { for (const [id, cb] of [...timers]) { timers.delete(id); cb(); } },
|
||||
async complete() { complete(); await completion; await Promise.resolve(); },
|
||||
nextCapture() { completion = new Promise(resolve => { complete = resolve; }); },
|
||||
timers: () => timers.size,
|
||||
};
|
||||
}
|
||||
|
||||
for (const partial of ['', 'Set a timer']) {
|
||||
test(`auto-send waits for slow native recognition with ${partial ? 'a committed segment' : 'no partial'}`, async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
if (partial) env.transcript(partial);
|
||||
env.layer.endCapture();
|
||||
env.timeout(); // well beyond the old fallback deadline
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.timers(), 0);
|
||||
env.transcript('Set a timer for five minutes', true);
|
||||
await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, ['Set a timer for five minutes']);
|
||||
assert.equal(env.timers(), 0);
|
||||
});
|
||||
}
|
||||
|
||||
async function continuation() {
|
||||
const env = dialog();
|
||||
env.layer.endCapture();
|
||||
env.transcript('Set a timer for five minutes', true);
|
||||
await env.complete();
|
||||
env.nextCapture();
|
||||
env.input('scroll-down'); env.input('click'); // Continue
|
||||
env.input('click'); // stop follow-up
|
||||
return env;
|
||||
}
|
||||
|
||||
test('Continue refines the complete late result instead of appending an editing instruction', async () => {
|
||||
const env = await continuation();
|
||||
env.timeout();
|
||||
assert.equal(env.refinements.length, 0);
|
||||
env.transcript('Change five to ten', true);
|
||||
await env.complete(); env.timeout();
|
||||
assert.equal(env.refinements.length, 1);
|
||||
assert.equal(env.refinements[0].original, 'Set a timer for five minutes');
|
||||
assert.equal(env.refinements[0].followup, 'Change five to ten');
|
||||
});
|
||||
|
||||
test('Send and Continue cannot consume partial text while native recognition is pending', async () => {
|
||||
const env = dialog();
|
||||
env.transcript('Set a timer'); env.layer.endCapture();
|
||||
assert.equal(env.frame().status, 'Finishing transcription...');
|
||||
env.input('click'); env.input('scroll-down'); env.input('click');
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.starts(), 1);
|
||||
env.transcript('Set a timer for five minutes', true); await env.complete();
|
||||
env.input('scroll-up'); env.input('click');
|
||||
assert.deepEqual(env.sent, ['Set a timer for five minutes']);
|
||||
});
|
||||
|
||||
test('cancelling a pending continuation ignores its late transcript', async () => {
|
||||
const env = await continuation();
|
||||
env.input('double-click');
|
||||
env.transcript('Change five to ten', true); await env.complete(); env.timeout();
|
||||
assert.equal(env.refinements.length, 0);
|
||||
assert.equal(env.frame().text, 'Set a timer for five minutes');
|
||||
});
|
||||
|
||||
test('closing while native recognition is pending never schedules a fallback or sends', async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
env.layer.endCapture(); env.layer.onRemoved();
|
||||
env.transcript('late', true); await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.timers(), 0);
|
||||
});
|
||||
|
||||
test('failed or empty native captures return to the menu without sending', async () => {
|
||||
const env = dialog({ autoSend: true });
|
||||
env.layer.endCapture(); await env.complete(); env.timeout();
|
||||
assert.deepEqual(env.sent, []);
|
||||
assert.equal(env.frame().status, 'Send, continue, or discard?');
|
||||
});
|
||||
|
||||
test('cloud fallback and a synchronous final still send exactly once', () => {
|
||||
for (const synchronousFinal of [undefined, 'complete']) {
|
||||
const env = dialog({ autoSend: true, deferred: false, synchronousFinal });
|
||||
env.transcript('partial'); env.layer.endCapture(); env.timeout();
|
||||
assert.deepEqual(env.sent, [synchronousFinal ?? 'partial']);
|
||||
}
|
||||
});
|
||||
|
||||
test('native completion is capture-specific, follows the final, and does not wait for shared capture', async () => {
|
||||
let controller;
|
||||
class Controller {
|
||||
constructor() { controller = this; }
|
||||
setListener(listener) { this.listener = listener; }
|
||||
setCommunicator() {} setUsePhoneMic() {} setSaveRecordings() {} setEndpointing() {}
|
||||
setNoiseSuppression() {} setBeamFilter() {} clearSpeakerVerification() {} setOnboardModelKind() {}
|
||||
start(_mode, id) { this.captureId = id; this.active = true; }
|
||||
stop() { this.active = false; }
|
||||
isCapturing() { return this.active; }
|
||||
}
|
||||
const { FaceclawVoiceControlBridge } = load('app/native/voice-control.ts', {
|
||||
'@nativescript/core': { Utils: { android: { getApplicationContext: () => ({}) } } },
|
||||
'./speech-pause': { SpeechPauseDetector: class { reset() {} } },
|
||||
}, { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawVoiceController: Controller,
|
||||
FaceclawVoiceControllerListener: function (listener) { return listener; },
|
||||
} } } });
|
||||
const bridge = new FaceclawVoiceControlBridge();
|
||||
const options = { provider: 'onboard-whisper' };
|
||||
const events = [];
|
||||
bridge.onTranscript(event => events.push(event.text));
|
||||
bridge.startPushToTalk(options);
|
||||
const stopped = bridge.stopPushToTalk().then(() => events.push('stopped'));
|
||||
controller.listener.onStopped(0); // an older raw/cloud capture's callback
|
||||
await Promise.resolve(); assert.deepEqual(events, []);
|
||||
controller.listener.onTranscript('complete', true);
|
||||
assert.deepEqual(events, ['complete']);
|
||||
controller.listener.onStopped(controller.captureId); await stopped;
|
||||
assert.deepEqual(events, ['complete', 'stopped']);
|
||||
bridge.startContinuousCapture(options); bridge.startPushToTalk(options);
|
||||
assert.equal(bridge.stopPushToTalk(), undefined);
|
||||
assert.equal(controller.active, true);
|
||||
bridge.stopContinuousCapture(); controller.listener.onStopped(controller.captureId);
|
||||
});
|
||||
Reference in new issue
Block a user