mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
197 lines
7.7 KiB
TypeScript
197 lines
7.7 KiB
TypeScript
import { GrayImage } from "../../graphics/image";
|
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
|
import { lineStep } from "../../ui/metrics";
|
|
import { MicLevelMeter, paintListeningHeader, RECORDING_FAILED_NOTICE } from "../../ui/listening-header";
|
|
import { writeTextToDownloads } from "../../native/file-access";
|
|
import { voiceControlBridge } from "../../native/voice-control";
|
|
import { Layer, type LayerContext } from "../../ui/layers";
|
|
import { type InputEvent } from "../../ui/gestures";
|
|
import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
|
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
|
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
|
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
|
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
|
|
|
const NO_MODEL_NOTICE = "No language model is set up.";
|
|
|
|
/**
|
|
* Live translation view. Continuous mic capture is owned by the app wrapper
|
|
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
|
* model one block at a time (provisionally while a block is still being
|
|
* spoken), shows the English as it streams in, and saves the session to
|
|
* Downloads from the app menu.
|
|
*/
|
|
export class TranslateLayer implements Layer {
|
|
private status = "Starting microphone...";
|
|
private listening = false;
|
|
private detail = "";
|
|
private saveNotice = "";
|
|
private translationError = "";
|
|
// Why the app can't run (no multilingual provider or no model); shown in
|
|
// place of the transcript, and a tap asks the app wrapper to check again.
|
|
private problem: string | null = null;
|
|
private readonly model = new TranslationModel();
|
|
private readonly scroll = new TranscriptScroll();
|
|
private readonly meter = new MicLevelMeter();
|
|
// The request in flight (the model tracks which job it is).
|
|
private activeHandle: LlmStreamHandle | null = null;
|
|
private requestRender: () => void = () => {};
|
|
private removed = false;
|
|
private unsubscribeTranscript: (() => void) | null = null;
|
|
private unsubscribeStatus: (() => void) | null = null;
|
|
|
|
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
|
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
|
|
|
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
|
start(requestRender: () => void, isVisible: () => boolean): void {
|
|
this.requestRender = requestRender;
|
|
this.meter.start(requestRender, isVisible);
|
|
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
|
if (this.model.accept(event)) {
|
|
// The request in flight translates text the transcript has replaced.
|
|
this.activeHandle?.cancel();
|
|
this.activeHandle = null;
|
|
}
|
|
this.pump();
|
|
requestRender();
|
|
});
|
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
|
this.status = state.status;
|
|
this.listening = state.listening;
|
|
this.detail = state.detail;
|
|
this.saveNotice = "";
|
|
requestRender();
|
|
});
|
|
}
|
|
|
|
setProblem(problem: string | null): void {
|
|
this.problem = problem;
|
|
this.requestRender();
|
|
}
|
|
|
|
paint(ctx: LayerContext): GrayImage {
|
|
const font = getDefaultSmallFont();
|
|
const { width, height } = ctx.stack.getBaseSize();
|
|
const image = new GrayImage(width, height, 0);
|
|
const measure = (line: string) => font.measureText(line);
|
|
const maxWidth = width - 64;
|
|
|
|
const status = this.problem
|
|
? "Setup needed"
|
|
: voiceControlBridge.recordingFailed ? RECORDING_FAILED_NOTICE : this.translationError || this.status;
|
|
const meter = this.listening && !this.problem ? this.meter : null;
|
|
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, voiceControlBridge.recording);
|
|
|
|
let lines: TranslationLine[];
|
|
if (this.problem) {
|
|
lines = wrapTranscribeText(measure, `${this.problem}\n\nTap to check again.`, maxWidth).map((text) => ({ text, gray: 230 }));
|
|
} else {
|
|
lines = this.model.lines(measure, maxWidth, translateShowOriginalSetting.get());
|
|
if (!lines.length) {
|
|
const placeholder = this.listening ? "Listening..." : this.detail;
|
|
lines = wrapTranscribeText(measure, placeholder, maxWidth).map((text) => ({ text, gray: 230 }));
|
|
}
|
|
}
|
|
|
|
const step = lineStep(font) + 2;
|
|
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
|
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
|
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
|
const y = bodyTop + (index - firstLine) * step;
|
|
image.drawText(font, 32, y, lines[index]!.text, lines[index]!.gray);
|
|
}
|
|
|
|
if (lines.length > bodyLines) {
|
|
const trackHeight = height - bodyTop - 6;
|
|
const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / lines.length));
|
|
const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (lines.length - bodyLines));
|
|
image.fillRect(width - 14, bodyTop, 2, trackHeight, 45);
|
|
image.fillRect(width - 14, thumbY, 2, thumbHeight, 140);
|
|
}
|
|
return image;
|
|
}
|
|
|
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
|
if (event.type === "scroll-up" || event.type === "scroll-down") {
|
|
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
|
|
ctx.actions.requestRender();
|
|
return;
|
|
}
|
|
if (event.type === "click" && this.problem) {
|
|
this.retry();
|
|
return;
|
|
}
|
|
if (event.type === "double-click") {
|
|
ctx.stack.pop();
|
|
}
|
|
}
|
|
|
|
onRemoved(): void {
|
|
this.removed = true;
|
|
this.meter.stop();
|
|
this.activeHandle?.cancel();
|
|
this.activeHandle = null;
|
|
this.unsubscribeTranscript?.();
|
|
this.unsubscribeTranscript = null;
|
|
this.unsubscribeStatus?.();
|
|
this.unsubscribeStatus = null;
|
|
}
|
|
|
|
saveTranscript(): void {
|
|
const text = this.model.savedText().trim();
|
|
if (!text) {
|
|
this.saveNotice = "Nothing to save yet.";
|
|
return;
|
|
}
|
|
const filename = `translation-${this.stamp}.txt`;
|
|
const path = writeTextToDownloads(filename, `${text}\n`);
|
|
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
|
}
|
|
|
|
/**
|
|
* Start the next translation request (a final block, else the live block's
|
|
* provisional one), unless one is in flight.
|
|
*/
|
|
private pump(): void {
|
|
if (this.removed) return;
|
|
const job = this.model.nextJob();
|
|
if (!job) return;
|
|
const llm = resolveTranslationModel();
|
|
if (!llm) {
|
|
// The model went away mid-session (a key was cleared): keep the original.
|
|
this.model.fail(job);
|
|
this.translationError = NO_MODEL_NOTICE;
|
|
this.pump();
|
|
return;
|
|
}
|
|
const language = resolveTranslateLanguage();
|
|
const handle = streamTranslation(llm, {
|
|
speech: job.source,
|
|
context: job.context,
|
|
language: language ? spokenLanguageName(language) : "",
|
|
onProgress: (textSoFar) => {
|
|
if (this.model.progress(job, textSoFar)) this.requestRender();
|
|
},
|
|
onDone: (translation) => this.settle(job, () => {
|
|
this.model.complete(job, translation);
|
|
this.translationError = "";
|
|
}),
|
|
onError: (message) => this.settle(job, () => {
|
|
this.model.fail(job);
|
|
this.translationError = `Translation failed: ${message}`;
|
|
}),
|
|
});
|
|
// A provider that fails synchronously has already settled (and moved on).
|
|
if (this.model.isActive(job)) this.activeHandle = handle;
|
|
}
|
|
|
|
private settle(job: TranslationJob, apply: () => void): void {
|
|
if (!this.model.isActive(job)) return;
|
|
this.activeHandle = null;
|
|
apply();
|
|
this.pump();
|
|
this.requestRender();
|
|
}
|
|
}
|