mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Most of the builtin apps had some padding between the window edge and their content, because they expect to be displayed with a line border on their edge. We added a customization option to remove the border, so, that padding is not always necessary. Make the layouts dynamically adjust, to omit the padding when it isn't needed.
195 lines
7.7 KiB
TypeScript
195 lines
7.7 KiB
TypeScript
import { GrayImage } from "../../graphics/image";
|
|
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
|
import { lineStep } from "../../ui/metrics";
|
|
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
|
import { writeTextToDownloads } from "../../native/file-access";
|
|
import { voiceControlBridge } from "../../native/voice-control";
|
|
import { Layer, type LayerContext } from "../../ui/layers";
|
|
import { type InputEvent } from "../../ui/gestures";
|
|
import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
|
import { TranscriptScroll, transcriptLayout, wrapTranscribeText } from "../transcribe/transcript-model";
|
|
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
|
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
|
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
|
|
|
const NO_MODEL_NOTICE = "No language model is set up.";
|
|
|
|
/**
|
|
* Live translation view. Continuous mic capture is owned by the app wrapper
|
|
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
|
* model one block at a time (provisionally while a block is still being
|
|
* spoken), shows the English as it streams in, and saves the session to
|
|
* Downloads from the app menu.
|
|
*/
|
|
export class TranslateLayer implements Layer {
|
|
private status = "Starting microphone...";
|
|
private listening = false;
|
|
private detail = "";
|
|
private saveNotice = "";
|
|
private translationError = "";
|
|
// Why the app can't run (no multilingual provider or no model); shown in
|
|
// place of the transcript, and a tap asks the app wrapper to check again.
|
|
private problem: string | null = null;
|
|
private readonly model = new TranslationModel();
|
|
private readonly scroll = new TranscriptScroll();
|
|
private readonly meter = new MicLevelMeter();
|
|
// The request in flight (the model tracks which job it is).
|
|
private activeHandle: LlmStreamHandle | null = null;
|
|
private requestRender: () => void = () => {};
|
|
private removed = false;
|
|
private unsubscribeTranscript: (() => void) | null = null;
|
|
private unsubscribeStatus: (() => void) | null = null;
|
|
|
|
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
|
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
|
|
|
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
|
start(requestRender: () => void, isVisible: () => boolean): void {
|
|
this.requestRender = requestRender;
|
|
this.meter.start(requestRender, isVisible);
|
|
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
|
if (this.model.accept(event)) {
|
|
// The request in flight translates text the transcript has replaced.
|
|
this.activeHandle?.cancel();
|
|
this.activeHandle = null;
|
|
}
|
|
this.pump();
|
|
requestRender();
|
|
});
|
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
|
this.status = state.status;
|
|
this.listening = state.listening;
|
|
this.detail = state.detail;
|
|
this.saveNotice = "";
|
|
requestRender();
|
|
});
|
|
}
|
|
|
|
setProblem(problem: string | null): void {
|
|
this.problem = problem;
|
|
this.requestRender();
|
|
}
|
|
|
|
paint(ctx: LayerContext): GrayImage {
|
|
const font = getDefaultSmallFont();
|
|
const viewport = ctx.stack.getBaseSize();
|
|
const image = new GrayImage(viewport.width, viewport.height, 0);
|
|
const measure = (line: string) => font.measureText(line);
|
|
|
|
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
|
const meter = this.listening && !this.problem ? this.meter : null;
|
|
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter, viewport.padding);
|
|
const step = lineStep(font) + 2;
|
|
const layout = transcriptLayout(viewport, bodyTop, font.lineHeight, step);
|
|
const maxWidth = layout.textWidth;
|
|
|
|
let lines: TranslationLine[];
|
|
if (this.problem) {
|
|
lines = wrapTranscribeText(measure, `${this.problem}\n\nTap to check again.`, maxWidth).map((text) => ({ text, gray: 230 }));
|
|
} else {
|
|
lines = this.model.lines(measure, maxWidth, translateShowOriginalSetting.get());
|
|
if (!lines.length) {
|
|
const placeholder = this.listening ? "Listening..." : this.detail;
|
|
lines = wrapTranscribeText(measure, placeholder, maxWidth).map((text) => ({ text, gray: 230 }));
|
|
}
|
|
}
|
|
|
|
const firstLine = this.scroll.layout(lines.length, layout.pageLines);
|
|
for (let index = firstLine; index < Math.min(lines.length, firstLine + layout.drawnLines); index++) {
|
|
const y = bodyTop + (index - firstLine) * step;
|
|
image.drawText(font, layout.textX, y, lines[index]!.text, lines[index]!.gray);
|
|
}
|
|
|
|
if (lines.length > layout.pageLines) {
|
|
const track = layout.scrollbar;
|
|
const thumbHeight = Math.max(8, Math.round(track.height * layout.pageLines / lines.length));
|
|
const thumbY = track.y + Math.round((track.height - thumbHeight) * firstLine / (lines.length - layout.pageLines));
|
|
image.fillRect(track.x, track.y, track.width, track.height, 45);
|
|
image.fillRect(track.x, thumbY, track.width, thumbHeight, 140);
|
|
}
|
|
return image;
|
|
}
|
|
|
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
|
if (event.type === "scroll-up" || event.type === "scroll-down") {
|
|
this.scroll.scroll(event.type === "scroll-up" ? -3 : 3);
|
|
ctx.actions.requestRender();
|
|
return;
|
|
}
|
|
if (event.type === "click" && this.problem) {
|
|
this.retry();
|
|
return;
|
|
}
|
|
if (event.type === "double-click") {
|
|
ctx.stack.pop();
|
|
}
|
|
}
|
|
|
|
onRemoved(): void {
|
|
this.removed = true;
|
|
this.meter.stop();
|
|
this.activeHandle?.cancel();
|
|
this.activeHandle = null;
|
|
this.unsubscribeTranscript?.();
|
|
this.unsubscribeTranscript = null;
|
|
this.unsubscribeStatus?.();
|
|
this.unsubscribeStatus = null;
|
|
}
|
|
|
|
saveTranscript(): void {
|
|
const text = this.model.savedText().trim();
|
|
if (!text) {
|
|
this.saveNotice = "Nothing to save yet.";
|
|
return;
|
|
}
|
|
const filename = `translation-${this.stamp}.txt`;
|
|
const path = writeTextToDownloads(filename, `${text}\n`);
|
|
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
|
}
|
|
|
|
/**
|
|
* Start the next translation request (a final block, else the live block's
|
|
* provisional one), unless one is in flight.
|
|
*/
|
|
private pump(): void {
|
|
if (this.removed) return;
|
|
const job = this.model.nextJob();
|
|
if (!job) return;
|
|
const llm = resolveTranslationModel();
|
|
if (!llm) {
|
|
// The model went away mid-session (a key was cleared): keep the original.
|
|
this.model.fail(job);
|
|
this.translationError = NO_MODEL_NOTICE;
|
|
this.pump();
|
|
return;
|
|
}
|
|
const language = resolveTranslateLanguage();
|
|
const handle = streamTranslation(llm, {
|
|
speech: job.source,
|
|
context: job.context,
|
|
language: language ? spokenLanguageName(language) : "",
|
|
onProgress: (textSoFar) => {
|
|
if (this.model.progress(job, textSoFar)) this.requestRender();
|
|
},
|
|
onDone: (translation) => this.settle(job, () => {
|
|
this.model.complete(job, translation);
|
|
this.translationError = "";
|
|
}),
|
|
onError: (message) => this.settle(job, () => {
|
|
this.model.fail(job);
|
|
this.translationError = `Translation failed: ${message}`;
|
|
}),
|
|
});
|
|
// A provider that fails synchronously has already settled (and moved on).
|
|
if (this.model.isActive(job)) this.activeHandle = handle;
|
|
}
|
|
|
|
private settle(job: TranslationJob, apply: () => void): void {
|
|
if (!this.model.isActive(job)) return;
|
|
this.activeHandle = null;
|
|
apply();
|
|
this.pump();
|
|
this.requestRender();
|
|
}
|
|
}
|