mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Most of the builtin apps had some padding between the window edge and their content, because they expect to be displayed with a line border on their edge. We added a customization option to remove the border, so, that padding is not always necessary. Make the layouts dynamically adjust, to omit the padding when it isn't needed.
314 lines
11 KiB
TypeScript
314 lines
11 KiB
TypeScript
import { GrayImage, type UiFont } from "../../graphics/image";
|
|
import { truncateText } from "../../graphics/textwrap";
|
|
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
|
|
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
|
import { type InputEvent } from "../../ui/gestures";
|
|
import { Layer, type LayerContext } from "../../ui/layers";
|
|
import { type MenuItem } from "../../ui/menu";
|
|
import { lineStep } from "../../ui/metrics";
|
|
import { clamp } from "../../util/numeric-util";
|
|
import { ScriptTracker, tokenizeScript, type TokenizedScript } from "./script-tracker";
|
|
|
|
/** Gap between the last body line and the footer. */
|
|
const FOOTER_GAP = 4;
|
|
/** Lines per ring scroll tick. */
|
|
const SCROLL_STEP_LINES = 2;
|
|
|
|
/** Brightness of words already spoken, still to come, and the next one. */
|
|
const PAST_VALUE = 90;
|
|
const UPCOMING_VALUE = 200;
|
|
const NEXT_VALUE = 255;
|
|
|
|
type LayoutSpan = { tokenIndex: number; x: number; width: number; text: string };
|
|
type LayoutLine = { spans: LayoutSpan[] };
|
|
|
|
export type ReaderHooks = {
|
|
requestRender: () => void;
|
|
/** Hold continuous mic capture while tracking is on. */
|
|
startCapture: () => void;
|
|
stopCapture: () => void;
|
|
/** The reader closed (or paused) at this word position. */
|
|
savePosition: (position: number) => void;
|
|
};
|
|
|
|
/**
|
|
* The teleprompter's reading view. Lays the script out word by word in the
|
|
* medium UI font, follows the speaker through it from the live transcript
|
|
* (see ScriptTracker), auto-scrolls to keep the next word about a third of
|
|
* the way down, and highlights that word. Ring scrolls move the view by hand
|
|
* and re-anchor the tracker to what is now at the anchor row, so scrolling
|
|
* doubles as "I'm here now". Click pauses/resumes tracking; double-click
|
|
* goes back to the file list.
|
|
*
|
|
* The script's column and the footer line keep inside the padding suggested
|
|
* for the window's edges. The lines scroll, so they run from the window's
|
|
* top edge itself: only the script's first line keeps that edge's padding
|
|
* above it.
|
|
*/
|
|
export class TeleprompterReaderLayer implements Layer {
|
|
private readonly script: TokenizedScript;
|
|
private readonly tracker: ScriptTracker;
|
|
private lines: LayoutLine[] | null = null;
|
|
/** Line index of each word (parallel to script.words). */
|
|
private wordLines: number[] = [];
|
|
private layoutKey = "";
|
|
private firstLine = 0;
|
|
/** True after the tracker moved: the next paint re-centers on the position. */
|
|
private autoScrollPending = true;
|
|
private tracking = true;
|
|
private status = "";
|
|
private unsubscribeTranscript: (() => void) | null = null;
|
|
private unsubscribeStatus: (() => void) | null = null;
|
|
// Geometry from the last paint, for the input handlers.
|
|
private visibleRows = 10;
|
|
private anchorRow = 3;
|
|
|
|
constructor(
|
|
text: string,
|
|
private readonly title: string,
|
|
private readonly hooks: ReaderHooks,
|
|
initialPosition = 0,
|
|
) {
|
|
this.script = tokenizeScript(text);
|
|
this.tracker = new ScriptTracker(this.script.words);
|
|
this.tracker.anchor(initialPosition);
|
|
}
|
|
|
|
/** Start listening. Called once the layer is on its stack. */
|
|
attach(): void {
|
|
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
|
|
this.status = state.status;
|
|
this.hooks.requestRender();
|
|
});
|
|
this.startTracking();
|
|
}
|
|
|
|
onRemoved(): void {
|
|
this.hooks.savePosition(this.resumePosition());
|
|
this.stopTracking();
|
|
this.unsubscribeStatus?.();
|
|
this.unsubscribeStatus = null;
|
|
}
|
|
|
|
get position(): number {
|
|
return this.tracker.position;
|
|
}
|
|
|
|
buildMenuItems(): MenuItem[] {
|
|
return [
|
|
{
|
|
label: this.tracking ? "Pause voice tracking" : "Resume voice tracking",
|
|
onSelect: (ctx) => {
|
|
this.toggleTracking();
|
|
ctx.stack.pop();
|
|
},
|
|
},
|
|
{
|
|
label: "Restart from the top",
|
|
onSelect: (ctx) => {
|
|
this.tracker.anchor(0);
|
|
this.firstLine = 0;
|
|
this.autoScrollPending = true;
|
|
ctx.stack.pop();
|
|
},
|
|
},
|
|
];
|
|
}
|
|
|
|
paint(ctx: LayerContext): GrayImage {
|
|
const font = getDefaultMediumFont();
|
|
const smallFont = getDefaultSmallFont();
|
|
const step = lineStep(font);
|
|
const { width, height, padding } = ctx.stack.getBaseSize();
|
|
const image = new GrayImage(width, height, 0);
|
|
const bodyX = padding.left;
|
|
const bodyWidth = Math.max(0, width - padding.left - padding.right);
|
|
|
|
const footerY = height - padding.bottom - smallFont.lineHeight;
|
|
const bodyBottom = footerY - FOOTER_GAP;
|
|
// Rows that fit wherever the script is scrolled to, which is what
|
|
// scrolling and the anchor row go by.
|
|
this.visibleRows = Math.max(1, Math.floor((bodyBottom - padding.top) / step));
|
|
this.anchorRow = Math.floor(this.visibleRows / 3);
|
|
const lines = this.layout(font, bodyWidth);
|
|
|
|
if (this.autoScrollPending) {
|
|
this.autoScrollPending = false;
|
|
this.firstLine = this.clampFirstLine(this.lineOfPosition() - this.anchorRow);
|
|
} else {
|
|
this.firstLine = this.clampFirstLine(this.firstLine);
|
|
}
|
|
|
|
const position = this.tracker.position;
|
|
const nextToken = position < this.script.words.length ? this.script.wordTokens[position]! : this.script.tokens.length;
|
|
// Scrolled past the script's start, the lines begin at the window's top
|
|
// edge and the top padding's room may hold one more of them.
|
|
const bodyTop = this.firstLine === 0 ? padding.top : 0;
|
|
const shownRows = Math.max(1, Math.floor((bodyBottom - bodyTop) / step));
|
|
const lastLine = Math.min(lines.length, this.firstLine + shownRows);
|
|
for (let lineIndex = this.firstLine; lineIndex < lastLine; lineIndex++) {
|
|
const y = bodyTop + (lineIndex - this.firstLine) * step;
|
|
for (const span of lines[lineIndex]!.spans) {
|
|
const isNext = span.tokenIndex === nextToken;
|
|
const value = isNext ? NEXT_VALUE : span.tokenIndex < nextToken ? PAST_VALUE : UPCOMING_VALUE;
|
|
image.drawText(font, bodyX + span.x, y, span.text, value);
|
|
if (isNext) {
|
|
image.fillRect(bodyX + span.x, y + font.lineHeight - 2, span.width, 2, NEXT_VALUE);
|
|
}
|
|
}
|
|
}
|
|
|
|
this.paintFooter(image, smallFont, bodyX, bodyWidth, footerY);
|
|
return image;
|
|
}
|
|
|
|
handleInput(event: InputEvent, ctx: LayerContext): void {
|
|
switch (event.type) {
|
|
case "scroll-down":
|
|
this.scrollBy(SCROLL_STEP_LINES);
|
|
return;
|
|
case "scroll-up":
|
|
this.scrollBy(-SCROLL_STEP_LINES);
|
|
return;
|
|
case "click":
|
|
this.toggleTracking();
|
|
return;
|
|
case "double-click":
|
|
ctx.stack.pop();
|
|
return;
|
|
default:
|
|
return;
|
|
}
|
|
}
|
|
|
|
/** The footer line across [x, x + width): the script's name at its left, status and progress at its right. */
|
|
private paintFooter(image: GrayImage, font: UiFont, x: number, width: number, y: number): void {
|
|
const total = this.script.words.length;
|
|
const percent = total === 0 ? 0 : Math.round((100 * this.tracker.position) / total);
|
|
const right = `${this.tracking ? this.status || "Listening" : "Paused"} ${percent}%`;
|
|
const rightWidth = font.measureText(right);
|
|
image.drawText(font, x + width - rightWidth, y, right, 110);
|
|
const titleMax = width - rightWidth - 12;
|
|
if (titleMax > 20) {
|
|
image.drawText(font, x, y, truncateText(font, this.title, titleMax), 110);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Hand-scroll by whole lines. The tracker is re-anchored to the first word
|
|
* at (or after) the anchor row, so the highlight follows the view and
|
|
* speech matching resumes from there.
|
|
*/
|
|
private scrollBy(deltaLines: number): void {
|
|
const lines = this.lines;
|
|
if (!lines) return;
|
|
this.autoScrollPending = false;
|
|
this.firstLine = this.clampFirstLine(this.firstLine + deltaLines);
|
|
const anchorLine = this.firstLine + this.anchorRow;
|
|
const word = this.firstWordAtOrAfterLine(anchorLine);
|
|
this.tracker.anchor(word);
|
|
}
|
|
|
|
private clampFirstLine(value: number): number {
|
|
const lineCount = this.lines?.length ?? 0;
|
|
// Scroll far enough that the last line can reach the anchor row.
|
|
return clamp(value, 0, Math.max(0, lineCount - this.anchorRow - 1));
|
|
}
|
|
|
|
private lineOfPosition(): number {
|
|
const position = this.tracker.position;
|
|
if (this.wordLines.length === 0) return 0;
|
|
if (position >= this.wordLines.length) return this.wordLines[this.wordLines.length - 1]!;
|
|
return this.wordLines[position]!;
|
|
}
|
|
|
|
private firstWordAtOrAfterLine(line: number): number {
|
|
for (let word = 0; word < this.wordLines.length; word++) {
|
|
if (this.wordLines[word]! >= line) return word;
|
|
}
|
|
return this.wordLines.length;
|
|
}
|
|
|
|
/** Where to resume next time: a script read to the end restarts from the top. */
|
|
private resumePosition(): number {
|
|
return this.tracker.position >= this.script.words.length ? 0 : this.tracker.position;
|
|
}
|
|
|
|
private toggleTracking(): void {
|
|
if (this.tracking) {
|
|
this.stopTracking();
|
|
this.hooks.savePosition(this.resumePosition());
|
|
} else {
|
|
this.startTracking();
|
|
}
|
|
}
|
|
|
|
private startTracking(): void {
|
|
this.tracking = true;
|
|
if (!this.unsubscribeTranscript) {
|
|
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
|
}
|
|
this.hooks.startCapture();
|
|
}
|
|
|
|
private stopTracking(): void {
|
|
this.tracking = false;
|
|
this.unsubscribeTranscript?.();
|
|
this.unsubscribeTranscript = null;
|
|
this.hooks.stopCapture();
|
|
}
|
|
|
|
private onTranscript(event: VoiceTranscriptEvent): void {
|
|
if (!this.tracking) return;
|
|
if (this.tracker.feed(event.text)) {
|
|
this.autoScrollPending = true;
|
|
this.hooks.requestRender();
|
|
}
|
|
}
|
|
|
|
/** Word-by-word greedy line layout; cached per font and width. */
|
|
private layout(font: UiFont, maxWidth: number): LayoutLine[] {
|
|
const key = `${font.fingerprintId}:${maxWidth}`;
|
|
if (this.lines && this.layoutKey === key) return this.lines;
|
|
const spaceWidth = font.measureText(" ");
|
|
const lines: LayoutLine[] = [];
|
|
const wordLines: number[] = [];
|
|
let current: LayoutLine = { spans: [] };
|
|
let currentWidth = 0;
|
|
let paragraph = -1;
|
|
const flush = () => {
|
|
lines.push(current);
|
|
current = { spans: [] };
|
|
currentWidth = 0;
|
|
};
|
|
for (let tokenIndex = 0; tokenIndex < this.script.tokens.length; tokenIndex++) {
|
|
const token = this.script.tokens[tokenIndex]!;
|
|
if (token.paragraph !== paragraph) {
|
|
// New paragraph: end the current line, and leave one empty line per
|
|
// blank source line (paragraph numbers count every newline).
|
|
if (paragraph >= 0) {
|
|
flush();
|
|
for (let blank = paragraph + 1; blank < token.paragraph; blank++) flush();
|
|
}
|
|
paragraph = token.paragraph;
|
|
}
|
|
const wordWidth = font.measureText(token.text);
|
|
const x = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
|
if (current.spans.length > 0 && x + wordWidth > maxWidth) {
|
|
flush();
|
|
}
|
|
const spanX = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
|
|
current.spans.push({ tokenIndex, x: spanX, width: wordWidth, text: token.text });
|
|
currentWidth = spanX + wordWidth;
|
|
if (token.wordIndex >= 0) {
|
|
wordLines[token.wordIndex] = lines.length;
|
|
}
|
|
}
|
|
if (current.spans.length > 0 || lines.length === 0) flush();
|
|
this.lines = lines;
|
|
this.wordLines = wordLines;
|
|
this.layoutKey = key;
|
|
return lines;
|
|
}
|
|
}
|