Add teleprompter app

This commit is contained in:
James Babcock committed 2026-09-04 13:26:21 -07:00
1 parent b69fdc220a
commit 758cf7de3b
10 files changed
+986 -1

No files matched your search

+1
View File
@@ -14,6 +14,7 @@
* Navigate app: Darken the map background color
* EvenHub app: Add New, Search, and Updates tabs
* Timer app: Fully rewritten, with new better UI
* Add a teleprompter app
## 0.6.4 - 2026-09-02
* Context menus are centered and include window title
+2
View File
@@ -7,6 +7,7 @@ import filesApp from "./files";
import musicApp from "./music";
import nightscoutApp from "./nightscout";
import transcribeApp from "./transcribe";
import teleprompterApp from "./teleprompter";
import microphonesApp from "./microphones";
import notificationsApp from "./notifications";
import calendarApp from "./calendar";
@@ -36,6 +37,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
musicApp,
nightscoutApp,
transcribeApp,
teleprompterApp,
microphonesApp,
notificationsApp,
calendarApp,
+19
View File
@@ -0,0 +1,19 @@
import { type AppDefinition } from "../app-definition";
import { createTeleprompterAppWindow, TELEPROMPTER_SURFACE_ID, TELEPROMPTER_WINDOW_ID } from "./teleprompter-app";
const teleprompterApp: AppDefinition = {
appId: "teleprompter",
title: "Teleprompter",
icon: "scroll-text",
launch: (ctx) =>
ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) =>
createTeleprompterAppWindow({
...options,
startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(),
stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(),
appendLog: (message) => ctx.appendLog(message),
}),
),
};
export default teleprompterApp;
+300
View File
@@ -0,0 +1,300 @@
import { GrayImage, type UiFont } from "../../graphics/image";
import { truncateText } from "../../graphics/textwrap";
import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts";
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
import { type InputEvent } from "../../ui/gestures";
import { Layer, type LayerContext } from "../../ui/layers";
import { type MenuItem } from "../../ui/menu";
import { lineStep } from "../../ui/metrics";
import { clamp } from "../../util/numeric-util";
import { ScriptTracker, tokenizeScript, type TokenizedScript } from "./script-tracker";
const MARGIN_X = 18;
const BODY_TOP = 6;
/** Gap between the last body line and the footer. */
const FOOTER_GAP = 4;
/** Lines per ring scroll tick. */
const SCROLL_STEP_LINES = 2;
/** Brightness of words already spoken, still to come, and the next one. */
const PAST_VALUE = 90;
const UPCOMING_VALUE = 200;
const NEXT_VALUE = 255;
type LayoutSpan = { tokenIndex: number; x: number; width: number; text: string };
type LayoutLine = { spans: LayoutSpan[] };
export type ReaderHooks = {
requestRender: () => void;
/** Hold continuous mic capture while tracking is on. */
startCapture: () => void;
stopCapture: () => void;
/** The reader closed (or paused) at this word position. */
savePosition: (position: number) => void;
};
/**
* The teleprompter's reading view. Lays the script out word by word in the
* medium UI font, follows the speaker through it from the live transcript
* (see ScriptTracker), auto-scrolls to keep the next word about a third of
* the way down, and highlights that word. Ring scrolls move the view by hand
* and re-anchor the tracker to what is now at the anchor row, so scrolling
* doubles as "I'm here now". Click pauses/resumes tracking; double-click
* goes back to the file list.
*/
export class TeleprompterReaderLayer implements Layer {
private readonly script: TokenizedScript;
private readonly tracker: ScriptTracker;
private lines: LayoutLine[] | null = null;
/** Line index of each word (parallel to script.words). */
private wordLines: number[] = [];
private layoutKey = "";
private firstLine = 0;
/** True after the tracker moved: the next paint re-centers on the position. */
private autoScrollPending = true;
private tracking = true;
private status = "";
private unsubscribeTranscript: (() => void) | null = null;
private unsubscribeStatus: (() => void) | null = null;
// Geometry from the last paint, for the input handlers.
private visibleRows = 10;
private anchorRow = 3;
constructor(
text: string,
private readonly title: string,
private readonly hooks: ReaderHooks,
initialPosition = 0,
) {
this.script = tokenizeScript(text);
this.tracker = new ScriptTracker(this.script.words);
this.tracker.anchor(initialPosition);
}
/** Start listening. Called once the layer is on its stack. */
attach(): void {
this.unsubscribeStatus = voiceControlBridge.onStatus((state) => {
this.status = state.status;
this.hooks.requestRender();
});
this.startTracking();
}
onRemoved(): void {
this.hooks.savePosition(this.resumePosition());
this.stopTracking();
this.unsubscribeStatus?.();
this.unsubscribeStatus = null;
}
get position(): number {
return this.tracker.position;
}
buildMenuItems(): MenuItem[] {
return [
{
label: this.tracking ? "Pause voice tracking" : "Resume voice tracking",
onSelect: (ctx) => {
this.toggleTracking();
ctx.stack.pop();
},
},
{
label: "Restart from the top",
onSelect: (ctx) => {
this.tracker.anchor(0);
this.firstLine = 0;
this.autoScrollPending = true;
ctx.stack.pop();
},
},
];
}
paint(ctx: LayerContext): GrayImage {
const font = getDefaultMediumFont();
const smallFont = getDefaultSmallFont();
const step = lineStep(font);
const { width, height } = ctx.stack.getBaseSize();
const image = new GrayImage(width, height, 0);
const footerY = height - smallFont.lineHeight - 4;
this.visibleRows = Math.max(1, Math.floor((footerY - FOOTER_GAP - BODY_TOP) / step));
this.anchorRow = Math.floor(this.visibleRows / 3);
const lines = this.layout(font, width - 2 * MARGIN_X);
if (this.autoScrollPending) {
this.autoScrollPending = false;
this.firstLine = this.clampFirstLine(this.lineOfPosition() - this.anchorRow);
} else {
this.firstLine = this.clampFirstLine(this.firstLine);
}
const position = this.tracker.position;
const nextToken = position < this.script.words.length ? this.script.wordTokens[position]! : this.script.tokens.length;
const lastLine = Math.min(lines.length, this.firstLine + this.visibleRows);
for (let lineIndex = this.firstLine; lineIndex < lastLine; lineIndex++) {
const y = BODY_TOP + (lineIndex - this.firstLine) * step;
for (const span of lines[lineIndex]!.spans) {
const isNext = span.tokenIndex === nextToken;
const value = isNext ? NEXT_VALUE : span.tokenIndex < nextToken ? PAST_VALUE : UPCOMING_VALUE;
image.drawText(font, MARGIN_X + span.x, y, span.text, value);
if (isNext) {
image.fillRect(MARGIN_X + span.x, y + font.lineHeight - 2, span.width, 2, NEXT_VALUE);
}
}
}
this.paintFooter(image, smallFont, width, footerY);
return image;
}
handleInput(event: InputEvent, ctx: LayerContext): void {
switch (event.type) {
case "scroll-down":
this.scrollBy(SCROLL_STEP_LINES);
return;
case "scroll-up":
this.scrollBy(-SCROLL_STEP_LINES);
return;
case "click":
this.toggleTracking();
return;
case "double-click":
ctx.stack.pop();
return;
default:
return;
}
}
private paintFooter(image: GrayImage, font: UiFont, width: number, y: number): void {
const total = this.script.words.length;
const percent = total === 0 ? 0 : Math.round((100 * this.tracker.position) / total);
const right = `${this.tracking ? this.status || "Listening" : "Paused"} ${percent}%`;
const rightWidth = font.measureText(right);
image.drawText(font, width - MARGIN_X - rightWidth, y, right, 110);
const titleMax = width - 2 * MARGIN_X - rightWidth - 12;
if (titleMax > 20) {
image.drawText(font, MARGIN_X, y, truncateText(font, this.title, titleMax), 110);
}
}
/**
* Hand-scroll by whole lines. The tracker is re-anchored to the first word
* at (or after) the anchor row, so the highlight follows the view and
* speech matching resumes from there.
*/
private scrollBy(deltaLines: number): void {
const lines = this.lines;
if (!lines) return;
this.autoScrollPending = false;
this.firstLine = this.clampFirstLine(this.firstLine + deltaLines);
const anchorLine = this.firstLine + this.anchorRow;
const word = this.firstWordAtOrAfterLine(anchorLine);
this.tracker.anchor(word);
}
private clampFirstLine(value: number): number {
const lineCount = this.lines?.length ?? 0;
// Scroll far enough that the last line can reach the anchor row.
return clamp(value, 0, Math.max(0, lineCount - this.anchorRow - 1));
}
private lineOfPosition(): number {
const position = this.tracker.position;
if (this.wordLines.length === 0) return 0;
if (position >= this.wordLines.length) return this.wordLines[this.wordLines.length - 1]!;
return this.wordLines[position]!;
}
private firstWordAtOrAfterLine(line: number): number {
for (let word = 0; word < this.wordLines.length; word++) {
if (this.wordLines[word]! >= line) return word;
}
return this.wordLines.length;
}
/** Where to resume next time: a script read to the end restarts from the top. */
private resumePosition(): number {
return this.tracker.position >= this.script.words.length ? 0 : this.tracker.position;
}
private toggleTracking(): void {
if (this.tracking) {
this.stopTracking();
this.hooks.savePosition(this.resumePosition());
} else {
this.startTracking();
}
}
private startTracking(): void {
this.tracking = true;
if (!this.unsubscribeTranscript) {
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
}
this.hooks.startCapture();
}
private stopTracking(): void {
this.tracking = false;
this.unsubscribeTranscript?.();
this.unsubscribeTranscript = null;
this.hooks.stopCapture();
}
private onTranscript(event: VoiceTranscriptEvent): void {
if (!this.tracking) return;
if (this.tracker.feed(event.text)) {
this.autoScrollPending = true;
this.hooks.requestRender();
}
}
/** Word-by-word greedy line layout; cached per font and width. */
private layout(font: UiFont, maxWidth: number): LayoutLine[] {
const key = `${font.fingerprintId}:${maxWidth}`;
if (this.lines && this.layoutKey === key) return this.lines;
const spaceWidth = font.measureText(" ");
const lines: LayoutLine[] = [];
const wordLines: number[] = [];
let current: LayoutLine = { spans: [] };
let currentWidth = 0;
let paragraph = -1;
const flush = () => {
lines.push(current);
current = { spans: [] };
currentWidth = 0;
};
for (let tokenIndex = 0; tokenIndex < this.script.tokens.length; tokenIndex++) {
const token = this.script.tokens[tokenIndex]!;
if (token.paragraph !== paragraph) {
// New paragraph: end the current line, and leave one empty line per
// blank source line (paragraph numbers count every newline).
if (paragraph >= 0) {
flush();
for (let blank = paragraph + 1; blank < token.paragraph; blank++) flush();
}
paragraph = token.paragraph;
}
const wordWidth = font.measureText(token.text);
const x = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
if (current.spans.length > 0 && x + wordWidth > maxWidth) {
flush();
}
const spanX = current.spans.length === 0 ? 0 : currentWidth + spaceWidth;
current.spans.push({ tokenIndex, x: spanX, width: wordWidth, text: token.text });
currentWidth = spanX + wordWidth;
if (token.wordIndex >= 0) {
wordLines[token.wordIndex] = lines.length;
}
}
if (current.spans.length > 0 || lines.length === 0) flush();
this.lines = lines;
this.wordLines = wordLines;
this.layoutKey = key;
return lines;
}
}
+80
View File
@@ -0,0 +1,80 @@
import { getStringSetting, setStringSetting } from "../../native/settings-store";
/**
* The Teleprompter's recently opened scripts, most recent first, with the
* word position the reader was at when it closed (so reopening resumes
* there). Stored as a JSON array in the settings store.
*/
export type RecentScript = {
path: string;
name: string;
/** Word index the reader was at when last closed; 0 = start. */
position: number;
openedMs: number;
};
const RECENTS_KEY = "teleprompter.recents";
const MAX_RECENTS = 12;
export function getRecentScripts(): RecentScript[] {
try {
const parsed: unknown = JSON.parse(getStringSetting(RECENTS_KEY, "[]"));
if (!Array.isArray(parsed)) return [];
return parsed
.filter((entry): entry is RecentScript => !!entry && typeof entry === "object" && typeof entry.path === "string")
.map((entry) => ({
path: entry.path,
name: typeof entry.name === "string" && entry.name ? entry.name : baseName(entry.path),
position: typeof entry.position === "number" && entry.position >= 0 ? entry.position : 0,
openedMs: typeof entry.openedMs === "number" ? entry.openedMs : 0,
}));
} catch {
return [];
}
}
function saveRecentScripts(recents: RecentScript[]): void {
setStringSetting(RECENTS_KEY, JSON.stringify(recents.slice(0, MAX_RECENTS)));
}
/** Move (or add) a script to the top of the list, keeping its saved position. */
export function touchRecentScript(path: string, name: string): RecentScript {
const recents = getRecentScripts();
const existing = recents.find((entry) => entry.path === path);
const entry: RecentScript = {
path,
name,
position: existing?.position ?? 0,
openedMs: Date.now(),
};
saveRecentScripts([entry, ...recents.filter((other) => other.path !== path)]);
return entry;
}
export function saveRecentScriptPosition(path: string, position: number): void {
const recents = getRecentScripts();
const entry = recents.find((other) => other.path === path);
if (!entry) return;
entry.position = Math.max(0, position);
saveRecentScripts(recents);
}
export function removeRecentScript(path: string): void {
saveRecentScripts(getRecentScripts().filter((entry) => entry.path !== path));
}
export function clearRecentScripts(): void {
saveRecentScripts([]);
}
export function baseName(path: string): string {
const slash = path.lastIndexOf("/");
return slash >= 0 ? path.slice(slash + 1) : path;
}
/** The last directory segment of a file path, for a dim "where it lives" hint. */
export function parentDirName(path: string): string {
const slash = path.lastIndexOf("/");
if (slash <= 0) return "";
return baseName(path.slice(0, slash));
}
+260
View File
@@ -0,0 +1,260 @@
/**
* Follows a speaker through a script from live speech-to-text. Pure logic
* (no NativeScript imports) so it runs under plain node for the tests.
*
* The script is tokenized once into display tokens; the subset with a
* non-empty normalized key are the "words" the tracker aligns against. The
* tracker keeps `position`: the index (in word space) of the next word the
* speaker is expected to say. Each transcript update (replace semantics: the
* whole current utterance, partial or final) re-aligns the tail of the
* transcript against a window of script words around the current position,
* with a local alignment that tolerates misrecognized, skipped and ad-libbed
* words, and moves the position to just after the last aligned script word.
*/
export type ScriptToken = {
/** The token as written, for display. */
text: string;
/** Paragraph number (newline-separated), for layout. */
paragraph: number;
/** Lowercased alphanumeric key, or "" for punctuation-only tokens. */
key: string;
/** Index into the tracker's word list, or -1 when key is empty. */
wordIndex: number;
};
export type TokenizedScript = {
tokens: ScriptToken[];
/** Normalized keys of the trackable tokens, in order. */
words: string[];
/** Token index of each word. */
wordTokens: number[];
};
/** Lowercase, strip everything but letters/digits, fold common apostrophes. */
export function normalizeWord(raw: string): string {
return raw
.normalize("NFKD")
.replace(/[\u0300-\u036f]/g, "")
.toLowerCase()
.replace(/[^a-z0-9]+/g, "");
}
export function tokenizeScript(text: string): TokenizedScript {
const tokens: ScriptToken[] = [];
const words: string[] = [];
const wordTokens: number[] = [];
const paragraphs = text.replace(/\r/g, "").split("\n");
for (let paragraph = 0; paragraph < paragraphs.length; paragraph++) {
for (const raw of paragraphs[paragraph]!.split(/\s+/)) {
if (!raw) continue;
const key = normalizeWord(raw);
const wordIndex = key ? words.length : -1;
if (key) {
words.push(key);
wordTokens.push(tokens.length);
}
tokens.push({ text: raw, paragraph, key, wordIndex });
}
}
return { tokens, words, wordTokens };
}
/** Split a transcript into normalized keys, dropping empties. */
export function transcriptKeys(text: string): string[] {
return text
.split(/\s+/)
.map(normalizeWord)
.filter((key) => key.length > 0);
}
// Function words match so often that on their own they say little about
// where the speaker is; they count for less than content words.
const STOP_WORDS = new Set([
"a", "an", "the", "and", "or", "but", "of", "to", "in", "on", "at", "for", "is", "it", "its",
"i", "we", "you", "he", "she", "they", "that", "this", "as", "be", "are", "was", "were",
"with", "by", "so", "if", "not", "no", "do", "have", "has", "had", "will", "can",
]);
const MATCH_SCORE = 3;
const STOP_WORD_MATCH_SCORE = 1.5;
/** A fuzzy (edit-distance 1) or prefix match: real evidence, but weaker. */
const FUZZY_MATCH_SCORE = 2;
const MISMATCH_SCORE = -2;
const GAP_SCORE = -1.5;
/** The alignment must be worth at least this to move the position at all. */
const MIN_ALIGNMENT_SCORE = 4.5;
const MIN_MATCHED_WORDS = 2;
/** Per-word cost of ending an alignment away from the expected position (a tie-breaker). */
const FORWARD_DISTANCE_COST = 0.02;
const BACKWARD_DISTANCE_COST = 0.1;
export type TrackerOptions = {
/** Script words before the current position that stay in play (repeats, backtracking). */
back?: number;
/** Script words after the current position that stay in play (skipping ahead). */
ahead?: number;
/** Only the last this-many transcript words are aligned. */
tailWords?: number;
};
const DEFAULT_OPTIONS: Required<TrackerOptions> = { back: 30, ahead: 160, tailWords: 20 };
/**
* Edit distance capped at 1 (true when the strings are within one edit),
* which covers the typical STT slip: a dropped/extra letter or a substitution.
*/
export function withinOneEdit(a: string, b: string): boolean {
if (a === b) return true;
const lengthDelta = a.length - b.length;
if (lengthDelta > 1 || lengthDelta < -1) return false;
const [short, long] = a.length <= b.length ? [a, b] : [b, a];
let i = 0;
let j = 0;
let edits = 0;
while (i < short.length && j < long.length) {
if (short[i] === long[j]) {
i++;
j++;
continue;
}
if (++edits > 1) return false;
if (short.length === long.length) {
i++;
}
j++;
}
return edits + (long.length - j) <= 1;
}
/** Score of aligning transcript key `spoken` with script key `expected`. */
function matchScore(spoken: string, expected: string, isLastSpoken: boolean): number {
if (spoken === expected) {
return STOP_WORDS.has(expected) ? STOP_WORD_MATCH_SCORE : MATCH_SCORE;
}
// The final transcript word is often still being recognized: a prefix of
// the script word counts, once it is long enough not to be noise.
if (isLastSpoken && spoken.length >= 2 && expected.length > spoken.length && expected.startsWith(spoken)) {
return FUZZY_MATCH_SCORE;
}
if (spoken.length >= 5 && expected.length >= 5 && withinOneEdit(spoken, expected)) {
return FUZZY_MATCH_SCORE;
}
return MISMATCH_SCORE;
}
export type AlignmentResult = {
/** Index (in the script word list) of the word after the last aligned one; the new position. */
position: number;
score: number;
matchedWords: number;
};
/**
* Align the transcript tail against script words [windowStart, windowEnd).
* Local alignment (free start on both sides) that must consume the last
* transcript word and may end anywhere in the window. Returns null when the
* best alignment is too weak to trust.
*/
export function alignTail(
words: readonly string[],
spoken: readonly string[],
windowStart: number,
windowEnd: number,
expectedPosition: number,
): AlignmentResult | null {
const m = spoken.length;
const n = windowEnd - windowStart;
if (m === 0 || n <= 0) return null;
// H[i][j]: best score of an alignment ending with spoken[i-1] / window[j-1]
// (or a gap), clamped at 0 (free start). Matches counts how many spoken
// words matched along the path that produced H[i][j].
const width = n + 1;
const score = new Float64Array((m + 1) * width);
const matches = new Uint16Array((m + 1) * width);
for (let i = 1; i <= m; i++) {
const spokenWord = spoken[i - 1]!;
const isLast = i === m;
for (let j = 1; j <= n; j++) {
const here = i * width + j;
const s = matchScore(spokenWord, words[windowStart + j - 1]!, isLast);
const diagonal = score[here - width - 1]! + s;
const skipScript = score[here - 1]! + GAP_SCORE;
const skipSpoken = score[here - width]! + GAP_SCORE;
let best = 0;
let bestMatches = 0;
if (diagonal > best) {
best = diagonal;
bestMatches = matches[here - width - 1]! + (s > 0 ? 1 : 0);
}
if (skipScript > best) {
best = skipScript;
bestMatches = matches[here - 1]!;
}
if (skipSpoken > best) {
best = skipSpoken;
bestMatches = matches[here - width]!;
}
score[here] = best;
matches[here] = bestMatches;
}
}
let bestColumn = -1;
let bestAdjusted = -Infinity;
for (let j = 1; j <= n; j++) {
const raw = score[m * width + j]!;
if (raw <= 0) continue;
const position = windowStart + j;
const distance = position - expectedPosition;
const adjusted = raw - (distance >= 0 ? distance * FORWARD_DISTANCE_COST : -distance * BACKWARD_DISTANCE_COST);
if (adjusted > bestAdjusted) {
bestAdjusted = adjusted;
bestColumn = j;
}
}
if (bestColumn < 0) return null;
const rawScore = score[m * width + bestColumn]!;
const matchedWords = matches[m * width + bestColumn]!;
if (rawScore < MIN_ALIGNMENT_SCORE || matchedWords < MIN_MATCHED_WORDS) return null;
return { position: windowStart + bestColumn, score: rawScore, matchedWords };
}
export class ScriptTracker {
private readonly options: Required<TrackerOptions>;
/** Index of the next word the speaker is expected to say (0..words.length). */
position = 0;
constructor(
private readonly words: readonly string[],
options: TrackerOptions = {},
) {
this.options = { ...DEFAULT_OPTIONS, ...options };
}
get wordCount(): number {
return this.words.length;
}
/** Jump the expected position (manual scroll, restart). */
anchor(position: number): void {
this.position = Math.max(0, Math.min(this.words.length, position));
}
/**
* Apply a transcript update (the whole current utterance, partial or
* final). Returns true when the position moved.
*/
feed(text: string): boolean {
const keys = transcriptKeys(text);
if (keys.length === 0) return false;
const tail = keys.length > this.options.tailWords ? keys.slice(-this.options.tailWords) : keys;
const windowStart = Math.max(0, this.position - this.options.back);
const windowEnd = Math.min(this.words.length, this.position + this.options.ahead);
const result = alignTail(this.words, tail, windowStart, windowEnd, this.position);
if (!result || result.position === this.position) return false;
this.position = result.position;
return true;
}
}
+188
View File
@@ -0,0 +1,188 @@
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
import { truncateText } from "../../graphics/textwrap";
import { readTextFile, type DirectoryEntry } from "../../native/file-access";
import { LIST_ROW_TEXT_INSET } from "../../ui/metrics";
import { MenuLayer, type MenuItem } from "../../ui/menu";
import { type Layer, type LayerContext } from "../../ui/layers";
import { appViewportSize } from "../../ui/shell/geometry";
import {
createInProcessWindow,
YieldAtRootLayer,
type InProcessAppOptions,
type InProcessWindow,
} from "../../ui/shell/in-process-window";
import { FileBrowserLayer } from "../files/file-browser";
import { TeleprompterReaderLayer } from "./reader";
import {
baseName,
clearRecentScripts,
getRecentScripts,
parentDirName,
removeRecentScript,
saveRecentScriptPosition,
touchRecentScript,
} from "./recent-files";
export const TELEPROMPTER_WINDOW_ID = "teleprompter";
export const TELEPROMPTER_SURFACE_ID = "window:teleprompter";
const SCRIPT_FILE = /\.(txt|md|markdown|text|log)$/i;
const HOME_LAYOUT = {
x: 8,
y: 8,
width: 272,
showBorder: false,
minHeight: 0,
maxHeight: appViewportSize("min").height - 16,
opaque: true,
};
export type TeleprompterAppOptions = InProcessAppOptions & {
/** Hold continuous mic capture while a script is being tracked. */
startContinuousVoiceCapture: () => void;
stopContinuousVoiceCapture: () => void;
appendLog: (message: string) => void;
};
/**
* The Teleprompter app window. The home page is a Browse entry plus the
* recently opened scripts; Browse pushes the Files app's browser to pick a
* text file, and either path pushes the reader (see TeleprompterReaderLayer)
* over the home page.
*/
export function createTeleprompterAppWindow(options: TeleprompterAppOptions): InProcessWindow {
// The home menu's items are rebuilt in place whenever the recents change
// (MenuLayer reads the array each paint).
const homeItems: MenuItem[] = [];
const home = new MenuLayer("Teleprompter", homeItems, HOME_LAYOUT);
let created: InProcessWindow | null = null;
let notice = "";
const requestRender = () => created?.requestRender();
/** The layer currently on top of the window's stack (the home menu at base). */
const topLayer = (): Layer => {
let top: Layer = home;
created?.stack.topMatches((layer) => {
top = layer;
return true;
});
return top;
};
const openScript = (ctx: LayerContext, path: string, name: string) => {
const text = readTextFile(path);
if (text === null) {
removeRecentScript(path);
notice = `Could not read ${name}`;
options.appendLog(`teleprompter: could not read ${path}`);
refreshHome();
return;
}
const recent = touchRecentScript(path, name);
notice = "";
refreshHome();
const layer = new TeleprompterReaderLayer(
text,
name,
{
requestRender,
startCapture: () => options.startContinuousVoiceCapture(),
stopCapture: () => options.stopContinuousVoiceCapture(),
savePosition: (position) => saveRecentScriptPosition(path, position),
},
recent.position,
);
ctx.stack.push(layer);
layer.attach();
};
const openBrowser = (ctx: LayerContext) => {
const stack = ctx.stack;
const picker = new FileBrowserLayer({
isSupportedFile: (name) => SCRIPT_FILE.test(name),
onLeave: () => stack.pop(),
onFilePicked: (entry: DirectoryEntry, pickCtx) => {
if (!SCRIPT_FILE.test(entry.name)) {
notice = "Pick a text file (.txt, .md)";
return;
}
pickCtx.stack.pop();
openScript(pickCtx, entry.path, entry.name);
},
});
stack.push(picker);
};
function refreshHome(): void {
const font = getDefaultSmallFont();
const items: MenuItem[] = [
{
label: "Browse for a script...",
onSelect: (ctx) => openBrowser(ctx),
},
];
for (const recent of getRecentScripts()) {
const folder = parentDirName(recent.path);
items.push({
label: recent.name || baseName(recent.path),
onSelect: (ctx) => openScript(ctx, recent.path, recent.name),
render: ({ image, x, y, width, selected, text }) => {
const value = selected ? 255 : 200;
const folderWidth = folder ? font.measureText(folder) : 0;
const nameMax = folder ? width - folderWidth - 12 : width;
image.drawText(font, x, y + LIST_ROW_TEXT_INSET, truncateText(font, text, Math.max(40, nameMax)), value);
if (folder && nameMax >= 40) {
image.drawText(font, x + width - folderWidth, y + LIST_ROW_TEXT_INSET, folder, 90);
}
},
});
}
if (notice) {
items.push({ label: notice, disabled: true, onSelect: () => {} });
}
homeItems.length = 0;
homeItems.push(...items);
// Land on the most recent script (one click resumes it), else on Browse.
home.selectItem(getRecentScripts().length > 0 ? 1 : 0);
}
refreshHome();
created = createInProcessWindow({
appId: "teleprompter",
windowId: TELEPROMPTER_WINDOW_ID,
title: "Teleprompter",
iconLetter: "Tp",
icon: "scroll-text",
closeable: true,
actions: options.actions,
menuItems: () => {
const top = topLayer();
if (top instanceof TeleprompterReaderLayer) return top.buildMenuItems();
if (top instanceof FileBrowserLayer) return top.buildMenuItems();
if (top !== home) return [];
return [
{
label: "Clear recent scripts",
onSelect: (ctx) => {
clearRecentScripts();
refreshHome();
ctx.stack.pop();
},
},
];
},
onFocus: (lastInput) => {
const top = topLayer();
if (top instanceof FileBrowserLayer) top.onFocus(lastInput);
},
baseLayer: new YieldAtRootLayer(home),
submitFrame: options.submitFrame,
setSurfaceVisible: options.setSurfaceVisible,
removeSurface: options.removeSurface,
onClosed: options.onClosed,
});
return created;
}
+2
View File
@@ -36,6 +36,8 @@ export const ICON_SVGS = {
'<svg viewBox="0 0 24 24" fill="none"><path d="M12 18v4"/><path d="M2 14.499a5.5 5.5 0 0 0 9.591 3.675.6.6 0 0 1 .818.001A5.5 5.5 0 0 0 22 14.5c0-2.29-1.5-4-3-5.5l-5.492-5.312a2 2 0 0 0-3-.02L5 8.999c-1.5 1.5-3 3.2-3 5.5"/></svg>',
terminal:
'<svg viewBox="0 0 24 24" fill="none"><path d="m7 11 2-2-2-2"/><path d="M11 13h4"/><rect width="18" height="18" x="3" y="3" rx="2" ry="2"/></svg>',
"scroll-text":
'<svg viewBox="0 0 24 24" fill="none"><path d="M15 12h-5"/><path d="M15 8h-5"/><path d="M19 17V5a2 2 0 0 0-2-2H4"/><path d="M8 21h12a2 2 0 0 0 2-2v-1a1 1 0 0 0-1-1H11a1 1 0 0 0-1 1v1a2 2 0 1 1-4 0V5a2 2 0 1 0-4 0v2a1 1 0 0 0 1 1h3"/></svg>',
"file-text":
'<svg viewBox="0 0 24 24" fill="none"><path d="M6 22a2 2 0 0 1-2-2V4a2 2 0 0 1 2-2h8a2.4 2.4 0 0 1 1.704.706l3.588 3.588A2.4 2.4 0 0 1 20 8v12a2 2 0 0 1-2 2z"/><path d="M14 2v5a1 1 0 0 0 1 1h5"/><path d="M10 9H8"/><path d="M16 13H8"/><path d="M16 17H8"/></svg>',
file:
+132
View File
@@ -0,0 +1,132 @@
// Pins the Teleprompter's speech-to-script tracker: tokenization, the fuzzy
// word matcher, and the position tracking under realistic partial-transcript
// streams (replace semantics), including misrecognitions, ad-libs, skipped
// sentences and repeated phrases.
const test = require("node:test");
const assert = require("node:assert/strict");
const tracker = require("../.test-build/app/apps/teleprompter/script-tracker.js");
const SCRIPT = `Good evening, everyone. Thank you for coming tonight.
Four score and seven years ago our fathers brought forth on this continent
a new nation, conceived in liberty, and dedicated to the proposition that
all men are created equal.
Now we are engaged in a great civil war, testing whether that nation, or any
nation so conceived and so dedicated, can long endure. We are met on a great
battle-field of that war.
We have come to dedicate a portion of that field, as a final resting place
for those who here gave their lives that that nation might live.`;
/** Word index of the first occurrence of a normalized key, from `from`. */
function wordAt(script, key, from = 0) {
const index = script.words.indexOf(key, from);
assert.notEqual(index, -1, `script has ${key}`);
return index;
}
/**
* Feed an utterance the way a streaming recognizer delivers it: a growing
* partial after every word, then the final. Returns the tracker position.
*/
function speak(t, utterance) {
const words = utterance.split(/\s+/);
for (let i = 1; i <= words.length; i++) {
t.feed(words.slice(0, i).join(" "));
}
t.feed(utterance);
return t.position;
}
test("tokenizeScript separates display tokens from trackable words", () => {
const script = tracker.tokenizeScript("Hello, world — it's 2026.\nNext line");
assert.deepEqual(
script.tokens.map((token) => token.text),
["Hello,", "world", "—", "it's", "2026.", "Next", "line"],
);
assert.deepEqual(script.words, ["hello", "world", "its", "2026", "next", "line"]);
assert.equal(script.tokens[2].wordIndex, -1);
assert.equal(script.tokens[5].paragraph, 1);
assert.deepEqual(script.wordTokens, [0, 1, 3, 4, 5, 6]);
});
test("withinOneEdit accepts single slips and rejects more", () => {
assert.ok(tracker.withinOneEdit("conceived", "conceived"));
assert.ok(tracker.withinOneEdit("conceived", "concieved") === false, "transposition is two edits");
assert.ok(tracker.withinOneEdit("liberty", "libery"));
assert.ok(tracker.withinOneEdit("liberty", "libertyy"));
assert.ok(tracker.withinOneEdit("liberty", "libarty"));
assert.ok(!tracker.withinOneEdit("liberty", "library"));
assert.ok(tracker.withinOneEdit("nation", "notion"), "one substitution");
});
test("follows a clean read-through, word by word", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
assert.equal(t.position, 0);
// A lone first word is not enough evidence to move.
t.feed("Good");
assert.equal(t.position, 0);
t.feed("Good evening");
assert.equal(t.position, 2);
speak(t, "Good evening everyone thank you for coming tonight");
assert.equal(t.position, wordAt(script, "four"));
speak(t, "Four score and seven years ago our fathers brought forth on this continent");
assert.equal(t.position, wordAt(script, "a", wordAt(script, "continent")));
});
test("tolerates misrecognized and ad-libbed words", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
t.anchor(wordAt(script, "four"));
// "score" heard as "scored", "fathers" as "father's", plus an inserted "um".
speak(t, "for scored and seven years ago um our father's brought forth");
assert.equal(t.position, wordAt(script, "on"));
});
test("a partial last word counts as a prefix of the next script word", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
t.anchor(wordAt(script, "four"));
t.feed("four score and seven years ago our fath");
assert.equal(t.position, wordAt(script, "brought"));
});
test("catches up when the speaker skips a sentence", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
t.anchor(wordAt(script, "four"));
speak(t, "Now we are engaged in a great civil war");
assert.equal(t.position, wordAt(script, "testing"));
});
test("prefers the occurrence nearest the current position for repeated phrases", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
// "that nation" appears in paragraphs 3 and 4; anchored in paragraph 4 it
// should pick the later one.
const laterThat = wordAt(script, "gave");
t.anchor(laterThat);
speak(t, "their lives that that nation might live");
assert.equal(t.position, script.words.length);
});
test("does not move on transcripts that match nothing nearby", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words);
t.anchor(wordAt(script, "four"));
assert.equal(speak(t, "hang on let me find my glasses"), wordAt(script, "four"));
});
test("manual anchoring re-bases the search window", () => {
const script = tracker.tokenizeScript(SCRIPT);
const t = new tracker.ScriptTracker(script.words, { ahead: 10, back: 5 });
// Far outside the window from position 0.
speak(t, "final resting place for those who here gave");
assert.equal(t.position, 0);
t.anchor(wordAt(script, "final"));
speak(t, "final resting place for those who here gave");
assert.equal(t.position, wordAt(script, "their"));
});
+2 -1
View File
@@ -25,6 +25,7 @@
"../app/apps/microphones/radar-dots.ts",
"../app/apps/microphones/name-guess.ts",
"../app/apps/evenhub/containers.ts",
"../app/apps/timer/timer-model.ts"
"../app/apps/timer/timer-model.ts",
"../app/apps/teleprompter/script-tracker.ts"
]
}