From 758cf7de3b29a30bb9676b938fcbb2730bad6330 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Fri, 4 Sep 2026 12:58:31 -0700 Subject: [PATCH] Add teleprompter app --- CHANGELOG | 1 + app/apps/all-apps.ts | 2 + app/apps/teleprompter/index.ts | 19 ++ app/apps/teleprompter/reader.ts | 300 ++++++++++++++++++++++ app/apps/teleprompter/recent-files.ts | 80 ++++++ app/apps/teleprompter/script-tracker.ts | 260 +++++++++++++++++++ app/apps/teleprompter/teleprompter-app.ts | 188 ++++++++++++++ app/graphics/icons.ts | 2 + tests/teleprompter-tracker.test.cjs | 132 ++++++++++ tests/tsconfig.json | 3 +- 10 files changed, 986 insertions(+), 1 deletion(-) create mode 100644 app/apps/teleprompter/index.ts create mode 100644 app/apps/teleprompter/reader.ts create mode 100644 app/apps/teleprompter/recent-files.ts create mode 100644 app/apps/teleprompter/script-tracker.ts create mode 100644 app/apps/teleprompter/teleprompter-app.ts create mode 100644 tests/teleprompter-tracker.test.cjs diff --git a/CHANGELOG b/CHANGELOG index 8f7fa134..125fded8 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -14,6 +14,7 @@ * Navigate app: Darken the map background color * EvenHub app: Add New, Search, and Updates tabs * Timer app: Fully rewritten, with new better UI +* Add a teleprompter app ## 0.6.4 - 2026-09-02 * Context menus are centered and include window title diff --git a/app/apps/all-apps.ts b/app/apps/all-apps.ts index 96982703..6dbb80c1 100644 --- a/app/apps/all-apps.ts +++ b/app/apps/all-apps.ts @@ -7,6 +7,7 @@ import filesApp from "./files"; import musicApp from "./music"; import nightscoutApp from "./nightscout"; import transcribeApp from "./transcribe"; +import teleprompterApp from "./teleprompter"; import microphonesApp from "./microphones"; import notificationsApp from "./notifications"; import calendarApp from "./calendar"; @@ -36,6 +37,7 @@ export const ALL_APPS: readonly AppDefinition[] = [ musicApp, nightscoutApp, transcribeApp, + teleprompterApp, microphonesApp, notificationsApp, calendarApp, diff --git a/app/apps/teleprompter/index.ts b/app/apps/teleprompter/index.ts new file mode 100644 index 00000000..f2abbcf1 --- /dev/null +++ b/app/apps/teleprompter/index.ts @@ -0,0 +1,19 @@ +import { type AppDefinition } from "../app-definition"; +import { createTeleprompterAppWindow, TELEPROMPTER_SURFACE_ID, TELEPROMPTER_WINDOW_ID } from "./teleprompter-app"; + +const teleprompterApp: AppDefinition = { + appId: "teleprompter", + title: "Teleprompter", + icon: "scroll-text", + launch: (ctx) => + ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) => + createTeleprompterAppWindow({ + ...options, + startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), + appendLog: (message) => ctx.appendLog(message), + }), + ), +}; + +export default teleprompterApp; diff --git a/app/apps/teleprompter/reader.ts b/app/apps/teleprompter/reader.ts new file mode 100644 index 00000000..a3c39205 --- /dev/null +++ b/app/apps/teleprompter/reader.ts @@ -0,0 +1,300 @@ +import { GrayImage, type UiFont } from "../../graphics/image"; +import { truncateText } from "../../graphics/textwrap"; +import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { type InputEvent } from "../../ui/gestures"; +import { Layer, type LayerContext } from "../../ui/layers"; +import { type MenuItem } from "../../ui/menu"; +import { lineStep } from "../../ui/metrics"; +import { clamp } from "../../util/numeric-util"; +import { ScriptTracker, tokenizeScript, type TokenizedScript } from "./script-tracker"; + +const MARGIN_X = 18; +const BODY_TOP = 6; +/** Gap between the last body line and the footer. */ +const FOOTER_GAP = 4; +/** Lines per ring scroll tick. */ +const SCROLL_STEP_LINES = 2; + +/** Brightness of words already spoken, still to come, and the next one. */ +const PAST_VALUE = 90; +const UPCOMING_VALUE = 200; +const NEXT_VALUE = 255; + +type LayoutSpan = { tokenIndex: number; x: number; width: number; text: string }; +type LayoutLine = { spans: LayoutSpan[] }; + +export type ReaderHooks = { + requestRender: () => void; + /** Hold continuous mic capture while tracking is on. */ + startCapture: () => void; + stopCapture: () => void; + /** The reader closed (or paused) at this word position. */ + savePosition: (position: number) => void; +}; + +/** + * The teleprompter's reading view. Lays the script out word by word in the + * medium UI font, follows the speaker through it from the live transcript + * (see ScriptTracker), auto-scrolls to keep the next word about a third of + * the way down, and highlights that word. Ring scrolls move the view by hand + * and re-anchor the tracker to what is now at the anchor row, so scrolling + * doubles as "I'm here now". Click pauses/resumes tracking; double-click + * goes back to the file list. + */ +export class TeleprompterReaderLayer implements Layer { + private readonly script: TokenizedScript; + private readonly tracker: ScriptTracker; + private lines: LayoutLine[] | null = null; + /** Line index of each word (parallel to script.words). */ + private wordLines: number[] = []; + private layoutKey = ""; + private firstLine = 0; + /** True after the tracker moved: the next paint re-centers on the position. */ + private autoScrollPending = true; + private tracking = true; + private status = ""; + private unsubscribeTranscript: (() => void) | null = null; + private unsubscribeStatus: (() => void) | null = null; + // Geometry from the last paint, for the input handlers. + private visibleRows = 10; + private anchorRow = 3; + + constructor( + text: string, + private readonly title: string, + private readonly hooks: ReaderHooks, + initialPosition = 0, + ) { + this.script = tokenizeScript(text); + this.tracker = new ScriptTracker(this.script.words); + this.tracker.anchor(initialPosition); + } + + /** Start listening. Called once the layer is on its stack. */ + attach(): void { + this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + this.status = state.status; + this.hooks.requestRender(); + }); + this.startTracking(); + } + + onRemoved(): void { + this.hooks.savePosition(this.resumePosition()); + this.stopTracking(); + this.unsubscribeStatus?.(); + this.unsubscribeStatus = null; + } + + get position(): number { + return this.tracker.position; + } + + buildMenuItems(): MenuItem[] { + return [ + { + label: this.tracking ? "Pause voice tracking" : "Resume voice tracking", + onSelect: (ctx) => { + this.toggleTracking(); + ctx.stack.pop(); + }, + }, + { + label: "Restart from the top", + onSelect: (ctx) => { + this.tracker.anchor(0); + this.firstLine = 0; + this.autoScrollPending = true; + ctx.stack.pop(); + }, + }, + ]; + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultMediumFont(); + const smallFont = getDefaultSmallFont(); + const step = lineStep(font); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + + const footerY = height - smallFont.lineHeight - 4; + this.visibleRows = Math.max(1, Math.floor((footerY - FOOTER_GAP - BODY_TOP) / step)); + this.anchorRow = Math.floor(this.visibleRows / 3); + const lines = this.layout(font, width - 2 * MARGIN_X); + + if (this.autoScrollPending) { + this.autoScrollPending = false; + this.firstLine = this.clampFirstLine(this.lineOfPosition() - this.anchorRow); + } else { + this.firstLine = this.clampFirstLine(this.firstLine); + } + + const position = this.tracker.position; + const nextToken = position < this.script.words.length ? this.script.wordTokens[position]! : this.script.tokens.length; + const lastLine = Math.min(lines.length, this.firstLine + this.visibleRows); + for (let lineIndex = this.firstLine; lineIndex < lastLine; lineIndex++) { + const y = BODY_TOP + (lineIndex - this.firstLine) * step; + for (const span of lines[lineIndex]!.spans) { + const isNext = span.tokenIndex === nextToken; + const value = isNext ? NEXT_VALUE : span.tokenIndex < nextToken ? PAST_VALUE : UPCOMING_VALUE; + image.drawText(font, MARGIN_X + span.x, y, span.text, value); + if (isNext) { + image.fillRect(MARGIN_X + span.x, y + font.lineHeight - 2, span.width, 2, NEXT_VALUE); + } + } + } + + this.paintFooter(image, smallFont, width, footerY); + return image; + } + + handleInput(event: InputEvent, ctx: LayerContext): void { + switch (event.type) { + case "scroll-down": + this.scrollBy(SCROLL_STEP_LINES); + return; + case "scroll-up": + this.scrollBy(-SCROLL_STEP_LINES); + return; + case "click": + this.toggleTracking(); + return; + case "double-click": + ctx.stack.pop(); + return; + default: + return; + } + } + + private paintFooter(image: GrayImage, font: UiFont, width: number, y: number): void { + const total = this.script.words.length; + const percent = total === 0 ? 0 : Math.round((100 * this.tracker.position) / total); + const right = `${this.tracking ? this.status || "Listening" : "Paused"} ${percent}%`; + const rightWidth = font.measureText(right); + image.drawText(font, width - MARGIN_X - rightWidth, y, right, 110); + const titleMax = width - 2 * MARGIN_X - rightWidth - 12; + if (titleMax > 20) { + image.drawText(font, MARGIN_X, y, truncateText(font, this.title, titleMax), 110); + } + } + + /** + * Hand-scroll by whole lines. The tracker is re-anchored to the first word + * at (or after) the anchor row, so the highlight follows the view and + * speech matching resumes from there. + */ + private scrollBy(deltaLines: number): void { + const lines = this.lines; + if (!lines) return; + this.autoScrollPending = false; + this.firstLine = this.clampFirstLine(this.firstLine + deltaLines); + const anchorLine = this.firstLine + this.anchorRow; + const word = this.firstWordAtOrAfterLine(anchorLine); + this.tracker.anchor(word); + } + + private clampFirstLine(value: number): number { + const lineCount = this.lines?.length ?? 0; + // Scroll far enough that the last line can reach the anchor row. + return clamp(value, 0, Math.max(0, lineCount - this.anchorRow - 1)); + } + + private lineOfPosition(): number { + const position = this.tracker.position; + if (this.wordLines.length === 0) return 0; + if (position >= this.wordLines.length) return this.wordLines[this.wordLines.length - 1]!; + return this.wordLines[position]!; + } + + private firstWordAtOrAfterLine(line: number): number { + for (let word = 0; word < this.wordLines.length; word++) { + if (this.wordLines[word]! >= line) return word; + } + return this.wordLines.length; + } + + /** Where to resume next time: a script read to the end restarts from the top. */ + private resumePosition(): number { + return this.tracker.position >= this.script.words.length ? 0 : this.tracker.position; + } + + private toggleTracking(): void { + if (this.tracking) { + this.stopTracking(); + this.hooks.savePosition(this.resumePosition()); + } else { + this.startTracking(); + } + } + + private startTracking(): void { + this.tracking = true; + if (!this.unsubscribeTranscript) { + this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); + } + this.hooks.startCapture(); + } + + private stopTracking(): void { + this.tracking = false; + this.unsubscribeTranscript?.(); + this.unsubscribeTranscript = null; + this.hooks.stopCapture(); + } + + private onTranscript(event: VoiceTranscriptEvent): void { + if (!this.tracking) return; + if (this.tracker.feed(event.text)) { + this.autoScrollPending = true; + this.hooks.requestRender(); + } + } + + /** Word-by-word greedy line layout; cached per font and width. */ + private layout(font: UiFont, maxWidth: number): LayoutLine[] { + const key = `${font.fingerprintId}:${maxWidth}`; + if (this.lines && this.layoutKey === key) return this.lines; + const spaceWidth = font.measureText(" "); + const lines: LayoutLine[] = []; + const wordLines: number[] = []; + let current: LayoutLine = { spans: [] }; + let currentWidth = 0; + let paragraph = -1; + const flush = () => { + lines.push(current); + current = { spans: [] }; + currentWidth = 0; + }; + for (let tokenIndex = 0; tokenIndex < this.script.tokens.length; tokenIndex++) { + const token = this.script.tokens[tokenIndex]!; + if (token.paragraph !== paragraph) { + // New paragraph: end the current line, and leave one empty line per + // blank source line (paragraph numbers count every newline). + if (paragraph >= 0) { + flush(); + for (let blank = paragraph + 1; blank < token.paragraph; blank++) flush(); + } + paragraph = token.paragraph; + } + const wordWidth = font.measureText(token.text); + const x = current.spans.length === 0 ? 0 : currentWidth + spaceWidth; + if (current.spans.length > 0 && x + wordWidth > maxWidth) { + flush(); + } + const spanX = current.spans.length === 0 ? 0 : currentWidth + spaceWidth; + current.spans.push({ tokenIndex, x: spanX, width: wordWidth, text: token.text }); + currentWidth = spanX + wordWidth; + if (token.wordIndex >= 0) { + wordLines[token.wordIndex] = lines.length; + } + } + if (current.spans.length > 0 || lines.length === 0) flush(); + this.lines = lines; + this.wordLines = wordLines; + this.layoutKey = key; + return lines; + } +} diff --git a/app/apps/teleprompter/recent-files.ts b/app/apps/teleprompter/recent-files.ts new file mode 100644 index 00000000..61b1d809 --- /dev/null +++ b/app/apps/teleprompter/recent-files.ts @@ -0,0 +1,80 @@ +import { getStringSetting, setStringSetting } from "../../native/settings-store"; + +/** + * The Teleprompter's recently opened scripts, most recent first, with the + * word position the reader was at when it closed (so reopening resumes + * there). Stored as a JSON array in the settings store. + */ +export type RecentScript = { + path: string; + name: string; + /** Word index the reader was at when last closed; 0 = start. */ + position: number; + openedMs: number; +}; + +const RECENTS_KEY = "teleprompter.recents"; +const MAX_RECENTS = 12; + +export function getRecentScripts(): RecentScript[] { + try { + const parsed: unknown = JSON.parse(getStringSetting(RECENTS_KEY, "[]")); + if (!Array.isArray(parsed)) return []; + return parsed + .filter((entry): entry is RecentScript => !!entry && typeof entry === "object" && typeof entry.path === "string") + .map((entry) => ({ + path: entry.path, + name: typeof entry.name === "string" && entry.name ? entry.name : baseName(entry.path), + position: typeof entry.position === "number" && entry.position >= 0 ? entry.position : 0, + openedMs: typeof entry.openedMs === "number" ? entry.openedMs : 0, + })); + } catch { + return []; + } +} + +function saveRecentScripts(recents: RecentScript[]): void { + setStringSetting(RECENTS_KEY, JSON.stringify(recents.slice(0, MAX_RECENTS))); +} + +/** Move (or add) a script to the top of the list, keeping its saved position. */ +export function touchRecentScript(path: string, name: string): RecentScript { + const recents = getRecentScripts(); + const existing = recents.find((entry) => entry.path === path); + const entry: RecentScript = { + path, + name, + position: existing?.position ?? 0, + openedMs: Date.now(), + }; + saveRecentScripts([entry, ...recents.filter((other) => other.path !== path)]); + return entry; +} + +export function saveRecentScriptPosition(path: string, position: number): void { + const recents = getRecentScripts(); + const entry = recents.find((other) => other.path === path); + if (!entry) return; + entry.position = Math.max(0, position); + saveRecentScripts(recents); +} + +export function removeRecentScript(path: string): void { + saveRecentScripts(getRecentScripts().filter((entry) => entry.path !== path)); +} + +export function clearRecentScripts(): void { + saveRecentScripts([]); +} + +export function baseName(path: string): string { + const slash = path.lastIndexOf("/"); + return slash >= 0 ? path.slice(slash + 1) : path; +} + +/** The last directory segment of a file path, for a dim "where it lives" hint. */ +export function parentDirName(path: string): string { + const slash = path.lastIndexOf("/"); + if (slash <= 0) return ""; + return baseName(path.slice(0, slash)); +} diff --git a/app/apps/teleprompter/script-tracker.ts b/app/apps/teleprompter/script-tracker.ts new file mode 100644 index 00000000..8cdb4e41 --- /dev/null +++ b/app/apps/teleprompter/script-tracker.ts @@ -0,0 +1,260 @@ +/** + * Follows a speaker through a script from live speech-to-text. Pure logic + * (no NativeScript imports) so it runs under plain node for the tests. + * + * The script is tokenized once into display tokens; the subset with a + * non-empty normalized key are the "words" the tracker aligns against. The + * tracker keeps `position`: the index (in word space) of the next word the + * speaker is expected to say. Each transcript update (replace semantics: the + * whole current utterance, partial or final) re-aligns the tail of the + * transcript against a window of script words around the current position, + * with a local alignment that tolerates misrecognized, skipped and ad-libbed + * words, and moves the position to just after the last aligned script word. + */ + +export type ScriptToken = { + /** The token as written, for display. */ + text: string; + /** Paragraph number (newline-separated), for layout. */ + paragraph: number; + /** Lowercased alphanumeric key, or "" for punctuation-only tokens. */ + key: string; + /** Index into the tracker's word list, or -1 when key is empty. */ + wordIndex: number; +}; + +export type TokenizedScript = { + tokens: ScriptToken[]; + /** Normalized keys of the trackable tokens, in order. */ + words: string[]; + /** Token index of each word. */ + wordTokens: number[]; +}; + +/** Lowercase, strip everything but letters/digits, fold common apostrophes. */ +export function normalizeWord(raw: string): string { + return raw + .normalize("NFKD") + .replace(/[\u0300-\u036f]/g, "") + .toLowerCase() + .replace(/[^a-z0-9]+/g, ""); +} + +export function tokenizeScript(text: string): TokenizedScript { + const tokens: ScriptToken[] = []; + const words: string[] = []; + const wordTokens: number[] = []; + const paragraphs = text.replace(/\r/g, "").split("\n"); + for (let paragraph = 0; paragraph < paragraphs.length; paragraph++) { + for (const raw of paragraphs[paragraph]!.split(/\s+/)) { + if (!raw) continue; + const key = normalizeWord(raw); + const wordIndex = key ? words.length : -1; + if (key) { + words.push(key); + wordTokens.push(tokens.length); + } + tokens.push({ text: raw, paragraph, key, wordIndex }); + } + } + return { tokens, words, wordTokens }; +} + +/** Split a transcript into normalized keys, dropping empties. */ +export function transcriptKeys(text: string): string[] { + return text + .split(/\s+/) + .map(normalizeWord) + .filter((key) => key.length > 0); +} + +// Function words match so often that on their own they say little about +// where the speaker is; they count for less than content words. +const STOP_WORDS = new Set([ + "a", "an", "the", "and", "or", "but", "of", "to", "in", "on", "at", "for", "is", "it", "its", + "i", "we", "you", "he", "she", "they", "that", "this", "as", "be", "are", "was", "were", + "with", "by", "so", "if", "not", "no", "do", "have", "has", "had", "will", "can", +]); + +const MATCH_SCORE = 3; +const STOP_WORD_MATCH_SCORE = 1.5; +/** A fuzzy (edit-distance 1) or prefix match: real evidence, but weaker. */ +const FUZZY_MATCH_SCORE = 2; +const MISMATCH_SCORE = -2; +const GAP_SCORE = -1.5; +/** The alignment must be worth at least this to move the position at all. */ +const MIN_ALIGNMENT_SCORE = 4.5; +const MIN_MATCHED_WORDS = 2; +/** Per-word cost of ending an alignment away from the expected position (a tie-breaker). */ +const FORWARD_DISTANCE_COST = 0.02; +const BACKWARD_DISTANCE_COST = 0.1; + +export type TrackerOptions = { + /** Script words before the current position that stay in play (repeats, backtracking). */ + back?: number; + /** Script words after the current position that stay in play (skipping ahead). */ + ahead?: number; + /** Only the last this-many transcript words are aligned. */ + tailWords?: number; +}; + +const DEFAULT_OPTIONS: Required = { back: 30, ahead: 160, tailWords: 20 }; + +/** + * Edit distance capped at 1 (true when the strings are within one edit), + * which covers the typical STT slip: a dropped/extra letter or a substitution. + */ +export function withinOneEdit(a: string, b: string): boolean { + if (a === b) return true; + const lengthDelta = a.length - b.length; + if (lengthDelta > 1 || lengthDelta < -1) return false; + const [short, long] = a.length <= b.length ? [a, b] : [b, a]; + let i = 0; + let j = 0; + let edits = 0; + while (i < short.length && j < long.length) { + if (short[i] === long[j]) { + i++; + j++; + continue; + } + if (++edits > 1) return false; + if (short.length === long.length) { + i++; + } + j++; + } + return edits + (long.length - j) <= 1; +} + +/** Score of aligning transcript key `spoken` with script key `expected`. */ +function matchScore(spoken: string, expected: string, isLastSpoken: boolean): number { + if (spoken === expected) { + return STOP_WORDS.has(expected) ? STOP_WORD_MATCH_SCORE : MATCH_SCORE; + } + // The final transcript word is often still being recognized: a prefix of + // the script word counts, once it is long enough not to be noise. + if (isLastSpoken && spoken.length >= 2 && expected.length > spoken.length && expected.startsWith(spoken)) { + return FUZZY_MATCH_SCORE; + } + if (spoken.length >= 5 && expected.length >= 5 && withinOneEdit(spoken, expected)) { + return FUZZY_MATCH_SCORE; + } + return MISMATCH_SCORE; +} + +export type AlignmentResult = { + /** Index (in the script word list) of the word after the last aligned one; the new position. */ + position: number; + score: number; + matchedWords: number; +}; + +/** + * Align the transcript tail against script words [windowStart, windowEnd). + * Local alignment (free start on both sides) that must consume the last + * transcript word and may end anywhere in the window. Returns null when the + * best alignment is too weak to trust. + */ +export function alignTail( + words: readonly string[], + spoken: readonly string[], + windowStart: number, + windowEnd: number, + expectedPosition: number, +): AlignmentResult | null { + const m = spoken.length; + const n = windowEnd - windowStart; + if (m === 0 || n <= 0) return null; + + // H[i][j]: best score of an alignment ending with spoken[i-1] / window[j-1] + // (or a gap), clamped at 0 (free start). Matches counts how many spoken + // words matched along the path that produced H[i][j]. + const width = n + 1; + const score = new Float64Array((m + 1) * width); + const matches = new Uint16Array((m + 1) * width); + for (let i = 1; i <= m; i++) { + const spokenWord = spoken[i - 1]!; + const isLast = i === m; + for (let j = 1; j <= n; j++) { + const here = i * width + j; + const s = matchScore(spokenWord, words[windowStart + j - 1]!, isLast); + const diagonal = score[here - width - 1]! + s; + const skipScript = score[here - 1]! + GAP_SCORE; + const skipSpoken = score[here - width]! + GAP_SCORE; + let best = 0; + let bestMatches = 0; + if (diagonal > best) { + best = diagonal; + bestMatches = matches[here - width - 1]! + (s > 0 ? 1 : 0); + } + if (skipScript > best) { + best = skipScript; + bestMatches = matches[here - 1]!; + } + if (skipSpoken > best) { + best = skipSpoken; + bestMatches = matches[here - width]!; + } + score[here] = best; + matches[here] = bestMatches; + } + } + + let bestColumn = -1; + let bestAdjusted = -Infinity; + for (let j = 1; j <= n; j++) { + const raw = score[m * width + j]!; + if (raw <= 0) continue; + const position = windowStart + j; + const distance = position - expectedPosition; + const adjusted = raw - (distance >= 0 ? distance * FORWARD_DISTANCE_COST : -distance * BACKWARD_DISTANCE_COST); + if (adjusted > bestAdjusted) { + bestAdjusted = adjusted; + bestColumn = j; + } + } + if (bestColumn < 0) return null; + const rawScore = score[m * width + bestColumn]!; + const matchedWords = matches[m * width + bestColumn]!; + if (rawScore < MIN_ALIGNMENT_SCORE || matchedWords < MIN_MATCHED_WORDS) return null; + return { position: windowStart + bestColumn, score: rawScore, matchedWords }; +} + +export class ScriptTracker { + private readonly options: Required; + /** Index of the next word the speaker is expected to say (0..words.length). */ + position = 0; + + constructor( + private readonly words: readonly string[], + options: TrackerOptions = {}, + ) { + this.options = { ...DEFAULT_OPTIONS, ...options }; + } + + get wordCount(): number { + return this.words.length; + } + + /** Jump the expected position (manual scroll, restart). */ + anchor(position: number): void { + this.position = Math.max(0, Math.min(this.words.length, position)); + } + + /** + * Apply a transcript update (the whole current utterance, partial or + * final). Returns true when the position moved. + */ + feed(text: string): boolean { + const keys = transcriptKeys(text); + if (keys.length === 0) return false; + const tail = keys.length > this.options.tailWords ? keys.slice(-this.options.tailWords) : keys; + const windowStart = Math.max(0, this.position - this.options.back); + const windowEnd = Math.min(this.words.length, this.position + this.options.ahead); + const result = alignTail(this.words, tail, windowStart, windowEnd, this.position); + if (!result || result.position === this.position) return false; + this.position = result.position; + return true; + } +} diff --git a/app/apps/teleprompter/teleprompter-app.ts b/app/apps/teleprompter/teleprompter-app.ts new file mode 100644 index 00000000..bbd3be46 --- /dev/null +++ b/app/apps/teleprompter/teleprompter-app.ts @@ -0,0 +1,188 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { truncateText } from "../../graphics/textwrap"; +import { readTextFile, type DirectoryEntry } from "../../native/file-access"; +import { LIST_ROW_TEXT_INSET } from "../../ui/metrics"; +import { MenuLayer, type MenuItem } from "../../ui/menu"; +import { type Layer, type LayerContext } from "../../ui/layers"; +import { appViewportSize } from "../../ui/shell/geometry"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "../../ui/shell/in-process-window"; +import { FileBrowserLayer } from "../files/file-browser"; +import { TeleprompterReaderLayer } from "./reader"; +import { + baseName, + clearRecentScripts, + getRecentScripts, + parentDirName, + removeRecentScript, + saveRecentScriptPosition, + touchRecentScript, +} from "./recent-files"; + +export const TELEPROMPTER_WINDOW_ID = "teleprompter"; +export const TELEPROMPTER_SURFACE_ID = "window:teleprompter"; + +const SCRIPT_FILE = /\.(txt|md|markdown|text|log)$/i; + +const HOME_LAYOUT = { + x: 8, + y: 8, + width: 272, + showBorder: false, + minHeight: 0, + maxHeight: appViewportSize("min").height - 16, + opaque: true, +}; + +export type TeleprompterAppOptions = InProcessAppOptions & { + /** Hold continuous mic capture while a script is being tracked. */ + startContinuousVoiceCapture: () => void; + stopContinuousVoiceCapture: () => void; + appendLog: (message: string) => void; +}; + +/** + * The Teleprompter app window. The home page is a Browse entry plus the + * recently opened scripts; Browse pushes the Files app's browser to pick a + * text file, and either path pushes the reader (see TeleprompterReaderLayer) + * over the home page. + */ +export function createTeleprompterAppWindow(options: TeleprompterAppOptions): InProcessWindow { + // The home menu's items are rebuilt in place whenever the recents change + // (MenuLayer reads the array each paint). + const homeItems: MenuItem[] = []; + const home = new MenuLayer("Teleprompter", homeItems, HOME_LAYOUT); + let created: InProcessWindow | null = null; + let notice = ""; + + const requestRender = () => created?.requestRender(); + + /** The layer currently on top of the window's stack (the home menu at base). */ + const topLayer = (): Layer => { + let top: Layer = home; + created?.stack.topMatches((layer) => { + top = layer; + return true; + }); + return top; + }; + + const openScript = (ctx: LayerContext, path: string, name: string) => { + const text = readTextFile(path); + if (text === null) { + removeRecentScript(path); + notice = `Could not read ${name}`; + options.appendLog(`teleprompter: could not read ${path}`); + refreshHome(); + return; + } + const recent = touchRecentScript(path, name); + notice = ""; + refreshHome(); + const layer = new TeleprompterReaderLayer( + text, + name, + { + requestRender, + startCapture: () => options.startContinuousVoiceCapture(), + stopCapture: () => options.stopContinuousVoiceCapture(), + savePosition: (position) => saveRecentScriptPosition(path, position), + }, + recent.position, + ); + ctx.stack.push(layer); + layer.attach(); + }; + + const openBrowser = (ctx: LayerContext) => { + const stack = ctx.stack; + const picker = new FileBrowserLayer({ + isSupportedFile: (name) => SCRIPT_FILE.test(name), + onLeave: () => stack.pop(), + onFilePicked: (entry: DirectoryEntry, pickCtx) => { + if (!SCRIPT_FILE.test(entry.name)) { + notice = "Pick a text file (.txt, .md)"; + return; + } + pickCtx.stack.pop(); + openScript(pickCtx, entry.path, entry.name); + }, + }); + stack.push(picker); + }; + + function refreshHome(): void { + const font = getDefaultSmallFont(); + const items: MenuItem[] = [ + { + label: "Browse for a script...", + onSelect: (ctx) => openBrowser(ctx), + }, + ]; + for (const recent of getRecentScripts()) { + const folder = parentDirName(recent.path); + items.push({ + label: recent.name || baseName(recent.path), + onSelect: (ctx) => openScript(ctx, recent.path, recent.name), + render: ({ image, x, y, width, selected, text }) => { + const value = selected ? 255 : 200; + const folderWidth = folder ? font.measureText(folder) : 0; + const nameMax = folder ? width - folderWidth - 12 : width; + image.drawText(font, x, y + LIST_ROW_TEXT_INSET, truncateText(font, text, Math.max(40, nameMax)), value); + if (folder && nameMax >= 40) { + image.drawText(font, x + width - folderWidth, y + LIST_ROW_TEXT_INSET, folder, 90); + } + }, + }); + } + if (notice) { + items.push({ label: notice, disabled: true, onSelect: () => {} }); + } + homeItems.length = 0; + homeItems.push(...items); + // Land on the most recent script (one click resumes it), else on Browse. + home.selectItem(getRecentScripts().length > 0 ? 1 : 0); + } + + refreshHome(); + + created = createInProcessWindow({ + appId: "teleprompter", + windowId: TELEPROMPTER_WINDOW_ID, + title: "Teleprompter", + iconLetter: "Tp", + icon: "scroll-text", + closeable: true, + actions: options.actions, + menuItems: () => { + const top = topLayer(); + if (top instanceof TeleprompterReaderLayer) return top.buildMenuItems(); + if (top instanceof FileBrowserLayer) return top.buildMenuItems(); + if (top !== home) return []; + return [ + { + label: "Clear recent scripts", + onSelect: (ctx) => { + clearRecentScripts(); + refreshHome(); + ctx.stack.pop(); + }, + }, + ]; + }, + onFocus: (lastInput) => { + const top = topLayer(); + if (top instanceof FileBrowserLayer) top.onFocus(lastInput); + }, + baseLayer: new YieldAtRootLayer(home), + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + onClosed: options.onClosed, + }); + return created; +} diff --git a/app/graphics/icons.ts b/app/graphics/icons.ts index af478732..7c59d667 100644 --- a/app/graphics/icons.ts +++ b/app/graphics/icons.ts @@ -36,6 +36,8 @@ export const ICON_SVGS = { '', terminal: '', + "scroll-text": + '', "file-text": '', file: diff --git a/tests/teleprompter-tracker.test.cjs b/tests/teleprompter-tracker.test.cjs new file mode 100644 index 00000000..1891ed92 --- /dev/null +++ b/tests/teleprompter-tracker.test.cjs @@ -0,0 +1,132 @@ +// Pins the Teleprompter's speech-to-script tracker: tokenization, the fuzzy +// word matcher, and the position tracking under realistic partial-transcript +// streams (replace semantics), including misrecognitions, ad-libs, skipped +// sentences and repeated phrases. +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const tracker = require("../.test-build/app/apps/teleprompter/script-tracker.js"); + +const SCRIPT = `Good evening, everyone. Thank you for coming tonight. + +Four score and seven years ago our fathers brought forth on this continent +a new nation, conceived in liberty, and dedicated to the proposition that +all men are created equal. + +Now we are engaged in a great civil war, testing whether that nation, or any +nation so conceived and so dedicated, can long endure. We are met on a great +battle-field of that war. + +We have come to dedicate a portion of that field, as a final resting place +for those who here gave their lives that that nation might live.`; + +/** Word index of the first occurrence of a normalized key, from `from`. */ +function wordAt(script, key, from = 0) { + const index = script.words.indexOf(key, from); + assert.notEqual(index, -1, `script has ${key}`); + return index; +} + +/** + * Feed an utterance the way a streaming recognizer delivers it: a growing + * partial after every word, then the final. Returns the tracker position. + */ +function speak(t, utterance) { + const words = utterance.split(/\s+/); + for (let i = 1; i <= words.length; i++) { + t.feed(words.slice(0, i).join(" ")); + } + t.feed(utterance); + return t.position; +} + +test("tokenizeScript separates display tokens from trackable words", () => { + const script = tracker.tokenizeScript("Hello, world — it's 2026.\nNext line"); + assert.deepEqual( + script.tokens.map((token) => token.text), + ["Hello,", "world", "—", "it's", "2026.", "Next", "line"], + ); + assert.deepEqual(script.words, ["hello", "world", "its", "2026", "next", "line"]); + assert.equal(script.tokens[2].wordIndex, -1); + assert.equal(script.tokens[5].paragraph, 1); + assert.deepEqual(script.wordTokens, [0, 1, 3, 4, 5, 6]); +}); + +test("withinOneEdit accepts single slips and rejects more", () => { + assert.ok(tracker.withinOneEdit("conceived", "conceived")); + assert.ok(tracker.withinOneEdit("conceived", "concieved") === false, "transposition is two edits"); + assert.ok(tracker.withinOneEdit("liberty", "libery")); + assert.ok(tracker.withinOneEdit("liberty", "libertyy")); + assert.ok(tracker.withinOneEdit("liberty", "libarty")); + assert.ok(!tracker.withinOneEdit("liberty", "library")); + assert.ok(tracker.withinOneEdit("nation", "notion"), "one substitution"); +}); + +test("follows a clean read-through, word by word", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + assert.equal(t.position, 0); + // A lone first word is not enough evidence to move. + t.feed("Good"); + assert.equal(t.position, 0); + t.feed("Good evening"); + assert.equal(t.position, 2); + speak(t, "Good evening everyone thank you for coming tonight"); + assert.equal(t.position, wordAt(script, "four")); + speak(t, "Four score and seven years ago our fathers brought forth on this continent"); + assert.equal(t.position, wordAt(script, "a", wordAt(script, "continent"))); +}); + +test("tolerates misrecognized and ad-libbed words", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + t.anchor(wordAt(script, "four")); + // "score" heard as "scored", "fathers" as "father's", plus an inserted "um". + speak(t, "for scored and seven years ago um our father's brought forth"); + assert.equal(t.position, wordAt(script, "on")); +}); + +test("a partial last word counts as a prefix of the next script word", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + t.anchor(wordAt(script, "four")); + t.feed("four score and seven years ago our fath"); + assert.equal(t.position, wordAt(script, "brought")); +}); + +test("catches up when the speaker skips a sentence", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + t.anchor(wordAt(script, "four")); + speak(t, "Now we are engaged in a great civil war"); + assert.equal(t.position, wordAt(script, "testing")); +}); + +test("prefers the occurrence nearest the current position for repeated phrases", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + // "that nation" appears in paragraphs 3 and 4; anchored in paragraph 4 it + // should pick the later one. + const laterThat = wordAt(script, "gave"); + t.anchor(laterThat); + speak(t, "their lives that that nation might live"); + assert.equal(t.position, script.words.length); +}); + +test("does not move on transcripts that match nothing nearby", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words); + t.anchor(wordAt(script, "four")); + assert.equal(speak(t, "hang on let me find my glasses"), wordAt(script, "four")); +}); + +test("manual anchoring re-bases the search window", () => { + const script = tracker.tokenizeScript(SCRIPT); + const t = new tracker.ScriptTracker(script.words, { ahead: 10, back: 5 }); + // Far outside the window from position 0. + speak(t, "final resting place for those who here gave"); + assert.equal(t.position, 0); + t.anchor(wordAt(script, "final")); + speak(t, "final resting place for those who here gave"); + assert.equal(t.position, wordAt(script, "their")); +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index cbc4c641..cba8aa69 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -25,6 +25,7 @@ "../app/apps/microphones/radar-dots.ts", "../app/apps/microphones/name-guess.ts", "../app/apps/evenhub/containers.ts", - "../app/apps/timer/timer-model.ts" + "../app/apps/timer/timer-model.ts", + "../app/apps/teleprompter/script-tracker.ts" ] }