import { type NavigationSensorRequest, type NavigationSensorEvent } from "../../apps/navigate/navigation-sensors-messages"; import { GrayImage } from "../../graphics/image"; import { windowIcon } from "./chrome-layer"; import { type IconActivity, type IconName } from "../../graphics/icons"; import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry"; import { appViewport, type WindowHeightMode } from "./geometry"; import { type Viewport } from "../viewport"; import * as frameTimings from "../../native/frame-timings"; import { shell, type ShellWindow } from "./shell"; import { publishWorkerState } from "./worker-state"; /** * Messages between the shell (main thread) and an app worker. One worker * hosts one app, which may have several windows; messages are routed by * windowId. Everything crossing this boundary is small JSON; pixels go * worker→Java directly on Android; iOS posts baked frames to the main host. */ export type WorkerAppMessage = /** Last host window closed: report worker-idle unless background work still owns the worker. */ | { type: "check-idle" } /** Release app-wide resources, then acknowledge with worker-stopped. */ | { type: "shutdown" } | { type: "navigation-sensors"; event: NavigationSensorEvent } /** `viewport` carries what the window's edges touch and the padding suggested for them (viewport.ts). */ | { type: "open-window"; windowId: string; surfaceId: string; title: string; viewport: Viewport } /** The window's size changed, or what its edges touch did (so its padding). */ | { type: "resize-window"; windowId: string; viewport: Viewport } | { type: "close-window"; windowId: string } | { type: "input"; windowId: string; event: unknown; frameId: number; focused: boolean } | { type: "text-input"; windowId: string; text: string; submit?: boolean } | { type: "render"; windowId: string; focused: boolean } | { type: "foreground"; windowId: string; foreground: boolean; focused: boolean } /** * Input focus arrived at or left the window, counting shell overlays and * screen-off (see ShellWindow.setInputFocus). Sent on change only; the * per-message `focused` flags above stay the source of truth for painting. */ | { type: "input-focus"; windowId: string; focused: boolean } | { type: "screen"; on: boolean } /** Assistant tool invocation aimed at a window; reply with tool-result. */ | { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown }; export type WorkerAppReply = | { type: "worker-idle" } | { type: "worker-stopped" } | { type: "buzzer-sequence"; payload: number[] } | { type: "navigation-sensors"; request: NavigationSensorRequest } | { type: "open-url"; url: string } | { type: "surface-frame"; surfaceId: string; width: number; height: number; pixels: string; draws?: string } | { /** * The worker's bundle has evaluated and its onmessage handler is * installed. Messages posted to a still-loading worker can be silently * dropped, so the host queues everything until this arrives. */ type: "worker-ready"; } | { type: "yield-focus"; windowId: string } | { /** Foreground and focus one of the app's existing windows. */ type: "focus-window"; windowId: string; } | { /** * Wake the glasses and focus one of the app's windows (e.g. a terminal * bell with wake-on-bell enabled). Ignored unless the screen is off, so * it can never steal focus from an in-use screen. */ type: "wake-window"; windowId: string; } | { /** App-initiated window (e.g. a terminal view opened from the hub list). */ type: "open-window-request"; windowId: string; title: string; iconLetter: string; icon?: IconName; iconGlyph?: string; focus?: boolean; /** Window height: the standard 288px band ("min", default) or full screen ("max"). */ heightMode?: WindowHeightMode; } | { /** * Move a window to another height band at runtime. The host resizes its * surface and then always answers with resize-window, even if the size * came out the same, so the worker can wait for the new viewport. */ type: "set-height-mode"; windowId: string; heightMode: WindowHeightMode; } | { type: "set-title"; windowId: string; title: string } | { type: "set-attention"; windowId: string; attention: boolean } /** App-driven animation phase for sidebar icons that support an activity cursor. */ | { type: "set-icon-activity"; windowId: string; activity: IconActivity } | { /** Open the shell's voice dialog aimed at this window (a menu pick). */ type: "start-voice-input"; windowId: string; } | { /** Close this window (the shell owns the close path). */ type: "close-window-request"; windowId: string; } | { /** * The window's answer to tap-then-hold when it has no context menu of * its own: open the shell's system menu in its place. */ type: "open-system-menu"; windowId: string; } | { /** * The window's current gesture bindings, posted on change. hasAppMenu: * tap-then-hold opens a context menu with something in it (the system * menu shows its app-menu hint only then). claimsLongPress: the app * gives long-press a meaning of its own, so the shell forwards it * rather than opening the system menu (see ShellWindow). */ type: "set-window-gestures"; windowId: string; hasAppMenu: boolean; claimsLongPress: boolean; } | { /** Open or focus the Settings app, optionally jumping to a section. */ type: "open-settings"; section?: string; } | { /** * Open the phone app's text editor on a string setting (by id, resolved * on the main thread). The worker paints its own "type on the phone" * UI and watches the setting for changes; it must post * end-text-setting-edit when its flow finishes. */ type: "start-text-setting-edit"; settingId: string; } | { type: "end-text-setting-edit" } | { /** * Set or clear the app's top-bar tray icon. Pixels ride the JSON * postMessage roundtrip — acceptable because tray icons are small and * infrequently updated. */ type: "set-tray-icon"; icon: { width: number; height: number; pixels: number[] } | null; } | { /** * Declare (replacing the prior set) the assistant tools this window * contributes. Names are unprefixed; the registry adds `app..`. */ type: "set-tools"; windowId: string; tools: ToolSpec[]; } | { /** Result of a tool-call, matched to the request by callId. */ type: "tool-result"; callId: string; result: ToolResult; } | { /** * Publish a small piece of app state for main-thread consumers outside * the app's windows (see app/ui/shell/worker-state.ts). JSON only. */ type: "publish-state"; key: string; state: unknown; }; export type WorkerWindowSpec = { /** Unique across the shell; namespace with the appId (e.g. "terminal:view:3"). */ windowId: string; title: string; iconLetter: string; /** Lucide icon name for the sidebar indicator; falls back to iconLetter. */ icon?: IconName; /** Per-window character variant of `icon` (e.g. ">3" terminal icons). */ iconGlyph?: string; /** Foreground and focus the window once its surface exists. */ focus?: boolean; /** Window height: the standard 288px band ("min", default) or full screen ("max"). */ heightMode?: WindowHeightMode; /** * Deliver watch swipes to the worker as swipe-* events instead of the * scroll / click / double-click fallback. The worker then owns the meaning * of all four directions in every window state (see ShellWindow). */ acceptsDirectional?: boolean; }; export type WorkerAppHostOptions = { appId: string; worker: Worker; /** Drop this host from the app cache as soon as shutdown begins. */ onStopping?: () => void; navigationSensors?: { handle(request: NavigationSensorRequest): void; stop(): void }; openUrl?: (url: string) => void; playBuzzerSequence?: (payload: Uint8Array) => Promise | void; /** * Create/refresh a window surface on the compositor (no-op when disconnected). * `visible` may be a function, read once the surface is configured, for a * window that can lose the foreground in the meantime. */ configureSurface: (surfaceId: string, visible: boolean | (() => boolean), heightMode: WindowHeightMode) => Promise; setSurfaceVisible: (surfaceId: string, visible: boolean) => void; removeSurface: (surfaceId: string) => void; requestShellRender: () => void; submitPixels?: (surfaceId: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null) => void; startTextInput?: () => void; /** Open or focus the Settings app, optionally selecting a section. */ openSettings: (section?: string) => void; /** Open the phone app's text editor on a string setting (by id). */ startTextSettingEdit: (settingId: string) => void; /** Close the phone text editor opened by startTextSettingEdit. */ endTextSettingEdit: () => void; }; /** * Owns the Worker for one app and adapts its windows to the shell's window * interface: forwards input and lifecycle over postMessage, relays worker * requests (yield-focus, new windows, attention flags) back to the shell, * and manages compositor surfaces for the app's windows. Android workers send * frames straight to Java; iOS workers use surface-frame replies. */ /** A tool-call awaiting its worker reply; also its own leak-safety timeout. */ type PendingToolCall = { windowId: string; resolve: (result: ToolResult) => void; timer: ReturnType; }; /** * How long the host waits for a worker's tool-result before giving up. The * registry applies its own (usually shorter) per-tool timeout; this is the * backstop that also frees the pending-call entry if the worker never replies. */ const TOOL_CALL_HOST_TIMEOUT_MS = 15_000; export class WorkerAppHost { private readonly openWindows = new Set(); private readonly windowIconActivity = new Map(); /** Per-window gesture bindings, as last reported (see set-window-gestures). */ private readonly windowGestures = new Map(); private readonly pendingToolCalls = new Map(); private nextCallSerial = 1; /** * Messages posted before the worker finishes evaluating its bundle can be * silently dropped (notably the launch-time foreground/render pair, leaving * a black window until the first input). Queue everything until the worker * posts worker-ready, then flush in order. */ private workerReady = false; private readonly queuedMessages: WorkerAppMessage[] = []; private stopping = false; private terminated = false; private shutdownTimer: ReturnType | null = null; private readonly publishedStateKeys = new Set(); constructor(private readonly options: WorkerAppHostOptions) { options.worker.onmessage = (event: MessageEvent) => { const message = event.data as WorkerAppReply | undefined; if (!message) return; if (this.stopping) { if (message.type === "worker-stopped") this.terminate(); return; } switch (message.type) { case "worker-idle": if (!this.openWindows.size) this.shutdown(); break; case "buzzer-sequence": if (this.openWindows.size) { void Promise.resolve(this.options.playBuzzerSequence?.(new Uint8Array(message.payload))) .catch(error => console.warn(`${this.options.appId} buzzer failed: ${error}`)); } break; case "navigation-sensors": if (this.openWindows.size) this.options.navigationSensors?.handle(message.request); break; case "open-url": if (this.openWindows.size && /^https?:\/\//i.test(message.url)) this.options.openUrl?.(message.url); break; case "surface-frame": { if (!global.isIOS || !this.options.submitPixels || !this.openWindows.has(message.surfaceId.replace(/^window:/, ""))) break; const { width, height } = message; if (!Number.isInteger(width) || !Number.isInteger(height) || width < 1 || height < 1 || width > 640 || height > 480) break; const data = NSData.alloc().initWithBase64EncodedStringOptions(message.pixels, 0 as NSDataBase64DecodingOptions); const draws = message.draws ? NSData.alloc().initWithBase64EncodedStringOptions(message.draws, 0 as NSDataBase64DecodingOptions) : null; if (data?.length === width * height) this.options.submitPixels(message.surfaceId, new Uint8Array(interop.bufferFromData(data)), width, height, draws ? interop.bufferFromData(draws) : null); break; } case "worker-ready": this.workerReady = true; for (const queued of this.queuedMessages.splice(0)) { this.options.worker.postMessage(queued); } break; case "yield-focus": // Only the focused window's yield is meaningful. if (shell.foregroundWindow()?.windowId === message.windowId) { shell.yieldFocusToSidebar(); } break; case "focus-window": if (this.openWindows.has(message.windowId)) { shell.focusWindow(message.windowId); this.options.requestShellRender(); } break; case "wake-window": if (this.openWindows.has(message.windowId) && !shell.isScreenOn()) { // Focus first so wake's foreground-window refresh hits the target. shell.focusWindow(message.windowId); shell.wake("window"); this.options.requestShellRender(); } break; case "open-window-request": this.openWindow({ windowId: message.windowId, title: message.title, iconLetter: message.iconLetter, icon: message.icon, iconGlyph: message.iconGlyph, focus: message.focus, heightMode: message.heightMode, }); break; case "set-height-mode": this.setWindowHeightMode(message.windowId, message.heightMode); break; case "set-attention": shell.setWindowAttention(message.windowId, message.attention); break; case "start-voice-input": // Menus only open on the focused window, but re-check foreground: // the reply crosses a thread boundary and focus may have moved. if (shell.foregroundWindow()?.windowId === message.windowId) { if (this.options.startTextInput) this.options.startTextInput(); else shell.startVoiceInput(); } break; case "close-window-request": if (this.openWindows.has(message.windowId)) { shell.closeWindow(message.windowId); this.options.requestShellRender(); } break; case "open-system-menu": if (this.openWindows.has(message.windowId)) { shell.openSystemMenu(message.windowId); } break; case "set-icon-activity": if (this.openWindows.has(message.windowId) && this.windowIconActivity.get(message.windowId) !== message.activity) { this.windowIconActivity.set(message.windowId, message.activity); this.options.requestShellRender(); } break; case "set-window-gestures": this.windowGestures.set(message.windowId, { hasAppMenu: message.hasAppMenu, claimsLongPress: message.claimsLongPress, }); break; case "open-settings": this.options.openSettings(message.section); break; case "start-text-setting-edit": this.options.startTextSettingEdit(message.settingId); break; case "end-text-setting-edit": this.options.endTextSettingEdit(); break; case "set-tray-icon": { let icon: GrayImage | null = null; if (message.icon) { icon = new GrayImage(message.icon.width, message.icon.height, 0); icon.pixels.set(message.icon.pixels.slice(0, icon.pixels.length)); } shell.setTrayIcon(this.options.appId, icon); break; } case "set-title": // Titles are informational for now (sidebar shows icons only). break; case "publish-state": this.publishedStateKeys.add(message.key); publishWorkerState(message.key, message.state); break; case "set-tools": // Only a window we actually have open may contribute tools. if (this.openWindows.has(message.windowId)) { toolRegistry.setAppTools({ windowId: message.windowId, appId: this.options.appId, specs: message.tools, invoke: (toolName, args) => this.callWindowTool(message.windowId, toolName, args), isForeground: () => shell.foregroundWindow()?.windowId === message.windowId, }); } break; case "tool-result": { const pending = this.pendingToolCalls.get(message.callId); if (pending) { clearTimeout(pending.timer); this.pendingToolCalls.delete(message.callId); pending.resolve(message.result); } break; } case "worker-stopped": // Only meaningful as the reply to shutdown(), handled above. break; } }; options.worker.onerror = (error) => { options.navigationSensors?.stop(); console.error(`worker app ${options.appId} error: ${JSON.stringify(error)}`); }; } windowCount(): number { return this.openWindows.size; } private shutdown(): void { if (this.stopping) return; this.stopping = true; this.options.onStopping?.(); this.queuedMessages.length = 0; this.options.navigationSensors?.stop(); shell.setTrayIcon(this.options.appId, null); for (const key of this.publishedStateKeys) publishWorkerState(key, undefined); this.publishedStateKeys.clear(); // The idle reply follows all close-window cleanup. Allow the worker to // release app-wide native subscriptions/sockets before killing its isolate. this.shutdownTimer = setTimeout(() => this.terminate(), 5000); this.options.worker.postMessage({ type: "shutdown" } satisfies WorkerAppMessage); } private terminate(): void { if (this.terminated) return; this.terminated = true; if (this.shutdownTimer !== null) clearTimeout(this.shutdownTimer); this.shutdownTimer = null; this.options.worker.terminate(); } /** Open a window of this app and register it with the shell. */ openWindow(spec: WorkerWindowSpec): ShellWindow { if (this.stopping) throw new Error(`Worker ${this.options.appId} is shutting down`); let closed = false; const surfaceId = `window:${spec.windowId}`; const heightMode = spec.heightMode ?? "min"; this.openWindows.add(spec.windowId); let viewport = appViewport(heightMode, this.options.appId); this.post({ type: "open-window", windowId: spec.windowId, surfaceId, title: spec.title, viewport, }); const window: ShellWindow = { appId: this.options.appId, windowId: spec.windowId, title: spec.title, surfaceId, closeable: true, acceptsDirectional: spec.acceptsDirectional, heightMode, // Every app worker resizes its live content in place, keeping its state // (a game in progress, a route): the bottom switcher's height changes // with what it holds, and full-height windows give way to it. relayout: () => { viewport = appViewport(window.heightMode, this.options.appId); this.post({ type: "resize-window", windowId: spec.windowId, viewport }); }, layoutSize: () => viewport, hasAppMenu: () => this.windowGestures.get(spec.windowId)?.hasAppMenu ?? false, claimsLongPress: () => this.windowGestures.get(spec.windowId)?.claimsLongPress ?? false, close: () => { if (closed) return; closed = true; this.openWindows.delete(spec.windowId); if (!this.openWindows.size) this.options.navigationSensors?.stop(); this.windowGestures.delete(spec.windowId); this.windowIconActivity.delete(spec.windowId); // Withdraw this window's tools and fail any in-flight calls to it. toolRegistry.removeAppTools(spec.windowId); this.failPendingToolCallsFor(spec.windowId); this.post({ type: "close-window", windowId: spec.windowId }); this.options.removeSurface(surfaceId); if (!this.openWindows.size) this.post({ type: "check-idle" }); }, drawIcon: windowIcon(spec.icon, spec.iconLetter, spec.iconGlyph, () => this.windowIconActivity.get(spec.windowId) ?? "idle"), handleInput: (event, frameId) => { frameTimings.logFrame(frameId, `input posted to the ${this.options.appId} worker`); this.post({ type: "input", windowId: spec.windowId, event, frameId, focused: shell.isWindowFocused(spec.windowId), }); }, requestRender: () => { this.post({ type: "render", windowId: spec.windowId, focused: shell.isWindowFocused(spec.windowId) }); }, receiveTextInput: (text, options) => { this.post({ type: "text-input", windowId: spec.windowId, text, ...(options?.submit === undefined ? {} : { submit: options.submit }) }); }, setForeground: (foreground) => { this.options.setSurfaceVisible(surfaceId, foreground); if (foreground) { shell.setWindowAttention(spec.windowId, false); } this.post({ type: "foreground", windowId: spec.windowId, foreground, focused: shell.isWindowFocused(spec.windowId), }); }, setScreenOn: (on) => { // Screen state is per-app, but sending per-window keeps the protocol // uniform; the worker treats it globally. this.post({ type: "screen", on }); }, setInputFocus: (focused) => { this.post({ type: "input-focus", windowId: spec.windowId, focused }); }, }; shell.registerWindow(window); // Configure the surface before any foregrounding so the worker's first // frame has somewhere to land. void this.options .configureSurface(surfaceId, false, heightMode) .then(() => { if (closed || this.stopping) return; if (spec.focus) { shell.focusWindow(spec.windowId); } this.options.requestShellRender(); }) .catch((error) => { console.error(`surface setup for ${spec.windowId} failed: ${error}`); }); return window; } /** The worker asked for another height band (see set-height-mode). */ private setWindowHeightMode(windowId: string, heightMode: WindowHeightMode): void { const window = this.openWindows.has(windowId) ? shell.getWindows().find((w) => w.windowId === windowId) : undefined; if (!window) return; window.heightMode = heightMode; // Resize the surface first, so the frame answering resize-window fits it. void this.options .configureSurface(window.surfaceId, () => shell.foregroundWindow()?.windowId === windowId, heightMode) .catch((error) => { console.error(`surface resize for ${windowId} failed: ${error}`); }) .then(() => { window.relayout?.(); // The chrome (top bar, bottom switcher) follows the foreground's band. this.options.requestShellRender(); }); } /** * Post a tool-call to the worker and resolve when its tool-result comes back. * Resolves with a tool error on timeout so a hung worker yields a tool error * rather than a stuck assistant turn. */ private callWindowTool(windowId: string, toolName: string, args: unknown): Promise { return new Promise((resolve) => { const callId = `${this.options.appId}:${this.nextCallSerial++}`; const timer = setTimeout(() => { this.pendingToolCalls.delete(callId); resolve({ ok: false, error: `App ${this.options.appId} did not respond to ${toolName}` }); }, TOOL_CALL_HOST_TIMEOUT_MS); this.pendingToolCalls.set(callId, { windowId, resolve, timer }); this.post({ type: "tool-call", callId, windowId, name: toolName, args }); }); } private failPendingToolCallsFor(windowId: string): void { for (const [callId, pending] of this.pendingToolCalls) { if (pending.windowId !== windowId) continue; clearTimeout(pending.timer); this.pendingToolCalls.delete(callId); pending.resolve({ ok: false, error: "The target window was closed" }); } } private post(message: WorkerAppMessage): void { if (this.stopping) return; if (!this.workerReady) { this.queuedMessages.push(message); return; } this.options.worker.postMessage(message); } }