import { startRemoteInput } from "../remote/service"; import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { Application, Dialogs, ImageSource } from "@nativescript/core"; import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events"; import { isValidMacAddress, loadDeviceAddresses } from "./device-addresses"; import { ensureBlePermissions, ensureVoicePermissions, hasMicrophonePermission, requestMicrophonePermission, } from "./android-permissions"; import { FaceclawCommunicatorBridge, type RawInputEvent } from "../native/faceclaw-communicator"; import * as frameTimings from "../native/frame-timings"; import { startForegroundNotification, stopForegroundNotification, updateForegroundNotification } from "../native/foreground-service"; import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout-bridge"; import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, readActiveNotifications } from "../native/notification-icons"; import { shouldShowNotificationOnGlasses } from "../native/notification-sources"; import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict"; import { grayImageToPreviewSource } from "../native/gray-image-preview"; import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from "./firmware-compat"; import { hasExtractedEvenHubFonts } from "./firmware-builder"; import { resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy"; import { WearRemote, type WearRemoteInputKind } from "./wear-remote"; /** Who a synthetic (non-firmware) input stands for. */ type SyntheticInputOrigin = "ring" | "watch"; /** Gestures the phone's mirror view reports (see handleMirrorTouch). */ export type MirrorTouchKind = | "tap" | "double-tap" | "long-press" | "swipe-up" | "swipe-down" | "swipe-left" | "swipe-right"; const MIRROR_TOUCH_GESTURES: Record, WearRemoteInputKind> = { "double-tap": "double-click", "long-press": "long-press", "swipe-up": "swipe-up", "swipe-down": "swipe-down", "swipe-left": "swipe-left", "swipe-right": "swipe-right", }; import { findSoundEffect, playSoundEffect } from "../ui/sound-effects"; import { GlanceHost } from "./glance-host"; import { type GlanceEvent } from "./glance-state"; import { isPreviewOnlyMode, isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { voiceControlBridge } from "../native/voice-control"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import { prepareFrameDraws } from "../graphics/glyph-wire"; import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen"; import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; import { type InputEvent } from "../ui/gestures"; import { registerSystemTools } from "../assistant/system-tools"; import { updateGlassesPresence } from "./glasses-presence"; import { timerEngine } from "../apps/timer/timer-engine"; import { registerNavigateTools } from "../assistant/navigate-tools"; import { registerRoamTools } from "../assistant/roam-tools"; import { assistantBridge } from "../assistant/bridge-client"; import { registerWindowTools } from "../assistant/window-tools"; import { WorkerAppHost } from "../ui/shell/worker-window"; import { ALL_APPS } from "../apps/all-apps"; import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayModeSetting, navigateDisplayModeSetting, navigateVerticalPositionSetting, terminalDisplayModeSetting, terminalVerticalPositionSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage, } from "../apps/evenhub/installed-apps"; import { closeRunningPackage, launchInstalledPackage, onPhoneShownChanged, phoneUiButton, showOnPhone as showEvenHubPhoneUi, type PhoneUiButton, } from "../apps/evenhub/manager"; import { openEvenHubStoreForPackage } from "../apps/evenhub"; import { isInstalledPackagePresent } from "../apps/evenhub/updates"; import { wearerVerificationOptions } from "../apps/microphones/speakers"; import { micSession } from "../apps/microphones/mic-session"; import { glassesDisplayLabel } from "./glasses-display-state"; import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting"; export type DashboardSnapshot = { phase: ConnectionPhase; status: string; displayPreview: ImageSource | null; /** * When non-empty, the phone UI shows this instead of the display preview: * the preview would be a black rectangle indistinguishable from dead * glasses, and this says which harmless thing is actually going on. */ displayPreviewMessage: string; activeTextSettingId: string | null; activeTextEditorTitle: string; activeTextSettingTitle: string; activeTextSettingValue: string; activeTextSettingInputKind: "text" | "email" | "password"; secondaryTextSettingId: string | null; secondaryTextSettingTitle: string; secondaryTextSettingValue: string; secondaryTextSettingInputKind: "text" | "email" | "password"; activeTextEditorToggleLabel: string; activeTextEditorToggleValue: boolean; activeTextEditorToggleVisible: boolean; /** * The keyboard dialog is up on the glasses (opened by the phone's keyboard * button): the phone shows its typing panel, with a send button per * destination in the glasses menu's row order. */ keyboardInputActive: boolean; keyboardInputTargets: ReadonlyArray<{ id: string; label: string }>; evenAppConflictMessage: string; evenAppConflictWarningVisible: boolean; firmwareWarningMessage: string; firmwareWarningVisible: boolean; screenRecordingActive: boolean; batteryOptimizationWarningVisible: boolean; fontsMissingWarningVisible: boolean; /** Why the phone may fail to ring for a set alarm; "" when nothing is wrong or armed. */ alarmReliabilityMessage: string; /** * True while the headless preview display is standing in for a glasses * connection (preview-only mode): the mirror is live and interactive, but * nothing is paired, so "Disconnected" would be the wrong label. */ previewMode: boolean; /** * The foreground glasses window's EvenHub phone UI, when it can be opened * (the action bar's app-icon button); null otherwise. */ evenHubPhoneUi: PhoneUiButton | null; }; type DashboardListener = (snapshot: DashboardSnapshot) => void; // The shell chrome (sidebar + top bar + overlays) composites above all app // window surfaces with color-key transparency. const SHELL_SURFACE_ID = "shell"; /** The shell chrome composites above every window surface (zOrder 0). */ const SHELL_SURFACE_Z_ORDER = 1; // Top-bar clock refresh; the phone-side preview polls the Java composite so // it reflects every app (including worker apps the TS side never renders). const SHELL_REFRESH_INTERVAL_MS = 60_000; // Safety-net poll only; the preview is event-driven (every composited frame // schedules a refresh), so this just covers anything that slips through. const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; // The lock screen is a short notice, not something to read at leisure; it // ignores the screen-timeout setting (including "never"). const LOCK_SCREEN_TIMEOUT_MS = 15_000; const EVENHUB_SCREEN_OFF_SUSPEND_DELAY_MS = 5_000; const EVENHUB_WAKE_READY_TIMEOUT_MS = 4_500; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000; // Preview refresh floor. Refreshes are scheduled per composited frame with a // trailing update, so the mirror tracks the glasses within this bound instead // of the old 1s poll (which lagged up to two polls behind). const CONNECTED_PREVIEW_MIN_UPDATE_MS = 150; // The GIF recorder keeps its old cadence; per-frame captures would balloon it. const RECORDING_MIN_CAPTURE_MS = 1_000; // Below this, a disconnect is more likely a flat battery than a BLE problem. const LOW_BATTERY_PERCENT = 5; const EVEN_APP_DETECTED_MESSAGE = "The Even Realities app appears to be running. If Faceclaw has trouble connecting, open its app settings and force stop it."; // The launcher grid's app list; also fixes the app ids apps.launch accepts. const LAUNCHABLE_APPS = ALL_APPS.filter((app) => app.showInLauncher !== false); function createInitialDisplayPreview(): ImageSource | null { return grayImageToPreviewSource(new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0)); } /** Echo the controller's appendLog() trace to the console. */ const VERBOSE_CONTROLLER_LOG = false; function formatTimestamp(date: Date): string { return date.toISOString().slice(11, 23); } function eventName(eventType: number): string { return OsEventTypeName[eventType] ?? `UNKNOWN_${eventType}`; } /** * Event-type label for logs. "even-ai" frames carry eEvenAIStatus, not an * OsEventTypeList value, so naming them with the OS table would be wrong * (status 1 would read as "SCROLL_TOP_EVENT"). */ function eventLabel(kind: string, eventType: number): string { if (kind === "even-ai") { return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`; } if (kind === "watch-gesture") { return WatchGestureTypeName[eventType] ?? `WATCH_UNKNOWN_${eventType}`; } return eventName(eventType); } function sourceName(eventSource: number): string { return EventSourceTypeName[eventSource] ?? `SOURCE_${eventSource}`; } class DashboardController { private phase: ConnectionPhase = "disconnected"; private status = "Disconnected."; private activeTextSettings: ConfigSettingString[] = []; private activeTextEditorTitle = ""; private activeTextEditorOnFinish: (() => void) | null = null; /** * The caller's cancel callback. LayerActions has always declared this fifth * parameter, but neither the wiring below nor startTextSettingsEdit accepted * it, so it was dropped silently -- a 4-parameter function is assignable to a * 5-parameter type, so nothing failed to compile. A caller that tracks its own * "editor is open" state therefore had no way to learn the editor closed. */ private activeTextEditorOnCancel: (() => void) | null = null; private activeTextEditorToggle: TextSettingsEditToggle | null = null; /** * The active settings' values (and the toggle's) when the edit began. The * phone fields write through on every keystroke, so this is what the * editor's Cancel button puts back. */ private activeTextSettingOriginalValues: string[] = []; private activeTextEditorToggleOriginalValue = false; private evenNotificationActive = false; private evenAppConflictMessage = ""; private firmwareWarningMessage = ""; private batteryOptimizationWarningVisible = false; private fontsMissingWarningVisible = false; private alarmReliabilityMessage = ""; // Sticky positive: the extracted-font file doesn't vanish mid-session, so a // successful check spares later refreshes the file read and JSON parse. private extractedFontsConfirmed = false; private screenRecordingActive = false; private displayPreview: ImageSource | null = createInitialDisplayPreview(); private silentMode = false; // Brightness value last pushed to the glasses; see pushBrightness. private lastPushedBrightness: string | null = null; // Last battery level the glasses reported, kept across disconnects so a // drop-off right after a low reading can be explained as a flat battery. private lastHeadsetBattery: number | null = null; private readonly listeners = new Set(); // Set at connect time from the persisted flag; the one-time post-onboarding // welcome sound plays on the first rendered frame (proof the session is warm). private welcomeSoundArmed = false; private communicator: FaceclawCommunicatorBridge | null = null; // Headless stand-in for the compositor when no glasses are paired // (preview-only mode); a real connection always outranks it. See `display`. private previewTarget: PreviewDisplayTarget | null = null; private shellRefreshTimer: ReturnType | null = null; private previewTimer: ReturnType | null = null; private screenTimeoutTimer: ReturnType | null = null; private evenHubSuspendTimer: ReturnType | null = null; private evenHubSessionSuspended = false; /** * The Glanceboard: the alternate sleep-time display, on its own surface * above the shell. Fed sleep-time input from handleInputEvent; it uses the * same wake barrier as the shell and hands the compositor back to the * screen-off path when it hides. */ private readonly glance = new GlanceHost({ getDisplay: () => this.display, getProvider: () => ALL_APPS.find((app) => app.glanceboard)?.glanceboard ?? null, canShow: () => (this.phase === "connected" || this.isPreviewDisplayActive()) && !this.glassesLocked, ensureSessionActive: (frameId) => this.ensureEvenHubSessionActive(frameId), onHiddenWhileAsleep: () => { if (!shell.isScreenOn()) this.handleScreenStateChanged(false); }, onVisibilityChanged: () => { this.schedulePreviewUpdate(); this.emit(); }, appendLog: (message) => this.appendLog(message), }); private evenHubResumePromise: Promise | null = null; /** Faceclaw's firmware at the required revision reported in; its extensions can be used. */ private customFirmwareConfirmed = false; private faceclawWakeLeaseState: boolean | null = null; private glassesWorn: boolean | null = null; /** Put on while charging: Java reconnects on that, and the wake waits for the new session. */ private wakeAfterChargingReconnect = false; private phoneLocked = false; private glassesLocked = false; // Wear OS watch remote; constructed last so it sees a fully wired shell. private wearRemote: WearRemote | null = null; private lockSurfaceConfigured = false; private lastLockScreenEnabled = lockScreenEnabledSetting.get(); private offState: (() => void) | null = null; private offRing: (() => void) | null = null; private offBattery: (() => void) | null = null; private offSilentMode: (() => void) | null = null; private offWearState: (() => void) | null = null; private offPhoneLockState: (() => void) | null = null; private offEvenAppConflict: (() => void) | null = null; private offFrameMetrics: (() => void) | null = null; private offFirmwareInfo: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; private lastInput = "waiting..."; private lastSys = "none yet"; private shellRenderInProgress = false; private shellRenderQueued = false; /** Input frame that requested the next shell render; see requestShellRender. */ private pendingShellRenderCauseFrameId = 0; private nextShellRenderWantsFreshData = false; /** The under-shell dim last sent to the display (see Shell.underlayDim); 1 = none. */ private appliedUnderlayDim = 1; // One shared worker per app hosts all its windows; spawned on first launch. private readonly appHosts = new Map(); // The window hosting the on-glasses text-setting editor (the Settings app // registers itself); edit flows from the phone UI reach it through this. private textEditorHost: TextEditorHost | null = null; // The open keyboard dialog's session (see Shell.startKeyboardInput); the // phone's typing panel feeds it. private keyboardInput: KeyboardInputSession | null = null; // Other in-process singleton apps, keyed by windowId. private readonly inProcessApps = new Map(); private sharedActions!: Omit; private lastForegroundNotificationUpdateAtMs = 0; private lastConnectedPreviewUpdateAtMs = 0; // Saving the open-app list is gated until the one-time restore has run, so // the boot-time registry (just the launcher) never clobbers the saved list. private openAppsRestored = false; private suppressOpenAppsPersist = false; // connect() is a long sequence of awaits against this.communicator; the // incompatible-firmware disconnect must not tear that communicator down // underneath it, so it waits for this to clear. private connectRunning = false; private incompatibleDisconnectPending = false; private unpairedDisconnectPending = false; constructor() { const sharedActions = { disconnect: () => this.disconnect(), startTextSettingEdit: (setting: ConfigSettingString) => this.startTextSettingEdit(setting), startTextSettingsEdit: ( settings: readonly ConfigSettingString[], title: string, onFinish?: () => void, toggle?: TextSettingsEditToggle, onCancel?: () => void, ) => this.startTextSettingsEdit(settings, title, onFinish, toggle, onCancel), endTextSettingEdit: () => this.endTextSettingEdit(), startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing), stopVoiceCapture: () => this.stopVoiceCapture(), startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; // The always-available assistant tools (calendar, media, notifications, // glasses state) register once at startup, independent of any connection. registerSystemTools(); // nav.* tools launch the Navigate app on demand, so they need launchApp. registerNavigateTools((appId) => this.launchApp(appId)); // roam.* tools launch the Roam app on demand likewise. registerRoamTools((appId) => this.launchApp(appId)); // apps.* tools mirror the launcher grid and sidebar (launch, focus, close). registerWindowTools({ apps: LAUNCHABLE_APPS, launchApp: (appId) => this.launchApp(appId), requestShellRender: () => this.requestShellRender(), }); // The action bar's EvenHub phone-UI button hides while that UI is showing. onPhoneShownChanged(() => this.emit()); shell.configure({ actions: { ...sharedActions, // Shell overlays (voice dialog, long-press menu) live on the shell // surface, so their repaints go through the shell render path. requestRender: () => this.requestShellRender(), }, getScreenTimeoutMs: () => this.glassesLocked ? LOCK_SCREEN_TIMEOUT_MS : screenTimeoutSettingToMs(screenTimeoutSetting.get()), requestShellRender: () => this.requestShellRender(), prepareVoiceCapture: () => this.prepareVoiceCapture(), onKeyboardInputChanged: (session) => { this.keyboardInput = session; this.emit(); }, onWindowsChanged: () => { this.persistOpenApps(); // The foreground title is mirrored on both remote-control faces. this.emit(); }, onScreenStateChanged: (on) => { // Any wake of the regular UI replaces a showing Glanceboard. if (on) this.glance.dismiss(); this.handleScreenStateChanged(on); if (on) this.requestShellRender(); this.emit(); }, }); // Boot hooks register windows that exist from startup (the launcher, // pinned first in the sidebar and the boot foreground). for (const app of ALL_APPS) { app.boot?.(this.buildAppContext(app)); } timerEngine.onChange(() => this.refreshAlarmReliabilityWarning()); this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => { void this.handleAndroidNotificationPosted(notificationKey).catch((error) => { this.appendLog(`notification wake failed: ${this.formatError(error)}`); }); }); readActiveNotifications(ALL_NOTIFICATIONS); // Settings toggled from the glasses can change what the phone UI shows // (e.g. the text-setting editor), so re-emit the snapshot on any change. onAnySettingChanged(() => { this.emit(); // Apply a firmware-debug-flags toggle live while connected (Java dedups). this.pushFirmwareDebugFlags(); this.pushBrightness(); this.syncEvenHubScreenOffSetting(); this.applyVerticalPositionIfChanged(); this.applyDisplayModeIfChanged(); this.applyAppLayoutsIfChanged(); this.syncAssistantBridgeIfChanged(); this.syncLockScreenSettingIfChanged(); // A Preview color change should show on the mirror at once, not at the // next composited frame (cheap: floor-limited, dedup'd on the Java side). this.schedulePreviewUpdate(); }); // Connect to the external agent bridge at boot if configured; the // connection stays up (with re-dial) so proactive tool calls work // outside voice turns. this.syncAssistantBridge(); // The watch drives the same synthetic-input path as the phone UI's test // buttons, plus app/window/lock commands; it mirrors the state below. startRemoteInput({ ready: () => this.phase === "connected" || this.phase === "charging", locked: () => this.glassesLocked, input: (gesture, source) => this.injectSyntheticRingInput(gesture, source), acceptsText: () => !!shell.foregroundWindow()?.receiveTextInput, text: (text, submit) => { if (!shell.isScreenOn()) shell.wake("window"); shell.sendTextToForegroundWindow(text, { submit }); this.requestShellRender(); }, assistantAvailable: () => shell.isAssistantAvailable(), assistant: text => shell.sendToAssistant(text), }); this.wearRemote = new WearRemote({ apps: LAUNCHABLE_APPS, injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"), launchApp: (appId) => this.launchApp(appId), connect: () => this.connect(), disconnect: () => this.disconnect(), setGlassesLocked: (locked, reason) => this.setGlassesLocked(locked, reason), requestShellRender: () => this.requestShellRender(), getState: () => ({ phase: this.phase, status: this.status, glassesLocked: this.glassesLocked, glassesWorn: this.glassesWorn, silentMode: this.silentMode, lastHeadsetBattery: this.lastHeadsetBattery, }), appendLog: (line) => this.appendLog(line), }); } // Bridge settings changes re-dial the connection; unrelated setting changes // must not, so the config it was last started with is tracked. private lastBridgeConfigKey = ""; private bridgeConfigKey(): string { return JSON.stringify([ assistantBackendSetting.get(), assistantBridgeHostSetting.get().trim(), assistantBridgePortSetting.get().trim(), assistantBridgeTokenSetting.get(), ]); } private syncAssistantBridgeIfChanged(): void { if (this.bridgeConfigKey() === this.lastBridgeConfigKey) return; this.syncAssistantBridge(); } private syncAssistantBridge(): void { this.lastBridgeConfigKey = this.bridgeConfigKey(); const host = assistantBridgeHostSetting.get().trim(); const token = assistantBridgeTokenSetting.get(); if (assistantBackendSetting.get() !== "external" || !host || !token) { assistantBridge.stop(); return; } assistantBridge.configure({ host, port: parseInt(assistantBridgePortSetting.get(), 10) || 8790, token, deviceName: "faceclaw", allowProactive: () => assistantAllowProactiveSetting.get(), }); } // Vertical-position changes move every window surface (and the chrome that // aligns with them); the value is tracked so unrelated setting changes // don't trigger a full reposition. private lastVerticalPosition = verticalPositionSetting.get(); private lastDisplayMode = displayModeSetting.get(); /** * Display mode changed (Settings > Display, or the phone page's picker): * every window's viewport size changes. In-process windows re-measure in * place; workers that support resizing receive the new viewport. Other * worker windows are closed and launched again at the new size. */ private applyDisplayModeIfChanged(): void { const mode = displayModeSetting.get(); if (mode === this.lastDisplayMode) return; this.lastDisplayMode = mode; this.appendLog(`display mode: ${mode}`); const foregroundWindowId = shell.foregroundWindow()?.windowId; void (async () => { const relaunch: string[] = []; for (const window of Array.from(shell.getWindows())) { if (window.relayout) { window.relayout(); await this.configureWindowSurface( window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode, ); } else if (window.closeable) { relaunch.push(window.appId); shell.closeWindow(window.windowId); } } for (const appId of relaunch) { await this.launchApp(appId); } if (foregroundWindowId && shell.getWindows().some((w) => w.windowId === foregroundWindowId)) { shell.focusWindow(foregroundWindowId); } shell.foregroundWindow()?.requestRender(); this.requestShellRender(); })().catch((error) => { this.appendLog(`display mode change failed: ${this.formatError(error)}`); }); } private lastAppLayouts = this.appLayouts(); private appLayouts(): Record { return { navigate: `${navigateDisplayModeSetting.get()}:${navigateVerticalPositionSetting.get()}`, terminal: `${terminalDisplayModeSetting.get()}:${terminalVerticalPositionSetting.get()}`, }; } private applyAppLayoutsIfChanged(): void { const layouts = this.appLayouts(); const changed = Object.keys(layouts).filter((appId) => layouts[appId] !== this.lastAppLayouts[appId]); if (!changed.length) return; this.lastAppLayouts = layouts; void (async () => { for (const window of Array.from(shell.getWindows())) { if (!changed.includes(window.appId)) continue; await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode); window.relayout?.(); } shell.foregroundWindow()?.requestRender(); this.requestShellRender(); })().catch((error) => this.appendLog(`app layout change failed: ${this.formatError(error)}`)); } private applyVerticalPositionIfChanged(): void { const position = verticalPositionSetting.get(); if (position === this.lastVerticalPosition) return; this.lastVerticalPosition = position; void (async () => { for (const window of shell.getWindows()) { await this.configureWindowSurface( window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode, ); } // Repaint after the surfaces have moved: window content is unchanged // but the moved surfaces need a recomposite, and the chrome band moved. shell.foregroundWindow()?.requestRender(); this.requestShellRender(); })().catch((error) => { this.appendLog(`vertical reposition failed: ${this.formatError(error)}`); }); } private handleScreenStateChanged(on: boolean): void { if (on) { this.cancelEvenHubSuspendTimer(); void this.ensureEvenHubSessionActive().catch((error) => { this.appendLog(`screen wake failed: ${this.formatError(error)}`); }); return; } const communicator = this.communicator; if (communicator) { // Blanking is a compositor-level flag so worker-window surfaces go dark // too; retained state survives while the EvenHub page is absent. void (async () => { await communicator.setScreenBlanked(true); await communicator.setG2ScreenOn(false); })().catch((error) => { this.appendLog(`screen sleep failed: ${this.formatError(error)}`); }); this.scheduleEvenHubSuspend(); return; } const preview = this.previewTarget; if (!preview) return; // Preview mode: the mirror goes dark like the lens would; retained // surfaces survive for the next wake. void preview .setScreenBlanked(true) .then(() => this.schedulePreviewUpdate()) .catch((error) => { this.appendLog(`preview screen sleep failed: ${this.formatError(error)}`); }); } private syncLockScreenSettingIfChanged(): void { const enabled = lockScreenEnabledSetting.get(); if (enabled === this.lastLockScreenEnabled) return; this.lastLockScreenEnabled = enabled; if (!enabled) { this.setGlassesLocked(false, "lock screen disabled"); return; } if (this.phoneLocked && this.glassesWorn === false) { this.setGlassesLocked(true, "lock screen enabled while glasses are off-head"); } this.ensureWearStateTracking(); } private handleWearState(wearing: boolean): void { // Only an observed OFF_HEAD -> ON_HEAD transition counts as putting the // glasses on; the first snapshot of a session (null) is just the status. const putOn = wearing && this.glassesWorn === false; this.glassesWorn = wearing; updateGlassesPresence({ worn: wearing }); this.appendLog(wearing ? "glasses wear state: ON_HEAD" : "glasses wear state: OFF_HEAD"); this.wearRemote?.schedulePublish(); if (!wearing && this.phoneLocked && lockScreenEnabledSetting.get()) { this.setGlassesLocked(true, "glasses removed while phone locked"); } if (!wearing) this.wakeAfterChargingReconnect = false; if (putOn && this.phase === "charging") { // They can't be worn in the case, so charging is over even though no // battery poll has said so yet. Java reconnects on this same report // (the wake barrier needs the rebuilt session), so wake after that. this.wakeAfterChargingReconnect = true; this.appendLog("glasses put on while charging; waking after reconnect"); } else if (putOn) { this.wakeForGlassesPutOn(); } } /** * Putting the glasses on wakes the screen, like a double-tap would. While * locked that shows the lock-screen notice (which then times out on its * own short timer); otherwise it lands on the sidebar as a normal wake. */ private wakeForGlassesPutOn(): void { if (this.phase !== "connected" || !this.communicator || shell.isScreenOn()) return; // wake() -> onScreenStateChanged(true) dismisses a showing Glanceboard, // runs the EvenHub wake barrier and repaints the shell. if (shell.wake("sidebar")) this.appendLog("screen woken: glasses put on"); } /** * Finish a put-on that arrived while charging. Each arm reports "connected" * as it links, before the prelude; the ready session reports it again. */ private maybeWakeAfterChargingReconnect(): void { if (!this.wakeAfterChargingReconnect || !this.communicator?.isSessionReady()) return; this.wakeAfterChargingReconnect = false; this.wakeForGlassesPutOn(); } private handlePhoneLockState(locked: boolean): void { if (locked === this.phoneLocked) return; this.phoneLocked = locked; this.appendLog(`phone lock state: ${locked ? "locked" : "unlocked"}`); if (!locked) { this.setGlassesLocked(false, "phone unlocked"); } else if (this.glassesWorn === false && lockScreenEnabledSetting.get()) { this.setGlassesLocked(true, "phone locked while glasses are off-head"); } } private setGlassesLocked(locked: boolean, reason: string): void { if (locked === this.glassesLocked) return; this.glassesLocked = locked; this.appendLog(`glasses ${locked ? "locked" : "unlocked"}: ${reason}`); this.wearRemote?.schedulePublish(); void this.syncLockSurface().catch((error) => { this.appendLog(`lock screen update failed: ${this.formatError(error)}`); }); } private ensureWearStateTracking(): void { const communicator = this.communicator; // Charging counts: a session that starts in the case (firmware confirmed // only after charging mode began) needs the OFF_HEAD snapshot so that // putting the glasses on registers as a put-on. if ( !lockScreenEnabledSetting.get() || !this.customFirmwareConfirmed || (this.phase !== "connected" && this.phase !== "charging") || !communicator ) { return; } void communicator.enableWearDetectionAndRequestState().catch((error) => { this.appendLog(`wear detector setup failed: ${this.formatError(error)}`); }); } private async configureLockSurface(communicator: FaceclawCommunicatorBridge): Promise { await communicator.configureSurface(LOCK_SCREEN_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: 1000, transparency: "opaque", }); await communicator.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, false); const image = createLockScreenImage(); await communicator.submitSurfaceFrame( LOCK_SCREEN_SURFACE_ID, image.to8bppBuffer(), { x: 0, y: 0, width: image.width, height: image.height }, image.fingerprint(), ); if (this.communicator === communicator) { this.lockSurfaceConfigured = true; } } private async syncLockSurface(): Promise { const communicator = this.communicator; if (!communicator || !this.lockSurfaceConfigured) return; await communicator.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, this.glassesLocked); this.updateCompositePreview(); } /** * The single wake barrier for every source: restore the EvenHub lifecycle, * unblank the retained compositor, and resolve only after that frame is * visible. Concurrent display-wake, shell, and wakeword callbacks share the * same operation. */ /** * frameId (0 when the caller has none) attributes the barrier's four * round trips to the input frame waiting on them; a wake barrier is the * largest single input-to-display term there is, so knowing which step * owns the second matters. Callers that join an in-flight barrier share * the first caller's spans. */ private ensureEvenHubSessionActive(frameId = 0): Promise { if (this.evenHubResumePromise) return this.evenHubResumePromise; const communicator = this.communicator; if (!communicator) { // Preview mode has no plugin session to restore; waking is just // unblanking the headless compositor so the mirror lights back up. const preview = this.previewTarget; if (!preview) return Promise.resolve(false); return preview.setScreenBlanked(false).then(() => { this.schedulePreviewUpdate(); return true; }); } if (this.phase === "charging" || this.phase === "disconnected") { return Promise.resolve(false); } const operation = (async () => { await frameTimings.spanAsync(frameId, "wake:screen-on", () => communicator.setG2ScreenOn(true)); const resumed = await frameTimings.spanAsync(frameId, "wake:resume-session", () => communicator.resumeEvenHubSession(), ); if (!resumed) { return false; } // Resume first: setScreenBlanked(false) then recomposites retained state // as the desired first frame for the fresh layout. await frameTimings.spanAsync(frameId, "wake:unblank", () => communicator.setScreenBlanked(false)); const ready = await frameTimings.spanAsync(frameId, "wake:await-ready", () => communicator.awaitEvenHubSessionReady(EVENHUB_WAKE_READY_TIMEOUT_MS), ); if (ready && this.communicator === communicator) { this.evenHubSessionSuspended = false; } return ready; })(); this.evenHubResumePromise = operation; const clearOperation = () => { if (this.evenHubResumePromise === operation) { this.evenHubResumePromise = null; } }; void operation.then(clearOperation, clearOperation); return operation; } private desiredFaceclawWakeLeaseState(): boolean { // The same firmware lease owns two stock wake paths: deferred dashboard // launch while EvenHub is suspended, and suppression of the stock Even AI // foreground app when Faceclaw handles the glasses wakeword. Without the // latter, Even AI displaces EvenHub before our first wake frame can ACK. return ( this.customFirmwareConfirmed && (suspendEvenHubWhenScreenOffSetting.get() || wakeWordActionSetting.get() !== "off") ); } private async syncFaceclawWakeLease( communicator = this.communicator, force = false, ): Promise { if (!communicator || communicator !== this.communicator) return false; const enabled = this.desiredFaceclawWakeLeaseState(); if (!force && this.faceclawWakeLeaseState === enabled) return true; const applied = await communicator.setFaceclawWakeLeaseEnabled(enabled); if (this.communicator === communicator) { this.faceclawWakeLeaseState = applied || !enabled ? enabled : null; } return applied; } private syncEvenHubScreenOffSetting(): void { void this.syncFaceclawWakeLease().catch((error) => { this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`); }); if (suspendEvenHubWhenScreenOffSetting.get() && !shell.isScreenOn()) { this.scheduleEvenHubSuspend(); return; } this.cancelEvenHubSuspendTimer(); if (!this.evenHubSessionSuspended) return; // Turning the experiment off restores the legacy all-black page even if // the screen remains asleep. const communicator = this.communicator; if (!communicator) return; void communicator .resumeEvenHubSession() .then((resumed) => { if (resumed && this.communicator === communicator) { this.evenHubSessionSuspended = false; } }) .catch((error) => { this.appendLog(`EvenHub resume failed: ${this.formatError(error)}`); }); } private scheduleEvenHubSuspend(): void { this.cancelEvenHubSuspendTimer(); if ( !suspendEvenHubWhenScreenOffSetting.get() || shell.isScreenOn() || this.glance.isVisible() || this.phase !== "connected" || !this.communicator || this.evenHubSessionSuspended ) { return; } const communicator = this.communicator; this.evenHubSuspendTimer = setTimeout(() => { this.evenHubSuspendTimer = null; if ( !suspendEvenHubWhenScreenOffSetting.get() || shell.isScreenOn() || this.glance.isVisible() || this.phase !== "connected" || this.communicator !== communicator ) { // A showing Glanceboard re-arms this when it hides. return; } if (voiceControlBridge.isCaptureHeld()) { // Ending the plugin task ends the mic with it, so a live capture (the // Transcribe window, typically) outranks the screen-off power save. // Check before the lease refresh below, which costs a BLE message. this.scheduleEvenHubSuspend(); return; } void (async () => { if (this.customFirmwareConfirmed) { // Refresh immediately before teardown so the first suspended wake // cannot race an old lease's expiry. const leaseReady = await this.syncFaceclawWakeLease(communicator, true); if (!leaseReady) { this.appendLog("EvenHub suspend deferred; wake takeover lease was not delivered"); this.scheduleEvenHubSuspend(); return; } } if ( !suspendEvenHubWhenScreenOffSetting.get() || shell.isScreenOn() || this.glance.isVisible() || this.phase !== "connected" || this.communicator !== communicator ) { return; } this.evenHubSessionSuspended = true; return communicator.suspendEvenHubSession(); })() .then((suspended) => { if (suspended === undefined) return; if (suspended) { // A capture that raced the suspend lost its mic with the plugin // task; park it for the resume path. voiceControlBridge.handleSessionEnded(); this.appendLog("EvenHub session suspended while screen is off"); return; } if (this.communicator !== communicator || shell.isScreenOn()) return; this.evenHubSessionSuspended = false; this.scheduleEvenHubSuspend(); }) .catch((error) => { if (this.communicator === communicator) { this.evenHubSessionSuspended = false; this.appendLog(`EvenHub suspend failed: ${this.formatError(error)}`); this.scheduleEvenHubSuspend(); } }); }, EVENHUB_SCREEN_OFF_SUSPEND_DELAY_MS); } private cancelEvenHubSuspendTimer(): void { if (this.evenHubSuspendTimer === null) return; clearTimeout(this.evenHubSuspendTimer); this.evenHubSuspendTimer = null; } private pushFirmwareDebugFlags(): void { if (!this.communicator) return; void this.communicator .setFirmwareDebugFlags(firmwareDebugFlagsSetting.get()) .catch(() => {}); } /** * Push the brightness setting to the glasses. Deduped TS-side (the change * listener fires for every setting, and each push queues a real BLE * message); `force` skips the dedup so a fresh connection always gets the * configured value regardless of what the firmware restored. */ private pushBrightness(force = false): void { if (!this.communicator) return; const preferences = getBrightnessPreferences(); const value = JSON.stringify(preferences); if (!force && value === this.lastPushedBrightness) return; this.lastPushedBrightness = value; void this.communicator.configureBrightness(preferences).catch((error) => { this.lastPushedBrightness = null; this.appendLog(`brightness configuration failed: ${this.formatError(error)}`); }); } /** Save the occupied part of the composited screen as a 4-bit grayscale PNG. */ saveScreenshot(): string { const path = this.display?.saveScreenshot(shell.screenshotCropRect()) ?? ""; this.appendLog(path ? `screenshot saved: ${path}` : "screenshot skipped: no display"); return path; } /** * Begin collecting composited frames for an animated-GIF screen recording. * Frames are captured at the same points the phone-side preview is * refreshed, so the recording matches what the phone display showed. */ startScreenRecording(): void { if (this.screenRecordingActive) return; const display = this.display; if (!display) { this.appendLog("screen recording skipped: no display"); return; } display.startScreenRecording(); // Capture the starting frame immediately rather than waiting for the // next preview flush. display.recordScreenFrame(); this.screenRecordingActive = true; this.appendLog("screen recording started"); this.emit(); } /** Finish the recording and save it as an animated GIF; returns the path or "". */ stopScreenRecording(): string { if (!this.screenRecordingActive) return ""; this.screenRecordingActive = false; let path = ""; try { path = this.display?.stopScreenRecording() ?? ""; } finally { this.emit(); } this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames"); return path; } subscribe(listener: DashboardListener): () => void { this.listeners.add(listener); listener(this.snapshot()); return () => { this.listeners.delete(listener); }; } snapshot(): DashboardSnapshot { const primaryTextSetting = this.activeTextSettings[0] ?? null; const secondaryTextSetting = this.activeTextSettings[1] ?? null; return { phase: this.phase, status: this.status, displayPreview: this.displayPreview, displayPreviewMessage: this.displayPreviewMessage(), activeTextSettingId: primaryTextSetting?.id ?? null, activeTextEditorTitle: this.activeTextEditorTitle, activeTextSettingTitle: primaryTextSetting?.editorTitle ?? "", activeTextSettingValue: primaryTextSetting?.get() ?? "", activeTextSettingInputKind: primaryTextSetting?.inputKind ?? "text", secondaryTextSettingId: secondaryTextSetting?.id ?? null, secondaryTextSettingTitle: secondaryTextSetting?.editorTitle ?? "", secondaryTextSettingValue: secondaryTextSetting?.get() ?? "", secondaryTextSettingInputKind: secondaryTextSetting?.inputKind ?? "text", activeTextEditorToggleLabel: this.activeTextEditorToggle?.label ?? "", activeTextEditorToggleValue: this.activeTextEditorToggle?.setting.get() ?? false, activeTextEditorToggleVisible: this.activeTextEditorToggle !== null, keyboardInputActive: this.keyboardInput !== null, keyboardInputTargets: this.keyboardInput?.targets ?? [], evenAppConflictMessage: this.evenAppConflictMessage, evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0, firmwareWarningMessage: this.firmwareWarningMessage, firmwareWarningVisible: this.firmwareWarningMessage.length > 0, screenRecordingActive: this.screenRecordingActive, batteryOptimizationWarningVisible: this.batteryOptimizationWarningVisible, fontsMissingWarningVisible: this.fontsMissingWarningVisible, alarmReliabilityMessage: this.alarmReliabilityMessage, previewMode: this.isPreviewDisplayActive(), evenHubPhoneUi: phoneUiButton(), }; } /** Overlay an EvenHub app's phone UI (the action bar's app-icon button). */ showEvenHubPhoneUi(windowId: string): void { showEvenHubPhoneUi(windowId); } /** * Message to show in place of the display preview, or "" to show the preview. * * These states look identical to a dead pair of glasses from the phone side: * charging and silent mode both make the display unavailable while BLE stays * up, and a battery that just ran out simply stops answering. */ private displayPreviewMessage(): string { if (this.phase === "charging") { return this.glassesDisplayLabel(); } if (this.silentMode && this.phase === "connected") { // Say how to leave it. Silent mode is entered and exited only by the // same gesture on the glasses, the firmware ignores all input and blanks // the display while it is on, and nothing here can turn it off -- there // is no setter, only onSilentMode. So a user who triggered it by // accident sees a connected pair of glasses that answers nothing, with // no way to find out why. The instruction belongs where they are looking. return "Connected (Silent mode — long-press both touchpads on the glasses to exit)"; } // The preview compositor keeps the mirror live (including as a black // frame while the simulated screen is off), so never cover it. if (this.isPreviewDisplayActive()) { return ""; } const connectionFailing = this.phase === "disconnected" || this.phase === "connecting"; if ( connectionFailing && this.lastHeadsetBattery !== null && this.lastHeadsetBattery < LOW_BATTERY_PERCENT ) { return "Disconnected (low battery)"; } return ""; } /** What occupies the foreground-title line on the watch-style controls. */ glassesDisplayLabel(): string { return glassesDisplayLabel({ phase: this.phase, silentMode: this.silentMode, screenOn: shell.isScreenOn(), battery: this.lastHeadsetBattery, foregroundTitle: shell.getForegroundApp()?.title ?? null, previewMode: this.isPreviewDisplayActive(), }); } /** * Re-check the Doze exemption (cheap system call, but cached so snapshot() * stays trivial). Called on connect, page load, and after the user answers * the system exemption dialog. */ refreshBatteryOptimizationStatus(): void { const warningVisible = !isIgnoringBatteryOptimizations(); if (warningVisible !== this.batteryOptimizationWarningVisible) { this.batteryOptimizationWarningVisible = warningVisible; this.emit(); } } requestBatteryOptimizationExemption(): void { requestIgnoreBatteryOptimizations(); // The system dialog is asynchronous and there is no result callback from // this context; poll briefly so the banner clears once granted. let checksLeft = 12; const poll = setInterval(() => { this.refreshBatteryOptimizationStatus(); if (!this.batteryOptimizationWarningVisible || --checksLeft <= 0) { clearInterval(poll); } }, 5_000); } /** * Warn when glasses are paired but the phone-side G2 fonts were never * extracted (the onboarding flash flow normally extracts them, but a dev * install or an imported config can arrive paired without them). Preview-only * users have no addresses configured and are never warned. */ refreshEvenHubFontStatus(): void { const addresses = loadDeviceAddresses(); const paired = isValidMacAddress(addresses.right) && isValidMacAddress(addresses.left); if (paired && !this.extractedFontsConfirmed) { this.extractedFontsConfirmed = hasExtractedEvenHubFonts(); } const warningVisible = paired && !this.extractedFontsConfirmed; if (warningVisible !== this.fontsMissingWarningVisible) { this.fontsMissingWarningVisible = warningVisible; this.emit(); } } /** * Mirror the Timers engine's phone self-check into the warnings modal. The * engine re-checks on its own schedule and on every timer/alarm change; * a page load forces a fresh check so a just-fixed setting clears at once. */ refreshAlarmReliabilityWarning(force = false): void { if (force) timerEngine.refreshReliability(true); const message = timerEngine.phoneReliabilityMessage(); if (message !== this.alarmReliabilityMessage) { this.alarmReliabilityMessage = message; this.emit(); } } openAlarmReliabilityFix(): void { timerEngine.openPhoneReliabilityFix(); // The system screen is asynchronous with no result; poll briefly so the // warning clears once the setting is changed. let checksLeft = 12; const poll = setInterval(() => { this.refreshAlarmReliabilityWarning(true); if (!this.alarmReliabilityMessage || --checksLeft <= 0) clearInterval(poll); }, 5_000); } refreshEvenAppStatus(): void { this.refreshBatteryOptimizationStatus(); this.refreshEvenHubFontStatus(); this.refreshAlarmReliabilityWarning(true); const state = readEvenAppNotificationState(); const wasActive = this.evenNotificationActive; this.evenNotificationActive = state.evenNotificationActive; if (state.evenNotificationActive && !wasActive) { this.appendLog("Even app notification is active."); } if (state.evenNotificationActive && !this.evenAppConflictMessage) { this.evenAppConflictMessage = EVEN_APP_DETECTED_MESSAGE; this.emit(); } if (!state.evenNotificationActive && this.evenAppConflictMessage) { this.evenAppConflictMessage = ""; this.emit(); } } openEvenAppSettings(): void { openEvenAppSettings(); } setActiveTextSettingValue(value: string, settingId?: string): void { shell.noteUserActivity(); const setting = settingId ? this.activeTextSettings.find((candidate) => candidate.id === settingId) ?? null : this.activeTextSettings[0] ?? null; if (!setting) return; if (setting.get() === value) return; setting.set(value); // Deliberately do NOT emit() here. The phone TextField is the source of // truth while typing; echoing activeTextSettingValue back into its two-way // binding on every keystroke drops fast/pasted characters (observed: a // 51-char API key stored as its first 46 chars). Just refresh the preview. this.previewOrRenderAfterTextSettingChange(); } /** * The phone's keyboard button: open the keyboard dialog on the glasses * (the typed twin of the mic button's wakeword). Needs a display to draw * on and, like ring input, does nothing while the glasses are locked. */ startKeyboardInput(): void { if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return; if (this.glassesLocked) { this.appendLog("keyboard input ignored while the glasses are locked"); return; } shell.startKeyboardInput(); } /** The phone's typing panel changed: mirror the text on the glasses. */ setKeyboardInputText(text: string): void { shell.noteUserActivity(); this.keyboardInput?.setText(text); } /** Send to one destination, or (no id: the IME's send key) the one highlighted on the glasses. */ sendKeyboardInput(targetId?: string): void { if (targetId) { this.keyboardInput?.sendTo(targetId); } else { this.keyboardInput?.send(); } } discardKeyboardInput(): void { this.keyboardInput?.discard(); } setActiveTextEditorToggleValue(value: boolean): void { if (!this.activeTextEditorToggle || this.activeTextEditorToggle.setting.get() === value) return; this.activeTextEditorToggle.setting.set(value); this.emit(); } /** * Where composited frames go and where the phone mirror, screenshots, and * recordings come from: the live glasses connection when there is one, * otherwise the headless preview compositor (preview-only mode), otherwise * nothing (frames are discarded). */ private get display(): DisplayTarget | null { return this.communicator ?? this.previewTarget; } private isPreviewDisplayActive(): boolean { return this.communicator === null && this.previewTarget !== null; } /** * Bring up the headless preview display for preview-only mode (onboarding's * "Preview Only" path): the same shell/app/compositor pipeline as a live * session minus the BLE transport, so the phone mirror and its touch/watch * controls work with no glasses paired. Idempotent; called from the main * page alongside autoConnect. A real connection outranks it — connect() * tears it down before creating the communicator. */ async ensurePreviewDisplay(): Promise { if (!isPreviewOnlyMode()) return; if (this.communicator || this.previewTarget || this.connectRunning) return; const target = new PreviewDisplayTarget(); // Published before the awaits below so a second call (or connect()) sees // it; the target's calls all complete in microtasks, so the screen size // is in place before any real render can reach the compositor. this.previewTarget = target; try { await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); await target.configureSurface(SHELL_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: SHELL_SURFACE_Z_ORDER, transparency: "color-key", }); // A fresh (or reused) compositor starts undimmed; the shell render // loop re-applies the current dim from here. await target.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1); this.appliedUnderlayDim = 1; for (const window of shell.getWindows()) { await this.configureWindowSurface( window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode, target, ); } } catch (error) { if (this.previewTarget === target) this.previewTarget = null; this.appendLog(`preview display setup failed: ${this.formatError(error)}`); return; } // Workers find the compositor through its Java static; the per-frame // callback keeps the mirror current (the connected path gets the same // from Java's frame metrics). target.activate(() => this.schedulePreviewUpdate()); this.setStatus("Preview mode (no glasses paired)."); this.appendLog("Preview-only display active; frames render to the phone mirror only."); shell.foregroundWindow()?.requestRender(); this.requestShellRender(); // The same recurring upkeep a live session gets: top-bar clock refresh, // the preview safety-net poll, and the screen timeout (a timed-out // preview screen wakes on double-tap, like the glasses would). this.shellRefreshTimer = setInterval(() => { this.requestShellRender(); this.updateCompositePreview(); }, SHELL_REFRESH_INTERVAL_MS); this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS); this.screenTimeoutTimer = setInterval(() => { if (!this.isPreviewDisplayActive()) return; if (!shell.applyScreenTimeout()) return; this.endTextSettingEdit(); this.requestShellRender(); }, SCREEN_TIMEOUT_CHECK_MS); this.emit(); } /** Drop the headless preview display (a real connection is taking over). */ private teardownPreviewDisplay(): void { const target = this.previewTarget; if (!target) return; this.previewTarget = null; this.glance.reset(); target.release(); this.clearDashboardTimer(); this.appendLog("Preview-only display released."); } async connect(): Promise { if (this.phase !== "disconnected") return; // Connecting is the explicit way out of the manual-disconnected state. resumeAutoReconnect(); const addresses = loadDeviceAddresses(); if (!addresses.right || !addresses.left) { const message = "Configure both left and right arm MAC addresses before connecting."; this.setPhase("disconnected"); this.setStatus(`Failed: ${message}`); this.appendLog(`error: ${message}`); throw new Error(message); } // Glasses are configured, so the real pipeline owns the display from here // (also frees the preview timers for connect() to replace). this.teardownPreviewDisplay(); this.lastInput = "waiting..."; this.lastSys = "none yet"; this.welcomeSoundArmed = isWelcomeSoundPending(); this.firmwareWarningMessage = ""; this.glassesWorn = null; this.wakeAfterChargingReconnect = false; this.glassesLocked = false; this.refreshBatteryOptimizationStatus(); this.refreshEvenAppStatus(); this.setPhase("connecting"); this.setStatus("Connecting to the glasses..."); // "Only via glasses" (the default) means the phone never opens its own // link to the ring: the glasses relay its gestures to us anyway, and the // direct link is currently unreliable. An empty address disables every // direct-ring code path in the communicator. const ringAddress = ringConnectionModeSetting.get() === "direct" ? addresses.ring : ""; this.appendLog( `Using configured arms: R=${addresses.right} L=${addresses.left}${ringAddress ? ` ring=${ringAddress}` : ""}`, ); let communicator: FaceclawCommunicatorBridge | null = null; this.customFirmwareConfirmed = false; this.faceclawWakeLeaseState = null; this.lockSurfaceConfigured = false; this.glance.reset(); this.evenHubResumePromise = null; this.connectRunning = true; try { await ensureBlePermissions(); startForegroundNotification("Connecting to the glasses"); communicator = new FaceclawCommunicatorBridge({ right: addresses.right, left: addresses.left, ring: ringAddress, }); this.communicator = communicator; // Size the compositor before anything else is awaited: windows opened // during the bridge start-ups below (the terminal hub, restored apps) // configure their surfaces through this.communicator, and the Java call // queue is FIFO, so enqueuing the screen size first guarantees it lands // ahead of them. Mirrors ensurePreviewDisplay. await communicator.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT); await communicator.configureBrightness(getBrightnessPreferences()); this.offState = communicator.onStateChange((state) => { if (state.phase !== "connected") resetRingInputFilter(); if (state.phase === "unpaired") { // Java parked its retry loop: an arm's Android bond is gone, so // every redial would fail the same way until the user re-pairs. // Tear down into the manual-disconnected state and keep the // re-pair instruction as the visible status. this.scheduleUnpairedDisconnect(state.status); return; } const mappedPhase = state.phase === "connected" ? "connected" : state.phase === "charging" ? "charging" : state.phase === "disconnecting" ? "disconnecting" : state.phase === "disconnected" ? "disconnected" : "connecting"; if (mappedPhase === "charging" && this.phase !== "charging") { // Nobody is wearing the glasses; drop the G2-screen wakelock so the // phone can sleep normally while they charge. void this.communicator?.setG2ScreenOn(false).catch(() => {}); } if (mappedPhase === "connected" && this.phase !== "connected") { // A transport reconnect starts with a live EvenHub lifecycle again; // if the shell is asleep, begin a fresh five-second grace period. this.evenHubSessionSuspended = false; // Firmware leases are volatile, and the Java transport may have been // rebuilt underneath this long-lived controller. Do not let a cached // TypeScript value suppress re-acquisition on the new session. this.faceclawWakeLeaseState = null; // Push the CFW firmware-debug-flags overlay preference; Java emits the // mode-7 control message once the dashboard container is warmed up. this.pushFirmwareDebugFlags(); this.pushBrightness(true); } if (mappedPhase !== "connected" && mappedPhase !== "charging") { // A wear snapshot is session-scoped. CFW reports a fresh value when // the transport comes back, so do not make lock decisions from a // stale pre-disconnect value in the meantime. Charging keeps the // link (and the value), so putting the glasses on registers. this.glassesWorn = null; updateGlassesPresence({ worn: null }); } if (mappedPhase !== "connected") { // The mic enable was session-scoped: park any live capture so // the next session restarts it, instead of leaving a holder that // makes every later request think the mic is already running. voiceControlBridge.handleSessionEnded(); } this.setPhase(mappedPhase); this.setStatus(state.status); if (mappedPhase === "connected") { this.syncEvenHubScreenOffSetting(); this.ensureWearStateTracking(); this.maybeWakeAfterChargingReconnect(); } else if (mappedPhase !== "charging") { this.cancelEvenHubSuspendTimer(); this.evenHubSessionSuspended = false; } }); this.offRing = communicator.onRingEvent((event) => { void this.handleInputEvent(event).catch((error) => { const message = this.formatError(error); this.appendLog(`input handler failed: ${message}`); }); }); this.offSilentMode = communicator.onSilentMode((silent) => { if (this.silentMode === silent) return; this.silentMode = silent; this.emit(); }); this.offWearState = communicator.onWearState((wearing) => { this.handleWearState(wearing); }); this.offPhoneLockState = communicator.onPhoneLockState((locked) => { this.handlePhoneLockState(locked); }); this.offBattery = communicator.onBatteryState((state) => { // An unavailable reading must not erase the useful last-known value, // especially while the glasses remain reachable in their case. const hasBatteryLevel = Number.isInteger(state.battery) && state.battery >= 0 && state.battery <= 100; if (hasBatteryLevel) this.lastHeadsetBattery = state.battery; const ringBattery = state.ringBattery; const hasRingBattery = typeof ringBattery === "number" && Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100; shell.setBatteryLevels({ headset: hasBatteryLevel ? state.battery : this.lastHeadsetBattery, headsetCharging: state.chargingStatus > 0, ring: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null, }); updateGlassesPresence({ charging: state.chargingStatus > 0 || this.phase === "charging" }); if ((this.phase === "connected" || this.phase === "charging") && this.communicator) { // Repaint the top bar (battery indicators live in the shell chrome). this.requestShellRender(); } // Battery belongs in the charging stand-in on the phone as well as in // the watch state, so refresh both consumers on every report. this.emit(); }); this.offEvenAppConflict = communicator.onEvenAppConflict((message) => { this.refreshEvenAppStatus(); if (!this.evenNotificationActive) { this.appendLog(`Even app conflict suspected, but notification was not active: ${message}`); return; } this.evenAppConflictMessage = message; this.appendLog(message); this.emit(); }); this.offFrameMetrics = communicator.onFrameMetrics(() => { // Every composited frame refreshes the phone-side mirror (bounded by // CONNECTED_PREVIEW_MIN_UPDATE_MS) so it tracks the glasses instead // of trailing the 1s safety-net poll. this.schedulePreviewUpdate(); if (this.phase === "connected") { this.setStatus("Connected."); // A rendered frame also means the display path is warm enough for // the glasses to accept a mic enable, so this is where a capture // parked by the previous session (or by an EvenHub suspend) resumes. this.resumeVoiceCapture(); // A rendered frame means the session is warmed up (fixedLayoutCreated), // so the buzzer won't be dropped. Play the one-time welcome sound now. if (this.welcomeSoundArmed) { this.welcomeSoundArmed = false; setWelcomeSoundPending(false); void this.playWelcomeSound(); } } }); this.offFirmwareInfo = communicator.onFirmwareInfo((info) => { this.appendLog( `firmware: L=${info.leftVersion || "?"} R=${info.rightVersion || "?"}` + (info.extension ? ` ext="${info.extension}"` : " (no firmware extension string)"), ); const warning = firmwareIncompatibilityMessage(info) ?? ""; // The firmware contract is all-or-nothing: a compatible revision has // every extension (wake lease, wear notifications, ...) we use. const customFirmwareConfirmed = !warning && hasCompatibleFirmware(info); if (customFirmwareConfirmed !== this.customFirmwareConfirmed) { this.customFirmwareConfirmed = customFirmwareConfirmed; this.faceclawWakeLeaseState = null; void this.syncFaceclawWakeLease().catch((error) => { this.appendLog(`wake takeover lease sync failed: ${this.formatError(error)}`); }); this.ensureWearStateTracking(); } if (warning !== this.firmwareWarningMessage) { this.firmwareWarningMessage = warning; if (warning) { this.appendLog(`firmware compatibility warning: ${warning}`); } this.emit(); } if (warning) { this.scheduleIncompatibleFirmwareDisconnect(); } }); // The Music and Nightscout apps subscribe to their bridges directly and // repaint their own windows, so bridge updates need no controller action. this.offVoiceStatus = voiceControlBridge.onStatus((state) => { this.appendLog(state.status); }); await mediaControllerBridge.start(); await nightscoutBridge.start(); // Register the compositor surfaces: the shell chrome above all windows, // and a surface per live window (only the foreground one is composited). await communicator.configureSurface(SHELL_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: SHELL_SURFACE_Z_ORDER, transparency: "color-key", }); // A fresh (or reused) compositor starts undimmed; the shell render // loop re-applies the current dim from here. await communicator.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1); this.appliedUnderlayDim = 1; await this.configureLockSurface(communicator); for (const window of shell.getWindows()) { await this.configureWindowSurface( window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode, ); } await communicator.start(); await this.syncLockSurface(); this.syncEvenHubScreenOffSetting(); shell.foregroundWindow()?.requestRender(); this.requestShellRender(); // Refresh the top-bar clock and the phone-side preview once a minute, // and keep the Android persistent notification current. this.shellRefreshTimer = setInterval(() => { this.requestShellRender(); this.updateCompositePreview(); this.updateConnectedForegroundNotification(); }, SHELL_REFRESH_INTERVAL_MS); this.previewTimer = setInterval(() => this.updateCompositePreview(), PREVIEW_INTERVAL_MS); this.screenTimeoutTimer = setInterval(() => { if (this.phase !== "connected" || !this.communicator) return; if (!shell.applyScreenTimeout()) return; this.endTextSettingEdit(); this.requestShellRender(); }, SCREEN_TIMEOUT_CHECK_MS); } catch (error) { const message = this.formatError(error); this.offState?.(); this.offState = null; this.offRing?.(); this.offRing = null; this.offBattery?.(); this.offBattery = null; this.offSilentMode?.(); this.offSilentMode = null; this.offWearState?.(); this.offWearState = null; this.offPhoneLockState?.(); this.offPhoneLockState = null; this.offEvenAppConflict?.(); this.offEvenAppConflict = null; this.offFrameMetrics?.(); this.offFrameMetrics = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.stop(); if (communicator) { await communicator.setFaceclawWakeLeaseEnabled(false).catch(() => false); await communicator.close().catch(() => {}); } this.communicator = null; this.lockSurfaceConfigured = false; this.glance.reset(); this.customFirmwareConfirmed = false; this.faceclawWakeLeaseState = null; this.evenHubResumePromise = null; this.clearDashboardTimer(); stopForegroundNotification(); this.setPhase("disconnected"); this.setStatus(`Failed: ${message}`); this.appendLog(`error: ${message}`); throw error; } finally { this.connectRunning = false; } } /** * Incompatible firmware means every message Faceclaw sends is one the * glasses may misinterpret, and a live session fights the flash flow's own * connection. Drop the connection (without the CFW-directed cleanup * messages) and hold in the manual-disconnected state until the user * connects explicitly or installs the custom firmware. */ private scheduleIncompatibleFirmwareDisconnect(): void { if (this.incompatibleDisconnectPending) return; this.incompatibleDisconnectPending = true; const attempt = () => { // Firmware info can arrive while connect() is still mid-flight; let it // finish so the teardown doesn't race its surface setup. if (this.connectRunning) { setTimeout(attempt, 200); return; } this.incompatibleDisconnectPending = false; if (this.phase === "disconnected" || this.phase === "disconnecting") { // The session ended some other way; still stop auto-reconnect from // re-dialing glasses we know can't run Faceclaw. suppressAutoReconnect(); return; } this.appendLog( "Disconnecting: the glasses firmware is incompatible. Auto-reconnect is disabled until you connect manually or install the custom firmware.", ); void this.disconnect({ skipFirmwareCleanup: true }) .then(() => this.setStatus("Disconnected (incompatible firmware).")) .catch((error) => { this.appendLog(`incompatible-firmware disconnect failed: ${this.formatError(error)}`); }); }; setTimeout(attempt, 0); } /** * A connect attempt found an arm whose Android bond is missing, so the Java * worker stopped retrying. Drop into the manual-disconnected state (no * auto-reconnect: it would just fail again) and leave the re-pair * instruction from Java as the status the user sees. */ private scheduleUnpairedDisconnect(message: string): void { if (this.unpairedDisconnectPending) return; this.unpairedDisconnectPending = true; const attempt = () => { // The unpaired report can arrive while connect() is still mid-flight; // let it finish so the teardown doesn't race its surface setup. if (this.connectRunning) { setTimeout(attempt, 200); return; } this.unpairedDisconnectPending = false; if (this.phase === "disconnected" || this.phase === "disconnecting") { // The session ended some other way; still stop auto-reconnect from // re-dialing glasses that are no longer paired. suppressAutoReconnect(); this.setStatus(message); return; } this.appendLog(`Disconnecting: ${message} Auto-reconnect is disabled until you connect manually.`); void this.disconnect({ skipFirmwareCleanup: true }) .then(() => this.setStatus(message)) .catch((error) => { this.appendLog(`unpaired disconnect failed: ${this.formatError(error)}`); }); }; setTimeout(attempt, 0); } /** * Tear down the connection and enter the manual-disconnected state: every * caller is deliberate (the phone/glasses Disconnect actions, entering the * flash flow, an incompatible-firmware bailout), so auto-reconnect stays * off until an explicit connect or a successful firmware install. * * skipFirmwareCleanup: don't send the CFW cleanup/shutdown or wake-lease * messages — used when the firmware is incompatible and would misread them. */ async disconnect(options?: { skipFirmwareCleanup?: boolean }): Promise { suppressAutoReconnect(); // The phone menu offers Disconnect during the connecting phase (it is the // only way out of a reconnection-attempt loop), so a call can land while // connect() is still mid-flight; let it finish so the teardown doesn't // race its surface setup. connect() returns once the Java worker thread // owns the retry loop, so this wait is short. while (this.connectRunning) { await new Promise((resolve) => setTimeout(resolve, 200)); } if (this.phase === "disconnected" || this.phase === "disconnecting") return; this.setPhase("disconnecting"); this.setStatus("Disconnecting..."); this.clearDashboardTimer(); this.offState?.(); this.offState = null; this.offRing?.(); this.offRing = null; this.offBattery?.(); this.offBattery = null; this.offSilentMode?.(); this.offSilentMode = null; this.offWearState?.(); this.offWearState = null; this.offPhoneLockState?.(); this.offPhoneLockState = null; this.offEvenAppConflict?.(); this.offEvenAppConflict = null; this.offFrameMetrics?.(); this.offFrameMetrics = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; this.offVoiceStatus?.(); this.offVoiceStatus = null; const communicator = this.communicator; this.communicator = null; this.lockSurfaceConfigured = false; this.glance.reset(); this.evenHubSessionSuspended = false; this.evenHubResumePromise = null; // The recording's frame store lives in the communicator, so save what has // accumulated rather than silently losing it with the connection. if (this.screenRecordingActive) { this.screenRecordingActive = false; try { const path = communicator?.stopScreenRecording() ?? ""; this.appendLog(path ? `screen recording saved: ${path}` : "screen recording discarded: no frames"); } catch (error) { this.appendLog(`screen recording save failed: ${this.formatError(error)}`); } } try { const skipFirmwareCleanup = options?.skipFirmwareCleanup === true; if (!skipFirmwareCleanup) { const leaseReleased = await communicator?.setFaceclawWakeLeaseEnabled(false).catch((error) => { this.appendLog(`wake takeover lease release failed: ${this.formatError(error)}`); return false; }); if (leaseReleased === true) { this.faceclawWakeLeaseState = false; } } // Quiesce every producer that can enqueue a glasses command before the // firmware cleanup barrier. On cleanup-capable CFW this leaves mode 11 as // Faceclaw's final BLE message before the transport closes. await mediaControllerBridge.stop().catch(() => {}); await nightscoutBridge.stop().catch(() => {}); voiceControlBridge.handleSessionEnded(); const cleanupAcked = skipFirmwareCleanup ? false : await communicator?.sendCfwCleanup().catch((error) => { this.appendLog(`CFW cleanup failed: ${this.formatError(error)}`); return false; }); if (cleanupAcked === true) { this.appendLog("CFW cleanup completed."); } else if (communicator && !skipFirmwareCleanup) { // Older CFWs do not advertise cleanup11. Preserve their established // teardown behavior; close() will also send the legacy FB lease release. const shutdownAcked = await communicator.sendShutdown(0).catch((error) => { this.appendLog(`shutdown command failed: ${this.formatError(error)}`); return false; }); if (shutdownAcked) { this.appendLog("Shutdown command completed (legacy cleanup fallback)."); } else { this.appendLog("Cleanup did not complete before disconnect."); } } await communicator?.close().catch(() => {}); } finally { stopForegroundNotification(); this.customFirmwareConfirmed = false; this.faceclawWakeLeaseState = null; this.setPhase("disconnected"); this.setStatus("Disconnected."); this.appendLog("Disconnected from the glasses."); } } /** * Feed a ring gesture from somewhere other than the ring (the phone UI's * test buttons, the watch). "long-press" is a complete short hold; * "long-press-start" / "long-press-release" let a source with a real * finger-down/finger-up (the watch) hold for as long as the user does. * Such a source also sends "ring-press" at each finger-down, as the ring * does before it knows which gesture the touch will become. */ async injectSyntheticRingInput(kind: WearRemoteInputKind, origin: SyntheticInputOrigin = "ring"): Promise { // Match the ring while the display is dark: a double-click wakes it // (handled by Shell.receiveInput) and a tap or hold shows the // Glanceboard; scrolls and the menu gesture mean nothing. This check also // protects against a watch acting on a stale state snapshot. if ( origin === "watch" && !shell.isScreenOn() && (kind === "scroll-up" || kind === "scroll-down" || kind === "short-then-long-press") ) { this.appendLog(`${kind} (watch scheme) ignored while display is off`); return; } if (kind === "long-press") { // Hardware delivers a press event and then a release when the finger // lifts; emulate a short hold so the escape-menu countdown never fires. await this.handleInputEvent(this.buildSyntheticRingInput("long-press", origin)); await this.handleInputEvent(this.buildSyntheticRingInput("long-press-release", origin)); return; } const event = this.buildSyntheticRingInput(kind === "long-press-start" ? "long-press" : kind, origin); await this.handleInputEvent(event); } /** * A touch on the phone's mirror of the glasses display. Gestures other than * a tap are the watch scheme (swipes navigate, double-tap is back, a hold * is the menu). A tap lands on what the mirror shows: a sidebar icon * switches to that window, a launcher cell opens, anything else selects. * Coordinates are fractions of the mirror (0..1). */ async handleMirrorTouch(kind: MirrorTouchKind, nx: number, ny: number): Promise { if (this.phase !== "connected" && this.phase !== "charging" && !this.isPreviewDisplayActive()) return; if (this.glassesLocked) { // Same as the ring: only a double-tap reaches a locked display. if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch"); return; } if (!shell.isScreenOn()) { // Asleep, the mirror offers the sleep gestures: double-tap wakes, a // tap shows the Glanceboard for its timeout. The mirror's hold has no // real release (the synthetic long-press releases at once, which would // only flash the board), so it counts as a tap here too. if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch"); else if (kind === "tap" || kind === "long-press") await this.injectSyntheticRingInput("click", "watch"); return; } if (kind !== "tap") { await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch"); return; } const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH); const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT); const stripShown = sidebarStripVisible(shell.getFocus(), shell.foregroundWindow()?.appId); this.appendLog(`mirror tap at ${x},${y}`); if (!shell.hasOverlay() && stripShown && x < SIDEBAR_WIDTH) { const target = shell.windowAtSidebarPoint(x, y); if (target) { this.appendLog(`mirror tap: sidebar -> ${target.title}`); shell.focusWindow(target.windowId); shell.foregroundWindow()?.requestRender(); this.requestShellRender(); } return; } const window = shell.foregroundWindow(); if (window && !shell.hasOverlay()) { const rect = appViewportRect(window.heightMode, window.appId); const inside = x >= rect.x && x < rect.x + rect.width && y >= rect.y && y < rect.y + rect.height; if (shell.getFocus() !== "window") { shell.focusWindow(window.windowId); this.requestShellRender(); } if (inside && window.hitTest && (await window.hitTest(x - rect.x, y - rect.y))) { this.appendLog(`mirror tap: ${window.title} at ${x - rect.x},${y - rect.y}`); this.requestShellRender(); return; } } await this.injectSyntheticRingInput("click", "watch"); } /** A document arrived via Android's Share intent: open it as a new window. */ async openSharedTextDocument(text: string): Promise { this.appendLog(`shared text document received (${text.length} chars)`); if (!shell.isScreenOn()) { shell.wake("sidebar"); } const handler = ALL_APPS.find((app) => app.openSharedText); if (!handler?.openSharedText) { this.appendLog("no app handles shared text"); return; } handler.openSharedText(this.buildAppContext(handler), "Shared text", text); } private startTextSettingEdit(setting: ConfigSettingString): void { this.startTextSettingsEdit([setting], setting.editorTitle); } private startTextSettingsEdit( settings: readonly ConfigSettingString[], title: string, onFinish?: () => void, toggle?: TextSettingsEditToggle, onCancel?: () => void, ): void { this.activeTextSettings = Array.from(settings.slice(0, 2)); this.activeTextSettingOriginalValues = this.activeTextSettings.map((setting) => setting.get()); this.activeTextEditorTitle = title; this.activeTextEditorOnFinish = onFinish ?? null; this.activeTextEditorToggle = toggle ?? null; this.activeTextEditorToggleOriginalValue = toggle?.setting.get() ?? false; this.activeTextEditorOnCancel = onCancel ?? null; this.emit(); } /** * Begin voice capture with the provider chosen in settings. Used by both * push-to-talk and the Transcribe app. Android mic permission is the consent * gate even though the audio source is the G2 mic over BLE. */ private pttCaptureGeneration = 0; private startVoiceCapture(endpointing = false): Promise { return this.beginVoiceCapture("ptt", endpointing); } private stopVoiceCapture(): Promise | void { ++this.pttCaptureGeneration; return voiceControlBridge.stopPushToTalk(); } private startContinuousVoiceCapture(): void { void this.beginVoiceCapture("continuous"); } private stopContinuousVoiceCapture(): void { voiceControlBridge.stopContinuousCapture(); } /** * Provider and key settings for a capture on the given connection; a null * communicator means preview-only mode, where the phone's own microphone * stands in for the G2 mic. */ private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, provider: voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances // against the enrolled wearer voice-print; non-matching speakers' // finals are suppressed by the bridge. speakerVerification: wearerVerificationOptions() ?? undefined, // The Microphones app's processing config applies to every capture: // spectral noise suppression and the Sonic Radar listening beam. ...micSession.captureProcessingOptions(), }; } /** * Restart the mic for a capture that a previous session left holding. Cheap * to call per frame: it does nothing unless one is actually parked. */ private resumeVoiceCapture(): void { if (!voiceControlBridge.hasSuspendedCapture()) return; const communicator = this.communicator; if (!communicator || this.phase !== "connected") return; this.appendLog("resuming voice capture on the new glasses session"); voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); } /** * Gate for opening the voice dialog (the shell asks before pushing the * layer). Connected sessions always proceed — the mic permission prompt, * if needed, appears over the open dialog as it always has. Preview mode * captures from the phone mic, so a missing permission turns the tap into * the system permission prompt instead of a dialog that would listen to * nothing; a grant lets the dialog open right away. */ private async prepareVoiceCapture(): Promise { if (!this.isPreviewDisplayActive()) return true; if (hasMicrophonePermission()) return true; const granted = await requestMicrophonePermission(); if (!granted) { this.appendLog("voice input blocked: microphone permission denied"); } return granted; } private async beginVoiceCapture(kind: "ptt" | "continuous", endpointing = false): Promise { const pttGeneration = kind === "ptt" ? ++this.pttCaptureGeneration : 0; // Preview mode captures from the phone mic (voiceCaptureOptions with a // null communicator); otherwise a live glasses session must be the source. const previewCapture = this.isPreviewDisplayActive(); if (!previewCapture && (this.phase !== "connected" || !this.communicator)) { return; } const communicator = this.communicator; // The system prompt appears on the phone; tell the glasses dialog so it // sends the user there rather than claiming to listen. if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionPrompt(); await ensureVoicePermissions() .then(() => { if (kind === "ptt" && pttGeneration !== this.pttCaptureGeneration) return; if (previewCapture) { if (!this.isPreviewDisplayActive()) return; } else if (this.phase !== "connected" || this.communicator !== communicator) { return; } const options = { ...this.voiceCaptureOptions(communicator), endpointing }; if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); } else { voiceControlBridge.startContinuousCapture(options); } }) .catch((error) => { this.appendLog(`voice permission failed: ${this.formatError(error)}`); if (!hasMicrophonePermission()) voiceControlBridge.reportPermissionDenied(); }); } private endTextSettingEdit(): void { const finishedSettings = this.activeTextSettings; // Closing without the done key is a cancel from the caller's point of view. // finishTextSettingEdit clears this first, so a completed edit never fires it. const onCancel = this.activeTextEditorOnCancel; this.activeTextSettings = []; this.activeTextSettingOriginalValues = []; this.activeTextEditorTitle = ""; this.activeTextEditorOnFinish = null; this.activeTextEditorToggle = null; this.activeTextEditorOnCancel = null; this.emit(); onCancel?.(); if (finishedSettings.includes(nightscoutSiteUrlSetting) || finishedSettings.includes(nightscoutApiTokenSetting)) { void this.refreshNightscoutAfterSettingsChange(); } } /** * Finish the active edit from the phone side (e.g. the IME's done key): * ends the edit session and navigates the Settings app's glasses editor * out of the edit page. */ finishActiveTextSettingEdit(): void { if (!this.activeTextSettings.length) return; const invalid = this.activeTextSettings.find(setting => setting.validationError()); if (invalid) { void Dialogs.alert({ title: invalid.editorTitle, message: invalid.validationError()!, okButtonText: "OK" }); return; } const onFinish = this.activeTextEditorOnFinish; // Completing is not cancelling: drop the cancel callback before // endTextSettingEdit, which fires whatever is still set. this.activeTextEditorOnCancel = null; const closesGlassesEditor = this.activeTextSettings.length === 1; this.endTextSettingEdit(); if (closesGlassesEditor && this.textEditorHost?.closeTextEditor()) { this.textEditorHost.requestRender(); } onFinish?.(); } /** * The phone editor's Cancel button: put back the values the edit started * with, end the edit as a cancel (the caller's onCancel fires), and * navigate the Settings app's glasses editor out of the edit page. */ cancelActiveTextSettingEdit(): void { if (!this.activeTextSettings.length) return; this.activeTextSettings.forEach((setting, index) => { const original = this.activeTextSettingOriginalValues[index] ?? ""; if (setting.get() !== original) setting.set(original); }); const toggle = this.activeTextEditorToggle; if (toggle && toggle.setting.get() !== this.activeTextEditorToggleOriginalValue) { toggle.setting.set(this.activeTextEditorToggleOriginalValue); } const closesGlassesEditor = this.activeTextSettings.length === 1; this.endTextSettingEdit(); if (closesGlassesEditor && this.textEditorHost?.closeTextEditor()) { this.textEditorHost.requestRender(); } } private updateTextSetting(setting: ConfigSettingString, value: string): void { if (setting.get() !== value) { setting.set(value); this.emit(); this.previewOrRenderAfterTextSettingChange(); } } private async refreshNightscoutAfterSettingsChange(): Promise { await nightscoutBridge.refreshNow().catch((error) => { this.appendLog(`nightscout settings refresh failed: ${this.formatError(error)}`); }); } private previewOrRenderAfterTextSettingChange(): void { // Echo phone-side keystrokes into the Settings app's glasses editor. if (this.textEditorHost?.isTextEditorOnTop()) { this.textEditorHost.requestRender(); } } private async handleInputEvent(event: RawInputEvent): Promise { const frameId = event.frameId > 0 ? event.frameId : frameTimings.startFrame(`input:${event.kind} (untracked source)`); frameTimings.logFrame(frameId, `TS input handler start: ${event.kind} ${eventLabel(event.kind, event.eventType)}`); let frameOwned = false; try { const inputEvent = rawInputEventToInputEvent(event); if (!acceptInput(inputEvent)) return; // The gesture, plus which app is on screen and whether input goes to it, // the sidebar, or a shell overlay. Java only knows the raw event codes, // and without the target the export says what was pressed but not who // spent the time drawing the answer. frameTimings.annotateFrame(frameId, `${inputEvent.type} ${shell.describeInputTarget()}`); if (this.glassesLocked) { const ringDoubleTap = inputEvent.type === "double-click" && (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING || event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH); const suspendedDisplayWake = inputEvent.type === "display-wake"; if (ringDoubleTap || suspendedDisplayWake) { if (shell.isScreenOn()) { // A normal ring double-tap still turns the locked display off. // display-wake is directional and can only turn an off display on. if (ringDoubleTap) shell.sleep(); } else { shell.wake("sidebar"); const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () => this.ensureEvenHubSessionActive(frameId), ); if (!ready) { this.appendLog("EvenHub wake barrier timed out for locked display"); } } frameTimings.finishFrame(frameId, "locked display toggled"); frameOwned = true; } return; } if (!shell.isScreenOn()) { // Sleep-time gestures belong to the Glanceboard: a tap or a hold // shows it (the shell would ignore them), and a double-tap while it // is up dismisses it and falls through to the normal wake below. const glanceEvent = this.glanceEventFor(inputEvent, event); if (glanceEvent?.type === "dismiss") { this.glance.dismiss(); } else if (glanceEvent) { frameTimings.annotateFrame(frameId, `glanceboard ${glanceEvent.type}`); await this.glance.handleEvent(glanceEvent, frameId); frameOwned = true; return; } } const wakewordShouldWake = event.kind === "even-ai" && event.eventType === EvenAIStatus.EVEN_AI_WAKE_UP && wakeWordActionSetting.get() !== "off"; const displayShouldWake = event.kind === "display-wake"; let shellPreWoke = false; if (wakewordShouldWake || displayShouldWake) { if (!shell.isScreenOn()) { shellPreWoke = shell.wake("sidebar"); } if (shellPreWoke || this.evenHubSessionSuspended || this.evenHubResumePromise) { const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () => this.ensureEvenHubSessionActive(frameId), ); if (!ready) { this.appendLog(`EvenHub wake barrier timed out for ${event.kind}`); } } } frameTimings.spanStart(frameId, "handle-input"); let outcome: ShellInputOutcome; try { // The shell consumes its reserved gestures and forwards the rest to // the focused window; the outcome says which surfaces changed. outcome = await shell.receiveInput(inputEvent, frameId); } finally { frameTimings.spanEnd(frameId, "handle-input"); } if (shellPreWoke && !outcome.shell) { outcome = { ...outcome, shell: true }; } if (event.kind === "watch-gesture") { this.appendLog(`watch-gesture ${eventLabel(event.kind, event.eventType)}`); } if (event.kind === "sys-event") { this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`; this.appendLog(`sys-event ${this.lastSys}`); if ( event.eventType === OsEventTypeList.FOREGROUND_EXIT_EVENT || event.eventType === OsEventTypeList.ABNORMAL_EXIT_EVENT || event.eventType === OsEventTypeList.SYSTEM_EXIT_EVENT ) { this.appendLog("display state invalidated by firmware exit event"); } if ( event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING || event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH ) { this.lastInput = eventName(event.eventType); } } if (outcome.shell) { // Pass frameId as the cause, not the owner: the chrome render gets its // own child frame under this one, so the export shows the input and // every render it triggered on a single timeline. this.requestShellRender(frameId); } if (outcome.window) { // The window adapter owned frameId (render or explicit finish). frameOwned = true; } else if (outcome.shell) { frameOwned = true; frameTimings.finishFrame(frameId, "handled by shell; chrome render spawned"); } } finally { if (!frameOwned) { frameTimings.finishFrame(frameId, "discarded: input did not trigger a render"); } } } /** * The Glanceboard event a sleep-time input means, or null when the shell * should see the input as usual. A display-wake carries the head-tilt * gesture (HEAD_UP_EVENT) as a press; the double-tap display-wake stays the * regular UI's wake. */ private glanceEventFor(inputEvent: InputEvent, event: RawInputEvent): GlanceEvent | null { if (inputEvent.type === "display-wake") { return event.eventType === OsEventTypeList.HEAD_UP_EVENT ? this.glance.eventForGesture("head-tilt") : null; } return this.glance.eventForGesture(inputEvent.type); } /** Launch or focus an in-process singleton app (notifications, debug tests). */ private async launchInProcessApp( windowId: string, surfaceId: string, create: (options: InProcessAppOptions) => InProcessWindow, ): Promise { const existing = this.inProcessApps.get(windowId); if (existing) { shell.focusWindow(windowId); this.requestShellRender(); return; } const app = create({ actions: { ...this.sharedActions, requestRender: () => {}, // rebound by createInProcessWindow }, submitFrame: (planes, paintMs, frameId) => this.submitWindowFrame(surfaceId, planes, paintMs, frameId), setSurfaceVisible: (visible) => this.setWindowSurfaceVisible(surfaceId, visible), removeSurface: () => this.removeWindowSurface(surfaceId), reconfigureSurface: (heightMode) => { // Resize the surface rect to the new band; a foreground window stays // visible. The shell re-renders so the chrome (top bar) follows. void this.configureWindowSurface(surfaceId, this.isForegroundWindow(windowId), heightMode); this.requestShellRender(); }, onClosed: () => { this.inProcessApps.delete(windowId); }, }); this.inProcessApps.set(windowId, app); shell.registerWindow(app.window); await this.configureWindowSurface(surfaceId, false, app.window.heightMode); shell.focusWindow(windowId); this.requestShellRender(); this.appendLog(`launched ${windowId}`); } /** Get or spawn the worker host for an app. */ private ensureWorkerHost(appId: string, createWorker: () => Worker): WorkerAppHost { const existing = this.appHosts.get(appId); if (existing) return existing; const host = new WorkerAppHost({ appId, worker: createWorker(), onStopping: () => { if (this.appHosts.get(appId) === host) this.appHosts.delete(appId); }, configureSurface: (surfaceId, visible, heightMode) => this.configureWindowSurface(surfaceId, visible, heightMode), setSurfaceVisible: (surfaceId, visible) => this.setWindowSurfaceVisible(surfaceId, visible), removeSurface: (surfaceId) => this.removeWindowSurface(surfaceId), requestShellRender: () => this.requestShellRender(), openSettings: (section) => { void this.launchApp("settings", { section }).catch((error) => { this.appendLog(`settings launch failed: ${this.formatError(error)}`); }); }, startTextSettingEdit: (settingId) => { const setting = getStringSettingById(settingId); if (setting) { this.startTextSettingEdit(setting); } else { this.appendLog(`worker requested edit of unknown setting ${settingId}`); } }, endTextSettingEdit: () => this.endTextSettingEdit(), }); this.appHosts.set(appId, host); return host; } /** The services an app's launch/boot callbacks may use; see AppContext. */ private buildAppContext(app: AppDefinition): AppContext { return { appId: app.appId, apps: ALL_APPS, actions: this.sharedActions, launchApp: (appId, params) => this.launchApp(appId, params), uninstallApp: (appId) => this.uninstallApp(appId), launchInProcessApp: (windowId, surfaceId, create) => this.launchInProcessApp(windowId, surfaceId, create), ensureWorkerHost: (createWorker) => this.ensureWorkerHost(app.appId, createWorker), submitWindowFrame: (surfaceId, planes, paintMs, frameId) => this.submitWindowFrame(surfaceId, planes, paintMs, frameId), setWindowSurfaceVisible: (surfaceId, visible) => this.setWindowSurfaceVisible(surfaceId, visible), requestShellRender: () => this.requestShellRender(), appendLog: (message) => this.appendLog(message), setTextEditorHost: (host) => { this.textEditorHost = host; }, }; } /** Save which apps are open (and which is foreground) for restoreOpenApps. */ private persistOpenApps(): void { if (!this.openAppsRestored || this.suppressOpenAppsPersist) return; const open: string[] = []; for (const window of shell.getWindows()) { if (window.appId === "launcher") continue; if (!open.includes(window.appId)) open.push(window.appId); } savePersistedOpenApps({ open, foreground: shell.foregroundWindow()?.appId ?? null }); } /** * Reopen the apps that were open when the app last ran — a development * convenience so installing a new build restores the working set. Shallow by * design: apps relaunch fresh (no within-app state), and multi-window apps * reopen only their primary window. Runs once, on reaching the main page; * windows opened here get compositor surfaces at connect like any launch. */ async restoreOpenApps(): Promise { if (this.openAppsRestored) return; this.openAppsRestored = true; const saved = loadPersistedOpenApps(); if (!saved.open.length) return; const known = new Set(ALL_APPS.map((app) => app.appId)); this.suppressOpenAppsPersist = true; try { for (const appId of saved.open) { if (!known.has(appId) && !getInstalledEvenHubAppById(appId)) continue; try { await this.launchApp(appId); } catch (error) { this.appendLog(`restore of ${appId} failed: ${this.formatError(error)}`); } } const foreground = saved.foreground ? shell.getWindows().find((window) => window.appId === saved.foreground) : undefined; if (foreground) shell.focusWindow(foreground.windowId); } finally { this.suppressOpenAppsPersist = false; } // Re-save the canonical post-restore state (drops apps that failed to open). this.persistOpenApps(); this.requestShellRender(); } /** * Launch an app by id: the launcher grid, assistant tools, window-menu * Settings item, and open-apps restore all come through here. Apps with an * open window focus it instead of opening another. */ private async launchApp(appId: string, params?: AppLaunchParams): Promise { const app = ALL_APPS.find((entry) => entry.appId === appId); if (app) { await app.launch(this.buildAppContext(app), params); return; } const installed = getInstalledEvenHubAppById(appId); if (installed) { const host = ALL_APPS.find((entry) => entry.appId === "evenhub")!; if (!isInstalledPackagePresent(installed.packageId)) { // The registry knows the app but its package is gone (a settings // import after a reinstall): offer the store's Reinstall page. this.appendLog(`evenhub: package missing for ${installed.packageId}; opening store page`); await openEvenHubStoreForPackage(this.buildAppContext(host), installed); return; } await launchInstalledPackage(this.buildAppContext({ ...host, appId }), installed); return; } this.appendLog(`unknown app: ${appId}`); } private async uninstallApp(appId: string): Promise { const packageId = installedEvenHubPackageId(appId); if (!packageId) { this.appendLog(`app is not uninstallable: ${appId}`); return; } closeRunningPackage(packageId); if (uninstallEvenHubPackage(packageId)) { this.appendLog(`evenhub: uninstalled ${packageId}`); } } /** * Create/refresh a window surface on the compositor, if a display target exists. * * `visible` may be a function, asked only after the surface is configured, * right when the visibility call is queued. Callers that loop over windows * or resize one in place must pass one (isForegroundWindow): the configure * call yields, the foreground can change meanwhile, and a visibility decided * before the await then lands AFTER the focus change's own calls, since the * bridge runs Java calls in order. At startup, restoreOpenApps focuses * windows while connect() registers surfaces, so a foreground read before * that loop left the old window visible and the new foreground hidden. */ private async configureWindowSurface( surfaceId: string, visible: boolean | (() => boolean), heightMode: WindowHeightMode = "min", // ensurePreviewDisplay passes its not-yet-published target explicitly. target: DisplayTarget | null = this.display, ): Promise { if (!target) return; await target.configureSurface(surfaceId, { ...appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId), zOrder: 0, transparency: "opaque", }); await target.setSurfaceVisible(surfaceId, typeof visible === "function" ? visible() : visible); } /** A late answer to "is this window in front?", for configureWindowSurface. */ private isForegroundWindow(windowId: string): () => boolean { return () => shell.foregroundWindow()?.windowId === windowId; } private removeWindowSurface(surfaceId: string): void { const display = this.display; if (!display) return; void display.removeSurface(surfaceId).catch((error) => { this.appendLog(`surface removal failed: ${this.formatError(error)}`); }); } /** Submit a painted frame for an in-process window (e.g. the launcher). */ private async submitWindowFrame(surfaceId: string, planes: Plane[], paintMs: number, frameId: number): Promise { const display = this.display; if (!display || this.phase === "charging") { frameTimings.finishFrame(frameId, "discarded: window frame with no display target"); return; } const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes)); // An identical repaint (e.g. every in-process window re-renders when the // switcher brings it to the foreground) would flatten, encode and // composite only to be deduped before sending. if (display.isSurfaceCurrent(surfaceId, fingerprint)) { frameTimings.finishFrame(frameId, "discarded: surface unchanged"); return; } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)); await frameTimings.spanAsync(frameId, "submit", () => display.submitSurfaceFrame( surfaceId, buffer, { x: 0, y: 0, width: image.width, height: image.height }, fingerprint, paintMs, frameId, preparedDraws, ), ); } /** Flip a window surface's compositor visibility; fire-and-forget. */ private setWindowSurfaceVisible(surfaceId: string, visible: boolean): void { const display = this.display; if (!display) return; void display.setSurfaceVisible(surfaceId, visible).catch((error) => { this.appendLog(`surface visibility change failed: ${this.formatError(error)}`); }); } /** * Re-render and resubmit the shell surface (sidebar, top bar, shell * overlays). Coalesces like requestRender: one render in flight, at most * one queued. */ requestShellRender(causeFrameId = 0): void { // Remember the input frame that asked for this chrome repaint so // renderShell can nest its frame under it; without that link an input // event and the shell render it caused show up as two unrelated frames // and the real input-to-pixels latency is never measured. if (causeFrameId > 0 && this.pendingShellRenderCauseFrameId === 0) { this.pendingShellRenderCauseFrameId = causeFrameId; } if (this.shellRenderInProgress) { this.shellRenderQueued = true; return; } this.shellRenderInProgress = true; void (async () => { try { do { this.shellRenderQueued = false; await this.renderShell(); } while (this.shellRenderQueued); } catch (error) { this.appendLog(`shell render failed: ${this.formatError(error)}`); } finally { this.shellRenderInProgress = false; } })(); } private async renderShell(): Promise { const causeFrameId = this.pendingShellRenderCauseFrameId; this.pendingShellRenderCauseFrameId = 0; const frameId = frameTimings.startFrame("render:shell", causeFrameId); frameTimings.annotateFrame(frameId, shell.describeInputTarget()); // Nothing to send a frame to: stop here, BEFORE painting. This check used // to run only after the paint, which made a no-display state a // main-thread busy loop: paint (~55 ms) -> endRenderPass reports stale // data -> requestShellRender re-queues (shellRenderQueued, since a render // is in progress) -> the display check discards the work -> the do/while // in requestShellRender goes straight round again. Measured on a device // with the glasses not presenting: 40 of 40 frames discarded, ~19 a // second, enough to starve input and ANR. Leaving early costs nothing and // breaks the loop, because the re-request lives downstream of this point. if (!this.display || this.phase === "charging") { frameTimings.finishFrame(frameId, "discarded: shell render with no display target"); return; } const wantFreshData = this.nextShellRenderWantsFreshData; this.nextShellRenderWantsFreshData = false; if (wantFreshData) frameTimings.logFrame(frameId, "follow-up repaint that must not use cached data"); beginRenderPass(!wantFreshData); const paintStartedAtMs = Date.now(); const planes = frameTimings.span(frameId, "paint", () => frameTimings.runWithFrame(frameId, () => shell.paintScene()), ); const paintMs = Date.now() - paintStartedAtMs; const paintUsedStaleData = endRenderPass(); if (paintUsedStaleData) { // Repaint with fresh data (e.g. notification icons) once this frame is // out; mirrors renderDashboard's stale-data contract. Linked to this // frame so the export shows one input costing two round trips instead of // an unexplained extra render. this.nextShellRenderWantsFreshData = true; this.requestShellRender(frameId); } // The display-target check that used to live here now runs before the // paint, at the top of this function. Nothing between the two can change // it -- paintScene is synchronous and there is no await in between -- so // repeating it here is dead code, and the compiler agrees: the early // return narrows this.phase, making the "charging" comparison impossible. const display = this.display; await frameTimings.spanAsync(frameId, "submit", () => display.submitShellScene(planes, paintMs, frameId)); // Backpressure: the next shell render waits for this one to reach the // glasses. Timing out here means the loop was blocked for the full timeout // and any input arriving meanwhile had its chrome repaint delayed, so say // so in the frame rather than leaving a silent stall. (The preview target // resolves immediately; nothing transmits.) const outcome = await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS); if (outcome === null) { frameTimings.logFrame( frameId, `still unsent after ${FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS}ms; shell render loop released`, ); } this.schedulePreviewUpdate(); } private async handleAndroidNotificationPosted(notificationKey: string): Promise { const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey); if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) { this.requestShellRender(); return; } // New notifications open a shell modal over the app viewport; if the // screen was off, wake for it and go back to sleep when it is closed. // Waking while already on would steal focus, so only wake from sleep. const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar"); if (wokeScreen) { this.appendLog("android notification woke the screen"); } shell.openNotificationModal(notificationKey, wokeScreen); this.requestShellRender(); } private async playBuzzerSequence(payload: Uint8Array): Promise { if (this.phase !== "connected" || !this.communicator) { return; } await this.communicator.playBuzzerSequence(payload); } /** One-time celebratory jingle on the first connection after onboarding. */ private async playWelcomeSound(): Promise { const effect = findSoundEffect("questcomplete"); if (!effect) return; try { await playSoundEffect( effect, (payload) => this.playBuzzerSequence(payload), (ms) => new Promise((resolve) => setTimeout(resolve, ms)), ); } catch (error) { this.appendLog(`welcome sound failed: ${this.formatError(error)}`); } } private clearDashboardTimer(): void { this.cancelEvenHubSuspendTimer(); if (this.shellRefreshTimer) { clearInterval(this.shellRefreshTimer); this.shellRefreshTimer = null; } if (this.previewTimer) { clearInterval(this.previewTimer); this.previewTimer = null; } if (this.previewTrailingTimer) { clearTimeout(this.previewTrailingTimer); this.previewTrailingTimer = null; } if (this.screenTimeoutTimer) { clearInterval(this.screenTimeoutTimer); this.screenTimeoutTimer = null; } } private setPhase(phase: ConnectionPhase): void { if (this.phase === phase) return; this.phase = phase; if (phase !== "connected" && phase !== "charging") { shell.setBatteryLevels({ ring: null, ringCharging: null }); } // "charging" is a live BLE link with the glasses in their case. updateGlassesPresence({ connected: phase === "connected" || phase === "charging", charging: phase === "charging" || (shell.getBatteryLevels().headsetCharging ?? false), }); if (phase === "disconnected") { // Kept across "connecting": silent mode blocks app launches, so it can // itself cause the reconnect churn, and Java re-reports it either way. this.silentMode = false; } this.emit(); } private setStatus(status: string): void { if (this.status === status) return; this.status = status; this.emit(); } private updateConnectedForegroundNotification(): void { if (this.phase !== "connected") return; const now = Date.now(); if (now - this.lastForegroundNotificationUpdateAtMs < FOREGROUND_NOTIFICATION_MIN_UPDATE_MS) return; this.lastForegroundNotificationUpdateAtMs = now; updateForegroundNotification("Connected"); } private appendLog(line: string): void { if (VERBOSE_CONTROLLER_LOG) console.log(`[${formatTimestamp(new Date())}] ${line}`); } private setDisplayPreview(preview: ImageSource | null): void { if (this.displayPreview === preview) return; this.displayPreview = preview; this.emit(); } /** * Phone-side preview of what is on the glasses, fetched from the Java * compositor so it reflects every surface (chrome + whichever app is * foreground, including worker apps the TS side never renders). Throttled * to avoid rebuilding the bitmap faster than the phone UI needs it. */ private previewTrailingTimer: ReturnType | null = null; private lastRecordCaptureAtMs = 0; private phonePreviewVisible: (() => boolean) | null = null; /** The main page owns the mirror; other phone pages must not keep it rendering. */ attachPhonePreview(isVisible: () => boolean): () => void { this.phonePreviewVisible = isVisible; this.lastConnectedPreviewUpdateAtMs = 0; this.updateCompositePreview(); return () => { if (this.phonePreviewVisible !== isVisible) return; this.phonePreviewVisible = null; if (this.previewTrailingTimer) { clearTimeout(this.previewTrailingTimer); this.previewTrailingTimer = null; } }; } /** * Refresh the preview now if the floor allows, otherwise once the floor * expires — so the last frame of a burst always reaches the mirror instead * of waiting for the safety-net poll. */ private schedulePreviewUpdate(): void { if (!this.phonePreviewVisible && !this.screenRecordingActive) return; if (this.previewTrailingTimer) return; const wait = this.lastConnectedPreviewUpdateAtMs + CONNECTED_PREVIEW_MIN_UPDATE_MS - Date.now(); if (wait <= 0) { this.updateCompositePreview(); return; } this.previewTrailingTimer = setTimeout(() => { this.previewTrailingTimer = null; this.updateCompositePreview(); }, wait); } private updateCompositePreview(): void { const display = this.display; if (!display) return; // The floor comes first (and stamps even when backgrounded below): with // it after the background check, per-frame refresh requests from a // backgrounded app never advance the timestamp and every one of them // pays the foreground-activity lookup. const now = Date.now(); if ( this.lastConnectedPreviewUpdateAtMs > 0 && now - this.lastConnectedPreviewUpdateAtMs < CONNECTED_PREVIEW_MIN_UPDATE_MS ) { return; } this.lastConnectedPreviewUpdateAtMs = now; // A user-requested GIF is an independent consumer, including while the // phone is asleep or another page/app is foregrounded. if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) { this.lastRecordCaptureAtMs = now; display.recordScreenFrame(); } if (!this.phonePreviewVisible?.()) return; // The connected foreground service intentionally keeps this controller // alive after the phone UI is backgrounded. Do not keep constructing // 640x480 Android Bitmaps for a window that cannot display them: besides // being wasted work, paused NativeScript views can retain queued image // updates long enough to put severe pressure on the native heap. if (global.isAndroid) { const activity = Application.android.foregroundActivity; if (!activity || !activity.hasWindowFocus()) return; const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager; if (!power?.isInteractive()) return; } const preview = display.getCompositePreview(previewColorSetting.get() === "green"); if (preview) { this.setDisplayPreview(preview); } } private formatError(error: unknown): string { const raw = (error as Error)?.message ?? String(error); const sanitized = raw.replace(/[\x00-\x1f]+/g, " ").replace(/\s+/g, " ").trim(); if (sanitized.length <= 240) return sanitized; return `${sanitized.slice(0, 237)}...`; } private buildSyntheticRingInput( kind: | "ring-press" | "click" | "double-click" | "scroll-up" | "scroll-down" | "long-press" | "long-press-release" | "short-then-long-press" | "wakeword" | "swipe-left" | "swipe-right" | "swipe-up" | "swipe-down", origin: SyntheticInputOrigin = "ring", ): RawInputEvent { const frameId = frameTimings.startFrame(`input:synthetic:${kind}`); // The phone UI's test buttons stand in for the ring; the watch is its own // source so the UI can give it a richer scheme (see InputSource). const eventSource = origin === "watch" ? EventSourceType.TOUCH_EVENT_FROM_WATCH : EventSourceType.TOUCH_EVENT_FROM_RING; switch (kind) { case "swipe-left": case "swipe-right": case "swipe-up": case "swipe-down": // Watch-only directional input; see WatchGestureType. return { kind: "watch-gesture", containerName: "", eventType: { "swipe-left": WatchGestureType.SWIPE_LEFT, "swipe-right": WatchGestureType.SWIPE_RIGHT, "swipe-up": WatchGestureType.SWIPE_UP, "swipe-down": WatchGestureType.SWIPE_DOWN, }[kind], eventSource: 0, systemExitReasonCode: 0, frameId, }; case "wakeword": // Same event the glasses report for the spoken wakeword (sid 0x07). return { kind: "even-ai", containerName: "", eventType: EvenAIStatus.EVEN_AI_WAKE_UP, eventSource: 0, systemExitReasonCode: 0, frameId, }; case "ring-press": return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.RING_PRESS_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "long-press": return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.RING_LONG_PRESS_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "long-press-release": return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "short-then-long-press": // Discrete, unlike a hold: the firmware's later generic release is a // hardware artifact the shell ignores, so synthetic sources send none. return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.SHORT_THEN_LONG_PRESS_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "click": return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.CLICK_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "double-click": return { kind: "sys-event", containerName: "", eventType: OsEventTypeList.DOUBLE_CLICK_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "scroll-up": return { kind: "text-click", containerName: "", eventType: OsEventTypeList.SCROLL_TOP_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; case "scroll-down": default: return { kind: "text-click", containerName: "", eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT, eventSource, systemExitReasonCode: 0, frameId, }; } } private emit(): void { const snapshot = this.snapshot(); for (const listener of this.listeners) { listener(snapshot); } this.wearRemote?.schedulePublish(); } } export const dashboardController = new DashboardController();