mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Most of the builtin apps had some padding between the window edge and their content, because they expect to be displayed with a line border on their edge. We added a customization option to remove the border, so, that padding is not always necessary. Make the layouts dynamically adjust, to omit the padding when it isn't needed.
1260 lines
69 KiB
TypeScript
1260 lines
69 KiB
TypeScript
import { startRemoteInput } from "../remote/service"
|
|
import type { KeyboardInputSession } from '../ui/shell/keyboard-input'
|
|
import { acceptInput, resetRingInputFilter } from "../ui/input-monitor";
|
|
import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup, readActiveNotifications } from '../native/notification-icons.ios'
|
|
import { launcherEntries } from '../apps/launcher'
|
|
import { discardZeroedEvenHubPackage, getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
|
import { isInstalledPackagePresent } from '../apps/evenhub/updates'
|
|
import { openEvenHubStoreForPackage } from '../apps/evenhub'
|
|
import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager'
|
|
import { registerSystemTools } from '../assistant/system-tools'
|
|
import { registerWindowTools } from '../assistant/window-tools'
|
|
import { registerNavigateTools } from '../assistant/navigate-tools'
|
|
import { registerRoamTools } from '../assistant/roam-tools'
|
|
import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
|
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
|
import { bindBandwidthBenchmarkSession } from '../native/bandwidth-benchmark.ios'
|
|
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
|
import { iosVoiceInput } from '../native/ios-voice-input'
|
|
import { voiceActivity } from '../ui/shell/voice-activity'
|
|
import { nightscoutBridge } from '../native/nightscout-bridge'
|
|
import { weatherBridge } from '../native/weather'
|
|
import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios'
|
|
import { PreviewDisplayTarget, type DisplayTarget } from '../native/preview-display.ios'
|
|
import { AncsClient, ANCS_FIRMWARE_VERSION } from './ancs-client'
|
|
import { G1V0_IMAGE } from '../g1/firmware/g1v0-image'
|
|
import { GlanceHost, type GlanceDisplay } from './glance-host'
|
|
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen'
|
|
import { OsEventTypeList } from './events'
|
|
import { loadDeviceAddresses } from './device-addresses'
|
|
import { updateGlassesPresence } from './glasses-presence'
|
|
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
|
|
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
|
|
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
|
|
import { deviceAddressError } from './ios-peripheral-identity'
|
|
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
|
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
|
|
import { ALL_APPS } from '../apps/all-apps'
|
|
import type { AppContext, AppDefinition, AppLaunchParams } from '../apps/app-definition'
|
|
import { WorkerAppHost } from '../ui/shell/worker-window'
|
|
import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from '../ui/shell/in-process-window'
|
|
import { readPhoneBatteryState } from '../native/phone-battery'
|
|
import { iosAppUnavailableReason } from '../apps/ios-availability'
|
|
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphics/plane'
|
|
import { prepareFrameDraws } from '../graphics/glyph-wire'
|
|
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
|
import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures'
|
|
import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers'
|
|
import { TextViewerLayer } from '../apps/files/text-viewer'
|
|
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
|
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, windowEdges, type ScreenRect } from '../ui/shell/geometry'
|
|
import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
|
import { onAnySettingChanged,
|
|
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
|
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
|
openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting,
|
|
whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting, glassesTypeSetting, shouldPopUpNewNotification } from '../ui/dashboard-settings'
|
|
import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider'
|
|
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
|
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
|
|
|
const SHELL_SURFACE_ID = 'shell'
|
|
// voiceActivity holder for continuous capture (Transcribe), as on Android.
|
|
const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture'
|
|
const SHELL_SURFACE_Z_ORDER = 1
|
|
const LOCK_SURFACE_Z_ORDER = 1000
|
|
/** The shell render loop waits this long for the glasses to take a frame before moving on. */
|
|
const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 2000
|
|
|
|
export type SessionPhase = 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error'
|
|
export type SessionState = { phase: SessionPhase; status: string; battery: number | null; charging: boolean | null;
|
|
ring: boolean; ringBattery: number | null; ringCharging: boolean | null; frames: number; capabilities: string; leftVersion: string; rightVersion: string;
|
|
/** Why the connected firmware can't run Faceclaw (firmware-compat.ts), or '' when it can or is unknown. */
|
|
firmwareWarning: string }
|
|
|
|
function ancsCapable(capabilities: string): boolean {
|
|
return /^Faceclaw\/(\d+)/.test(capabilities) && Number(capabilities.split('/')[1]) >= ANCS_FIRMWARE_VERSION
|
|
}
|
|
|
|
/**
|
|
* iOS host for the shared app registry, shell and BLE session. The session,
|
|
* compositor and transport live in shared Kotlin (FaceclawCommunicatorBridge
|
|
* over FaceclawKitIosGlassesSession); with no glasses connected the same
|
|
* compositor runs headless (PreviewDisplayTarget) so the phone mirror works.
|
|
*/
|
|
export class IosPreviewController {
|
|
private previewTarget: PreviewDisplayTarget | null = null
|
|
private communicator: FaceclawCommunicatorBridge | null = null
|
|
private readonly startedCommunicators = new WeakSet<FaceclawCommunicatorBridge>()
|
|
state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false,
|
|
ringBattery: null, ringCharging: null, frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' }
|
|
private notifications: AncsClient | null = null
|
|
private readonly sessionOffs: (() => void)[] = []
|
|
private stopping: Promise<void> | null = null
|
|
/** Resolves once the current display target has every surface configured. */
|
|
private displayReady: Promise<void> = Promise.resolve()
|
|
private connecting = false
|
|
private sessionCreated = false
|
|
private wearDetectionRequested = false
|
|
/** The last firmware report was Faceclaw's firmware at the required revision. */
|
|
private firmwareCompatible = false
|
|
private readonly glanceDisplay: GlanceDisplay = {
|
|
configureSurface: async (id, options) => { await this.display?.configureSurface(id, options) },
|
|
setSurfaceVisible: async (id, visible) => { await this.display?.setSurfaceVisible(id, visible); this.schedulePreviewUpdate() },
|
|
setSurfaceDepth: async (id, depth) => { await this.display?.setSurfaceDepth(id, depth) },
|
|
setScreenBlanked: async blanked => { await this.display?.setScreenBlanked(blanked); this.schedulePreviewUpdate() },
|
|
submitSurfaceFrame: async (id, pixels, rect, fingerprint, paintMs, frameId, draws) => {
|
|
await this.display?.submitSurfaceFrame(id, pixels, rect, fingerprint, paintMs, frameId, draws); this.schedulePreviewUpdate()
|
|
},
|
|
}
|
|
private readonly glance = new GlanceHost({
|
|
getDisplay: () => this.display ? this.glanceDisplay : null,
|
|
getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null,
|
|
canShow: () => this.runtimeNeeded && !this.glassesLocked,
|
|
// iOS retains the EvenHub session while the shell sleeps. The board's
|
|
// opaque first frame is ready before we unblank the compositor.
|
|
ensureSessionActive: async () => {
|
|
await this.display?.setScreenBlanked(false); this.schedulePreviewUpdate(); return true
|
|
},
|
|
onHiddenWhileAsleep: () => {
|
|
if (!shell.isScreenOn()) void this.display?.setScreenBlanked(true).then(() => this.schedulePreviewUpdate())
|
|
else this.schedulePreviewUpdate()
|
|
},
|
|
onVisibilityChanged: () => this.schedulePreviewUpdate(),
|
|
appendLog: message => this.logBluetooth(message),
|
|
})
|
|
private previewTimer: ReturnType<typeof setTimeout> | null = null
|
|
/** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */
|
|
private previewCrop: ScreenRect | null = null
|
|
private clockTimer: ReturnType<typeof setInterval> | null = null
|
|
private offSettings: (() => void) | null = null
|
|
private lastBrightness: string | null = null
|
|
private active = false
|
|
private runtimeRunning = false
|
|
private clockMinute = -1
|
|
private shellRenderInProgress = false
|
|
private shellRenderQueued = false
|
|
/** The shell woke; the next shell render unblanks the compositor after submitting. */
|
|
private unblankAfterShellRender = false
|
|
private nextFrameId = 1
|
|
private nextWorkerFrame = 0
|
|
private inputQueue: Promise<void> = Promise.resolve()
|
|
private lastLayout = ''
|
|
/** The phone-UI button last pushed to the page (onEvenHubPhoneUiChanged). */
|
|
private evenHubPhoneUi: PhoneUiButton | null = null
|
|
private readonly appHosts = new Map<string, WorkerAppHost>()
|
|
private readonly inProcessApps = new Map<string, InProcessWindow>()
|
|
private readonly batteryObservers: any[] = []
|
|
private readonly lockObservers: any[] = []
|
|
private phoneLocked = false
|
|
private glassesWorn: boolean | null = null
|
|
private glassesLocked = false
|
|
private lockEnabled = lockScreenEnabledSetting.get()
|
|
private acknowledgedFrames = 0
|
|
private logLines: string[] = []
|
|
private logTimer: ReturnType<typeof setTimeout> | null = null
|
|
// Windows that want continuous capture, and what each needs of it (a
|
|
// recording, a provider and language). They share one capture, which runs
|
|
// while any of them wants it.
|
|
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
|
// transcriptionKey of the running continuous capture (provider and
|
|
// language); one that no longer matches what its windows need is drained
|
|
// and restarted.
|
|
private continuousTranscription = ''
|
|
// Continuous capture should (re)start once audio is available: a window
|
|
// asked, the glasses connected or disconnected, or the glasses lock or the
|
|
// app lifecycle cut it short. A capture that fails doesn't set it, so a
|
|
// broken start is retried on the next such change rather than in a loop.
|
|
private continuousPending = false
|
|
private continuousCaptureToken = 0
|
|
private sessionConnected = false
|
|
/** The Apple Watch remote; constructed last so it sees a fully wired shell. */
|
|
private wearRemote: WearRemote | null = null
|
|
private readonly actions: LayerActions = {
|
|
...noopLayerActions,
|
|
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
|
|
requestRender: () => this.requestShellRender(),
|
|
disconnect: () => this.disconnect(),
|
|
startVoiceCapture: endpointing => this.startVoiceCapture(endpointing),
|
|
stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(),
|
|
startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options),
|
|
stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client),
|
|
startTextSettingEdit: async setting => { await this.editSetting(setting) },
|
|
startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => {
|
|
for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return }
|
|
if (toggle) {
|
|
const choice = await Dialogs.action({ title, cancelButtonText: 'Cancel', actions: [toggle.label, 'This session only'] })
|
|
if (choice === 'Cancel') { cancelled?.(); return }
|
|
toggle.setting.set(choice === toggle.label)
|
|
}
|
|
finished?.()
|
|
},
|
|
}
|
|
|
|
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
|
|
private readonly onError: (message: string) => void,
|
|
private readonly onConnectionState: (state: SessionState) => void = () => {},
|
|
private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {},
|
|
private readonly onEvenHubPhoneUiChanged: (button: PhoneUiButton | null) => void = () => {}) {
|
|
this.ensurePreviewDisplay()
|
|
shell.configure({
|
|
actions: this.actions,
|
|
voiceInputEnabled: true,
|
|
onKeyboardInputChanged: session => this.onKeyboardInputChanged(session ? {
|
|
targets: session.targets,
|
|
setText: text => { if (this.active && !this.glassesLocked) session.setText(text) },
|
|
send: () => { if (this.active && !this.glassesLocked) session.send() },
|
|
sendTo: id => { if (this.active && !this.glassesLocked) session.sendTo(id) },
|
|
discard: () => session.discard(),
|
|
get visibility() { return session.visibility },
|
|
setVisibility: visibility => { if (this.active && !this.glassesLocked) session.setVisibility(visibility) },
|
|
get history() { return session.history },
|
|
} : null),
|
|
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
|
getScreenTimeoutMs: () => null,
|
|
requestShellRender: () => this.requestShellRender(),
|
|
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() },
|
|
openNotificationsWindow: () =>
|
|
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID,
|
|
options => createNotificationsAppWindow(options, section => { void this.launchApp('settings', { section }) })),
|
|
onSwitcherBarHeightChanged: () => {
|
|
this.relayout()
|
|
shell.foregroundWindow()?.requestRender()
|
|
this.requestShellRender()
|
|
},
|
|
onScreenStateChanged: on => {
|
|
if (on) this.glance.dismiss()
|
|
// A wake unblanks only once the next shell render has staged an awake
|
|
// scene: unblanking composites the retained one, painted while asleep
|
|
// with no chrome and depth 0, and that first frame is the one the
|
|
// brightness fade-in starts on.
|
|
this.unblankAfterShellRender = on
|
|
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
|
|
this.requestShellRender()
|
|
this.wearRemote?.schedulePublish()
|
|
},
|
|
})
|
|
const launcher = createLauncherWindow({
|
|
actions: this.actions,
|
|
apps: () => launcherEntries(ALL_APPS),
|
|
launchApp: id => this.launchApp(id),
|
|
uninstallApp: id => this.uninstallApp(id),
|
|
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
|
|
setSurfaceVisible: visible => this.setWindowSurfaceVisible(LAUNCHER_SURFACE_ID, visible),
|
|
})
|
|
void this.configureWindow(launcher)
|
|
shell.registerWindow(launcher)
|
|
shell.wake('window')
|
|
shell.focusWindow(launcher.windowId)
|
|
onIosNotificationPopup(key => {
|
|
const notification = readActiveNotifications(128).find(n => n.key === key)
|
|
if (this.glassesLocked || !notification || !shouldPopUpNewNotification(notification.packageName)) return
|
|
const woke = !shell.isScreenOn() && shell.wake('sidebar')
|
|
shell.openNotificationModal(key, woke)
|
|
this.requestShellRender()
|
|
})
|
|
startRemoteInput({
|
|
ready: () => this.runtimeRunning,
|
|
locked: () => this.glassesLocked,
|
|
input: (gesture, source) => {
|
|
// Share the controller queue with physical input. A discrete hold includes its release.
|
|
const task = this.inputQueue.then(async () => {
|
|
await this.receiveInput(makeInputEvent({ type: gesture, source } as InputEventPayload))
|
|
if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source }))
|
|
})
|
|
this.inputQueue = task.catch(error => this.fail(error))
|
|
return task
|
|
},
|
|
acceptsText: () => !!shell.foregroundWindow()?.receiveTextInput,
|
|
text: (text, submit) => { if (!shell.isScreenOn()) shell.wake('window'); shell.sendTextToForegroundWindow(text, { submit }); this.requestShellRender() },
|
|
assistantAvailable: () => shell.isAssistantAvailable(),
|
|
assistant: text => shell.sendToAssistant(text),
|
|
})
|
|
// The header's EvenHub phone-UI button hides while that UI is showing.
|
|
onPhoneShownChanged(() => this.syncEvenHubPhoneUi())
|
|
registerSystemTools()
|
|
registerWindowTools({ apps: () => ALL_APPS.filter(app => app.isSupported?.() !== false && !iosAppUnavailableReason(app.appId)),
|
|
launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() })
|
|
registerNavigateTools(id => this.launchApp(id))
|
|
registerRoamTools(id => this.launchApp(id))
|
|
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
|
|
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
|
|
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
|
|
this.wearRemote = new WearRemote({
|
|
apps: () => ALL_APPS.filter(app => app.showInLauncher !== false && app.isSupported?.() !== false && !iosAppUnavailableReason(app.appId)),
|
|
injectInput: kind => this.injectWatchInput(kind),
|
|
launchApp: id => this.launchApp(id),
|
|
// The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons.
|
|
connect: () => { resumeAutoReconnect(); return this.connect(true) },
|
|
disconnect: () => { suppressAutoReconnect(); return this.disconnect() },
|
|
setGlassesLocked: locked => this.setGlassesLocked(locked),
|
|
requestShellRender: () => this.requestShellRender(),
|
|
previewInputActive: () => !this.communicator && this.runtimeRunning,
|
|
getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked,
|
|
glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }),
|
|
appendLog: line => this.logBluetooth(line),
|
|
})
|
|
}
|
|
|
|
/** The compositor frames go to: the live session, else the headless preview. */
|
|
private get display(): DisplayTarget | null { return this.communicator ?? this.previewTarget }
|
|
|
|
private ensurePreviewDisplay(): void {
|
|
if (this.communicator || this.previewTarget) return
|
|
const target = new PreviewDisplayTarget()
|
|
this.previewTarget = target
|
|
target.activate(() => this.schedulePreviewUpdate())
|
|
this.displayReady = this.registerSurfaces(target).catch(error => this.fail(error))
|
|
}
|
|
|
|
private teardownPreviewDisplay(): void {
|
|
const target = this.previewTarget
|
|
if (!target) return
|
|
this.previewTarget = null
|
|
this.glance.reset()
|
|
target.release()
|
|
}
|
|
|
|
/** (Re)create every compositor surface on a fresh display target. */
|
|
private async registerSurfaces(target: DisplayTarget): Promise<void> {
|
|
await target.configureCompositorScreen(G2_LENS_WIDTH, G2_LENS_HEIGHT)
|
|
await target.configureSurface(SHELL_SURFACE_ID, { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT,
|
|
zOrder: SHELL_SURFACE_Z_ORDER, transparency: 'color-key' })
|
|
await target.setUnderlayDim(SHELL_SURFACE_Z_ORDER, 1)
|
|
await target.configureSurface(LOCK_SCREEN_SURFACE_ID, {
|
|
x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: LOCK_SURFACE_Z_ORDER, transparency: 'opaque',
|
|
})
|
|
if (this.glassesLocked) await this.submitLockImage(target)
|
|
await target.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, this.glassesLocked)
|
|
await target.setScreenBlanked(!shell.isScreenOn())
|
|
this.lastLayout = ''
|
|
for (const window of shell.getWindows()) await this.configureWindow(window, target)
|
|
}
|
|
|
|
private async submitLockImage(target: DisplayTarget): Promise<void> {
|
|
const image = createLockScreenImage()
|
|
await target.submitSurfaceFrame(LOCK_SCREEN_SURFACE_ID, image.to8bppBuffer(),
|
|
{ x: 0, y: 0, width: image.width, height: image.height }, 'lock-screen', 0, 0)
|
|
}
|
|
|
|
resume(): void {
|
|
if (this.active) return
|
|
this.active = true
|
|
this.syncRuntime()
|
|
this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable)
|
|
this.logBluetooth('Phone foreground')
|
|
if (this.communicator) this.onConnectionState({ ...this.state })
|
|
this.relayout()
|
|
shell.foregroundWindow()?.requestRender()
|
|
this.requestShellRender()
|
|
this.syncEvenHubPhoneUi()
|
|
this.syncContinuousCapture()
|
|
}
|
|
pause(): void {
|
|
if (!this.active) return
|
|
this.active = false
|
|
if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true
|
|
iosVoiceInput.stopPhoneCapture()
|
|
// The phone preview is hidden; the glasses still own their screen and input.
|
|
this.syncRuntime()
|
|
this.logBluetooth(`Phone background; glasses ${this.state.phase}`)
|
|
this.flushBluetoothLog()
|
|
}
|
|
/** Overlay an EvenHub app's phone UI (the header's app-icon button). */
|
|
showEvenHubPhoneUi(windowId: string): void {
|
|
showOnPhone(windowId)
|
|
}
|
|
/**
|
|
* Tell the page when the foreground window's EvenHub phone-UI button
|
|
* changes. A paused controller skips this and catches up on resume.
|
|
*/
|
|
private syncEvenHubPhoneUi(): void {
|
|
if (!this.active) return
|
|
const button = phoneUiButton()
|
|
const current = this.evenHubPhoneUi
|
|
if (current?.windowId === button?.windowId && current?.icon === button?.icon) return
|
|
this.evenHubPhoneUi = button
|
|
this.onEvenHubPhoneUiChanged(button)
|
|
}
|
|
private get runtimeNeeded(): boolean {
|
|
return this.active || ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.state.phase)
|
|
}
|
|
private syncRuntime(): void {
|
|
const running = this.runtimeNeeded
|
|
if (running === this.runtimeRunning) return
|
|
this.runtimeRunning = running
|
|
for (const window of shell.getWindows()) window.setScreenOn?.(running && shell.isScreenOn())
|
|
if (!running) {
|
|
void nightscoutBridge.stop().catch(error => this.fail(error))
|
|
weatherBridge.setSessionActive(false)
|
|
this.glance.dismiss()
|
|
void this.display?.setScreenBlanked(!shell.isScreenOn()).catch(error => this.fail(error))
|
|
this.offSettings?.(); this.offSettings = null
|
|
for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
|
|
for (const observer of this.lockObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
|
|
UIDevice.currentDevice.batteryMonitoringEnabled = false
|
|
if (this.clockTimer !== null) clearInterval(this.clockTimer)
|
|
if (this.previewTimer !== null) clearTimeout(this.previewTimer)
|
|
this.clockTimer = this.previewTimer = null
|
|
return
|
|
}
|
|
UIDevice.currentDevice.batteryMonitoringEnabled = true
|
|
// Keep these observers alive while BLE owns the runtime, including background.
|
|
for (const [name, locked] of [
|
|
[UIApplicationProtectedDataWillBecomeUnavailable, true],
|
|
[UIApplicationProtectedDataDidBecomeAvailable, false],
|
|
] as const) {
|
|
this.lockObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(
|
|
name, null, NSOperationQueue.mainQueue, () => this.handlePhoneLockState(locked)))
|
|
}
|
|
this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable)
|
|
this.syncLockSetting()
|
|
void nightscoutBridge.start().catch(error => this.fail(error))
|
|
weatherBridge.setSessionActive(true)
|
|
for (const name of [UIDeviceBatteryLevelDidChangeNotification, UIDeviceBatteryStateDidChangeNotification]) {
|
|
this.batteryObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(name, null, NSOperationQueue.mainQueue, () => this.requestShellRender()))
|
|
}
|
|
const phone = readPhoneBatteryState()
|
|
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
|
this.offSettings = onAnySettingChanged(() => {
|
|
this.syncLockSetting()
|
|
this.pushBrightness()
|
|
this.relayout()
|
|
shell.foregroundWindow()?.requestRender()
|
|
this.requestShellRender()
|
|
})
|
|
this.clockTimer = setInterval(() => this.refreshClock(), 60_000)
|
|
}
|
|
private pushBrightness(): void {
|
|
const communicator = this.communicator
|
|
if (!communicator || this.state.phase !== 'connected') { this.lastBrightness = null; return }
|
|
const preferences = getBrightnessPreferences()
|
|
const value = JSON.stringify(preferences)
|
|
if (value === this.lastBrightness) return
|
|
this.lastBrightness = value
|
|
void communicator.configureBrightness(preferences).catch(error => {
|
|
this.lastBrightness = null
|
|
this.logBluetooth(`Brightness: ${String(error)}`)
|
|
})
|
|
}
|
|
private syncLockSetting(): void {
|
|
const enabled = lockScreenEnabledSetting.get()
|
|
if (enabled === this.lockEnabled) return
|
|
this.lockEnabled = enabled
|
|
if (!enabled) this.setGlassesLocked(false)
|
|
else {
|
|
if (this.phoneLocked && this.glassesWorn === false) this.setGlassesLocked(true)
|
|
if (this.communicator && this.state.phase === 'connected' && this.firmwareCompatible)
|
|
void this.communicator.enableWearDetectionAndRequestState().catch(error => this.fail(error))
|
|
}
|
|
}
|
|
private handlePhoneLockState(locked: boolean): void {
|
|
this.phoneLocked = locked
|
|
this.communicator?.onPhoneLockSignal()
|
|
if (!locked) this.setGlassesLocked(false)
|
|
else if (this.lockEnabled && this.glassesWorn === false) this.setGlassesLocked(true)
|
|
}
|
|
private handleWearState(wearing: boolean): void {
|
|
// Sample false to catch a notification missed during suspension. Do not
|
|
// sample true here: WillBecomeUnavailable precedes the property transition.
|
|
if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true)
|
|
// Putting the glasses on (an observed OFF_HEAD -> ON_HEAD transition, not
|
|
// a session's first snapshot) wakes the screen, to the lock notice if locked.
|
|
const putOn = wearing && this.glassesWorn === false
|
|
this.glassesWorn = wearing
|
|
this.wearRemote?.schedulePublish()
|
|
this.syncPresence()
|
|
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
|
|
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
|
|
if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar')
|
|
}
|
|
private setGlassesLocked(locked: boolean): void {
|
|
if (locked === this.glassesLocked) return
|
|
this.glassesLocked = locked
|
|
this.wearRemote?.schedulePublish()
|
|
if (locked) {
|
|
this.glance.dismiss()
|
|
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
|
|
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
|
'Glasses locked. Transcription resumes when you unlock your phone.')
|
|
} else this.syncContinuousCapture()
|
|
this.syncRawMicSession()
|
|
void (async () => {
|
|
await this.displayReady
|
|
const display = this.display
|
|
if (!display) return
|
|
{
|
|
if (locked) await this.submitLockImage(display)
|
|
await display.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, locked)
|
|
this.schedulePreviewUpdate()
|
|
}
|
|
})().catch(error => this.fail(error))
|
|
this.logBluetooth(`Glasses ${locked ? 'locked' : 'unlocked'}`)
|
|
this.requestShellRender()
|
|
}
|
|
private refreshClock(): void {
|
|
const minute = Math.floor(Date.now() / 60_000)
|
|
if (minute !== this.clockMinute) { this.clockMinute = minute; this.requestShellRender() }
|
|
}
|
|
private async configureWindow(window: ShellWindow, target: DisplayTarget | null = this.display): Promise<void> {
|
|
if (!target) return
|
|
await target.configureSurface(window.surfaceId, {
|
|
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
|
|
})
|
|
await target.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
|
|
}
|
|
private relayout(): void {
|
|
// Each window's rect, and what its edges touch (which sets its padding).
|
|
const layout = shell.getWindows().map(window =>
|
|
JSON.stringify([appViewportRect(window.heightMode, window.appId), windowEdges(window.appId)])).join(";")
|
|
if (layout === this.lastLayout) return
|
|
this.lastLayout = layout
|
|
for (const window of shell.getWindows()) {
|
|
void this.configureWindow(window).catch(error => this.fail(error))
|
|
window.relayout?.()
|
|
}
|
|
}
|
|
/** Submit a painted frame for an in-process window (e.g. the launcher). */
|
|
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
|
|
await this.displayReady
|
|
const display = this.display
|
|
// The window can close (removing its surface) while this frame waits.
|
|
if (!display || !live()) return
|
|
const fingerprint = planesFingerprint(planes)
|
|
const { image, draws } = flattenPlanesWithDraws(planes)
|
|
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
|
|
fingerprint, -1, 0, prepareFrameDraws(draws))
|
|
this.schedulePreviewUpdate()
|
|
}
|
|
/** Worker windows ship baked pixels; every submit is new content to the compositor. */
|
|
private async submitWorkerPixels(id: string, pixels: Uint8Array, width: number, height: number, draws: ArrayBuffer | null): Promise<void> {
|
|
await this.displayReady
|
|
const display = this.display
|
|
if (!display) return
|
|
await display.submitSurfaceFrame(id, pixels, { x: 0, y: 0, width, height }, `${id}#${++this.nextWorkerFrame}`, -1, 0, draws)
|
|
this.schedulePreviewUpdate()
|
|
}
|
|
private setWindowSurfaceVisible(id: string, visible: boolean): void {
|
|
void this.display?.setSurfaceVisible(id, visible).then(() => this.schedulePreviewUpdate()).catch(error => this.fail(error))
|
|
}
|
|
private removeWindowSurface(id: string): void {
|
|
void this.display?.removeSurface(id).then(() => this.schedulePreviewUpdate()).catch(error => this.fail(error))
|
|
}
|
|
/** Re-render and resubmit the shell surface; one render in flight, at most one queued. */
|
|
private requestShellRender(): void {
|
|
if (this.shellRenderInProgress) { this.shellRenderQueued = true; return }
|
|
this.shellRenderInProgress = true
|
|
void (async () => {
|
|
try {
|
|
do {
|
|
this.shellRenderQueued = false
|
|
await this.renderShell()
|
|
} while (this.shellRenderQueued)
|
|
} catch (error) { this.fail(error) }
|
|
finally { this.shellRenderInProgress = false }
|
|
})()
|
|
}
|
|
private async renderShell(): Promise<void> {
|
|
await this.displayReady
|
|
const display = this.display
|
|
if (!display) return
|
|
// Taken with the paint, so a render that painted before the wake can't unblank.
|
|
const unblank = this.unblankAfterShellRender
|
|
this.unblankAfterShellRender = false
|
|
const frameId = this.allocateFrameId()
|
|
try {
|
|
const startedAt = Date.now()
|
|
const bytes = shell.paintScene()
|
|
await display.submitShellScene(bytes, Date.now() - startedAt, frameId)
|
|
} finally {
|
|
// Even if the render failed, so the screen doesn't stay dark; not if it
|
|
// went back to sleep meanwhile (that sleep re-blanked).
|
|
if (unblank && shell.isScreenOn()) await display.setScreenBlanked(false)
|
|
}
|
|
// Backpressure: the next shell render waits for this one to reach the
|
|
// glasses (the preview target resolves immediately; nothing transmits).
|
|
await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS)
|
|
this.schedulePreviewUpdate()
|
|
}
|
|
private allocateFrameId(): number {
|
|
const id = this.nextFrameId
|
|
this.nextFrameId = this.nextFrameId >= 0x7fffffff ? 1 : this.nextFrameId + 1
|
|
return id
|
|
}
|
|
/** Refresh the phone mirror from the composited screen, coalesced at 30 fps. */
|
|
private schedulePreviewUpdate(): void {
|
|
if (!this.active || this.previewTimer !== null) return
|
|
this.previewTimer = setTimeout(() => {
|
|
this.previewTimer = null
|
|
if (!this.active) return
|
|
try {
|
|
const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect())
|
|
const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop)
|
|
if (!image) return
|
|
this.previewCrop = crop
|
|
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
|
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
|
} catch (error) { this.fail(error) }
|
|
}, 33)
|
|
}
|
|
private fail(error: unknown): void {
|
|
const message = error instanceof Error ? error.stack ?? error.message : String(error)
|
|
console.error(`[ios-preview] ${message}`)
|
|
this.logBluetooth(`App error: ${message}`)
|
|
if (this.active) this.onError(error instanceof Error ? error.message : String(error))
|
|
}
|
|
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
|
|
this.inputQueue = this.inputQueue.then(async () => {
|
|
if (!this.active) return
|
|
// A mirror tap may land on a hit target rather than being a gesture.
|
|
if (gesture === 'press' && origin === 'mirror') return
|
|
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn() && !this.glassesLocked) {
|
|
await this.mirrorTap(nx, ny)
|
|
} else {
|
|
let type: string = ({ press: 'ring-press', tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
|
|
if (origin === 'ring' && type.startsWith('swipe-')) {
|
|
if (type === 'swipe-left' || type === 'swipe-right') return
|
|
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
|
|
}
|
|
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
|
|
await this.receiveInput(event)
|
|
}
|
|
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
|
|
}).catch(error => this.fail(error))
|
|
}
|
|
/** Input from the Apple Watch: shares the controller queue with physical input. */
|
|
private injectWatchInput(kind: WearRemoteInputKind): Promise<void> {
|
|
const task = this.inputQueue.then(async () => {
|
|
this.onActivity()
|
|
// As on Android, and because the watch may be acting on a stale state snapshot: a dark
|
|
// display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing.
|
|
if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return
|
|
// A real hold releases when the finger lifts; a discrete long-press releases at once.
|
|
const type = kind === 'long-press-start' ? 'long-press' : kind
|
|
await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload))
|
|
if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' }))
|
|
console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`)
|
|
})
|
|
this.inputQueue = task.catch(error => this.fail(error))
|
|
return task
|
|
}
|
|
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
|
|
// Input can change the foreground app or the display state, and the shell doesn't announce those.
|
|
this.wearRemote?.schedulePublish()
|
|
if (!acceptInput(event)) return
|
|
if (this.glassesLocked) {
|
|
// Preserve the locked display's sleep/wake controls without dispatching
|
|
// gestures to apps, shell menus, voice input or the Glanceboard.
|
|
if (event.type === 'double-click' && (event.source === 'ring' || event.source === 'watch')) {
|
|
if (shell.isScreenOn()) shell.sleep()
|
|
else shell.wake('sidebar')
|
|
} else if (event.type === 'display-wake' && !shell.isScreenOn()) shell.wake('sidebar')
|
|
this.requestShellRender()
|
|
return
|
|
}
|
|
if (!shell.isScreenOn()) {
|
|
const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type)
|
|
if (glanceEvent?.type === 'dismiss') this.glance.dismiss()
|
|
else if (glanceEvent) {
|
|
await this.glance.handleEvent(glanceEvent, 0)
|
|
return
|
|
}
|
|
}
|
|
await shell.receiveInput(event)
|
|
this.requestShellRender()
|
|
}
|
|
private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise<void> {
|
|
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
|
|
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
|
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
|
if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) { this.requestShellRender(); return }
|
|
const window = shell.foregroundWindow()
|
|
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId)
|
|
&& isOnSwitcherEdge(x, y, window?.heightMode ?? 'min', window?.appId)) {
|
|
const target = shell.windowAtSidebarPoint(x, y)
|
|
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
|
|
return
|
|
}
|
|
if (window && !shell.hasOverlay()) {
|
|
const rect = appViewportRect(window.heightMode, window.appId)
|
|
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
|
|
shell.focusWindow(window.windowId)
|
|
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
|
|
}
|
|
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
|
|
this.requestShellRender()
|
|
}
|
|
async launchApp(id: string, params?: AppLaunchParams): Promise<void> {
|
|
const app = ALL_APPS.find(candidate => candidate.appId === id)
|
|
if (!app) {
|
|
const installed = getInstalledEvenHubAppById(id)
|
|
const host = ALL_APPS.find(candidate => candidate.appId === 'evenhub')
|
|
if (installed && host) {
|
|
try {
|
|
if (isInstalledPackagePresent(installed.packageId)) {
|
|
try { await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed); return }
|
|
catch (error) {
|
|
// Checked only after a failure: it reads the whole package again.
|
|
if (!discardZeroedEvenHubPackage(installed.packageId)) throw error
|
|
console.log(`[ios-preview] Discarded zeroed package for ${installed.packageId}; opening store page`)
|
|
}
|
|
}
|
|
await openEvenHubStoreForPackage(this.buildAppContext(host), installed)
|
|
}
|
|
catch (error) { this.fail(error) }
|
|
}
|
|
return
|
|
}
|
|
try {
|
|
const reason = iosAppUnavailableReason(id)
|
|
if (reason) {
|
|
await this.launchInProcessApp(id, `window:${id}`, options => createInProcessWindow({
|
|
...options, appId: id, windowId: id, title: app.title, icon: app.icon,
|
|
iconLetter: app.title[0], closeable: true,
|
|
baseLayer: new YieldAtRootLayer(new TextViewerLayer(reason, app.title)),
|
|
}))
|
|
} else await app.launch(this.buildAppContext(app), params)
|
|
console.log(`[ios-preview] Launched ${id}`)
|
|
} catch (error) { this.fail(error) }
|
|
}
|
|
private async uninstallApp(id: string): Promise<void> {
|
|
const packageId = installedEvenHubPackageId(id)
|
|
if (!packageId) return
|
|
closeRunningPackage(packageId)
|
|
uninstallEvenHubPackage(packageId)
|
|
this.requestShellRender()
|
|
}
|
|
private buildAppContext(app: AppDefinition): AppContext {
|
|
return {
|
|
appId: app.appId, apps: ALL_APPS, actions: this.actions,
|
|
launchApp: (id, params) => this.launchApp(id, params), uninstallApp: id => this.uninstallApp(id),
|
|
launchInProcessApp: (id, surface, create) => this.launchInProcessApp(id, surface, create),
|
|
openInProcessAppInBackground: (id, surface, create) => this.openInProcessAppInBackground(id, surface, create),
|
|
ensureWorkerHost: create => this.ensureWorkerHost(app.appId, create),
|
|
submitWindowFrame: (id, planes) => this.submit(id, planes),
|
|
setWindowSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
|
|
requestShellRender: () => this.requestShellRender(), appendLog: message => console.log(`[ios-app] ${message}`),
|
|
setTextEditorHost: () => {},
|
|
}
|
|
}
|
|
private async launchInProcessApp(windowId: string, surfaceId: string,
|
|
create: (options: InProcessAppOptions) => InProcessWindow): Promise<void> {
|
|
const existing = this.inProcessApps.get(windowId)
|
|
if (existing) { shell.focusWindow(windowId); existing.requestRender(); this.requestShellRender(); return }
|
|
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
|
await this.configureWindow(app.window)
|
|
surface.ready = true
|
|
shell.registerWindow(app.window); shell.focusWindow(windowId)
|
|
app.requestRender(); this.requestShellRender()
|
|
}
|
|
/**
|
|
* An in-process singleton app's window, opened in the background (neither
|
|
* foregrounded nor focused) unless it is open already, for the shell to
|
|
* bring forward itself: the Notifications window under a notification
|
|
* selected in the switcher, or an app widget's window under the widget.
|
|
* Display calls run in order and frames wait on displayReady before
|
|
* queueing, so the surface configure queued here lands ahead of any frame
|
|
* and of the shell foregrounding the window, which it may do at once.
|
|
*/
|
|
private openInProcessAppInBackground(windowId: string, surfaceId: string,
|
|
create: (options: InProcessAppOptions) => InProcessWindow): { window: InProcessWindow; opened: boolean } {
|
|
const existing = this.inProcessApps.get(windowId)
|
|
if (existing) return { window: existing, opened: false }
|
|
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
|
void this.configureWindow(app.window).catch(error => this.fail(error))
|
|
surface.ready = true
|
|
shell.registerWindow(app.window)
|
|
return { window: app, opened: true }
|
|
}
|
|
/**
|
|
* Create an in-process singleton app's window (not yet registered with the
|
|
* shell). Its frames reach the compositor only while `surface.ready`: apps
|
|
* can request a render while their factory runs (Nightscout's tray
|
|
* subscription does), before the caller has configured the surface.
|
|
*/
|
|
private createInProcessApp(windowId: string, surfaceId: string,
|
|
create: (options: InProcessAppOptions) => InProcessWindow): { app: InProcessWindow; surface: { ready: boolean } } {
|
|
const surface = { ready: false }
|
|
const app = create({
|
|
actions: this.actions, submitFrame: planes => surface.ready ? this.submit(surfaceId, planes, () => surface.ready) : Promise.resolve(),
|
|
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
|
removeSurface: () => { surface.ready = false; this.removeWindowSurface(surfaceId) },
|
|
reconfigureSurface: () => {
|
|
const window = shell.getWindows().find(w => w.windowId === windowId)
|
|
if (window) void this.configureWindow(window).catch(error => this.fail(error))
|
|
this.requestShellRender()
|
|
},
|
|
onClosed: () => { this.inProcessApps.delete(windowId) },
|
|
})
|
|
this.inProcessApps.set(windowId, app)
|
|
return { app, surface }
|
|
}
|
|
private ensureWorkerHost(appId: string, create: () => Worker): WorkerAppHost {
|
|
const existing = this.appHosts.get(appId)
|
|
if (existing) return existing
|
|
const worker = create()
|
|
const navigationSensors = appId === 'navigate'
|
|
? new IosNavigationSensors(event => worker.postMessage({ type: 'navigation-sensors', event })) : undefined
|
|
const host = new WorkerAppHost({
|
|
appId, worker, navigationSensors,
|
|
onStopping: () => { if (this.appHosts.get(appId) === host) this.appHosts.delete(appId) },
|
|
playBuzzerSequence: payload => this.actions.playBuzzerSequence(payload),
|
|
openUrl: url => { void Utils.openUrl(url) },
|
|
configureSurface: async (id, visible, heightMode) => {
|
|
const display = this.display
|
|
if (!display) return
|
|
await display.configureSurface(id, { ...appViewportRect(heightMode, appId), zOrder: 0, transparency: 'opaque' })
|
|
await display.setSurfaceVisible(id, typeof visible === 'function' ? visible() : visible)
|
|
},
|
|
setSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
|
|
removeSurface: id => this.removeWindowSurface(id),
|
|
submitPixels: (id, pixels, width, height, draws) => {
|
|
void this.submitWorkerPixels(id, pixels, width, height, draws).catch(error => this.fail(error))
|
|
},
|
|
requestShellRender: () => this.requestShellRender(),
|
|
openSettings: section => { void this.launchApp('settings', { section }) },
|
|
startTextSettingEdit: id => { const setting = getStringSettingById(id); if (setting) void this.editSetting(setting) },
|
|
endTextSettingEdit: () => {},
|
|
startTextInput: () => shell.startVoiceInput(),
|
|
})
|
|
this.appHosts.set(appId, host)
|
|
return host
|
|
}
|
|
get connectionState(): SessionState | null { return this.sessionCreated ? this.state : null }
|
|
get connectionDetails(): string {
|
|
const state = this.connectionState
|
|
return state ? `${state.status}\nL: ${state.leftVersion || 'unknown'}\nR: ${state.rightVersion || 'unknown'}\nBattery: ${state.battery ?? '?'}%\nRing: ${state.ring ? 'connected' : 'disconnected'}\nFrames acknowledged: ${state.frames}\n${state.capabilities}\n\n${this.logLines.slice(-12).join('\n')}` : 'Preview only. Configure devices, then connect.'
|
|
}
|
|
private update(phase: SessionPhase, status: string): void {
|
|
this.state = { ...this.state, phase, status }
|
|
this.logBluetooth(status)
|
|
this.emitState()
|
|
}
|
|
/** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */
|
|
private syncPresence(): void {
|
|
updateGlassesPresence({ connected: this.state.phase === 'connected', worn: this.glassesWorn, charging: this.state.charging ?? false })
|
|
}
|
|
private emitState(): void {
|
|
if (this.state.phase !== 'connected') resetRingInputFilter()
|
|
this.pushBrightness()
|
|
this.maybeLaunchOnboarding(this.state)
|
|
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
|
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
|
|
this.syncRawMicSession()
|
|
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
|
this.syncContinuousSession()
|
|
this.pushBatteryLevels()
|
|
this.syncPresence()
|
|
this.syncRuntime()
|
|
if (this.active) this.onConnectionState({ ...this.state })
|
|
this.requestShellRender()
|
|
this.wearRemote?.schedulePublish()
|
|
}
|
|
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
|
|
private pushBatteryLevels(): void {
|
|
const connected = this.state.phase === 'connected'
|
|
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
|
|
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
|
|
}
|
|
/** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */
|
|
async connect(fromBackground = false): Promise<void> {
|
|
if ((!this.active && !fromBackground) || this.connecting || this.stopping) return
|
|
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
|
|
const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses)
|
|
if (error) {
|
|
if (this.active) this.onError(error)
|
|
else this.logBluetooth(error)
|
|
return
|
|
}
|
|
this.connecting = true
|
|
this.sessionCreated = true
|
|
try {
|
|
this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false,
|
|
ringBattery: null, ringCharging: null }
|
|
this.firmwareCompatible = false
|
|
this.update('connecting', 'Finding configured devices…')
|
|
let identifiers
|
|
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
|
|
// fails and retries every few seconds, starving the glasses link.
|
|
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
|
|
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
|
|
catch (failure) {
|
|
this.update('error', failure instanceof Error ? failure.message : String(failure))
|
|
return
|
|
}
|
|
if (this.communicator) await this.closeCommunicator()
|
|
const communicator = new FaceclawCommunicatorBridge({ ...identifiers, glassesType: glassesTypeSetting.get() })
|
|
this.teardownPreviewDisplay()
|
|
this.communicator = communicator
|
|
this.wearDetectionRequested = false
|
|
this.bindCommunicator(communicator)
|
|
this.displayReady = this.registerSurfaces(communicator)
|
|
await this.displayReady
|
|
this.update('connecting', 'Connecting to the glasses…')
|
|
await communicator.configureBrightness(getBrightnessPreferences())
|
|
this.startedCommunicators.add(communicator)
|
|
await communicator.start()
|
|
for (const window of shell.getWindows()) window.requestRender()
|
|
this.requestShellRender()
|
|
} catch (failure) {
|
|
this.fail(failure)
|
|
await this.closeCommunicator().catch(() => {})
|
|
this.ensurePreviewDisplay()
|
|
this.update('error', failure instanceof Error ? failure.message : String(failure))
|
|
} finally {
|
|
this.connecting = false
|
|
}
|
|
}
|
|
private bindCommunicator(communicator: FaceclawCommunicatorBridge): void {
|
|
const notifications = new AncsClient(packet => communicator.writeRawToRight(packet), (key, popup) => {
|
|
iosNotificationsChanged(key, popup); this.requestShellRender()
|
|
}, message => this.logBluetooth('ANCS: ' + message))
|
|
this.notifications = notifications
|
|
bindIosNotifications(notifications)
|
|
bindCompassSession(communicator)
|
|
bindBandwidthBenchmarkSession(communicator)
|
|
communicator.setRequiresAncs(true)
|
|
this.sessionOffs.push(
|
|
communicator.onStateChange(state => this.handleSessionState(communicator, state)),
|
|
communicator.onRingEvent(input => this.handleRingEvent(input)),
|
|
communicator.onBatteryState(state => {
|
|
if (this.communicator !== communicator) return
|
|
// Unlike the headset level, an unavailable ring reading (-1: ring away or old
|
|
// firmware) clears the indicator, as on Android.
|
|
const ringBattery = state.ringBattery ?? -1, hasRingBattery = Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100
|
|
this.state = { ...this.state, battery: state.battery >= 0 && state.battery <= 100 ? state.battery : this.state.battery,
|
|
charging: state.chargingStatus >= 0 ? state.chargingStatus > 0 : this.state.charging,
|
|
ring: hasRingBattery || this.state.ring,
|
|
ringBattery: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null }
|
|
this.pushBatteryLevels()
|
|
if (this.active) this.onConnectionState({ ...this.state })
|
|
this.requestShellRender()
|
|
this.wearRemote?.schedulePublish()
|
|
}),
|
|
communicator.onCaseState(state => {
|
|
if (this.communicator !== communicator) return
|
|
// As on Android: keep the last level when a reading has none (out of the case);
|
|
// the case's own charging is only known while the glasses are docked.
|
|
shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}),
|
|
caseCharging: state.docked ? state.charging : null })
|
|
this.wearRemote?.schedulePublish()
|
|
}),
|
|
communicator.onFirmwareInfo(info => {
|
|
if (this.communicator !== communicator) return
|
|
this.logBluetooth(`Firmware: L=${info.leftVersion || '?'} R=${info.rightVersion || '?'}` +
|
|
(info.extension ? ` ext="${info.extension}"` : ' (no firmware extension string)'))
|
|
// The shared session halts on incompatible firmware by itself ("incompatible-firmware"
|
|
// phase, handled in handleSessionState); this only records why, for the warning.
|
|
const firmwareWarning = firmwareIncompatibilityMessage(info) ?? ''
|
|
if (firmwareWarning && firmwareWarning !== this.state.firmwareWarning) this.logBluetooth(`Firmware compatibility warning: ${firmwareWarning}`)
|
|
this.firmwareCompatible = !firmwareWarning && hasCompatibleFirmware(info)
|
|
this.state = { ...this.state, leftVersion: info.leftVersion || this.state.leftVersion,
|
|
rightVersion: info.rightVersion || this.state.rightVersion, capabilities: info.extension || this.state.capabilities,
|
|
firmwareWarning }
|
|
if (this.active) this.onConnectionState({ ...this.state })
|
|
this.syncNotifications(communicator)
|
|
this.ensureWearDetection(communicator)
|
|
}),
|
|
communicator.onFrameMetrics(() => {
|
|
if (this.communicator !== communicator) return
|
|
this.state = { ...this.state, frames: this.state.frames + 1 }
|
|
this.onActivity()
|
|
this.maybeLaunchOnboarding(this.state)
|
|
if (this.active) this.onConnectionState({ ...this.state })
|
|
this.schedulePreviewUpdate()
|
|
}),
|
|
communicator.onPreviewAnimationFrame(() => { if (this.communicator === communicator) this.schedulePreviewUpdate() }),
|
|
communicator.onWearState(wearing => { if (this.communicator === communicator) this.handleWearState(wearing) }),
|
|
communicator.addCompassListener(receiveCompassEvent),
|
|
communicator.onAncsRelayFrame(frame => { if (this.communicator === communicator) notifications.receive(frame) }),
|
|
communicator.onAncsAuthorization(authorized => {
|
|
if (this.communicator !== communicator) return
|
|
if (authorized) { this.syncNotifications(communicator); return }
|
|
const active = notifications.state !== 'disconnected'
|
|
const command = notifications.stopCommand()
|
|
notifications.stop('Enable Share System Notifications in iPhone Settings → Bluetooth → right lens.')
|
|
if (active && this.state.phase === 'connected') void communicator.writeRawToRight(command).catch(() => {})
|
|
}),
|
|
)
|
|
}
|
|
private handleSessionState(communicator: FaceclawCommunicatorBridge, state: CommunicatorState): void {
|
|
if (this.communicator !== communicator) return
|
|
// Registering the listener makes the Kotlin session report its current state, which is
|
|
// "disconnected" before start(); that must not be mistaken for the session ending.
|
|
if (!this.startedCommunicators.has(communicator)) return
|
|
// The shared session parked its retry loop (an arm unpaired, or firmware Faceclaw can't
|
|
// run): a redial would fail the same way, so stay down until the user connects explicitly
|
|
// (or re-pairs / installs the custom firmware, which lift the suppression).
|
|
if (isHaltedSessionPhase(state.phase)) suppressAutoReconnect()
|
|
const phase: SessionPhase = state.phase === 'charging' ? 'connected' : isHaltedSessionPhase(state.phase) ? 'error' : state.phase
|
|
// The relay ends with the link, so a reconnected session has to ask for it again (syncNotifications).
|
|
if (phase !== 'connected' && this.notifications && this.notifications.state !== 'disconnected') this.notifications.stop()
|
|
// Wear state is a per-transport snapshot: ask again on every new session.
|
|
if (phase !== 'connected') this.wearDetectionRequested = false
|
|
this.update(phase, state.status)
|
|
if (phase === 'connected') {
|
|
this.syncNotifications(communicator)
|
|
this.ensureWearDetection(communicator)
|
|
} else if ((phase === 'disconnected' || phase === 'error') && !this.stopping) {
|
|
// The session ended on its own (an arm unpaired, incompatible firmware): release it and go headless.
|
|
void this.disconnect(state.status)
|
|
}
|
|
}
|
|
/**
|
|
* Wear reports come from the custom firmware, so like Android only ask once the session
|
|
* is up and the firmware is confirmed compatible (stock firmware never hears the query).
|
|
*/
|
|
private ensureWearDetection(communicator: FaceclawCommunicatorBridge): void {
|
|
if (this.wearDetectionRequested || !this.firmwareCompatible || this.state.phase !== 'connected') return
|
|
this.wearDetectionRequested = true
|
|
void communicator.enableWearDetectionAndRequestState().catch(error => this.logBluetooth(`Wear detection: ${String(error)}`))
|
|
}
|
|
private handleRingEvent(input: RawInputEvent): void {
|
|
this.inputQueue = this.inputQueue.then(async () => {
|
|
if (this.state.phase !== 'connected') return
|
|
this.onActivity()
|
|
if (input.ringInput) this.state = { ...this.state, ring: true }
|
|
await this.receiveInput(rawInputEventToInputEvent(input),
|
|
input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT)
|
|
this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`)
|
|
}).catch(error => this.fail(error))
|
|
}
|
|
/** Any glasses traffic: CoreBluetooth can wake a suspended process, so refresh what timers would have. */
|
|
private onActivity(): void {
|
|
if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true)
|
|
this.refreshClock()
|
|
}
|
|
/** Start iPhone notification relay once the firmware is known to support it and ANCS is authorized. */
|
|
private syncNotifications(communicator: FaceclawCommunicatorBridge): void {
|
|
const notifications = this.notifications
|
|
if (!notifications || this.state.phase !== 'connected') return
|
|
if (!ancsCapable(this.state.capabilities)) {
|
|
if (this.state.capabilities) notifications.stop(`Update glasses to Faceclaw firmware ${ANCS_FIRMWARE_VERSION} or newer for iPhone notifications.`)
|
|
return
|
|
}
|
|
// A G1 reports no revision that says so: its session knows from what the right arm answered.
|
|
if (!communicator.ancsRelayAvailable()) {
|
|
notifications.stop(`Update the glasses to Faceclaw's firmware ${G1V0_IMAGE.version} for iPhone notifications, then reconnect.`)
|
|
return
|
|
}
|
|
if (notifications.state !== 'disconnected') return
|
|
notifications.start((Date.now() ^ Math.floor(Math.random() * 0xffffffff)) >>> 0, communicator.rightWriteLimit())
|
|
}
|
|
async disconnect(reason?: string): Promise<void> {
|
|
if (this.stopping) return this.stopping
|
|
const communicator = this.communicator
|
|
if (!communicator) return
|
|
// Marked before any bridge call: the session may report "disconnected" from inside disconnect().
|
|
let finished!: () => void
|
|
this.stopping = new Promise<void>(resolve => { finished = resolve })
|
|
try {
|
|
const notifications = this.notifications
|
|
if (notifications && notifications.state !== 'disconnected') {
|
|
const command = notifications.stopCommand()
|
|
notifications.stop()
|
|
if (this.state.phase === 'connected') { try { await communicator.writeRawToRight(command) } catch {} }
|
|
}
|
|
if (this.state.phase === 'connected' || this.state.phase === 'connecting' || this.state.phase === 'retrying')
|
|
this.update('disconnecting', 'Disconnecting…')
|
|
await communicator.disconnect().catch(error => this.logBluetooth(`Disconnect: ${String(error)}`))
|
|
} finally {
|
|
await this.closeCommunicator()
|
|
this.ensurePreviewDisplay()
|
|
this.update(reason ? 'error' : 'disconnected', reason ?? 'Preview only')
|
|
this.stopping = null
|
|
finished()
|
|
}
|
|
}
|
|
private async closeCommunicator(): Promise<void> {
|
|
const communicator = this.communicator
|
|
if (!communicator) return
|
|
for (const off of this.sessionOffs.splice(0)) off()
|
|
this.notifications = null
|
|
this.communicator = null
|
|
bindBandwidthBenchmarkSession(null)
|
|
// Releases an EvenHub app's mic before the session closes.
|
|
this.syncRawMicSession()
|
|
this.glance.reset()
|
|
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
|
}
|
|
private maybeLaunchOnboarding(state: SessionState): void {
|
|
const firstNewFrame = state.frames > this.acknowledgedFrames
|
|
this.acknowledgedFrames = state.frames
|
|
// Match Android: wait for an acknowledged frame (a warm session, so the
|
|
// onboarding's welcome jingle isn't dropped) before opening it.
|
|
if (!firstNewFrame || state.phase !== 'connected' || !claimGlassesOnboardingAutoLaunch()) return
|
|
void this.launchApp(ONBOARDING_APP_ID)
|
|
}
|
|
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
|
|
private async prepareVoiceCapture(): Promise<boolean> {
|
|
if (this.glassesLocked) return false
|
|
const usePhoneMic = this.state.phase !== 'connected'
|
|
if (usePhoneMic && !this.active) return false
|
|
const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider())
|
|
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
|
return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active)
|
|
}
|
|
private async startVoiceCapture(endpointing = false): Promise<void> {
|
|
if (this.glassesLocked) return
|
|
const log = (message: string) => this.logBluetooth(message)
|
|
const communicator = this.communicator
|
|
const voice = this.voiceProvider()
|
|
if (communicator && this.state.phase === 'connected') {
|
|
await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing)
|
|
} else if (this.active) {
|
|
// Recheck microphone permission if the glasses disconnected after prepare.
|
|
if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return
|
|
await iosVoiceInput.startPhoneCapture(voice, log, endpointing)
|
|
}
|
|
}
|
|
/**
|
|
* Continuous capture on behalf of `client` (one id per window: Transcribe,
|
|
* Translate, Teleprompter, Calculator); it runs while any client wants it, as
|
|
* on Android. `recordAs` names a WAV of the audio in Downloads when Save
|
|
* Transcribe audio is on.
|
|
*/
|
|
private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void {
|
|
this.continuousClients.set(client, options)
|
|
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
|
this.continuousClients.delete(client)
|
|
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
|
return
|
|
}
|
|
this.syncContinuousRecording()
|
|
// A listening capture is shared as it is; otherwise start one (after any drain).
|
|
if (!iosVoiceInput.continuousSource) this.continuousPending = true
|
|
this.syncContinuousCapture()
|
|
}
|
|
private stopContinuousVoiceCapture(client: string): void {
|
|
if (!this.continuousClients.delete(client)) return
|
|
this.syncContinuousRecording()
|
|
// The window that needed its own provider may be the one that left.
|
|
if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return }
|
|
// A start still awaiting permission or the mic backs off.
|
|
++this.continuousCaptureToken
|
|
this.continuousPending = false
|
|
this.syncContinuousClaim()
|
|
void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`))
|
|
}
|
|
/** Connecting or losing the glasses moves continuous capture to the new audio source. */
|
|
private syncContinuousSession(): void {
|
|
const connected = this.state.phase === 'connected'
|
|
if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true
|
|
this.sessionConnected = connected
|
|
this.syncContinuousCapture()
|
|
}
|
|
/**
|
|
* Keep continuous capture running while a window wants it: start a pending
|
|
* one once audio is available, and move a phone capture to the glasses once
|
|
* they connect. A glasses session supplies the audio, even while it
|
|
* reconnects (as on Android); the phone mic stands in only without one, while
|
|
* Faceclaw is in the foreground. Cheap enough to call on every state change.
|
|
*/
|
|
private syncContinuousCapture(): void {
|
|
this.syncContinuousClaim()
|
|
if (this.continuousClients.size === 0 || this.glassesLocked) return
|
|
const glassesReady = !!this.communicator && this.state.phase === 'connected'
|
|
if (iosVoiceInput.continuousActive) {
|
|
// Draining or listening already. A phone capture from before the glasses
|
|
// connected is drained here, and its end restarts it on the glasses; so
|
|
// is one transcribing with another provider than its windows now need.
|
|
const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider())
|
|
if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) {
|
|
this.continuousPending = true
|
|
void iosVoiceInput.stopContinuousCapture()
|
|
}
|
|
return
|
|
}
|
|
if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return
|
|
this.continuousPending = false
|
|
void this.beginContinuousCapture()
|
|
}
|
|
private async beginContinuousCapture(): Promise<void> {
|
|
const voice = this.voiceProvider()
|
|
if (!iosVoiceInput.supportsContinuous(voice)) return
|
|
this.continuousTranscription = transcriptionKey(voice)
|
|
// A stop during an await below bumps the token; this start then backs off.
|
|
const token = ++this.continuousCaptureToken
|
|
const current = () => token === this.continuousCaptureToken
|
|
const log = (message: string) => this.logBluetooth(message)
|
|
// Claimed before the capture starts so other mic users stand down first, as on Android.
|
|
voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER)
|
|
const communicator = this.communicator
|
|
if (communicator) {
|
|
await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true)
|
|
} else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) {
|
|
await iosVoiceInput.startPhoneCapture(voice, log, false, true)
|
|
}
|
|
if (current()) this.syncContinuousClaim()
|
|
}
|
|
/**
|
|
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
|
|
* unlocked glasses session, as speech capture does; the EvenHub mic router
|
|
* decides which app hears it and when.
|
|
*/
|
|
private syncRawMicSession(): void {
|
|
const ready = this.state.phase === 'connected' && !this.glassesLocked
|
|
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
|
|
}
|
|
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
|
private syncContinuousClaim(): void {
|
|
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
|
}
|
|
/** Record to the first client's requested name while Save Transcribe audio is on. */
|
|
private syncContinuousRecording(): void {
|
|
const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : ''
|
|
iosVoiceInput.recordTo(name)
|
|
}
|
|
/**
|
|
* The transcription provider setting and the keys the cloud ones need. A
|
|
* window that needs its own provider and language (Translate) sets them for
|
|
* every capture while it is open; otherwise only Whisper server is told a
|
|
* language (its own setting).
|
|
*/
|
|
private voiceProvider(): VoiceProviderSelection {
|
|
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
|
const provider = choice?.provider ?? voiceProviderSetting.get()
|
|
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
|
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
|
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
|
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
|
}
|
|
private logBluetooth(message: string): void {
|
|
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
|
console.log(`[ios-ble] ${line}`); this.logLines.push(line)
|
|
if (this.logLines.length > 1000) this.logLines.shift()
|
|
if (this.logTimer !== null) return
|
|
this.logTimer = setTimeout(() => {
|
|
this.logTimer = null
|
|
this.flushBluetoothLog()
|
|
}, 500)
|
|
}
|
|
private flushBluetoothLog(): void {
|
|
if (this.logTimer !== null) clearTimeout(this.logTimer)
|
|
this.logTimer = null
|
|
try { File.fromPath(path.join(knownFolders.documents().path, 'bluetooth.log')).writeTextSync(this.logLines.join('\n')) }
|
|
catch (error) { console.warn(`Bluetooth log: ${error}`) }
|
|
}
|
|
typeIntoApp(): void {
|
|
if (this.glassesLocked || !this.active) return
|
|
shell.startKeyboardInput()
|
|
}
|
|
private async editSetting(setting: { editorTitle: string; inputKind?: string; get(): string; set(value: string): void; validationError?(value?: string): string | null }): Promise<boolean> {
|
|
if (!this.active) { this.logBluetooth('Editing text settings requires opening Faceclaw on the phone'); return false }
|
|
let draft = setting.get()
|
|
while (true) {
|
|
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: draft, inputType: setting.inputKind, okButtonText: 'Save', cancelButtonText: 'Cancel' })
|
|
if (!result.result) return false
|
|
draft = result.text
|
|
const error = setting.validationError?.(draft)
|
|
if (error) {
|
|
await Dialogs.alert({ title: setting.editorTitle, message: error, okButtonText: 'OK' })
|
|
continue
|
|
}
|
|
setting.set(draft)
|
|
if (setting === nightscoutSiteUrlSetting || setting === nightscoutApiTokenSetting) await nightscoutBridge.refreshNow()
|
|
return true
|
|
}
|
|
}
|
|
}
|