mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Port lock screen to iOS
This commit is contained in:
1 parent
ef7878359f
commit
e2b35ada1a
12 files changed
+273
-46
No files matched your search
@@ -51,8 +51,7 @@ import { voiceControlBridge } from "../native/voice-control";
|
||||
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane";
|
||||
import { prepareFrameDraws } from "../graphics/glyph-wire";
|
||||
import { getDefaultMediumFont } from "../graphics/ui-fonts";
|
||||
import { wrapText } from "../graphics/textwrap";
|
||||
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen";
|
||||
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { type InputEvent } from "../ui/gestures";
|
||||
import { registerSystemTools } from "../assistant/system-tools";
|
||||
@@ -141,8 +140,6 @@ type DashboardListener = (snapshot: DashboardSnapshot) => void;
|
||||
const SHELL_SURFACE_ID = "shell";
|
||||
/** The shell chrome composites above every window surface (zOrder 0). */
|
||||
const SHELL_SURFACE_Z_ORDER = 1;
|
||||
const LOCK_SCREEN_SURFACE_ID = "lock-screen";
|
||||
const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glasses.";
|
||||
// Top-bar clock refresh; the phone-side preview polls the Java composite so
|
||||
// it reflects every app (including worker apps the TS side never renders).
|
||||
const SHELL_REFRESH_INTERVAL_MS = 60_000;
|
||||
@@ -172,24 +169,6 @@ function createInitialDisplayPreview(): ImageSource | null {
|
||||
return grayImageToPreviewSource(new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0));
|
||||
}
|
||||
|
||||
function createLockScreenImage(): GrayImage {
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const font = getDefaultMediumFont();
|
||||
const boxWidth = 480;
|
||||
const boxHeight = 150;
|
||||
const boxX = Math.round((G2_LENS_WIDTH - boxWidth) / 2);
|
||||
const boxY = Math.round((G2_LENS_HEIGHT - boxHeight) / 2);
|
||||
image.drawRoundedRect(boxX, boxY, boxWidth, boxHeight, 150, 12);
|
||||
const lines = wrapText(font, LOCK_SCREEN_MESSAGE, boxWidth - 64);
|
||||
const textHeight = lines.length * font.lineHeight;
|
||||
const firstY = boxY + Math.round((boxHeight - textHeight) / 2);
|
||||
lines.forEach((line, index) => {
|
||||
const x = Math.round((G2_LENS_WIDTH - font.measureText(line)) / 2);
|
||||
image.drawText(font, x, firstY + index * font.lineHeight, line, 230);
|
||||
});
|
||||
return image;
|
||||
}
|
||||
|
||||
function formatTimestamp(date: Date): string {
|
||||
return date.toISOString().slice(11, 23);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import { CfwTransport, CFW_MAX_MESSAGE, parseCfwAcks } from './cfw-transport'
|
||||
import { hasCompatibleFirmware, firmwareIncompatibilityMessage, REQUIRED_FACECLAW_FIRMWARE_VERSION } from './firmware-compat'
|
||||
import { hexToBytes } from '../util/hex-util'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
import { decodeWearState, enableWearDetection, queryWearState } from './wear-protocol'
|
||||
|
||||
declare function setTimeout(callback: () => void, ms: number): number
|
||||
declare function clearTimeout(id: number): void
|
||||
@@ -84,7 +85,8 @@ export class GlassesSession {
|
||||
private readonly onInput: (input: protocol.GlassesInput) => void,
|
||||
private readonly log: (message: string) => void = () => {},
|
||||
private readonly onActivity: () => void = () => {},
|
||||
private readonly onCompass: (event: CompassEvent) => void = () => {}) {
|
||||
private readonly onCompass: (event: CompassEvent) => void = () => {},
|
||||
private readonly onWearState: (wearing: boolean) => void = () => {}) {
|
||||
this.off = transport.onEvent(event => this.receive(event))
|
||||
}
|
||||
private update(phase: SessionState['phase'], status: string): void {
|
||||
@@ -134,6 +136,7 @@ export class GlassesSession {
|
||||
await this.lease(true)
|
||||
await this.request('right', protocol.SID.hub, protocol.createLayout, 'Create display layout')
|
||||
this.check(generation); this.layoutCreated = true
|
||||
await this.enableWearDetectionAndRequestState(); this.check(generation)
|
||||
this.lastHeartbeat = this.lastLease = this.lastSettings = Date.now()
|
||||
this.displayed = null; this.retryCount = 0
|
||||
this.update('connected', 'Glasses connected')
|
||||
@@ -353,6 +356,8 @@ export class GlassesSession {
|
||||
}
|
||||
}
|
||||
if (message.sid === protocol.SID.settings) this.applySettings(message)
|
||||
const wearing = decodeWearState(message)
|
||||
if (wearing !== null) this.onWearState(wearing)
|
||||
if (event.identifier === this.ids.right) {
|
||||
if (this.state.phase === 'connected') {
|
||||
const compass = protocol.decodeCompassInput(message)
|
||||
@@ -389,6 +394,16 @@ export class GlassesSession {
|
||||
}
|
||||
this.onState({ ...this.state })
|
||||
}
|
||||
async enableWearDetectionAndRequestState(): Promise<void> {
|
||||
const generation = this.generation
|
||||
// Like Android, a missing enable ACK must not suppress the current-state query.
|
||||
try { await this.request('right', protocol.SID.settings, enableWearDetection, 'Enable wear detection') }
|
||||
catch (error) { if (!(error instanceof AckTimeout)) throw error; this.log(this.message(error)) }
|
||||
this.check(generation)
|
||||
await this.send('right', protocol.SID.settings, 0x20, queryWearState())
|
||||
this.check(generation)
|
||||
await this.send('left', protocol.SID.settings, 0x20, queryWearState())
|
||||
}
|
||||
setFrame(gray: Uint8Array): void {
|
||||
if (this.state.phase !== 'connected') return
|
||||
this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0)
|
||||
|
||||
@@ -16,6 +16,7 @@ import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { nightscoutBridge } from '../native/nightscout-bridge'
|
||||
import { GlassesSession, type SessionState } from './glasses-session'
|
||||
import { GlanceHost, type GlanceDisplay } from './glance-host'
|
||||
import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen'
|
||||
import { OsEventTypeList } from './events'
|
||||
import { loadDeviceAddresses } from './device-addresses'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
@@ -37,7 +38,7 @@ import { TextViewerLayer } from '../apps/files/text-viewer'
|
||||
import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell'
|
||||
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||
previewColorSetting } from '../ui/dashboard-settings'
|
||||
previewColorSetting, lockScreenEnabledSetting } from '../ui/dashboard-settings'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
|
||||
/** iOS host for the shared app registry, shell, compositor and BLE session. */
|
||||
@@ -60,7 +61,7 @@ export class IosPreviewController {
|
||||
private readonly glance = new GlanceHost({
|
||||
getDisplay: () => this.glanceDisplay,
|
||||
getProvider: () => ALL_APPS.find(app => app.glanceboard)?.glanceboard ?? null,
|
||||
canShow: () => this.runtimeNeeded,
|
||||
canShow: () => this.runtimeNeeded && !this.glassesLocked,
|
||||
// iOS retains the EvenHub session while the shell sleeps. The board's
|
||||
// opaque first frame is ready before we unblank the compositor.
|
||||
ensureSessionActive: async () => {
|
||||
@@ -86,6 +87,11 @@ export class IosPreviewController {
|
||||
private readonly appHosts = new Map<string, WorkerAppHost>()
|
||||
private readonly inProcessApps = new Map<string, InProcessWindow>()
|
||||
private readonly batteryObservers: any[] = []
|
||||
private readonly lockObservers: any[] = []
|
||||
private phoneLocked = false
|
||||
private glassesWorn: boolean | null = null
|
||||
private glassesLocked = false
|
||||
private lockEnabled = lockScreenEnabledSetting.get()
|
||||
private session: GlassesSession | null = null
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
@@ -112,6 +118,10 @@ export class IosPreviewController {
|
||||
private readonly onError: (message: string) => void,
|
||||
private readonly onConnectionState: (state: SessionState) => void = () => {}) {
|
||||
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||
this.compositor.configureSurface(LOCK_SCREEN_SURFACE_ID, {
|
||||
x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT, zOrder: 1000, transparency: 'opaque',
|
||||
})
|
||||
this.compositor.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, false)
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
voiceInputEnabled: true,
|
||||
@@ -148,6 +158,7 @@ export class IosPreviewController {
|
||||
if (this.active) return
|
||||
this.active = true
|
||||
this.syncRuntime()
|
||||
this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable)
|
||||
this.logBluetooth('Phone foreground')
|
||||
if (this.session) this.onConnectionState({ ...this.session.state })
|
||||
this.session?.wake()
|
||||
@@ -178,6 +189,7 @@ export class IosPreviewController {
|
||||
this.compositor.setScreenBlanked(!shell.isScreenOn())
|
||||
this.offSettings?.(); this.offSettings = null
|
||||
for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
|
||||
for (const observer of this.lockObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer)
|
||||
UIDevice.currentDevice.batteryMonitoringEnabled = false
|
||||
if (this.clockTimer !== null) clearInterval(this.clockTimer)
|
||||
if (this.renderTimer !== null) clearTimeout(this.renderTimer)
|
||||
@@ -185,6 +197,16 @@ export class IosPreviewController {
|
||||
return
|
||||
}
|
||||
UIDevice.currentDevice.batteryMonitoringEnabled = true
|
||||
// Keep these observers alive while BLE owns the runtime, including background.
|
||||
for (const [name, locked] of [
|
||||
[UIApplicationProtectedDataWillBecomeUnavailable, true],
|
||||
[UIApplicationProtectedDataDidBecomeAvailable, false],
|
||||
] as const) {
|
||||
this.lockObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(
|
||||
name, null, NSOperationQueue.mainQueue, () => this.handlePhoneLockState(locked)))
|
||||
}
|
||||
this.handlePhoneLockState(!UIApplication.sharedApplication.protectedDataAvailable)
|
||||
this.syncLockSetting()
|
||||
void nightscoutBridge.start().catch(error => this.fail(error))
|
||||
for (const name of [UIDeviceBatteryLevelDidChangeNotification, UIDeviceBatteryStateDidChangeNotification]) {
|
||||
this.batteryObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(name, null, NSOperationQueue.mainQueue, () => this.requestShellRender()))
|
||||
@@ -192,12 +214,53 @@ export class IosPreviewController {
|
||||
const phone = readPhoneBatteryState()
|
||||
this.logBluetooth(`Phone battery ${phone.battery ?? "unknown"}% charging=${phone.charging}`)
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.syncLockSetting()
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
})
|
||||
this.clockTimer = setInterval(() => this.refreshClock(), 60_000)
|
||||
}
|
||||
private syncLockSetting(): void {
|
||||
const enabled = lockScreenEnabledSetting.get()
|
||||
if (enabled === this.lockEnabled) return
|
||||
this.lockEnabled = enabled
|
||||
if (!enabled) this.setGlassesLocked(false)
|
||||
else {
|
||||
if (this.phoneLocked && this.glassesWorn === false) this.setGlassesLocked(true)
|
||||
if (this.session?.state.phase === 'connected')
|
||||
void this.session.enableWearDetectionAndRequestState().catch(error => this.fail(error))
|
||||
}
|
||||
}
|
||||
private handlePhoneLockState(locked: boolean): void {
|
||||
this.phoneLocked = locked
|
||||
if (!locked) this.setGlassesLocked(false)
|
||||
else if (this.lockEnabled && this.glassesWorn === false) this.setGlassesLocked(true)
|
||||
}
|
||||
private handleWearState(wearing: boolean): void {
|
||||
// Sample false to catch a notification missed during suspension. Do not
|
||||
// sample true here: WillBecomeUnavailable precedes the property transition.
|
||||
if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true)
|
||||
this.glassesWorn = wearing
|
||||
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
|
||||
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
|
||||
}
|
||||
private setGlassesLocked(locked: boolean): void {
|
||||
if (locked === this.glassesLocked) return
|
||||
if (locked) {
|
||||
const image = createLockScreenImage()
|
||||
this.compositor.submitSurfaceFrame(LOCK_SCREEN_SURFACE_ID, image.to8bppBuffer(),
|
||||
{ x: 0, y: 0, width: image.width, height: image.height })
|
||||
}
|
||||
this.glassesLocked = locked
|
||||
if (locked) {
|
||||
this.glance.dismiss()
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.')
|
||||
}
|
||||
this.compositor.setSurfaceVisible(LOCK_SCREEN_SURFACE_ID, locked)
|
||||
this.logBluetooth(`Glasses ${locked ? 'locked' : 'unlocked'}`)
|
||||
this.requestShellRender()
|
||||
}
|
||||
private refreshClock(): void {
|
||||
const minute = Math.floor(Date.now() / 60_000)
|
||||
if (minute !== this.clockMinute) { this.clockMinute = minute; this.requestShellRender() }
|
||||
@@ -240,7 +303,7 @@ export class IosPreviewController {
|
||||
this.session?.setFrame(pixels)
|
||||
if (this.active) {
|
||||
const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green')
|
||||
this.onFrame(image, this.glance.isVisible() ? 'Glanceboard'
|
||||
this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard'
|
||||
: `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||
}
|
||||
} catch (error) { this.fail(error) }
|
||||
@@ -255,7 +318,7 @@ export class IosPreviewController {
|
||||
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
|
||||
this.inputQueue = this.inputQueue.then(async () => {
|
||||
if (!this.active) return
|
||||
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
|
||||
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn() && !this.glassesLocked) {
|
||||
await this.mirrorTap(nx, ny)
|
||||
} else {
|
||||
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
|
||||
@@ -270,6 +333,16 @@ export class IosPreviewController {
|
||||
}).catch(error => this.fail(error))
|
||||
}
|
||||
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
|
||||
if (this.glassesLocked) {
|
||||
// Preserve the locked display's sleep/wake controls without dispatching
|
||||
// gestures to apps, shell menus, voice input or the Glanceboard.
|
||||
if (event.type === 'double-click' && (event.source === 'ring' || event.source === 'watch')) {
|
||||
if (shell.isScreenOn()) shell.sleep()
|
||||
else shell.wake('sidebar')
|
||||
} else if (event.type === 'display-wake' && !shell.isScreenOn()) shell.wake('sidebar')
|
||||
this.requestShellRender()
|
||||
return
|
||||
}
|
||||
if (!shell.isScreenOn()) {
|
||||
const glanceEvent = this.glance.eventForGesture(headTilt ? 'head-tilt' : event.type)
|
||||
if (glanceEvent?.type === 'dismiss') this.glance.dismiss()
|
||||
@@ -409,6 +482,7 @@ export class IosPreviewController {
|
||||
if (error) { this.onError(error); return }
|
||||
if (!this.session) {
|
||||
this.session = new GlassesSession(iosBluetooth(), state => {
|
||||
if (state.phase !== 'connected' && state.phase !== 'connecting') this.glassesWorn = null
|
||||
if (state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging })
|
||||
this.syncRuntime()
|
||||
@@ -421,24 +495,28 @@ export class IosPreviewController {
|
||||
input.kind === 'display-wake' && input.eventType === OsEventTypeList.HEAD_UP_EVENT)
|
||||
this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`)
|
||||
}).catch(error => this.fail(error))
|
||||
}, message => this.logBluetooth(message), () => this.refreshClock(), receiveCompassEvent)
|
||||
}, message => this.logBluetooth(message), () => {
|
||||
if (!UIApplication.sharedApplication.protectedDataAvailable) this.handlePhoneLockState(true)
|
||||
this.refreshClock()
|
||||
}, receiveCompassEvent, wearing => this.handleWearState(wearing))
|
||||
bindCompassSession(this.session)
|
||||
}
|
||||
await this.session.start(addresses)
|
||||
}
|
||||
async disconnect(): Promise<void> { await this.session?.stop() }
|
||||
startVoiceInput(): void { shell.startVoiceInput() }
|
||||
startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() }
|
||||
private async prepareVoiceCapture(): Promise<boolean> {
|
||||
if (this.glassesLocked) return false
|
||||
if (this.session?.state.phase !== 'connected') {
|
||||
if (this.active) this.onError('Connect the glasses to use their microphone.')
|
||||
return false
|
||||
}
|
||||
const ready = await iosVoiceInput.prepare(this.active)
|
||||
if (!ready && this.active) this.onError(iosVoiceInput.statusText)
|
||||
return ready && this.session?.state.phase === 'connected'
|
||||
return ready && !this.glassesLocked && this.session?.state.phase === 'connected'
|
||||
}
|
||||
private async startVoiceCapture(endpointing = false): Promise<void> {
|
||||
if (!this.session || this.session.state.phase !== 'connected') return
|
||||
if (this.glassesLocked || !this.session || this.session.state.phase !== 'connected') return
|
||||
await iosVoiceInput.startGlassesCapture(this.session, message => this.logBluetooth(message), endpointing)
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
@@ -458,13 +536,14 @@ export class IosPreviewController {
|
||||
catch (error) { console.warn(`Bluetooth log: ${error}`) }
|
||||
}
|
||||
async typeIntoApp(): Promise<void> {
|
||||
if (this.glassesLocked) return
|
||||
if (!this.active) { this.logBluetooth('Text input requires opening Faceclaw on the phone'); return }
|
||||
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
|
||||
this.prompting = true
|
||||
try {
|
||||
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
|
||||
message: 'Enter text to send to this app.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
|
||||
if (result.result) shell.sendTextToForegroundWindow(result.text)
|
||||
if (result.result && !this.glassesLocked) shell.sendTextToForegroundWindow(result.text)
|
||||
} catch (error) { this.fail(error) }
|
||||
finally { this.prompting = false }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
import { GrayImage, G2_LENS_WIDTH, G2_LENS_HEIGHT } from "../graphics/image";
|
||||
import { getDefaultMediumFont } from "../graphics/ui-fonts";
|
||||
import { wrapText } from "../graphics/textwrap";
|
||||
|
||||
export const LOCK_SCREEN_SURFACE_ID = "lock-screen";
|
||||
const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glasses.";
|
||||
|
||||
export function createLockScreenImage(): GrayImage {
|
||||
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
|
||||
const font = getDefaultMediumFont();
|
||||
const boxWidth = 480;
|
||||
const boxHeight = 150;
|
||||
const boxX = Math.round((G2_LENS_WIDTH - boxWidth) / 2);
|
||||
const boxY = Math.round((G2_LENS_HEIGHT - boxHeight) / 2);
|
||||
image.drawRoundedRect(boxX, boxY, boxWidth, boxHeight, 150, 12);
|
||||
const lines = wrapText(font, LOCK_SCREEN_MESSAGE, boxWidth - 64);
|
||||
const textHeight = lines.length * font.lineHeight;
|
||||
const firstY = boxY + Math.round((boxHeight - textHeight) / 2);
|
||||
lines.forEach((line, index) => {
|
||||
const x = Math.round((G2_LENS_WIDTH - font.measureText(line)) / 2);
|
||||
image.drawText(font, x, firstY + index * font.lineHeight, line, 230);
|
||||
});
|
||||
return image;
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
import { bytes, concat, integer, readBytes, readInteger, type ProtocolMessage } from './ble-protocol'
|
||||
|
||||
// Matches BleProtocol.buildSetWearDetection/buildFaceclawWakeControl on Android.
|
||||
export const enableWearDetection = (magic: number): Uint8Array => concat(
|
||||
integer(1, 1), integer(2, magic), bytes(3, bytes(5, integer(1, 1))))
|
||||
export const queryWearState = (): Uint8Array => concat(
|
||||
integer(1, 1), integer(2, 0), bytes(101, new Uint8Array([70, 67, 1, 7, 0, 0])))
|
||||
|
||||
export function decodeWearState(message: ProtocolMessage): boolean | null {
|
||||
if (message.sid !== 0x10 || message.command !== 3) return null
|
||||
const event = readBytes(message.payload, 5)
|
||||
if (!event || readInteger(event, 1, -1) !== 1) return null
|
||||
const state = readInteger(event, 2, -1)
|
||||
return state === 0 ? false : state === 1 ? true : null
|
||||
}
|
||||
@@ -91,9 +91,9 @@ export class IosVoiceControlBridge {
|
||||
this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel()
|
||||
this.finishCompletion()
|
||||
}
|
||||
handleSessionEnded(): void {
|
||||
handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void {
|
||||
if (!this.capturing && !this.finalizing) return
|
||||
this.stop(); this.setStatus('Glasses disconnected. Start voice input again after reconnecting.'); this.emitEnd()
|
||||
this.stop(); this.setStatus(message); this.emitEnd()
|
||||
}
|
||||
private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null }
|
||||
private releaseMicrophone(): void {
|
||||
|
||||
@@ -341,7 +341,8 @@ export const lockScreenEnabledSetting = new ConfigSettingBoolean({
|
||||
storageKey: "display.lockScreenEnabled",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses.",
|
||||
"Lock the glasses after they are taken off while the phone is locked. Unlocking the phone unlocks the glasses." +
|
||||
(global.isIOS ? " On iPhone, this requires a device passcode and follows iOS data-protection notifications, which may be delayed after the screen locks." : ""),
|
||||
});
|
||||
|
||||
// Phone display: the phone app's mirror of the glasses screen and the
|
||||
|
||||
@@ -254,7 +254,7 @@ function settingsSections(): SettingsSection[] {
|
||||
description: `${section.label} integration has not been ported to iOS.`,
|
||||
}] };
|
||||
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
|
||||
![brightnessSetting.label, screenTimeoutSetting.label, lockScreenEnabledSetting.label].includes(item.label)) };
|
||||
![brightnessSetting.label, screenTimeoutSetting.label].includes(item.label)) };
|
||||
if (section.label === "Phone display") return { ...section, items: section.items.filter(item => item.label !== mirrorTouchSetting.label) };
|
||||
return section;
|
||||
});
|
||||
|
||||
@@ -80,6 +80,7 @@ for (const isIOS of [true, false]) {
|
||||
uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) };
|
||||
} });
|
||||
const sections = menus.createSettingsPanelLayer().sections;
|
||||
assert.ok(sections.find(section => section.label === 'Display').items.some(item => item.label === settings.lockScreenEnabledSetting.label));
|
||||
const assistant = sections.find(section => section.label === 'Assistant');
|
||||
assert.equal(assistant.items.some(item => item.disabled === true), false);
|
||||
const ctx = { stack: { pop() {} }, actions: { requestRender() {} } };
|
||||
@@ -186,7 +187,7 @@ test('background glasses input still composites frames; phone resume preserves t
|
||||
setFrame(pixels) { if (this.state.phase === 'connected') frames.push(pixels); }
|
||||
wake() {}
|
||||
}
|
||||
const settings = { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
|
||||
const settings = { lockScreenEnabledSetting: { get: () => true }, onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } };
|
||||
const modules = {
|
||||
'../assistant/system-tools': { registerSystemTools() {} },
|
||||
'../assistant/window-tools': { registerWindowTools() {} },
|
||||
@@ -219,6 +220,7 @@ test('background glasses input still composites frames; phone resume preserves t
|
||||
setTimeout: fn => { tasks.set(++nextTask, fn); return nextTask; }, clearTimeout: id => tasks.delete(id),
|
||||
setInterval: () => ++nextTask, clearInterval() {},
|
||||
UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } },
|
||||
UIApplicationProtectedDataWillBecomeUnavailable: 'lock', UIApplicationProtectedDataDidBecomeAvailable: 'unlock',
|
||||
UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state',
|
||||
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } },
|
||||
NSOperationQueue: { mainQueue: {} },
|
||||
|
||||
+42
-3
@@ -159,10 +159,10 @@ async function until(predicate) {
|
||||
assert.fail('Timed out waiting for test state');
|
||||
}
|
||||
function harness(t, transport = new FakeTransport()) {
|
||||
const states = [], input = [], compass = [];
|
||||
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event));
|
||||
const states = [], input = [], compass = [], wear = [];
|
||||
const session = new GlassesSession(transport, state => states.push(state), event => input.push(event), undefined, undefined, event => compass.push(event), wearing => wear.push(wearing));
|
||||
t.after(async () => { transport.hold = false; await session.stop(); });
|
||||
return { transport, session, states, input, compass };
|
||||
return { transport, session, states, input, compass, wear };
|
||||
}
|
||||
test('session authenticates both arms, checks firmware and sends acknowledged display data to L', async t => {
|
||||
const h = harness(t); await h.session.start({ ...addresses, ring: 'AA:BB:CC:DD:EE:03' });
|
||||
@@ -511,3 +511,42 @@ test('compass notifications only come from the right arm and preserve diagnostic
|
||||
magneticAccuracy: 3, magneticAnomalies: 2, orientationSource: 3, flags: 140, sampleTimeMs: 0xfedcba98,
|
||||
} }, { command: 16, headingDegrees: -1 }, { command: 17, headingDegrees: -1 }]);
|
||||
});
|
||||
|
||||
|
||||
test('wear detection matches Android wire format and queries both arms on every connection', async t => {
|
||||
const wear = require('../.test-build/app/g2/wear-protocol.js');
|
||||
assert.equal(hex(wear.enableWearDetection(42)), '0801102a1a042a020801');
|
||||
assert.equal(hex(wear.queryWearState()), '08011000aa0606464301070000');
|
||||
const h = harness(t); await h.session.start(addresses);
|
||||
const queries = () => h.transport.sent.filter(({ message }) => hex(message.payload) === hex(wear.queryWearState()));
|
||||
assert.deepEqual(queries().map(q => q.id).sort(), ['L', 'R']);
|
||||
const enable = h.transport.sent.find(({ message }) => {
|
||||
const body = p.readBytes(message.payload, 3);
|
||||
return message.sid === p.SID.settings && message.command === 1 && body && p.readBytes(body, 5);
|
||||
});
|
||||
assert.equal(enable.id, 'R'); assert.equal(enable.message.flag, 0x20);
|
||||
await h.session.stop(); await h.session.start(addresses);
|
||||
assert.equal(queries().length, 4);
|
||||
});
|
||||
|
||||
test('wear notifications validate the event and peer and work during initial connection', async t => {
|
||||
const h = harness(t);
|
||||
const payload = state => p.concat(p.integer(1, 3), p.bytes(5, p.concat(p.integer(1, 1), p.integer(2, state))));
|
||||
const emit = (id, data, sid = 0x10) => h.transport.emit({ kind: 'notification', identifier: id,
|
||||
characteristic: p.G2_NOTIFY, data: hex(p.concat(...p.frameMessage(data, sid, 1, 12))) });
|
||||
const originalWrite = h.transport.write.bind(h.transport);
|
||||
h.transport.write = async (...args) => {
|
||||
await originalWrite(...args);
|
||||
if (h.transport.sent.at(-1)?.message.magic === 0) emit(args[0], payload(0));
|
||||
};
|
||||
await h.session.start(addresses);
|
||||
assert.ok(h.wear.length >= 2); assert.ok(h.wear.every(value => value === false));
|
||||
h.wear.length = 0;
|
||||
emit('stranger', payload(0)); emit('ring', payload(0)); emit('R', payload(2));
|
||||
emit('R', payload(0), p.SID.hub); emit('R', p.integer(1, 3));
|
||||
assert.deepEqual(h.wear, []);
|
||||
emit('L', payload(0)); emit('R', payload(1));
|
||||
assert.deepEqual(h.wear, [false, true]);
|
||||
await h.session.stop(); emit('R', payload(0));
|
||||
assert.deepEqual(h.wear, [false, true]);
|
||||
});
|
||||
@@ -23,10 +23,12 @@ const planes = load('app/graphics/plane.ts', { './image': images });
|
||||
const timings = load('app/native/frame-timings.ts', {});
|
||||
|
||||
function fixture() {
|
||||
const observers = new Map(), phoneState = { protectedDataAvailable: true };
|
||||
let settingsChanged;
|
||||
let now = 1000, nextTask = 0, screenOn = true, shellOptions, session;
|
||||
const tasks = new Map(), sent = [], previews = [], received = [], errors = [];
|
||||
const boardStats = { starts: 0, stops: 0, paints: 0 };
|
||||
const settings = { enabled: true, tap: true, hold: true, tilt: true, duration: 3000 };
|
||||
const settings = { lock: true, enabled: true, tap: true, hold: true, tilt: true, duration: 3000 };
|
||||
const clock = { Date: class extends Date { static now() { return now; } },
|
||||
setTimeout: (fn, ms) => { tasks.set(++nextTask, { fn, at: now + ms }); return nextTask; },
|
||||
clearTimeout: id => tasks.delete(id), setInterval: () => ++nextTask, clearInterval() {} };
|
||||
@@ -54,7 +56,8 @@ function fixture() {
|
||||
};
|
||||
class Session {
|
||||
state = { phase: 'disconnected' };
|
||||
constructor(_transport, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; }
|
||||
constructor(_transport, onState, onInput, _log, _activity, _compass, onWear) { session = this; this.onState = onState; this.onInput = onInput; this.onWear = onWear; }
|
||||
async enableWearDetectionAndRequestState() {}
|
||||
async start() { this.state = { phase: 'connected' }; this.onState(this.state); }
|
||||
async stop() { this.state = { phase: 'disconnected' }; this.onState(this.state); }
|
||||
setFrame(pixels) { if (this.state.phase === 'connected') sent.push(pixels); }
|
||||
@@ -81,13 +84,14 @@ function fixture() {
|
||||
'./glasses-session': { GlassesSession: Session },
|
||||
'../native/nightscout-bridge': { nightscoutBridge: { async start() {}, async stop() {} } },
|
||||
'./glance-host': { GlanceHost }, './events': events,
|
||||
'./lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) },
|
||||
'./device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null },
|
||||
'../apps/launcher': { launcherEntries: () => [] },
|
||||
'../apps/evenhub/installed-apps': {}, '../apps/evenhub/manager': {}, '../apps/evenhub/updates': {}, '../apps/evenhub': {},
|
||||
'../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' },
|
||||
'../apps/all-apps': { ALL_APPS: [{ appId: 'glanceboard', glanceboard: provider }] },
|
||||
'../ui/shell/worker-window': {}, '../ui/shell/in-process-window': {},
|
||||
'../ui/dashboard-settings': { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } },
|
||||
'../ui/dashboard-settings': { lockScreenEnabledSetting: { get: () => settings.lock }, onAnySettingChanged: fn => { settingsChanged = fn; return () => {}; }, previewColorSetting: { get: () => 'white' } },
|
||||
'../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) },
|
||||
'../apps/ios-availability': { iosAppUnavailableReason: () => null },
|
||||
'../graphics/surface-compositor': { SurfaceCompositor }, '../graphics/plane': planes, '../graphics/image': images,
|
||||
@@ -103,9 +107,10 @@ function fixture() {
|
||||
};
|
||||
const { IosPreviewController } = load('app/g2/ios-preview-controller.ts', modules, {
|
||||
...clock, console: { log() {}, warn() {}, error: text => errors.push(text) },
|
||||
UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } },
|
||||
UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: phoneState },
|
||||
UIApplicationProtectedDataWillBecomeUnavailable: 'lock', UIApplicationProtectedDataDidBecomeAvailable: 'unlock',
|
||||
UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state',
|
||||
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } },
|
||||
NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock(name, _object, _queue, fn) { observers.set(name, fn); return name; }, removeObserver(name) { observers.delete(name); } } },
|
||||
NSOperationQueue: { mainQueue: {} },
|
||||
});
|
||||
const controller = new IosPreviewController((pixels, title) => previews.push({ pixels, title }), message => errors.push(message));
|
||||
@@ -124,7 +129,7 @@ function fixture() {
|
||||
await render();
|
||||
}
|
||||
async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); }
|
||||
return { controller, shell, settings, sent, previews, received, boardStats, hardware, phone, advance, render };
|
||||
return { controller, shell, settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render };
|
||||
}
|
||||
const assertBlank = pixels => assert.ok(pixels.every(p => p === 0));
|
||||
const assertBoard = pixels => {
|
||||
@@ -197,3 +202,70 @@ test('phone preview gestures use Glanceboard; losing runtime cleans up holds and
|
||||
f.controller.resume(); await f.render(); assertBlank(f.previews.at(-1).pixels);
|
||||
assert.equal(f.boardStats.starts, f.boardStats.stops);
|
||||
});
|
||||
|
||||
|
||||
test('iOS lock hides apps and Glanceboard, blocks input, and stays locked when put back on', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.shell.sleep(); await f.hardware(0);
|
||||
f.controller.pause();
|
||||
// The notification precedes the property change: do not immediately undo it.
|
||||
f.observers.get('lock')(); f.wear(false); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
assert.equal(f.controller.glance.isVisible(), false);
|
||||
await f.hardware(3); // wake the lock screen
|
||||
assert.ok(f.sent.at(-1).every(p => p === 123));
|
||||
for (const type of [0, 1, 2, 9, 11]) await f.hardware(type);
|
||||
f.wear(true); await f.render();
|
||||
assert.equal(f.received.length, 0);
|
||||
assert.ok(f.sent.at(-1).every(p => p === 123));
|
||||
await f.hardware(3); assertBlank(f.sent.at(-1));
|
||||
await f.hardware(12, 1); assert.ok(f.sent.at(-1).every(p => p === 123));
|
||||
f.observers.get('unlock')(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, false);
|
||||
assert.equal(f.sent.at(-1)[0], 75);
|
||||
await f.hardware(0); assert.equal(f.received.length, 1);
|
||||
});
|
||||
|
||||
test('iOS removal before phone lock, setting changes, and observer lifetime', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.wear(false); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, false);
|
||||
f.observers.get('lock')(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
f.settings.lock = false; f.settingsChanged(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, false);
|
||||
f.settings.lock = true; f.settingsChanged(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
f.controller.pause(); assert.ok(f.observers.has('unlock'));
|
||||
await f.controller.disconnect(); assert.equal(f.observers.has('unlock'), false);
|
||||
f.controller.resume(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, false);
|
||||
assert.equal(f.observers.size, 4);
|
||||
});
|
||||
|
||||
test('iOS wear notifications catch protected-data changes missed while suspended', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.controller.pause();
|
||||
f.phoneState.protectedDataAvailable = false; f.wear(false); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
f.controller.resume(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
f.controller.pause(); f.phoneState.protectedDataAvailable = true;
|
||||
f.controller.resume(); await f.render();
|
||||
assert.equal(f.controller.glassesLocked, false);
|
||||
});
|
||||
|
||||
|
||||
test('iOS locked phone controls cannot dispatch mirror, voice, or keyboard input', async () => {
|
||||
const f = fixture(); await f.controller.connect(); f.wear(false);
|
||||
f.observers.get('lock')(); await f.render();
|
||||
await f.phone('tap', 'mirror'); await f.phone('long-press', 'watch');
|
||||
f.controller.startVoiceInput();
|
||||
assert.equal(await f.controller.prepareVoiceCapture(), false);
|
||||
await f.controller.startVoiceCapture(); await f.controller.typeIntoApp();
|
||||
assert.equal(f.received.length, 0);
|
||||
assert.equal(f.controller.glassesLocked, true);
|
||||
// Locked state also survives transport recovery until a phone unlock.
|
||||
f.controller.session.onState({ phase: 'retrying' });
|
||||
f.controller.session.onState({ phase: 'connecting' });
|
||||
f.controller.session.onState({ phase: 'connected' });
|
||||
await f.render();
|
||||
assert.ok(f.sent.at(-1).every(p => p === 123));
|
||||
});
|
||||
@@ -18,6 +18,7 @@
|
||||
"../app/g2/ios-peripheral-identity.ts",
|
||||
"../app/g2/glasses-session.ts",
|
||||
"../app/graphics/surface-compositor.ts",
|
||||
"../app/graphics/image.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/g2/firmware-compat.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
|
||||
Reference in new issue
Block a user