diff --git a/App_Resources/iOS/Info.plist b/App_Resources/iOS/Info.plist index 3740c24b..11d23bb4 100644 --- a/App_Resources/iOS/Info.plist +++ b/App_Resources/iOS/Info.plist @@ -20,6 +20,10 @@ NSBluetoothAlwaysUsageDescription Faceclaw uses Bluetooth to connect to your glasses and ring, display your apps, and receive gestures. + UIBackgroundModes + + bluetooth-central + NSLocalNetworkUsageDescription Faceclaw connects to the terminal servers you configure on your local network. UILaunchScreen diff --git a/app/g2/glasses-session.ts b/app/g2/glasses-session.ts index 697128eb..ec89e2da 100644 --- a/app/g2/glasses-session.ts +++ b/app/g2/glasses-session.ts @@ -55,7 +55,8 @@ export class GlassesSession { private readonly deflate: (data: Uint8Array) => Uint8Array, private readonly onState: (state: SessionState) => void, private readonly onInput: (input: protocol.GlassesInput) => void, - private readonly log: (message: string) => void = () => {}) { + private readonly log: (message: string) => void = () => {}, + private readonly onActivity: () => void = () => {}) { this.off = transport.onEvent(event => this.receive(event)) } private update(phase: SessionState['phase'], status: string): void { @@ -210,15 +211,15 @@ export class GlassesSession { return } if (event.kind !== 'notification' || !event.data) return + const ring = event.identifier === this.ids.ring + if (ring ? !protocol.RING_NOTIFY.includes(event.characteristic ?? '') : event.characteristic !== protocol.G2_NOTIFY) return try { const data = hexToBytes(event.data) - if (event.identifier === this.ids.ring) { - if (!protocol.RING_NOTIFY.includes(event.characteristic ?? '')) return + if (ring) { const input = protocol.decodeRingInput(data) if (input) { this.log(`Direct R1 gesture ${input.eventType}`); this.onInput(input) } return } - if (event.characteristic !== protocol.G2_NOTIFY) return for (const message of this.receiver.receive(event.identifier!, data)) { this.log(`RX ${event.identifier === this.ids.left ? 'L' : 'R'} sid=${message.sid.toString(16)} flag=${message.flag.toString(16)} cmd=${message.command} magic=${message.magic}`) if (![1, 6].includes(message.flag)) { @@ -243,6 +244,11 @@ export class GlassesSession { } } } catch (error) { this.log(`Ignored malformed BLE notification: ${this.message(error)}`) } + finally { + // CoreBluetooth can wake a suspended iOS process for a notification. + // Service wall-clock deadlines now, without waiting for a suspended timer. + if (this.state.phase === 'connected') { this.onActivity(); this.wake() } + } } private applySettings(message: protocol.ProtocolMessage): void { const values = protocol.readBytes(message.payload, 4) @@ -262,6 +268,10 @@ export class GlassesSession { if (this.state.phase !== 'connected') return this.latest = protocol.packGray4(gray, 640, 480); this.schedule(0) } + wake(): void { + if (this.timer !== null) { clearTimeout(this.timer); this.timer = null } + void this.pump() + } private schedule(delay = 1000): void { if (delay === 0 && this.timer !== null) { clearTimeout(this.timer); this.timer = null } if (this.timer !== null || this.pumping || this.state.phase !== 'connected') return @@ -273,7 +283,9 @@ export class GlassesSession { try { const now = Date.now() if (now - this.lastHeartbeat >= 4000) { + const gap = now - this.lastHeartbeat await this.request('right', protocol.SID.hub, protocol.heartbeat, 'Heartbeat', 1500); this.lastHeartbeat = Date.now() + this.log(`Heartbeat acknowledged (gap ${gap}ms)`) } if (now - this.lastLease >= 45_000) { await this.lease(true); this.lastLease = Date.now() } if (now - this.lastSettings >= (this.charging ? 30_000 : 300_000)) { diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 32c48277..6bab4f2f 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -31,6 +31,8 @@ export class IosPreviewController { private clockTimer: ReturnType | null = null private offSettings: (() => void) | null = null private active = false + private runtimeRunning = false + private clockMinute = -1 private shellDirty = true private inputQueue: Promise = Promise.resolve() private lastLayout = '' @@ -39,8 +41,6 @@ export class IosPreviewController { private readonly inProcessApps = new Map() private readonly batteryObservers: any[] = [] private session: GlassesSession | null = null - private resumeConnection = false - private suspendedDisconnect: Promise = Promise.resolve() private logLines: string[] = [] private logTimer: ReturnType | null = null private readonly actions: LayerActions = { @@ -88,6 +88,40 @@ export class IosPreviewController { resume(): void { if (this.active) return this.active = true + this.syncRuntime() + this.logBluetooth('Phone foreground') + if (this.session) this.onConnectionState({ ...this.session.state }) + this.session?.wake() + this.relayout() + shell.foregroundWindow()?.requestRender() + this.requestShellRender() + } + pause(): void { + if (!this.active) return + this.active = false + // The phone preview is hidden; the glasses still own their screen and input. + this.syncRuntime() + this.logBluetooth(`Phone background; glasses ${this.session?.state.phase ?? 'disconnected'}`) + this.flushBluetoothLog() + this.session?.wake() + } + private get runtimeNeeded(): boolean { + return this.active || !!this.session && ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.session.state.phase) + } + private syncRuntime(): void { + const running = this.runtimeNeeded + if (running === this.runtimeRunning) return + this.runtimeRunning = running + for (const window of shell.getWindows()) window.setScreenOn?.(running && shell.isScreenOn()) + if (!running) { + this.offSettings?.(); this.offSettings = null + for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) + UIDevice.currentDevice.batteryMonitoringEnabled = false + if (this.clockTimer !== null) clearInterval(this.clockTimer) + if (this.renderTimer !== null) clearTimeout(this.renderTimer) + this.clockTimer = this.renderTimer = null + return + } UIDevice.currentDevice.batteryMonitoringEnabled = true for (const name of [UIDeviceBatteryLevelDidChangeNotification, UIDeviceBatteryStateDidChangeNotification]) { this.batteryObservers.push(NSNotificationCenter.defaultCenter.addObserverForNameObjectQueueUsingBlock(name, null, NSOperationQueue.mainQueue, () => this.requestShellRender())) @@ -99,28 +133,11 @@ export class IosPreviewController { shell.foregroundWindow()?.requestRender() this.requestShellRender() }) - this.clockTimer = setInterval(() => this.requestShellRender(), 60_000) - for (const window of shell.getWindows()) window.setScreenOn?.(shell.isScreenOn()) - this.relayout() - shell.foregroundWindow()?.requestRender() - this.requestShellRender() - console.log('[ios-preview] Main screen active') - if (this.resumeConnection) { - this.resumeConnection = false - void this.suspendedDisconnect.then(() => { if (this.active) void this.connect() }) - } + this.clockTimer = setInterval(() => this.refreshClock(), 60_000) } - pause(): void { - this.resumeConnection = !!this.session && ['connected', 'connecting', 'retrying'].includes(this.session.state.phase) - if (this.session) this.suspendedDisconnect = this.session.stop() - this.active = false - for (const window of shell.getWindows()) window.setScreenOn?.(false) - this.offSettings?.(); this.offSettings = null - for (const observer of this.batteryObservers.splice(0)) NSNotificationCenter.defaultCenter.removeObserver(observer) - UIDevice.currentDevice.batteryMonitoringEnabled = false - if (this.clockTimer) clearInterval(this.clockTimer) - if (this.renderTimer) clearTimeout(this.renderTimer) - this.clockTimer = this.renderTimer = null + private refreshClock(): void { + const minute = Math.floor(Date.now() / 60_000) + if (minute !== this.clockMinute) { this.clockMinute = minute; this.requestShellRender() } } private configureWindow(window: ShellWindow): void { this.compositor.configureSurface(window.surfaceId, { @@ -144,10 +161,11 @@ export class IosPreviewController { } private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() } private scheduleFrame(): void { - if (!this.active || this.renderTimer !== null) return - // Coalesce window/chrome updates into one preview, capped at 30 fps. + if (!this.runtimeNeeded || this.renderTimer !== null) return + // Coalesce window/chrome updates into one glasses frame, capped at 30 fps. this.renderTimer = setTimeout(() => { this.renderTimer = null + if (!this.runtimeNeeded) return try { if (this.shellDirty) { this.shellDirty = false @@ -157,15 +175,18 @@ export class IosPreviewController { } const pixels = this.compositor.composite() this.session?.setFrame(pixels) - const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green') - this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) + if (this.active) { + const image = previewPixels(pixels, 640, 480, previewColorSetting.get() === 'green') + this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) + } } catch (error) { this.fail(error) } }, 33) } private fail(error: unknown): void { const message = error instanceof Error ? error.stack ?? error.message : String(error) console.error(`[ios-preview] ${message}`) - this.onError(error instanceof Error ? error.message : String(error)) + this.logBluetooth(`App error: ${message}`) + if (this.active) this.onError(error instanceof Error ? error.message : String(error)) } gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void { this.inputQueue = this.inputQueue.then(async () => { @@ -287,31 +308,38 @@ export class IosPreviewController { const { deflate } = require('pako') as { deflate: (data: Uint8Array) => Uint8Array } this.session = new GlassesSession(iosBluetooth(), deflate, state => { shell.setBatteryLevels({ headset: state.battery, headsetCharging: state.charging }) - this.onConnectionState(state) + this.syncRuntime() + if (this.active) this.onConnectionState(state) this.requestShellRender() }, input => { this.inputQueue = this.inputQueue.then(async () => { - if (!this.active) return + if (this.session?.state.phase !== 'connected') return await shell.receiveInput(rawInputEventToInputEvent(input)); this.requestShellRender() this.logBluetooth(`Input ${input.eventType} source ${input.eventSource}`) }).catch(error => this.fail(error)) - }, message => this.logBluetooth(message)) + }, message => this.logBluetooth(message), () => this.refreshClock()) } await this.session.start(addresses) } - async disconnect(): Promise { this.resumeConnection = false; await this.session?.stop() } + async disconnect(): Promise { await this.session?.stop() } private logBluetooth(message: string): void { - const line = `${new Date().toISOString()} ${message}` + const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` console.log(`[ios-ble] ${line}`); this.logLines.push(line) - if (this.logLines.length > 200) this.logLines.shift() + if (this.logLines.length > 1000) this.logLines.shift() if (this.logTimer !== null) return this.logTimer = setTimeout(() => { this.logTimer = null - try { File.fromPath(path.join(knownFolders.documents().path, 'bluetooth.log')).writeTextSync(this.logLines.join('\n')) } - catch (error) { console.warn(`Bluetooth log: ${error}`) } + this.flushBluetoothLog() }, 500) } + private flushBluetoothLog(): void { + if (this.logTimer !== null) clearTimeout(this.logTimer) + this.logTimer = null + try { File.fromPath(path.join(knownFolders.documents().path, 'bluetooth.log')).writeTextSync(this.logLines.join('\n')) } + catch (error) { console.warn(`Bluetooth log: ${error}`) } + } async typeIntoApp(): Promise { + if (!this.active) { this.logBluetooth('Text input requires opening Faceclaw on the phone'); return } if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return this.prompting = true try { @@ -322,6 +350,7 @@ export class IosPreviewController { finally { this.prompting = false } } private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise { + if (!this.active) { this.logBluetooth('Editing text settings requires opening Faceclaw on the phone'); return } const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' }) if (result.result) setting.set(result.text) } diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index ea75fff1..f937330d 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -54,3 +54,68 @@ test('iOS worker frames preserve baked grayscale bytes through the message bound assert.equal(messages[0].width, 3); assert.equal(messages[0].height, 2); assert.deepEqual(Buffer.from(messages[0].pixels, 'base64'), Buffer.from(pixels)); }); + +test('background glasses input still composites frames; phone resume preserves the session; explicit stop stays stopped', async () => { + const tasks = new Map(), screenStates = [], inputs = [], frames = [], previews = [], states = []; + let nextTask = 0, starts = 0, stops = 0, session; + const window = { windowId: 'launcher', surfaceId: 'launcher', appId: 'launcher', title: 'Apps', + setScreenOn: on => screenStates.push(on), requestRender() {} }; + const shell = { configure() {}, registerWindow() {}, wake() {}, focusWindow() {}, + getWindows: () => [window], foregroundWindow: () => window, isScreenOn: () => true, + setBatteryLevels() {}, paintSurface() {}, underlayDim: () => 0, getFocus: () => 'app', + receiveInput: async input => { inputs.push(input); } }; + class Session { + state = { phase: 'disconnected' }; + constructor(_transport, _deflate, onState, onInput) { session = this; this.onState = onState; this.onInput = onInput; } + async start() { starts++; this.state = { phase: 'connected' }; this.onState(this.state); } + async stop() { stops++; this.state = { phase: 'disconnected' }; this.onState(this.state); } + setFrame(pixels) { if (this.state.phase === 'connected') frames.push(pixels); } + wake() {} + } + const settings = { onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } }; + const modules = { + '@nativescript/core': { File: { fromPath: () => ({ writeTextSync() {} }) }, knownFolders: { documents: () => ({ path: '/tmp' }) }, path }, + '../native/ios-bluetooth': { iosBluetooth: () => ({}) }, './glasses-session': { GlassesSession: Session }, + './device-addresses': { loadDeviceAddresses: () => ({}) }, './ios-peripheral-identity': { deviceAddressError: () => null }, + '../apps/launcher/launcher-app': { createLauncherWindow: () => window, LAUNCHER_SURFACE_ID: 'launcher' }, + '../apps/all-apps': { ALL_APPS: [] }, '../ui/dashboard-settings': settings, + '../native/phone-battery': { readPhoneBatteryState: () => ({ battery: 80, charging: false }) }, + '../graphics/surface-compositor': { SurfaceCompositor: class { + configureSurface() {} setSurfaceVisible() {} submitSurfaceFrame() {} setUnderlayDim() {} + composite() { return new Uint8Array([1, 2]); } + } }, + '../graphics/plane': { flattenPlanes: () => ({ pixels: new Uint8Array([1, 2]), width: 2, height: 1 }) }, + '../native/ios-graphics': { previewPixels: pixels => pixels }, + '../ui/shell/shell': { shell, rawInputEventToInputEvent: input => input }, + '../ui/shell/geometry': { appViewportRect: () => ({ x: 0, y: 0, width: 640, height: 480 }) }, + pako: { deflate: x => x }, + }; + const api = load('app/g2/ios-preview-controller.ts', { + require: id => modules[id] ?? {}, console: { log() {}, warn() {}, error() {} }, + setTimeout: fn => { tasks.set(++nextTask, fn); return nextTask; }, clearTimeout: id => tasks.delete(id), + setInterval: () => ++nextTask, clearInterval() {}, + UIDevice: { currentDevice: {} }, UIApplication: { sharedApplication: { protectedDataAvailable: true } }, + UIDeviceBatteryLevelDidChangeNotification: 'level', UIDeviceBatteryStateDidChangeNotification: 'state', + NSNotificationCenter: { defaultCenter: { addObserverForNameObjectQueueUsingBlock() {}, removeObserver() {} } }, + NSOperationQueue: { mainQueue: {} }, + }); + const flush = () => { for (const [id, fn] of [...tasks]) { if (tasks.delete(id)) fn(); } }; + const controller = new api.IosPreviewController(image => previews.push(image), assert.fail, state => states.push(state)); + controller.resume(); await controller.connect(); flush(); + const previewCount = previews.length, stateCount = states.length; + controller.pause(); controller.pause(); // NativeScript also unloads its root page on background entry. + assert.equal(stops, 0); assert.ok(screenStates.every(Boolean)); + session.onInput({ eventType: 3, eventSource: 1 }); + await controller.inputQueue; flush(); + assert.equal(inputs.length, 1); assert.ok(frames.length >= 2); + assert.equal(previews.length, previewCount); assert.equal(states.length, stateCount); + controller.resume(); flush(); + assert.equal(starts, 1); assert.ok(previews.length > previewCount); + controller.pause(); await controller.disconnect(); flush(); + assert.equal(stops, 1); assert.equal(screenStates.at(-1), false); + const frameCount = frames.length; + session.onInput({ eventType: 3, eventSource: 1 }); await controller.inputQueue; flush(); + assert.equal(inputs.length, 1); assert.equal(frames.length, frameCount); + controller.resume(); flush(); + assert.equal(starts, 1); assert.equal(controller.connectionState.phase, 'disconnected'); +}); diff --git a/tests/ios-ble.test.cjs b/tests/ios-ble.test.cjs index 295f563a..eae347c4 100644 --- a/tests/ios-ble.test.cjs +++ b/tests/ios-ble.test.cjs @@ -198,3 +198,27 @@ test('ring disconnect retries independently and stop cancels further retries', a await h.session.stop(); assert.equal(h.session.ringRetryTimer, null); }); + +test('a BLE wake services overdue heartbeat and lease without waiting for a suspended poll timer', async t => { + const h = harness(t); await h.session.start(addresses); + clearTimeout(h.session.timer); h.session.timer = null; + h.session.lastHeartbeat = Date.now() - 12_000; + h.session.lastLease = Date.now() - 60_000; + const before = h.transport.sent.length; + const data = hex(p.concat(...p.frameMessage(p.integer(1, 6), p.SID.auth, 1, 22))); + h.transport.emit({ kind: 'notification', identifier: 'L', characteristic: p.G2_NOTIFY, data }); + // pump starts in the notification callback, before any timers can fire. + assert.equal(h.session.pumping, true); + await until(() => !h.session.pumping); + const sent = h.transport.sent.slice(before); + assert.equal(sent.filter(s => s.message.sid === p.SID.hub && s.message.command === 12).length, 1); + assert.deepEqual(sent.filter(s => s.message.sid === p.SID.settings).map(s => s.id).sort(), ['L', 'R']); + assert.equal(h.session.state.phase, 'connected'); + assert.deepEqual(h.transport.closed, []); + await h.session.stop(); + const count = h.transport.sent.length; + h.session.wake(); + h.transport.emit({ kind: 'notification', identifier: 'L', characteristic: p.G2_NOTIFY, data }); + assert.equal(h.transport.sent.length, count); + assert.equal(h.session.state.phase, 'disconnected'); +});