diff --git a/app/apps/developer/developer-app.ts b/app/apps/developer/developer-app.ts index 7030c02e..6e6ae8a4 100644 --- a/app/apps/developer/developer-app.ts +++ b/app/apps/developer/developer-app.ts @@ -50,32 +50,24 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M }, { label: "Buzzer demo", - disabled: global.isIOS, - description: global.isIOS ? "Glasses sound playback is not available on iOS yet." : undefined, onSelect: (ctx) => { ctx.stack.push(new BuzzerDemoLayer()); }, }, { label: "Accelerometer demo", - disabled: global.isIOS, - description: global.isIOS ? "Not available on iOS yet." : undefined, onSelect: (ctx) => { ctx.stack.push(new AccelerometerDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender)); }, }, { label: "Light sensor", - disabled: global.isIOS, - description: global.isIOS ? "Not available on iOS yet." : undefined, onSelect: (ctx) => { ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender)); }, }, { label: "Microphones", - disabled: global.isIOS, - description: global.isIOS ? "Not available on iOS yet." : undefined, onSelect: openMicrophones, }, { diff --git a/app/apps/microphones/mic-control.ts b/app/apps/microphones/mic-control.ts index 3ac8ddca..6f62b3ad 100644 --- a/app/apps/microphones/mic-control.ts +++ b/app/apps/microphones/mic-control.ts @@ -1,7 +1,6 @@ import { getStringSetting, setStringSetting } from "../../native/settings-store"; import { useMicControlSetting } from "../../ui/dashboard-settings"; -import { toJavaBytes } from "../../native/cloud-stt"; -import { toUint8Array } from "../../util/array-util"; +import { addMicStatusListener, micControlSession, sendMicControlRecord } from "../../native/mic-control-link"; import { MIC_LEASE_MS, decodeMicStatus, @@ -14,8 +13,6 @@ import { type MicStatus, } from "./mic-protocol"; -declare const com: any; - /** * Host-side controller for the CFW mic_control array: sends the same config * to both temples (each temple is its own endpoint and its own 2-mic array), @@ -30,15 +27,6 @@ const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3); export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null }; -function activeCommunicator(): any { - if (!global.isAndroid) return null; - try { - return com.faceclaw.app.FaceclawBleCommunicator.getActive(); - } catch { - return null; - } -} - /** * True when the "Use microphone control" developer setting is on. With the * setting off, Faceclaw behaves exactly as it would on firmware without the @@ -71,9 +59,9 @@ export function saveMicConfig(config: MicConfig): void { class MicArrayController { private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>(); private statuses: MicArrayStatuses = { left: null, right: null }; - // The Java listener proxy must stay referenced or it gets GC'd. - private statusProxy: any = null; - private proxyCommunicator: any = null; + // Holds the native listener proxy, which must stay referenced or it gets GC'd. + private unsubscribeStatus: (() => void) | null = null; + private statusSession: unknown = null; private leaseTimer: ReturnType | null = null; private armed = false; @@ -158,39 +146,28 @@ class MicArrayController { rightTemple = true, leftTemple = true, ): boolean { - const communicator = activeCommunicator(); - if (!communicator) return false; + const session = micControlSession(); + if (!session) return false; try { - this.ensureStatusProxy(communicator); - const record = encodeMicControl(op, config); - // False when the session wasn't ready and nothing was queued. - return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false; + this.ensureStatusListener(session); + return sendMicControlRecord(session, encodeMicControl(op, config), label, rightTemple, leftTemple); } catch (error) { console.warn(`mic control ${label} failed: ${error}`); return false; } } - private ensureStatusProxy(communicator: any): void { - if (this.statusProxy && this.proxyCommunicator === communicator) return; - if (this.statusProxy && this.proxyCommunicator) { - try { - this.proxyCommunicator.removeMicStatusListener(this.statusProxy); - } catch { - // The old communicator instance may be gone after a reconnect. - } - } - this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({ - onMicStatus: (body: any, arm: string) => { - const status = decodeMicStatus(toUint8Array(body)); - if (!status) return; - const side: MicSide = String(arm) === "L" ? "left" : "right"; - this.statuses = { ...this.statuses, [side]: { ...status, side } }; - this.statusListeners.forEach((listener) => listener(this.statuses)); - }, + private ensureStatusListener(session: unknown): void { + if (this.unsubscribeStatus && this.statusSession === session) return; + this.unsubscribeStatus?.(); + this.statusSession = session; + this.unsubscribeStatus = addMicStatusListener(session, (body, arm) => { + const status = decodeMicStatus(body); + if (!status) return; + const side: MicSide = arm === "L" ? "left" : "right"; + this.statuses = { ...this.statuses, [side]: { ...status, side } }; + this.statusListeners.forEach((listener) => listener(this.statuses)); }); - this.proxyCommunicator = communicator; - communicator.addMicStatusListener(this.statusProxy); } private syncLeaseTimer(): void { diff --git a/app/native/ambient-light-types.ts b/app/native/ambient-light-types.ts new file mode 100644 index 00000000..62ba1525 --- /dev/null +++ b/app/native/ambient-light-types.ts @@ -0,0 +1,93 @@ +/** + * Platform-neutral ambient-light types and the CFW report decoder (settings + * field 105), shared by ambient-light.ts and ambient-light.ios.ts. + */ + +export const ALS_REPORT_VERSION = 1; + +export type AmbientLightReason = "query" | "started" | "poll" | "stopped"; + +export type AmbientLightReport = { + reason: AmbientLightReason; + /** The sensor driver is open (readings are live). */ + opened: boolean; + /** CFW passive polling is installed (stock auto-brightness is not stepping). */ + passive: boolean; + /** The user's stock auto-brightness setting is on. */ + autoBrightness: boolean; + /** The last passive read succeeded (only meaningful while passive). */ + readOk: boolean; + /** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */ + status: number; + /** Ambient light in the stock driver's units (roughly tenths of a lux). */ + alsValue: number; + /** Max of the last five readings (what the stock curve is applied to). */ + peak: number; + /** Brightness level (0..100) the stock curve would choose for `peak`. */ + stockTarget: number; + /** Current brightness setting (0..100). */ + brightness: number; + /** Row of the stock brightness curve (0..5) for `peak`. */ + gear: number; + /** Stock learned scale factor, Q10 (1024 = 1.0). */ + scaleQ10: number; + /** Firmware millisecond tick when the report was built. */ + tickMs: number; +}; + +const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"]; + +function rd16(b: Uint8Array, i: number): number { + return b[i] | (b[i + 1] << 8); +} + +function rd32(b: Uint8Array, i: number): number { + return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0; +} + +/** Decode a field-105 record body, or null when it isn't one. */ +export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null { + if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) { + return null; + } + const flags = body[4]; + return { + reason: REASONS[body[3]] ?? "poll", + opened: (flags & 0x01) !== 0, + passive: (flags & 0x02) !== 0, + autoBrightness: (flags & 0x04) !== 0, + readOk: (flags & 0x08) !== 0, + status: body[5], + alsValue: rd32(body, 6), + peak: rd32(body, 10), + stockTarget: rd16(body, 14), + brightness: body[16], + gear: body[17], + scaleQ10: rd16(body, 18), + tickMs: rd32(body, 20), + }; +} + +export type AmbientLightPollingOptions = { + /** Poll period in ms; the firmware clamps to 100..5000. Default 500. */ + intervalMs?: number; + /** Report when the reading moved by at least this much. Default 1. */ + minDelta?: number; + /** Also report after this many ms without one (0 = never). Default 5000. */ + heartbeatMs?: number; + /** + * Stop automatically when the Faceclaw framebuffer lease is released or + * lapses (the phone went away). Default true. + */ + bindToLease?: boolean; +}; + +/** The options with their defaults filled in. */ +export function resolvePollingOptions(options: AmbientLightPollingOptions): Required { + return { + intervalMs: options.intervalMs ?? 500, + minDelta: options.minDelta ?? 1, + heartbeatMs: options.heartbeatMs ?? 5000, + bindToLease: options.bindToLease ?? true, + }; +} diff --git a/app/native/ambient-light.ios.ts b/app/native/ambient-light.ios.ts new file mode 100644 index 00000000..3a3c35ea --- /dev/null +++ b/app/native/ambient-light.ios.ts @@ -0,0 +1,51 @@ +import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios"; +import { + decodeAmbientLightReport, + resolvePollingOptions, + type AmbientLightPollingOptions, + type AmbientLightReport, +} from "./ambient-light-types"; + +export * from "./ambient-light-types"; + +/** + * Ambient light sensor access on iOS: the same API as ambient-light.ts (see + * there for the CFW query and passive-polling semantics) over the running + * shared-Kotlin session. No-ops while no session is running. + */ + +/** Ask the firmware for one ambient-light report (delivered to listeners). */ +export function queryAmbientLight(): void { + try { + FaceclawCommunicatorBridge.getActive()?.queryAmbientLight(); + } catch (error) { + console.warn(`queryAmbientLight failed: ${error}`); + } +} + +/** Start or stop CFW passive light-sensor polling. */ +export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void { + const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options); + try { + FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease); + } catch (error) { + console.warn(`setAmbientLightPolling failed: ${error}`); + } +} + +/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */ +export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void { + const active = FaceclawCommunicatorBridge.getActive(); + if (!active) return () => {}; + const unsubscribe = active.addAmbientLightListener((body) => { + const report = decodeAmbientLightReport(body); + if (report) listener(report); + }); + return () => { + try { + unsubscribe(); + } catch { + // The session may have been closed during a disconnect. + } + }; +} diff --git a/app/native/ambient-light.ts b/app/native/ambient-light.ts index 51334d46..412d926e 100644 --- a/app/native/ambient-light.ts +++ b/app/native/ambient-light.ts @@ -16,88 +16,17 @@ * Reports are 24-byte ['A','L',version,...] records on settings field 105. */ import { toUint8Array } from "../util/array-util"; +import { + decodeAmbientLightReport, + resolvePollingOptions, + type AmbientLightPollingOptions, + type AmbientLightReport, +} from "./ambient-light-types"; declare const com: any; declare const global: any; -export const ALS_REPORT_VERSION = 1; - -export type AmbientLightReason = "query" | "started" | "poll" | "stopped"; - -export type AmbientLightReport = { - reason: AmbientLightReason; - /** The sensor driver is open (readings are live). */ - opened: boolean; - /** CFW passive polling is installed (stock auto-brightness is not stepping). */ - passive: boolean; - /** The user's stock auto-brightness setting is on. */ - autoBrightness: boolean; - /** The last passive read succeeded (only meaningful while passive). */ - readOk: boolean; - /** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */ - status: number; - /** Ambient light in the stock driver's units (roughly tenths of a lux). */ - alsValue: number; - /** Max of the last five readings (what the stock curve is applied to). */ - peak: number; - /** Brightness level (0..100) the stock curve would choose for `peak`. */ - stockTarget: number; - /** Current brightness setting (0..100). */ - brightness: number; - /** Row of the stock brightness curve (0..5) for `peak`. */ - gear: number; - /** Stock learned scale factor, Q10 (1024 = 1.0). */ - scaleQ10: number; - /** Firmware millisecond tick when the report was built. */ - tickMs: number; -}; - -const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"]; - -function rd16(b: Uint8Array, i: number): number { - return b[i] | (b[i + 1] << 8); -} - -function rd32(b: Uint8Array, i: number): number { - return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0; -} - -/** Decode a field-105 record body, or null when it isn't one. */ -export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null { - if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) { - return null; - } - const flags = body[4]; - return { - reason: REASONS[body[3]] ?? "poll", - opened: (flags & 0x01) !== 0, - passive: (flags & 0x02) !== 0, - autoBrightness: (flags & 0x04) !== 0, - readOk: (flags & 0x08) !== 0, - status: body[5], - alsValue: rd32(body, 6), - peak: rd32(body, 10), - stockTarget: rd16(body, 14), - brightness: body[16], - gear: body[17], - scaleQ10: rd16(body, 18), - tickMs: rd32(body, 20), - }; -} - -export type AmbientLightPollingOptions = { - /** Poll period in ms; the firmware clamps to 100..5000. Default 500. */ - intervalMs?: number; - /** Report when the reading moved by at least this much. Default 1. */ - minDelta?: number; - /** Also report after this many ms without one (0 = never). Default 5000. */ - heartbeatMs?: number; - /** - * Stop automatically when the Faceclaw framebuffer lease is released or - * lapses (the phone went away). Default true. - */ - bindToLease?: boolean; -}; +export * from "./ambient-light-types"; function activeCommunicator(): any { if (!global.isAndroid) return null; @@ -123,14 +52,9 @@ export function queryAmbientLight(): void { * on every session start. */ export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void { + const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options); try { - activeCommunicator()?.setAmbientLightPolling( - enabled, - options.intervalMs ?? 500, - options.minDelta ?? 1, - options.heartbeatMs ?? 5000, - options.bindToLease ?? true, - ); + activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease); } catch (error) { console.warn(`setAmbientLightPolling failed: ${error}`); } diff --git a/app/native/audio-monitor.ios.ts b/app/native/audio-monitor.ios.ts new file mode 100644 index 00000000..48cf3ac6 --- /dev/null +++ b/app/native/audio-monitor.ios.ts @@ -0,0 +1,50 @@ +import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios"; + +/** + * Passive view of the glasses' audio traffic on iOS: the same API as + * audio-monitor.ts over the running shared-Kotlin session. Nothing here sends + * anything to the glasses. + */ + +export type AudioArm = "L" | "R" | "?"; + +/** The session listeners attach to; changes across reconnects. */ +export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null { + return FaceclawCommunicatorBridge.getActive(); +} + +/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */ +export function isFaceclawAudioCaptureActive(): boolean { + try { + return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive()); + } catch { + return false; + } +} + +/** + * Subscribe `communicator` to every audio-characteristic packet and every + * field-104 mic status record body (undecoded). Delivered on the main thread. + * Returns an unsubscribe function. + */ +export function addAudioMonitorListeners( + communicator: FaceclawCommunicatorBridge | null, + onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void, + onStatus: (body: Uint8Array, arm: AudioArm) => void, +): () => void { + if (!communicator) return () => {}; + const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs)); + const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm))); + return () => { + try { + offPackets(); + offStatus(); + } catch { + // The session may have been closed during a disconnect. + } + }; +} + +function toArm(arm: string): AudioArm { + return arm === "L" || arm === "R" ? arm : "?"; +} diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index 6df07c9b..3e0087c1 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input"; import { File, knownFolders, path, type ImageSource } from "@nativescript/core"; import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat"; import { type CompassEvent } from "./compass-types"; +import { type ImuReading } from "./imu-types"; import { fromData, toData } from "./kotlin-data"; import { previewPixels } from "./ios-graphics"; import { iosBluetooth } from "./ios-bluetooth"; @@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any; declare const FaceclawKitIosAncsListener: any; declare const FaceclawKitIosAudioPacketListener: any; declare const FaceclawKitFaceclawCompassListener: any; +declare const FaceclawKitFaceclawImuListener: any; +declare const FaceclawKitIosAmbientLightListener: any; +declare const FaceclawKitIosMicStatusListener: any; export type CommunicatorPhase = @@ -164,6 +168,29 @@ class CompassListener extends NSObject { } } +@NativeClass() +class ImuListener extends NSObject { + static ObjCProtocols = [FaceclawKitFaceclawImuListener]; + handler!: (reading: ImuReading) => void; + onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void { + this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) }); + } +} + +@NativeClass() +class AmbientLightListener extends NSObject { + static ObjCProtocols = [FaceclawKitIosAmbientLightListener]; + handler!: (body: Uint8Array) => void; + onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); } +} + +@NativeClass() +class MicStatusListener extends NSObject { + static ObjCProtocols = [FaceclawKitIosMicStatusListener]; + handler!: (body: Uint8Array, arm: string) => void; + onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); } +} + /** * The iOS twin of the Android communicator bridge: the same public API over * the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth @@ -173,6 +200,8 @@ class CompassListener extends NSObject { * Android so callers never re-enter the bridge from a Kotlin frame. */ export class FaceclawCommunicatorBridge { + // The started, not yet closed bridge (see getActive). + private static active: FaceclawCommunicatorBridge | null = null; private readonly communicator: any; private readonly listenerProxy: SessionListener; private ancsProxy: AncsListener | null = null; @@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge { private audioListener: AudioPacketListener | null = null; private microphoneWork: Promise = Promise.resolve(); private microphoneToken = 0; + private closed = false; + + /** + * The bridge whose session is running, for modules that reach the glasses + * without the bridge threaded through (imu, ambient-light, audio-monitor): + * the iOS stand-in for Android's FaceclawBleCommunicator.getActive(). + * Set by start(), cleared by close(). + */ + static getActive(): FaceclawCommunicatorBridge | null { + return FaceclawCommunicatorBridge.active; + } constructor(addresses: { right: string; left: string; ring?: string }) { this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress( @@ -450,7 +490,9 @@ export class FaceclawCommunicatorBridge { // ----- session --------------------------------------------------------------------- async start(): Promise { - await this.enqueueNativeCall(() => { this.communicator.start(); }); + await this.enqueueNativeCall(() => { + if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this; + }); } async setG2ScreenOn(screenOn: boolean): Promise { @@ -640,6 +682,8 @@ export class FaceclawCommunicatorBridge { } async close(): Promise { + this.closed = true; + if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null; this.setAncsListeners(false); this.previewAnimationListeners.clear(); await this.enqueueNativeCall(() => { @@ -663,6 +707,70 @@ export class FaceclawCommunicatorBridge { return () => { this.communicator.removeCompassListenerListener(proxy); }; } + // ----- sensors and passive audio, for the developer demos ----------------------------- + // Fire-and-forget, so they skip the native-call queue like the compass controls do. + + /** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */ + setImuReportEnabled(enable: boolean, reportFrq: number): void { + this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq))); + } + + /** Accelerometer samples, delivered on the main queue. */ + addImuListener(listener: (reading: ImuReading) => void): () => void { + const proxy = ImuListener.new() as ImuListener; + proxy.handler = listener; + this.communicator.addImuListenerListener(proxy); + return () => { this.communicator.removeImuListenerListener(proxy); }; + } + + /** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */ + queryAmbientLight(): void { + this.communicator.queryAmbientLight(); + } + + setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void { + this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable), + Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease)); + } + + /** Raw field-105 ambient-light records, delivered on the main queue. */ + addAmbientLightListener(listener: (body: Uint8Array) => void): () => void { + const proxy = AmbientLightListener.new() as AmbientLightListener; + proxy.handler = listener; + const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy); + return () => { this.communicator.removeAmbientLightListenerListener(handle); }; + } + + /** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */ + addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void { + const proxy = MicStatusListener.new() as MicStatusListener; + proxy.handler = listener; + const handle = this.communicator.addMicStatusListenerNsDataListener(proxy); + return () => { this.communicator.removeMicStatusListenerListener(handle); }; + } + + /** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */ + sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean { + return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple( + toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple))); + } + + /** + * Every audio-characteristic packet, whatever enabled the mics, on the + * main queue. Passive: sends nothing and doesn't affect capture. + */ + addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void { + const proxy = AudioPacketListener.new() as AudioPacketListener; + proxy.handler = listener; + const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy); + return () => { this.communicator.removeAudioMonitorListenerListener(handle); }; + } + + /** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */ + isAudioCaptureActive(): boolean { + return Boolean(this.communicator.isAudioCaptureActive()); + } + /** * Serialized glasses microphone enable/disable (the voice bridge's * IosMicrophoneSession contract). A release while an enable awaits its ACK diff --git a/app/native/imu-types.ts b/app/native/imu-types.ts new file mode 100644 index 00000000..bc0eb693 --- /dev/null +++ b/app/native/imu-types.ts @@ -0,0 +1,17 @@ +/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */ + +export type ImuReading = { x: number; y: number; z: number; source: number }; + +/** Map an IMU sample's EventSource to a short label. */ +export function imuSourceLabel(source: number): string { + switch (source) { + case 1: + return "R arm"; + case 2: + return "ring"; + case 3: + return "L arm"; + default: + return "device"; + } +} diff --git a/app/native/imu.ios.ts b/app/native/imu.ios.ts new file mode 100644 index 00000000..32288335 --- /dev/null +++ b/app/native/imu.ios.ts @@ -0,0 +1,34 @@ +import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios"; +import { type ImuReading } from "./imu-types"; + +export * from "./imu-types"; + +/** + * IMU (accelerometer) access on iOS: the same API as imu.ts over the running + * shared-Kotlin session. As there, the stream is experimental, the calls are + * no-ops while no session is running, and a listener stays with the session + * it subscribed to. + */ + +/** Enable/disable the accelerometer report stream. No-op if not connected. */ +export function setImuReportEnabled(enable: boolean, reportFreq: number): void { + try { + FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq); + } catch (error) { + console.warn(`setImuReportEnabled failed: ${error}`); + } +} + +/** Subscribe to IMU readings; returns an unsubscribe function. */ +export function addImuListener(listener: (reading: ImuReading) => void): () => void { + const active = FaceclawCommunicatorBridge.getActive(); + if (!active) return () => {}; + const unsubscribe = active.addImuListener(listener); + return () => { + try { + unsubscribe(); + } catch { + // The session may be gone after a disconnect; nothing to clean up. + } + }; +} diff --git a/app/native/imu.ts b/app/native/imu.ts index c01cd962..11a1cdc2 100644 --- a/app/native/imu.ts +++ b/app/native/imu.ts @@ -12,7 +12,9 @@ declare const com: any; * static, so it works from any isolate without threading the bridge through. */ -export type ImuReading = { x: number; y: number; z: number; source: number }; +import { type ImuReading } from "./imu-types"; + +export * from "./imu-types"; function activeCommunicator(): any { if (!global.isAndroid) return null; @@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v } }; } - -/** Map an IMU sample's EventSource to a short label. */ -export function imuSourceLabel(source: number): string { - switch (source) { - case 1: - return "R arm"; - case 2: - return "ring"; - case 3: - return "L arm"; - default: - return "device"; - } -} diff --git a/app/native/mic-control-link.ios.ts b/app/native/mic-control-link.ios.ts new file mode 100644 index 00000000..e19ea4a6 --- /dev/null +++ b/app/native/mic-control-link.ios.ts @@ -0,0 +1,30 @@ +import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios"; + +/** The iOS twin of mic-control-link.ts, over the running shared-Kotlin session. */ + +export function micControlSession(): unknown { + return FaceclawCommunicatorBridge.getActive(); +} + +/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */ +export function sendMicControlRecord( + session: unknown, + record: Uint8Array, + label: string, + rightTemple: boolean, + leftTemple: boolean, +): boolean { + return (session as FaceclawCommunicatorBridge).sendFaceclawMicControl(record, label, rightTemple, leftTemple); +} + +/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */ +export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void { + const unsubscribe = (session as FaceclawCommunicatorBridge).addMicStatusListener(listener); + return () => { + try { + unsubscribe(); + } catch { + // The session may have been closed during a reconnect. + } + }; +} diff --git a/app/native/mic-control-link.ts b/app/native/mic-control-link.ts new file mode 100644 index 00000000..1b5f238d --- /dev/null +++ b/app/native/mic-control-link.ts @@ -0,0 +1,46 @@ +import { toJavaBytes } from "./cloud-stt"; +import { toUint8Array } from "../util/array-util"; + +declare const com: any; + +/** + * Platform plumbing for the CFW mic_control channel (apps/microphones/mic-control.ts): + * the running session to send records on and to hear field-104 status replies + * from. The session is opaque and is a different object after a reconnect. + */ + +export function micControlSession(): unknown { + if (!global.isAndroid) return null; + try { + return com.faceclaw.app.FaceclawBleCommunicator.getActive(); + } catch { + return null; + } +} + +/** Queue a record for the chosen temples; false when the session wasn't ready and nothing was queued. */ +export function sendMicControlRecord( + session: unknown, + record: Uint8Array, + label: string, + rightTemple: boolean, + leftTemple: boolean, +): boolean { + return (session as any).sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false; +} + +/** Status record bodies (undecoded) and their temple: "L", "R" or "?". Returns an unsubscribe function. */ +export function addMicStatusListener(session: unknown, listener: (body: Uint8Array, arm: string) => void): () => void { + const communicator = session as any; + const proxy = new com.faceclaw.app.FaceclawMicStatusListener({ + onMicStatus: (body: any, arm: string) => listener(toUint8Array(body), String(arm)), + }); + communicator.addMicStatusListener(proxy); + return () => { + try { + communicator.removeMicStatusListener(proxy); + } catch { + // The old communicator instance may be gone after a reconnect. + } + }; +} diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt index b326bdbd..6114b182 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt @@ -24,6 +24,16 @@ interface IosAudioPacketListener { fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long) } +/** CFW ambient-light reports (the raw field-105 record) as NSData. Main queue. Named apart like [IosAudioPacketListener]. */ +interface IosAmbientLightListener { + fun onNsDataAmbientLight(body: NSData) +} + +/** CFW mic_control status records (the raw field-104 record) as NSData, with the temple: "L", "R" or "?". Main queue. */ +interface IosMicStatusListener { + fun onNsDataMicStatus(body: NSData, arm: String) +} + /** * ObjC-facing facade over the shared [GlassesSessionCore] for the NativeScript app: the iOS * counterpart of Android's FaceclawBleCommunicator with the same method names. GATT goes @@ -188,6 +198,39 @@ class IosGlassesSession internal constructor( fun sendFaceclawMicControl(record: NSData?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean = core.sendFaceclawMicControl(record?.byteArray(), label, rightTemple, leftTemple) + // NSData twins of the ByteArray listeners above (a Kotlin ByteArray crosses into JavaScript one + // byte per call). The core already delivers these on the main queue. Each returns the adapter it + // registered: pass that to the matching remove* method to unsubscribe. + + fun addAmbientLightListenerNsData(listener: IosAmbientLightListener): FaceclawAmbientLightListener { + val adapter = object : FaceclawAmbientLightListener { + override fun onAmbientLight(body: ByteArray?) = listener.onNsDataAmbientLight((body ?: ByteArray(0)).data()) + } + core.addAmbientLightListener(adapter) + return adapter + } + + fun addMicStatusListenerNsData(listener: IosMicStatusListener): FaceclawMicStatusListener { + val adapter = object : FaceclawMicStatusListener { + override fun onMicStatus(body: ByteArray?, arm: String?) = + listener.onNsDataMicStatus((body ?: ByteArray(0)).data(), arm ?: "?") + } + core.addMicStatusListener(adapter) + return adapter + } + + /** Every audio-characteristic packet, whether or not a capture is active (passive; sends nothing). */ + fun addAudioMonitorListenerNsData(listener: IosAudioPacketListener): FaceclawAudioPacketListener { + val adapter = object : FaceclawAudioPacketListener { + override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) = + listener.onNsDataAudioPacket((data ?: ByteArray(0)).data(), arm ?: "?", arrivalMs) + } + core.addAudioMonitorListener(adapter) + return adapter + } + + fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.removeAudioMonitorListener(listener) + fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener?.let { MainQueueAudioListener(it) }) diff --git a/tests/ios-communicator-surfaces.test.cjs b/tests/ios-communicator-surfaces.test.cjs index dc6bdc73..6e1a6cd9 100644 --- a/tests/ios-communicator-surfaces.test.cjs +++ b/tests/ios-communicator-surfaces.test.cjs @@ -32,7 +32,8 @@ function harness(refused = new Set()) { NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; }, FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) }, FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {}, - FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} }, + FaceclawKitFaceclawCompassListener: {}, FaceclawKitFaceclawImuListener: {}, FaceclawKitIosAmbientLightListener: {}, + FaceclawKitIosMicStatusListener: {}, console: { log() {}, warn() {} }, }, { '@nativescript/core': { File: {}, knownFolders: {}, path: {} }, '../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 }, diff --git a/tests/ios-sensors.test.cjs b/tests/ios-sensors.test.cjs new file mode 100644 index 00000000..94bec3b8 --- /dev/null +++ b/tests/ios-sensors.test.cjs @@ -0,0 +1,186 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +// The developer demos' sensor plumbing on iOS: FaceclawCommunicatorBridge over a +// fake IosGlassesSession, and the imu/ambient-light/audio-monitor/mic-control-link +// .ios.ts modules over that bridge. +function harness() { + const calls = []; + const listeners = { imu: new Set(), light: new Set(), status: new Set(), audio: new Set() }; + // Like the Kotlin facade, the NSData add* methods return an adapter handle for remove*. + const handleFor = (set, proxy) => { const handle = { proxy }; set.add(handle); return handle; }; + const session = { + setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {}, + start() { calls.push(['start']); return true; }, + close() { calls.push(['close']); }, + setImuReportEnabledEnableReportFrq(enable, freq) { calls.push(['imu', enable, freq]); }, + addImuListenerListener(proxy) { listeners.imu.add(proxy); }, + removeImuListenerListener(proxy) { listeners.imu.delete(proxy); }, + queryAmbientLight() { calls.push(['als-query']); }, + setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(...args) { calls.push(['als-poll', ...args]); }, + addAmbientLightListenerNsDataListener: proxy => handleFor(listeners.light, proxy), + removeAmbientLightListenerListener(handle) { listeners.light.delete(handle); }, + addMicStatusListenerNsDataListener: proxy => handleFor(listeners.status, proxy), + removeMicStatusListenerListener(handle) { listeners.status.delete(handle); }, + addAudioMonitorListenerNsDataListener: proxy => handleFor(listeners.audio, proxy), + removeAudioMonitorListenerListener(handle) { listeners.audio.delete(handle); }, + isAudioCaptureActive: () => true, + sendFaceclawMicControlRecordLabelRightTempleLeftTemple(record, label, right, left) { + calls.push(['mic-control', Array.from(record), label, right, left]); + return label !== 'refused'; + }, + }; + class NSObject { static new() { return new this(); } } + const load = loader({ + NSObject, NativeClass: () => cls => cls, setTimeout, clearTimeout, + FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) }, + FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {}, + FaceclawKitFaceclawCompassListener: {}, FaceclawKitFaceclawImuListener: {}, FaceclawKitIosAmbientLightListener: {}, + FaceclawKitIosMicStatusListener: {}, console: { log() {}, warn() {} }, + }, { + '@nativescript/core': { File: {}, knownFolders: {}, path: {} }, + '../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 }, + // NSData stands in as the bytes themselves. + './kotlin-data': { toData: bytes => bytes, fromData: data => new Uint8Array(data) }, + './ios-graphics': { previewPixels() {} }, + './ios-bluetooth': { iosBluetooth: {} }, + }); + const bridgeModule = load('app/native/faceclaw-communicator.ios.ts'); + return { load, session, calls, listeners, FaceclawCommunicatorBridge: bridgeModule.FaceclawCommunicatorBridge }; +} + +function alsRecord(alsValue) { + const body = new Uint8Array(24); + body.set([0x41, 0x4c, 1, 2, 0x0b]); + body[6] = alsValue & 0xff; body[7] = alsValue >> 8; + return body; +} + +test('the bridge is the active session from start() until close()', async () => { + const h = harness(); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + assert.equal(h.FaceclawCommunicatorBridge.getActive(), null, 'not before start'); + await bridge.start(); + assert.equal(h.FaceclawCommunicatorBridge.getActive(), bridge); + const closing = bridge.close(); + assert.equal(h.FaceclawCommunicatorBridge.getActive(), null, 'cleared as soon as close() is called'); + await closing; + assert.deepEqual(h.calls, [['start'], ['close']]); +}); + +test('a start queued behind close() does not make a closed bridge active', async () => { + const h = harness(); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + const started = bridge.start(); + const closed = bridge.close(); + await Promise.all([started, closed]); + assert.equal(h.FaceclawCommunicatorBridge.getActive(), null); +}); + +test('the sensor modules are no-ops without a running session', () => { + const h = harness(); + const imu = h.load('app/native/imu.ios.ts'); + const light = h.load('app/native/ambient-light.ios.ts'); + const monitor = h.load('app/native/audio-monitor.ios.ts'); + imu.setImuReportEnabled(true, 200); + light.setAmbientLightPolling(true); + light.queryAmbientLight(); + imu.addImuListener(() => assert.fail('no session'))(); + light.addAmbientLightListener(() => assert.fail('no session'))(); + assert.equal(monitor.audioMonitorCommunicator(), null); + assert.equal(monitor.isFaceclawAudioCaptureActive(), false); + monitor.addAudioMonitorListeners(null, () => {}, () => {})(); + assert.deepEqual(h.calls, []); +}); + +test('IMU control and samples go through the active session', async () => { + const h = harness(); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + await bridge.start(); + const imu = h.load('app/native/imu.ios.ts'); + imu.setImuReportEnabled(true, 199.6); + imu.setImuReportEnabled(false, -5); + const readings = []; + const off = imu.addImuListener(reading => readings.push({ ...reading })); + const [proxy] = h.listeners.imu; + proxy.onImuDataXYZEventSource(0.5, -1, 9.75, 3); + off(); + assert.equal(h.listeners.imu.size, 0); + assert.deepEqual(readings, [{ x: 0.5, y: -1, z: 9.75, source: 3 }]); + assert.equal(imu.imuSourceLabel(3), 'L arm'); + assert.deepEqual(h.calls.slice(1), [['imu', true, 200], ['imu', false, 0]]); +}); + +test('ambient light: defaults, query, and decoded reports over NSData', async () => { + const h = harness(); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + await bridge.start(); + const light = h.load('app/native/ambient-light.ios.ts'); + light.setAmbientLightPolling(true, { intervalMs: 250, minDelta: 0, heartbeatMs: 1000, bindToLease: true }); + light.setAmbientLightPolling(false); + light.queryAmbientLight(); + assert.deepEqual(h.calls.slice(1), [ + ['als-poll', true, 250, 0, 1000, true], + ['als-poll', false, 500, 1, 5000, true], + ['als-query'], + ]); + const reports = []; + const off = light.addAmbientLightListener(report => reports.push(report)); + const [handle] = h.listeners.light; + handle.proxy.onNsDataAmbientLightBody(alsRecord(1234)); + handle.proxy.onNsDataAmbientLightBody(new Uint8Array([1, 2, 3])); // not a report: dropped + off(); + assert.equal(h.listeners.light.size, 0, 'removed by the handle the session returned'); + assert.equal(reports.length, 1); + assert.equal(reports[0].alsValue, 1234); + assert.equal(reports[0].reason, 'poll'); + assert.equal(reports[0].opened, true); + assert.equal(reports[0].passive, true); + assert.equal(reports[0].readOk, true); +}); + +test('the passive audio monitor sees packets and mic status per temple', async () => { + const h = harness(); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + await bridge.start(); + const monitor = h.load('app/native/audio-monitor.ios.ts'); + const communicator = monitor.audioMonitorCommunicator(); + assert.equal(communicator, bridge); + assert.equal(monitor.isFaceclawAudioCaptureActive(), true); + const packets = [], statuses = []; + const off = monitor.addAudioMonitorListeners(communicator, + (data, arm, arrivalMs) => packets.push([Array.from(data), arm, arrivalMs]), + (body, arm) => statuses.push([Array.from(body), arm])); + const [audio] = h.listeners.audio, [status] = h.listeners.status; + audio.proxy.onNsDataAudioPacketDataArmArrivalMs(new Uint8Array([1, 2]), 'L', 42); + audio.proxy.onNsDataAudioPacketDataArmArrivalMs(new Uint8Array([3]), '', 43); + status.proxy.onNsDataMicStatusBodyArm(new Uint8Array([0x4d, 0x43]), 'R'); + off(); + assert.equal(h.listeners.audio.size + h.listeners.status.size, 0); + assert.deepEqual(packets, [[[1, 2], 'L', 42], [[3], '?', 43]]); + assert.deepEqual(statuses, [[[0x4d, 0x43], 'R']]); + assert.deepEqual(h.calls, [['start']], 'monitoring sends nothing'); +}); + +test('mic-control link sends records and hears status on iOS', async () => { + const h = harness(); + const link = h.load('app/native/mic-control-link.ios.ts'); + assert.equal(link.micControlSession(), null); + const bridge = new h.FaceclawCommunicatorBridge({ right: 'r', left: 'l' }); + await bridge.start(); + const session = link.micControlSession(); + assert.equal(session, bridge); + assert.equal(link.sendMicControlRecord(session, new Uint8Array([0x4d, 0x43, 1, 4]), 'query', true, true), true); + assert.equal(link.sendMicControlRecord(session, new Uint8Array([0x4d, 0x43, 1, 4]), 'refused', true, false), false); + const heard = []; + const off = link.addMicStatusListener(session, (body, arm) => heard.push([Array.from(body), arm])); + [...h.listeners.status][0].proxy.onNsDataMicStatusBodyArm(new Uint8Array([7]), 'L'); + off(); + assert.equal(h.listeners.status.size, 0); + assert.deepEqual(heard, [[[7], 'L']]); + assert.deepEqual(h.calls.slice(1), [ + ['mic-control', [0x4d, 0x43, 1, 4], 'query', true, true], + ['mic-control', [0x4d, 0x43, 1, 4], 'refused', true, false], + ]); +}); diff --git a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosSessionFacadeTest.kt b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosSessionFacadeTest.kt index 6067bfc8..17fd93a5 100644 --- a/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosSessionFacadeTest.kt +++ b/tests/kotlin/src/iosTest/kotlin/com/faceclaw/app/IosSessionFacadeTest.kt @@ -197,6 +197,52 @@ class IosSessionFacadeTest { assertFalse(pumpMainUntil(200) { relayed.size > 1 }) } + @Test + fun nsDataSensorListenersDeliverOnMainAndUnsubscribeByTheirHandles() { + val session = IosGlassesSession(FakeIosLink(), null, IosSessionHost(threadName = "test-session-sensors"), RIGHT, LEFT, "", IosProtocolPlatform) + val core = session.coreForTests() + val light = ArrayList>() + val status = ArrayList, String>>() + val audio = ArrayList, String, Long>>() + var offMain = 0 + val lightHandle = session.addAmbientLightListenerNsData(object : IosAmbientLightListener { + override fun onNsDataAmbientLight(body: platform.Foundation.NSData) { + if (!NSThread.isMainThread) offMain++ + light.add(body.byteArray().toList()) + } + }) + val statusHandle = session.addMicStatusListenerNsData(object : IosMicStatusListener { + override fun onNsDataMicStatus(body: platform.Foundation.NSData, arm: String) { + if (!NSThread.isMainThread) offMain++ + status.add(body.byteArray().toList() to arm) + } + }) + val audioHandle = session.addAudioMonitorListenerNsData(object : IosAudioPacketListener { + override fun onNsDataAudioPacket(data: platform.Foundation.NSData, arm: String, arrivalMs: Long) { + if (!NSThread.isMainThread) offMain++ + audio.add(Triple(data.byteArray().toList(), arm, arrivalMs)) + } + }) + val report = byteArrayOf(0x41, 0x4c, 1, 2) + val record = byteArrayOf(0x4d, 0x43, 1) + val packet = byteArrayOf(9, 8, 7) + core.emitAmbientLight(report) + core.emitMicStatus(record, LEFT) + core.emitAudioMonitorPacket(packet, "R", 1234L) + assertTrue(pumpMainUntil(2_000) { light.size == 1 && status.size == 1 && audio.size == 1 }) + assertEquals(report.toList(), light[0]) + assertEquals(record.toList() to "L", status[0]) + assertEquals(Triple(packet.toList(), "R", 1234L), audio[0]) + assertEquals(0, offMain) + session.removeAmbientLightListener(lightHandle) + session.removeMicStatusListener(statusHandle) + session.removeAudioMonitorListener(audioHandle) + core.emitAmbientLight(report) + core.emitMicStatus(record, RIGHT) + core.emitAudioMonitorPacket(packet, "L", 5678L) + assertFalse(pumpMainUntil(200) { light.size > 1 || status.size > 1 || audio.size > 1 }) + } + @Test fun hostDispatchesToTheCallingQueueAndRunsTheWorker() { val host = IosSessionHost(threadName = "test-host")