Merge commit 'ecb6c7' into fix-ios-crashes

This commit is contained in:
James Babcock committed 2026-10-17 12:20:23 -07:00
commit e12ea8ee4c
4 files changed
+151 -21

No files matched your search

+9 -5
View File
@@ -48,7 +48,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
import { onAnySettingChanged,
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
@@ -451,10 +451,11 @@ export class IosPreviewController {
}
}
/** Submit a painted frame for an in-process window (e.g. the launcher). */
private async submit(id: string, planes: Plane[]): Promise<void> {
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
await this.displayReady
const display = this.display
if (!display) return
// The window can close (removing its surface) while this frame waits.
if (!display || !live()) return
const fingerprint = planesFingerprint(planes)
const { image, draws } = flattenPlanesWithDraws(planes)
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
@@ -646,7 +647,7 @@ export class IosPreviewController {
// The explicit render below supplies a fresh frame after configuration.
let surfaceReady = false
const app = create({
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
reconfigureSurface: () => {
@@ -731,7 +732,10 @@ export class IosPreviewController {
this.firmwareCompatible = false
this.update('connecting', 'Finding configured devices…')
let identifiers
try { identifiers = await resolveIosPeripherals(addresses) }
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
// fails and retries every few seconds, starving the glasses link.
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
catch (failure) {
this.update('error', failure instanceof Error ? failure.message : String(failure))
return
+26 -5
View File
@@ -141,7 +141,7 @@ class AncsListener extends NSObject {
class AudioPacketListener extends NSObject {
static ObjCProtocols = [FaceclawKitIosAudioPacketListener];
handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void;
onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs));
}
}
@@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge {
private ancsProxy: AncsListener | null = null;
private nativeCallQueue: Promise<void> = Promise.resolve();
private queuedNativeCalls = 0;
// Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface).
private readonly nativeSurfaces = new Set<string>();
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
private readonly finishedFrameOutcomes = new Map<number, string>();
private readonly frameFinishedWaiters = new Map<number, Set<(outcome: string) => void>>();
@@ -518,15 +520,29 @@ export class FaceclawCommunicatorBridge {
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
const transparency = options.transparency === "color-key" ? 1 : 0;
await this.enqueueNativeCall(() => {
this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height),
Math.round(options.zOrder), transparency,
);
if (configured !== false) this.nativeSurfaces.add(id);
});
}
async removeSurface(id: string): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); });
await this.enqueueNativeCall(() => {
this.nativeSurfaces.delete(id);
this.communicator.removeSurfaceId(id);
});
}
/**
* Checked inside each queued call: a render in flight when its window closed lands
* behind the removal, and Kotlin aborts on a frame for an unknown surface.
*/
private hasNativeSurface(id: string, what: string): boolean {
if (this.nativeSurfaces.has(id)) return true;
console.log(`[ios-ble] ${what} for removed surface ${id} dropped`);
return false;
}
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
@@ -541,11 +557,15 @@ export class FaceclawCommunicatorBridge {
}
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible));
});
}
async setSurfaceDepth(id: string, depth: number): Promise<void> {
await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); });
await this.enqueueNativeCall(() => {
if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth));
});
}
async setScreenBlanked(blanked: boolean): Promise<void> {
@@ -572,6 +592,7 @@ export class FaceclawCommunicatorBridge {
const snapshot = new Uint8Array(pixels8bpp);
const draws = glyphs ? toData(new Uint8Array(glyphs)) : null;
await this.enqueueNativeCall(() => {
if (!this.hasNativeSurface(surfaceId, "frame")) return;
this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(
toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height),
fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws,
@@ -15,9 +15,13 @@ interface IosAncsListener {
fun onRelayFrame(data: NSData)
}
/** Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. */
/**
* Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue.
* Not named onAudioPacket: that selector collides with FaceclawAudioPacketListener's, and the
* mangled name Kotlin/Native picks broke the TypeScript proxy ("unrecognized selector").
*/
interface IosAudioPacketListener {
fun onAudioPacket(data: NSData, arm: String, arrivalMs: Long)
fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long)
}
/**
@@ -209,14 +213,21 @@ class IosGlassesSession internal constructor(
/** Set the compositor's output frame size. Call before configuring surfaces. */
fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height)
fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible)
// The compositor throws for an unknown surface or a bad rect, and without @Throws (which
// would rename the selectors) that aborts the process. Log and drop instead; false then.
fun setSurfaceDepth(id: String, depth: Int) = core.setSurfaceDepth(id, depth)
fun setSurfaceVisible(id: String, visible: Boolean): Boolean = surfaceCall("setSurfaceVisible $id") {
core.setSurfaceVisible(id, visible)
}
fun setSurfaceDepth(id: String, depth: Int): Boolean = surfaceCall("setSurfaceDepth $id") {
core.setSurfaceDepth(id, depth)
}
fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked)
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) =
core.configureSurface(id, x, y, width, height, zOrder, transparency)
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int): Boolean =
surfaceCall("configureSurface $id") { core.configureSurface(id, x, y, width, height, zOrder, transparency) }
fun removeSurface(id: String) = core.removeSurface(id)
@@ -235,10 +246,28 @@ class IosGlassesSession internal constructor(
paintMs: Int,
frameId: Int,
glyphs: NSData?,
) = core.submitSurfaceFrame(
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
glyphs?.let { IosByteReader(it) },
)
): Boolean {
val ok = surfaceCall("submitSurfaceFrame $surfaceId") {
core.submitSurfaceFrame(
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
glyphs?.let { IosByteReader(it) },
)
}
if (!ok) frameTimings.finishFrame(frameId, "discarded: surface $surfaceId rejected")
return ok
}
private inline fun surfaceCall(what: String, block: () -> Unit): Boolean =
try {
block()
true
} catch (e: IllegalArgumentException) {
core.logLine("$what dropped: ${e.message}")
false
} catch (e: IllegalStateException) {
core.logLine("$what dropped: ${e.message}")
false
}
/** The current composited screen (gray, 8bpp) or null before any surface was configured. */
fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) }
@@ -265,7 +294,7 @@ class IosGlassesSession internal constructor(
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
val packet = (data ?: ByteArray(0)).data()
val label = arm ?: ""
dispatch_async(dispatch_get_main_queue()) { delegate.onAudioPacket(packet, label, arrivalMs) }
dispatch_async(dispatch_get_main_queue()) { delegate.onNsDataAudioPacket(packet, label, arrivalMs) }
}
}
+76
View File
@@ -0,0 +1,76 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
// FaceclawCommunicatorBridge over a fake Kotlin session that, like the real
// SurfaceCompositor, rejects calls for surfaces it does not hold.
function harness() {
const calls = [];
const surfaces = new Set();
const session = {
setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {},
configureSurfaceIdXYWidthHeightZOrderTransparency(id) { calls.push(['configure', id]); surfaces.add(id); return true; },
removeSurfaceId(id) { calls.push(['remove', id]); surfaces.delete(id); },
setSurfaceVisibleIdVisible(id) { calls.push(['visible', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
setSurfaceDepthIdDepth(id) { calls.push(['depth', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(_pixels, id) {
calls.push(['frame', id]);
if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`);
return true;
},
isSurfaceCurrentSurfaceIdFingerprint: (id, fingerprint) => fingerprint === 'same',
};
class NSObject { static new() { return new this(); } }
const timers = [];
const load = loader({
NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; },
FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) },
FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {},
FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} },
}, {
'@nativescript/core': { File: {}, knownFolders: {}, path: {} },
'../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 },
'./kotlin-data': { toData: bytes => bytes, fromData: data => data },
'./ios-graphics': { previewPixels() {} },
'./ios-bluetooth': { iosBluetooth: {} },
});
const { FaceclawCommunicatorBridge } = load('app/native/faceclaw-communicator.ios.ts');
const bridge = new FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
// Run queued native calls (each is one setTimeout(0) hop) until none are left.
async function drain() {
for (let i = 0; i < 50; i++) {
await new Promise(resolve => setImmediate(resolve));
const next = timers.find(timer => !timer.ran && !timer.cleared && timer.delay === 0);
if (!next) { await new Promise(resolve => setImmediate(resolve)); if (!timers.some(t => !t.ran && !t.cleared && t.delay === 0)) return; continue; }
next.ran = true; next.fn();
}
}
return { bridge, calls, drain };
}
const rect = { x: 0, y: 0, width: 2, height: 1 };
const options = { x: 0, y: 0, width: 2, height: 1, zOrder: 0, transparency: 'opaque' };
test('a frame queued behind its surface removal is dropped instead of reaching Kotlin', async () => {
const h = harness();
const configured = h.bridge.configureSurface('window:transcribe', options); await h.drain(); await configured;
// Close-window order: the removal is queued first, the in-flight render's frame lands behind it.
const removed = h.bridge.removeSurface('window:transcribe');
const frame = h.bridge.submitSurfaceFrame('window:transcribe', new Uint8Array(2), rect, 'f');
const visible = h.bridge.setSurfaceVisible('window:transcribe', true);
const depth = h.bridge.setSurfaceDepth('window:transcribe', 1);
await h.drain();
await Promise.all([removed, frame, visible, depth]);
assert.deepEqual(h.calls, [['configure', 'window:transcribe'], ['remove', 'window:transcribe']]);
});
test('frames for a live surface still go through, including the idle inline path', async () => {
const h = harness();
const configured = h.bridge.configureSurface('window:calc', options); await h.drain(); await configured;
await h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'f');
assert.deepEqual(h.calls.at(-1), ['frame', 'window:calc']);
const reconfigured = h.bridge.configureSurface('window:calc', options);
const frame = h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'g');
await h.drain(); await Promise.all([reconfigured, frame]);
assert.deepEqual(h.calls.slice(-2), [['configure', 'window:calc'], ['frame', 'window:calc']]);
});