mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
iOS: fix the audio-packet and window-close crashes, stop retrying the ring
- Rename IosAudioPacketListener.onAudioPacket to onNsDataAudioPacket. The selector collided with FaceclawAudioPacketListener's, Kotlin/Native mangled one of the two depending on export order, and the first glasses audio packet aborted the app with "unrecognized selector". - Closing a window (e.g. Transcribe from its system menu) aborted the app: the menu pop re-renders the window, that frame is queued behind the surface removal, and Kotlin's compositor throws on the unknown surface. The bridge now tracks which surfaces Kotlin holds and drops stale calls inside the queue, submit re-checks the window is still open, and the IosGlassesSession surface wrappers log and drop compositor errors instead of aborting. - Only connect to the R1 ring from the phone in "Direct" mode, as Android does. iOS always tried; the failing connect retried every few seconds and starved the glasses link, so the glasses image lagged. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
2bb7b3ea1a
commit
ecb6c7f465
4 files changed
+151
-21
No files matched your search
@@ -48,7 +48,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
|
||||
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
@@ -451,10 +451,11 @@ export class IosPreviewController {
|
||||
}
|
||||
}
|
||||
/** Submit a painted frame for an in-process window (e.g. the launcher). */
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
if (!display) return
|
||||
// The window can close (removing its surface) while this frame waits.
|
||||
if (!display || !live()) return
|
||||
const fingerprint = planesFingerprint(planes)
|
||||
const { image, draws } = flattenPlanesWithDraws(planes)
|
||||
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
|
||||
@@ -646,7 +647,7 @@ export class IosPreviewController {
|
||||
// The explicit render below supplies a fresh frame after configuration.
|
||||
let surfaceReady = false
|
||||
const app = create({
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
|
||||
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
||||
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
|
||||
reconfigureSurface: () => {
|
||||
@@ -731,7 +732,10 @@ export class IosPreviewController {
|
||||
this.firmwareCompatible = false
|
||||
this.update('connecting', 'Finding configured devices…')
|
||||
let identifiers
|
||||
try { identifiers = await resolveIosPeripherals(addresses) }
|
||||
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
|
||||
// fails and retries every few seconds, starving the glasses link.
|
||||
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
|
||||
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
|
||||
catch (failure) {
|
||||
this.update('error', failure instanceof Error ? failure.message : String(failure))
|
||||
return
|
||||
|
||||
@@ -141,7 +141,7 @@ class AncsListener extends NSObject {
|
||||
class AudioPacketListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosAudioPacketListener];
|
||||
handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void;
|
||||
onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
|
||||
onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
|
||||
this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs));
|
||||
}
|
||||
}
|
||||
@@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge {
|
||||
private ancsProxy: AncsListener | null = null;
|
||||
private nativeCallQueue: Promise<void> = Promise.resolve();
|
||||
private queuedNativeCalls = 0;
|
||||
// Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface).
|
||||
private readonly nativeSurfaces = new Set<string>();
|
||||
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
|
||||
private readonly finishedFrameOutcomes = new Map<number, string>();
|
||||
private readonly frameFinishedWaiters = new Map<number, Set<(outcome: string) => void>>();
|
||||
@@ -518,15 +520,29 @@ export class FaceclawCommunicatorBridge {
|
||||
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
|
||||
const transparency = options.transparency === "color-key" ? 1 : 0;
|
||||
await this.enqueueNativeCall(() => {
|
||||
this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
|
||||
const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
|
||||
id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height),
|
||||
Math.round(options.zOrder), transparency,
|
||||
);
|
||||
if (configured !== false) this.nativeSurfaces.add(id);
|
||||
});
|
||||
}
|
||||
|
||||
async removeSurface(id: string): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
this.nativeSurfaces.delete(id);
|
||||
this.communicator.removeSurfaceId(id);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Checked inside each queued call: a render in flight when its window closed lands
|
||||
* behind the removal, and Kotlin aborts on a frame for an unknown surface.
|
||||
*/
|
||||
private hasNativeSurface(id: string, what: string): boolean {
|
||||
if (this.nativeSurfaces.has(id)) return true;
|
||||
console.log(`[ios-ble] ${what} for removed surface ${id} dropped`);
|
||||
return false;
|
||||
}
|
||||
|
||||
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
|
||||
@@ -541,11 +557,15 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible));
|
||||
});
|
||||
}
|
||||
|
||||
async setSurfaceDepth(id: string, depth: number): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth));
|
||||
});
|
||||
}
|
||||
|
||||
async setScreenBlanked(blanked: boolean): Promise<void> {
|
||||
@@ -572,6 +592,7 @@ export class FaceclawCommunicatorBridge {
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
const draws = glyphs ? toData(new Uint8Array(glyphs)) : null;
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (!this.hasNativeSurface(surfaceId, "frame")) return;
|
||||
this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(
|
||||
toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height),
|
||||
fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws,
|
||||
|
||||
@@ -15,9 +15,13 @@ interface IosAncsListener {
|
||||
fun onRelayFrame(data: NSData)
|
||||
}
|
||||
|
||||
/** Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. */
|
||||
/**
|
||||
* Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue.
|
||||
* Not named onAudioPacket: that selector collides with FaceclawAudioPacketListener's, and the
|
||||
* mangled name Kotlin/Native picks broke the TypeScript proxy ("unrecognized selector").
|
||||
*/
|
||||
interface IosAudioPacketListener {
|
||||
fun onAudioPacket(data: NSData, arm: String, arrivalMs: Long)
|
||||
fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -209,14 +213,21 @@ class IosGlassesSession internal constructor(
|
||||
/** Set the compositor's output frame size. Call before configuring surfaces. */
|
||||
fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height)
|
||||
|
||||
fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible)
|
||||
// The compositor throws for an unknown surface or a bad rect, and without @Throws (which
|
||||
// would rename the selectors) that aborts the process. Log and drop instead; false then.
|
||||
|
||||
fun setSurfaceDepth(id: String, depth: Int) = core.setSurfaceDepth(id, depth)
|
||||
fun setSurfaceVisible(id: String, visible: Boolean): Boolean = surfaceCall("setSurfaceVisible $id") {
|
||||
core.setSurfaceVisible(id, visible)
|
||||
}
|
||||
|
||||
fun setSurfaceDepth(id: String, depth: Int): Boolean = surfaceCall("setSurfaceDepth $id") {
|
||||
core.setSurfaceDepth(id, depth)
|
||||
}
|
||||
|
||||
fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked)
|
||||
|
||||
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) =
|
||||
core.configureSurface(id, x, y, width, height, zOrder, transparency)
|
||||
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int): Boolean =
|
||||
surfaceCall("configureSurface $id") { core.configureSurface(id, x, y, width, height, zOrder, transparency) }
|
||||
|
||||
fun removeSurface(id: String) = core.removeSurface(id)
|
||||
|
||||
@@ -235,10 +246,28 @@ class IosGlassesSession internal constructor(
|
||||
paintMs: Int,
|
||||
frameId: Int,
|
||||
glyphs: NSData?,
|
||||
) = core.submitSurfaceFrame(
|
||||
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
|
||||
glyphs?.let { IosByteReader(it) },
|
||||
)
|
||||
): Boolean {
|
||||
val ok = surfaceCall("submitSurfaceFrame $surfaceId") {
|
||||
core.submitSurfaceFrame(
|
||||
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
|
||||
glyphs?.let { IosByteReader(it) },
|
||||
)
|
||||
}
|
||||
if (!ok) frameTimings.finishFrame(frameId, "discarded: surface $surfaceId rejected")
|
||||
return ok
|
||||
}
|
||||
|
||||
private inline fun surfaceCall(what: String, block: () -> Unit): Boolean =
|
||||
try {
|
||||
block()
|
||||
true
|
||||
} catch (e: IllegalArgumentException) {
|
||||
core.logLine("$what dropped: ${e.message}")
|
||||
false
|
||||
} catch (e: IllegalStateException) {
|
||||
core.logLine("$what dropped: ${e.message}")
|
||||
false
|
||||
}
|
||||
|
||||
/** The current composited screen (gray, 8bpp) or null before any surface was configured. */
|
||||
fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) }
|
||||
@@ -265,7 +294,7 @@ class IosGlassesSession internal constructor(
|
||||
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
|
||||
val packet = (data ?: ByteArray(0)).data()
|
||||
val label = arm ?: ""
|
||||
dispatch_async(dispatch_get_main_queue()) { delegate.onAudioPacket(packet, label, arrivalMs) }
|
||||
dispatch_async(dispatch_get_main_queue()) { delegate.onNsDataAudioPacket(packet, label, arrivalMs) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
// FaceclawCommunicatorBridge over a fake Kotlin session that, like the real
|
||||
// SurfaceCompositor, rejects calls for surfaces it does not hold.
|
||||
function harness() {
|
||||
const calls = [];
|
||||
const surfaces = new Set();
|
||||
const session = {
|
||||
setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {},
|
||||
configureSurfaceIdXYWidthHeightZOrderTransparency(id) { calls.push(['configure', id]); surfaces.add(id); return true; },
|
||||
removeSurfaceId(id) { calls.push(['remove', id]); surfaces.delete(id); },
|
||||
setSurfaceVisibleIdVisible(id) { calls.push(['visible', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
|
||||
setSurfaceDepthIdDepth(id) { calls.push(['depth', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
|
||||
submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(_pixels, id) {
|
||||
calls.push(['frame', id]);
|
||||
if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`);
|
||||
return true;
|
||||
},
|
||||
isSurfaceCurrentSurfaceIdFingerprint: (id, fingerprint) => fingerprint === 'same',
|
||||
};
|
||||
class NSObject { static new() { return new this(); } }
|
||||
const timers = [];
|
||||
const load = loader({
|
||||
NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; },
|
||||
FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) },
|
||||
FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {},
|
||||
FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} },
|
||||
}, {
|
||||
'@nativescript/core': { File: {}, knownFolders: {}, path: {} },
|
||||
'../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 },
|
||||
'./kotlin-data': { toData: bytes => bytes, fromData: data => data },
|
||||
'./ios-graphics': { previewPixels() {} },
|
||||
'./ios-bluetooth': { iosBluetooth: {} },
|
||||
});
|
||||
const { FaceclawCommunicatorBridge } = load('app/native/faceclaw-communicator.ios.ts');
|
||||
const bridge = new FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
|
||||
// Run queued native calls (each is one setTimeout(0) hop) until none are left.
|
||||
async function drain() {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
const next = timers.find(timer => !timer.ran && !timer.cleared && timer.delay === 0);
|
||||
if (!next) { await new Promise(resolve => setImmediate(resolve)); if (!timers.some(t => !t.ran && !t.cleared && t.delay === 0)) return; continue; }
|
||||
next.ran = true; next.fn();
|
||||
}
|
||||
}
|
||||
return { bridge, calls, drain };
|
||||
}
|
||||
|
||||
const rect = { x: 0, y: 0, width: 2, height: 1 };
|
||||
const options = { x: 0, y: 0, width: 2, height: 1, zOrder: 0, transparency: 'opaque' };
|
||||
|
||||
test('a frame queued behind its surface removal is dropped instead of reaching Kotlin', async () => {
|
||||
const h = harness();
|
||||
const configured = h.bridge.configureSurface('window:transcribe', options); await h.drain(); await configured;
|
||||
// Close-window order: the removal is queued first, the in-flight render's frame lands behind it.
|
||||
const removed = h.bridge.removeSurface('window:transcribe');
|
||||
const frame = h.bridge.submitSurfaceFrame('window:transcribe', new Uint8Array(2), rect, 'f');
|
||||
const visible = h.bridge.setSurfaceVisible('window:transcribe', true);
|
||||
const depth = h.bridge.setSurfaceDepth('window:transcribe', 1);
|
||||
await h.drain();
|
||||
await Promise.all([removed, frame, visible, depth]);
|
||||
assert.deepEqual(h.calls, [['configure', 'window:transcribe'], ['remove', 'window:transcribe']]);
|
||||
});
|
||||
|
||||
test('frames for a live surface still go through, including the idle inline path', async () => {
|
||||
const h = harness();
|
||||
const configured = h.bridge.configureSurface('window:calc', options); await h.drain(); await configured;
|
||||
await h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'f');
|
||||
assert.deepEqual(h.calls.at(-1), ['frame', 'window:calc']);
|
||||
const reconfigured = h.bridge.configureSurface('window:calc', options);
|
||||
const frame = h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'g');
|
||||
await h.drain(); await Promise.all([reconfigured, frame]);
|
||||
assert.deepEqual(h.calls.slice(-2), [['configure', 'window:calc'], ['frame', 'window:calc']]);
|
||||
});
|
||||
Reference in new issue
Block a user