mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge commit 'ecb6c7' into fix-ios-crashes
This commit is contained in:
commit
e12ea8ee4c
4 files changed
+151
-21
No files matched your search
@@ -48,7 +48,7 @@ import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop'
|
||||
import { onAnySettingChanged,
|
||||
previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById,
|
||||
nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting,
|
||||
openAiApiKeySetting, sonioxApiKeySetting } from '../ui/dashboard-settings'
|
||||
openAiApiKeySetting, sonioxApiKeySetting, ringConnectionModeSetting } from '../ui/dashboard-settings'
|
||||
import type { VoiceProviderSelection } from '../native/cloud-stt-provider'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress'
|
||||
@@ -451,10 +451,11 @@ export class IosPreviewController {
|
||||
}
|
||||
}
|
||||
/** Submit a painted frame for an in-process window (e.g. the launcher). */
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise<void> {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
if (!display) return
|
||||
// The window can close (removing its surface) while this frame waits.
|
||||
if (!display || !live()) return
|
||||
const fingerprint = planesFingerprint(planes)
|
||||
const { image, draws } = flattenPlanesWithDraws(planes)
|
||||
await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height },
|
||||
@@ -646,7 +647,7 @@ export class IosPreviewController {
|
||||
// The explicit render below supplies a fresh frame after configuration.
|
||||
let surfaceReady = false
|
||||
const app = create({
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(),
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
|
||||
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
||||
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
|
||||
reconfigureSurface: () => {
|
||||
@@ -731,7 +732,10 @@ export class IosPreviewController {
|
||||
this.firmwareCompatible = false
|
||||
this.update('connecting', 'Finding configured devices…')
|
||||
let identifiers
|
||||
try { identifiers = await resolveIosPeripherals(addresses) }
|
||||
// As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect
|
||||
// fails and retries every few seconds, starving the glasses link.
|
||||
const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : ''
|
||||
try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) }
|
||||
catch (failure) {
|
||||
this.update('error', failure instanceof Error ? failure.message : String(failure))
|
||||
return
|
||||
|
||||
@@ -141,7 +141,7 @@ class AncsListener extends NSObject {
|
||||
class AudioPacketListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosAudioPacketListener];
|
||||
handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void;
|
||||
onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
|
||||
onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void {
|
||||
this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs));
|
||||
}
|
||||
}
|
||||
@@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge {
|
||||
private ancsProxy: AncsListener | null = null;
|
||||
private nativeCallQueue: Promise<void> = Promise.resolve();
|
||||
private queuedNativeCalls = 0;
|
||||
// Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface).
|
||||
private readonly nativeSurfaces = new Set<string>();
|
||||
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
|
||||
private readonly finishedFrameOutcomes = new Map<number, string>();
|
||||
private readonly frameFinishedWaiters = new Map<number, Set<(outcome: string) => void>>();
|
||||
@@ -518,15 +520,29 @@ export class FaceclawCommunicatorBridge {
|
||||
async configureSurface(id: string, options: SurfaceOptions): Promise<void> {
|
||||
const transparency = options.transparency === "color-key" ? 1 : 0;
|
||||
await this.enqueueNativeCall(() => {
|
||||
this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
|
||||
const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency(
|
||||
id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height),
|
||||
Math.round(options.zOrder), transparency,
|
||||
);
|
||||
if (configured !== false) this.nativeSurfaces.add(id);
|
||||
});
|
||||
}
|
||||
|
||||
async removeSurface(id: string): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
this.nativeSurfaces.delete(id);
|
||||
this.communicator.removeSurfaceId(id);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Checked inside each queued call: a render in flight when its window closed lands
|
||||
* behind the removal, and Kotlin aborts on a frame for an unknown surface.
|
||||
*/
|
||||
private hasNativeSurface(id: string, what: string): boolean {
|
||||
if (this.nativeSurfaces.has(id)) return true;
|
||||
console.log(`[ios-ble] ${what} for removed surface ${id} dropped`);
|
||||
return false;
|
||||
}
|
||||
|
||||
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
|
||||
@@ -541,11 +557,15 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible));
|
||||
});
|
||||
}
|
||||
|
||||
async setSurfaceDepth(id: string, depth: number): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth));
|
||||
});
|
||||
}
|
||||
|
||||
async setScreenBlanked(blanked: boolean): Promise<void> {
|
||||
@@ -572,6 +592,7 @@ export class FaceclawCommunicatorBridge {
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
const draws = glyphs ? toData(new Uint8Array(glyphs)) : null;
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (!this.hasNativeSurface(surfaceId, "frame")) return;
|
||||
this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(
|
||||
toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height),
|
||||
fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws,
|
||||
|
||||
@@ -15,9 +15,13 @@ interface IosAncsListener {
|
||||
fun onRelayFrame(data: NSData)
|
||||
}
|
||||
|
||||
/** Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. */
|
||||
/**
|
||||
* Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue.
|
||||
* Not named onAudioPacket: that selector collides with FaceclawAudioPacketListener's, and the
|
||||
* mangled name Kotlin/Native picks broke the TypeScript proxy ("unrecognized selector").
|
||||
*/
|
||||
interface IosAudioPacketListener {
|
||||
fun onAudioPacket(data: NSData, arm: String, arrivalMs: Long)
|
||||
fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long)
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -209,14 +213,21 @@ class IosGlassesSession internal constructor(
|
||||
/** Set the compositor's output frame size. Call before configuring surfaces. */
|
||||
fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height)
|
||||
|
||||
fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible)
|
||||
// The compositor throws for an unknown surface or a bad rect, and without @Throws (which
|
||||
// would rename the selectors) that aborts the process. Log and drop instead; false then.
|
||||
|
||||
fun setSurfaceDepth(id: String, depth: Int) = core.setSurfaceDepth(id, depth)
|
||||
fun setSurfaceVisible(id: String, visible: Boolean): Boolean = surfaceCall("setSurfaceVisible $id") {
|
||||
core.setSurfaceVisible(id, visible)
|
||||
}
|
||||
|
||||
fun setSurfaceDepth(id: String, depth: Int): Boolean = surfaceCall("setSurfaceDepth $id") {
|
||||
core.setSurfaceDepth(id, depth)
|
||||
}
|
||||
|
||||
fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked)
|
||||
|
||||
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) =
|
||||
core.configureSurface(id, x, y, width, height, zOrder, transparency)
|
||||
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int): Boolean =
|
||||
surfaceCall("configureSurface $id") { core.configureSurface(id, x, y, width, height, zOrder, transparency) }
|
||||
|
||||
fun removeSurface(id: String) = core.removeSurface(id)
|
||||
|
||||
@@ -235,10 +246,28 @@ class IosGlassesSession internal constructor(
|
||||
paintMs: Int,
|
||||
frameId: Int,
|
||||
glyphs: NSData?,
|
||||
) = core.submitSurfaceFrame(
|
||||
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
|
||||
glyphs?.let { IosByteReader(it) },
|
||||
)
|
||||
): Boolean {
|
||||
val ok = surfaceCall("submitSurfaceFrame $surfaceId") {
|
||||
core.submitSurfaceFrame(
|
||||
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
|
||||
glyphs?.let { IosByteReader(it) },
|
||||
)
|
||||
}
|
||||
if (!ok) frameTimings.finishFrame(frameId, "discarded: surface $surfaceId rejected")
|
||||
return ok
|
||||
}
|
||||
|
||||
private inline fun surfaceCall(what: String, block: () -> Unit): Boolean =
|
||||
try {
|
||||
block()
|
||||
true
|
||||
} catch (e: IllegalArgumentException) {
|
||||
core.logLine("$what dropped: ${e.message}")
|
||||
false
|
||||
} catch (e: IllegalStateException) {
|
||||
core.logLine("$what dropped: ${e.message}")
|
||||
false
|
||||
}
|
||||
|
||||
/** The current composited screen (gray, 8bpp) or null before any surface was configured. */
|
||||
fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) }
|
||||
@@ -265,7 +294,7 @@ class IosGlassesSession internal constructor(
|
||||
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
|
||||
val packet = (data ?: ByteArray(0)).data()
|
||||
val label = arm ?: ""
|
||||
dispatch_async(dispatch_get_main_queue()) { delegate.onAudioPacket(packet, label, arrivalMs) }
|
||||
dispatch_async(dispatch_get_main_queue()) { delegate.onNsDataAudioPacket(packet, label, arrivalMs) }
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
// FaceclawCommunicatorBridge over a fake Kotlin session that, like the real
|
||||
// SurfaceCompositor, rejects calls for surfaces it does not hold.
|
||||
function harness() {
|
||||
const calls = [];
|
||||
const surfaces = new Set();
|
||||
const session = {
|
||||
setRequiredFirmwareRevisionRevision() {}, setListenerListener() {}, setPreviewAnimationListenerListener() {},
|
||||
configureSurfaceIdXYWidthHeightZOrderTransparency(id) { calls.push(['configure', id]); surfaces.add(id); return true; },
|
||||
removeSurfaceId(id) { calls.push(['remove', id]); surfaces.delete(id); },
|
||||
setSurfaceVisibleIdVisible(id) { calls.push(['visible', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
|
||||
setSurfaceDepthIdDepth(id) { calls.push(['depth', id]); if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`); return true; },
|
||||
submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs(_pixels, id) {
|
||||
calls.push(['frame', id]);
|
||||
if (!surfaces.has(id)) throw new Error(`unknown surface ${id}`);
|
||||
return true;
|
||||
},
|
||||
isSurfaceCurrentSurfaceIdFingerprint: (id, fingerprint) => fingerprint === 'same',
|
||||
};
|
||||
class NSObject { static new() { return new this(); } }
|
||||
const timers = [];
|
||||
const load = loader({
|
||||
NSObject, NativeClass: () => cls => cls, setTimeout: (fn, delay) => { timers.push({ fn, delay }); return timers.length; }, clearTimeout: id => { timers[id - 1].cleared = true; },
|
||||
FaceclawKitIosGlassesSession: { alloc: () => ({ initWithRightAddressLeftAddressRingAddress: () => session }) },
|
||||
FaceclawKitFaceclawBleCommunicatorListener: {}, FaceclawKitIosAncsListener: {}, FaceclawKitIosAudioPacketListener: {},
|
||||
FaceclawKitFaceclawCompassListener: {}, console: { log() {}, warn() {} },
|
||||
}, {
|
||||
'@nativescript/core': { File: {}, knownFolders: {}, path: {} },
|
||||
'../g2/firmware-compat': { REQUIRED_FACECLAW_FIRMWARE_VERSION: 37 },
|
||||
'./kotlin-data': { toData: bytes => bytes, fromData: data => data },
|
||||
'./ios-graphics': { previewPixels() {} },
|
||||
'./ios-bluetooth': { iosBluetooth: {} },
|
||||
});
|
||||
const { FaceclawCommunicatorBridge } = load('app/native/faceclaw-communicator.ios.ts');
|
||||
const bridge = new FaceclawCommunicatorBridge({ right: 'r', left: 'l' });
|
||||
// Run queued native calls (each is one setTimeout(0) hop) until none are left.
|
||||
async function drain() {
|
||||
for (let i = 0; i < 50; i++) {
|
||||
await new Promise(resolve => setImmediate(resolve));
|
||||
const next = timers.find(timer => !timer.ran && !timer.cleared && timer.delay === 0);
|
||||
if (!next) { await new Promise(resolve => setImmediate(resolve)); if (!timers.some(t => !t.ran && !t.cleared && t.delay === 0)) return; continue; }
|
||||
next.ran = true; next.fn();
|
||||
}
|
||||
}
|
||||
return { bridge, calls, drain };
|
||||
}
|
||||
|
||||
const rect = { x: 0, y: 0, width: 2, height: 1 };
|
||||
const options = { x: 0, y: 0, width: 2, height: 1, zOrder: 0, transparency: 'opaque' };
|
||||
|
||||
test('a frame queued behind its surface removal is dropped instead of reaching Kotlin', async () => {
|
||||
const h = harness();
|
||||
const configured = h.bridge.configureSurface('window:transcribe', options); await h.drain(); await configured;
|
||||
// Close-window order: the removal is queued first, the in-flight render's frame lands behind it.
|
||||
const removed = h.bridge.removeSurface('window:transcribe');
|
||||
const frame = h.bridge.submitSurfaceFrame('window:transcribe', new Uint8Array(2), rect, 'f');
|
||||
const visible = h.bridge.setSurfaceVisible('window:transcribe', true);
|
||||
const depth = h.bridge.setSurfaceDepth('window:transcribe', 1);
|
||||
await h.drain();
|
||||
await Promise.all([removed, frame, visible, depth]);
|
||||
assert.deepEqual(h.calls, [['configure', 'window:transcribe'], ['remove', 'window:transcribe']]);
|
||||
});
|
||||
|
||||
test('frames for a live surface still go through, including the idle inline path', async () => {
|
||||
const h = harness();
|
||||
const configured = h.bridge.configureSurface('window:calc', options); await h.drain(); await configured;
|
||||
await h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'f');
|
||||
assert.deepEqual(h.calls.at(-1), ['frame', 'window:calc']);
|
||||
const reconfigured = h.bridge.configureSurface('window:calc', options);
|
||||
const frame = h.bridge.submitSurfaceFrame('window:calc', new Uint8Array(2), rect, 'g');
|
||||
await h.drain(); await Promise.all([reconfigured, frame]);
|
||||
assert.deepEqual(h.calls.slice(-2), [['configure', 'window:calc'], ['frame', 'window:calc']]);
|
||||
});
|
||||
Reference in new issue
Block a user