From 4234d3e9e950de837331279ee42c879ebb2aa1a6 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sat, 3 Oct 2026 15:54:57 -0700 Subject: [PATCH] Fix an issue where animations on the phone preview stopped prematurely --- .../faceclaw/app/FaceclawBleCommunicator.kt | 8 ++++ .../faceclaw/app/FaceclawPreviewCompositor.kt | 4 +- CHANGELOG | 11 +++++ app/g2/dashboard-controller.ts | 14 +++++- app/g2/ios-preview-controller.ts | 1 + app/graphics/surface-compositor.ios.ts | 2 + app/native/faceclaw-communicator.ios.ts | 20 ++++++++- app/native/faceclaw-communicator.ts | 24 +++++++++- app/native/preview-display.ios.ts | 10 ++++- .../g2protocol/session/GlassesSessionCore.kt | 3 ++ .../app/graphics/SurfaceCompositor.kt | 19 +++++++- .../kotlin/com/faceclaw/app/IosNativeApi.kt | 3 ++ .../com/faceclaw/app/ble/IosGlassesSession.kt | 3 ++ .../app/PreviewAnimationListenerTest.kt | 44 +++++++++++++++++++ 14 files changed, 160 insertions(+), 6 deletions(-) create mode 100644 tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/PreviewAnimationListenerTest.kt diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index d7c24320..24ce8c53 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -242,6 +242,14 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** + * Posted to the main looper after each step of an animation the preview is replaying (menu + * highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover + * frames sent to the glasses, which carry the whole animation in one display list. + */ + fun setPreviewAnimationListener(listener: Runnable?) = + core.setPreviewAnimationListener(listener?.let { { mainHandler.post(it); Unit } }) + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(): String { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt index 4f4c65fa..6ae3b8ab 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt @@ -51,11 +51,13 @@ class FaceclawPreviewCompositor(context: Context) { if (activeInstance === this) { activeInstance = null } - frameListener = null + setFrameListener(null) } + /** Also notified after each step of an animation the preview is replaying (see SurfaceCompositor). */ fun setFrameListener(listener: Runnable?) { this.frameListener = listener + compositor.setPreviewAnimationListener(listener?.let { { mainHandler.post(it); Unit } }) } /** Set the compositor's output frame size. Call before configuring surfaces. */ diff --git a/CHANGELOG b/CHANGELOG index 37428aa9..37132519 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,10 +2,21 @@ ## 0.8.2 - Future +### User Interface Customization + * Customizable app switcher: put it on the left, right, bottom, or make it a popup +* The status bar can be combined with the app switcher when it's on the bottom +* The display depth is configurable when the app switcher is on the bottom * Add Dense style for battery indicators + +### Other Features and Fixes + +* Animations have configurable speed +* Fix an issue where animations on the phone preview display stopped before finishing * Terminal app lists every configured host, including ones that are still connecting * Music app shows its library-browsing app picker on the front page when nothing is playing +* Fix iOS failing to connect when an incompatible firmware version is installed +* Mark stock firmware 2.3.2 as validated for installing custom firmware over ## 0.8.1 - 2026-09-30 diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 63548a57..7f6ab9e6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -331,6 +331,7 @@ class DashboardController { private offPhoneLockState: (() => void) | null = null; private offEvenAppConflict: (() => void) | null = null; private offFrameMetrics: (() => void) | null = null; + private offPreviewAnimation: (() => void) | null = null; private offFirmwareInfo: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; @@ -1586,6 +1587,10 @@ class DashboardController { } } }); + // A menu slide reaches the glasses as one frame whose display list + // animates on-device; the mirror replays it in the compositor and needs + // a pull per step, or it keeps whichever mid-motion frame it last got. + this.offPreviewAnimation = communicator.onPreviewAnimationFrame(() => this.schedulePreviewUpdate()); this.offFirmwareInfo = communicator.onFirmwareInfo((info) => { this.appendLog( `firmware: L=${info.leftVersion || "?"} R=${info.rightVersion || "?"}` + @@ -1680,6 +1685,8 @@ class DashboardController { this.offEvenAppConflict = null; this.offFrameMetrics?.(); this.offFrameMetrics = null; + this.offPreviewAnimation?.(); + this.offPreviewAnimation = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; this.offVoiceStatus?.(); @@ -1785,6 +1792,8 @@ class DashboardController { this.offEvenAppConflict = null; this.offFrameMetrics?.(); this.offFrameMetrics = null; + this.offPreviewAnimation?.(); + this.offPreviewAnimation = null; this.offFirmwareInfo?.(); this.offFirmwareInfo = null; this.offVoiceStatus?.(); @@ -2847,7 +2856,10 @@ class DashboardController { } this.previewTrailingTimer = setTimeout(() => { this.previewTrailingTimer = null; - this.updateCompositePreview(); + // Re-check rather than pull directly: a timer that fires a millisecond + // early would fail the floor in updateCompositePreview and drop the + // burst's last frame (an animation's settled frame) until the 1s poll. + this.schedulePreviewUpdate(); }, wait); } diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index b8a73e53..1c09eda8 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -764,6 +764,7 @@ export class IosPreviewController { if (this.active) this.onConnectionState({ ...this.state }) this.schedulePreviewUpdate() }), + communicator.onPreviewAnimationFrame(() => { if (this.communicator === communicator) this.schedulePreviewUpdate() }), communicator.onWearState(wearing => { if (this.communicator === communicator) this.handleWearState(wearing) }), communicator.addCompassListener(receiveCompassEvent), communicator.onAncsRelayFrame(frame => { if (this.communicator === communicator) notifications.receive(frame) }), diff --git a/app/graphics/surface-compositor.ios.ts b/app/graphics/surface-compositor.ios.ts index 065be122..17adfa02 100644 --- a/app/graphics/surface-compositor.ios.ts +++ b/app/graphics/surface-compositor.ios.ts @@ -38,4 +38,6 @@ export class SurfaceCompositor { } setShellScene(bytes: Uint8Array): void { this.native.shellData(toData(bytes)) } composite(): Uint8Array { return fromData(this.native.composite()) } + /** Called on the main queue after each step of an animation the preview is replaying. */ + setPreviewAnimationListener(listener: (() => void) | null): void { this.native.setPreviewAnimationListenerListener(listener) } } diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index a6b99e7f..9bb7ed63 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -191,6 +191,7 @@ export class FaceclawCommunicatorBridge { private latestPhoneLockState: boolean | null = null; private readonly evenAppConflictListeners = new Set<(message: string) => void>(); private readonly frameMetricsListeners = new Set<(metrics: FrameMetrics) => void>(); + private readonly previewAnimationListeners = new Set<() => void>(); private readonly firmwareInfoListeners = new Set<(info: FirmwareInfo) => void>(); private readonly ancsAuthorizationListeners = new Set<(authorized: boolean) => void>(); private readonly ancsRelayListeners = new Set<(frame: Uint8Array) => void>(); @@ -209,6 +210,10 @@ export class FaceclawCommunicatorBridge { this.listenerProxy = SessionListener.new() as SessionListener; this.listenerProxy.bridge = this; this.communicator.setListenerListener(this.listenerProxy); + // Kotlin calls this on the main queue. + this.communicator.setPreviewAnimationListenerListener(() => { + for (const listener of Array.from(this.previewAnimationListeners)) listener(); + }); } // ----- Kotlin listener entry points (main queue) ----------------------------------- @@ -357,6 +362,15 @@ export class FaceclawCommunicatorBridge { return () => this.frameMetricsListeners.delete(listener); } + /** + * Each step of an animation the phone preview is replaying (a menu slide + * reaches the glasses as one frame, so frame metrics fire only at its start). + */ + onPreviewAnimationFrame(listener: () => void): () => void { + this.previewAnimationListeners.add(listener); + return () => this.previewAnimationListeners.delete(listener); + } + onFirmwareInfo(listener: (info: FirmwareInfo) => void): () => void { this.firmwareInfoListeners.add(listener); return () => this.firmwareInfoListeners.delete(listener); @@ -603,7 +617,11 @@ export class FaceclawCommunicatorBridge { async close(): Promise { this.setAncsListeners(false); - await this.enqueueNativeCall(() => this.communicator.close()); + this.previewAnimationListeners.clear(); + await this.enqueueNativeCall(() => { + this.communicator.setPreviewAnimationListenerListener(null); + this.communicator.close(); + }); } // ----- iOS extras: compass, microphone, iPhone notifications (ANCS) ---------------------- diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index f6bc84f1..c469d2c2 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -159,6 +159,7 @@ export class FaceclawCommunicatorBridge { private readonly evenAppConflictListeners = new Set<(message: string) => void>(); private readonly frameMetricsListeners = new Set<(metrics: FrameMetrics) => void>(); private readonly firmwareInfoListeners = new Set<(info: FirmwareInfo) => void>(); + private readonly previewAnimationListeners = new Set<() => void>(); constructor(addresses: { right: string; left: string; ring?: string }) { const context = Utils.android.getApplicationContext(); @@ -251,6 +252,14 @@ export class FaceclawCommunicatorBridge { }, }); this.communicator.setListener(this.listenerProxy); + // Already posted to the main looper by the Java side. + this.communicator.setPreviewAnimationListener( + new java.lang.Runnable({ + run: () => { + for (const listener of Array.from(this.previewAnimationListeners)) listener(); + }, + }), + ); } private emitAsync(listeners: Set<(value: T) => void>, value: T): void { @@ -364,6 +373,15 @@ export class FaceclawCommunicatorBridge { return () => this.frameMetricsListeners.delete(listener); } + /** + * Each step of an animation the phone preview is replaying (a menu slide + * reaches the glasses as one frame, so frame metrics fire only at its start). + */ + onPreviewAnimationFrame(listener: () => void): () => void { + this.previewAnimationListeners.add(listener); + return () => this.previewAnimationListeners.delete(listener); + } + onFirmwareInfo(listener: (info: FirmwareInfo) => void): () => void { this.firmwareInfoListeners.add(listener); return () => this.firmwareInfoListeners.delete(listener); @@ -720,6 +738,10 @@ export class FaceclawCommunicatorBridge { } async close(): Promise { - await this.enqueueJavaCall(() => this.communicator.close()); + this.previewAnimationListeners.clear(); + await this.enqueueJavaCall(() => { + this.communicator.setPreviewAnimationListener(null); + this.communicator.close(); + }); } } diff --git a/app/native/preview-display.ios.ts b/app/native/preview-display.ios.ts index 10edad53..c25628a6 100644 --- a/app/native/preview-display.ios.ts +++ b/app/native/preview-display.ios.ts @@ -45,14 +45,20 @@ export class PreviewDisplayTarget implements DisplayTarget { private onFrameComposited: (() => void) | null = null; private released = false; - /** Receive a callback per applied frame, the stand-in for the connected path's frame-finished callback. */ + /** + * Receive a callback per applied frame, the stand-in for the connected + * path's frame-finished callback, and per step of an animation the + * preview is replaying. + */ activate(onFrameComposited: () => void): void { this.onFrameComposited = onFrameComposited; + this.compositor.setPreviewAnimationListener(() => this.composited()); } release(): void { this.released = true; this.onFrameComposited = null; + this.compositor.setPreviewAnimationListener(null); } private composited(): void { @@ -61,7 +67,9 @@ export class PreviewDisplayTarget implements DisplayTarget { async configureCompositorScreen(width: number, height: number): Promise { if (Math.round(width) !== this.compositor.width || Math.round(height) !== this.compositor.height) { + this.compositor.setPreviewAnimationListener(null); this.compositor = new SurfaceCompositor(Math.round(width), Math.round(height)); + if (this.onFrameComposited) this.compositor.setPreviewAnimationListener(() => this.composited()); } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index 3e37a8a0..2381369b 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -971,6 +971,9 @@ class GlassesSessionCore( /** The current composite for previews/screenshots, or null before any surface exists. */ fun previewComposite(): SurfaceCompositor.Composite? = compositor.previewComposite() + /** See SurfaceCompositor.setPreviewAnimationListener; runs on the redraw scheduler's thread. */ + fun setPreviewAnimationListener(listener: (() -> Unit)?) = compositor.setPreviewAnimationListener(listener) + /** * Show or hide a compositor surface, immediately submitting the resulting * frame. Recompositing here (rather than waiting for the next surface diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt index 19f60f16..475153d1 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt @@ -1,5 +1,6 @@ package com.faceclaw.app +import kotlin.concurrent.Volatile import kotlin.jvm.JvmField import kotlin.jvm.JvmStatic import kotlin.jvm.JvmOverloads @@ -315,6 +316,17 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr /** The last composite's scene; reused while its inputs are unchanged (see compositeLocked). */ private var lastScene: ShellScene? = null private var previewKey: String? = null + @Volatile private var previewAnimationListener: (() -> Unit)? = null + + /** + * Called after each timer redraw of an animated preview (including the one where it settles), + * outside the lock and on the redraw scheduler's thread. The mirror pulls previews on new + * frames only, and the animation advances between them, so without this the mirror keeps + * whichever mid-motion frame it last pulled. + */ + fun setPreviewAnimationListener(listener: (() -> Unit)?) { + previewAnimationListener = listener + } /** * The phone mirror's view of [scene]: without the whole-screen stereo shift, which would move @@ -324,7 +336,12 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr if (previewKey != key) { animatedPreview?.player?.stop() animatedPreview = scene.unshifted().animatedPreview(gray, screenWidth, screenHeight, - schedule = { delay, action -> scheduleDrawRedraw(delay) { lock.withLock(action) } }) + schedule = { delay, action -> + scheduleDrawRedraw(delay) { + lock.withLock(action) + previewAnimationListener?.invoke() + } + }) previewKey = key } return animatedPreview!!.pixels diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt index 5c56d330..ffe38e36 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/IosNativeApi.kt @@ -277,4 +277,7 @@ class IosSurfaceCompositor(width: Int, height: Int) { fun composite(): NSData = compositor.composite().gray.data() fun compositeFrame(): IosTextureFrame = IosTextureFrame(compositor.composite()) + + /** Called on the main queue after each step of an animation the preview is replaying (see SurfaceCompositor). */ + fun setPreviewAnimationListener(listener: (() -> Unit)?) = compositor.setPreviewAnimationListener(listener) } diff --git a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt index 761b8185..548c483f 100644 --- a/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt +++ b/native/kotlin/shared/src/iosMain/kotlin/com/faceclaw/app/ble/IosGlassesSession.kt @@ -243,6 +243,9 @@ class IosGlassesSession internal constructor( /** The current composited screen (gray, 8bpp) or null before any surface was configured. */ fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) } + /** Called on the main queue after each step of an animation the preview is replaying (see SurfaceCompositor). */ + fun setPreviewAnimationListener(listener: (() -> Unit)?) = core.setPreviewAnimationListener(listener) + fun compositeWidth(): Int = core.previewComposite()?.width ?: 0 fun compositeHeight(): Int = core.previewComposite()?.height ?: 0 diff --git a/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/PreviewAnimationListenerTest.kt b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/PreviewAnimationListenerTest.kt new file mode 100644 index 00000000..9a573d88 --- /dev/null +++ b/tests/kotlin/src/androidHostTest/kotlin/com/faceclaw/app/PreviewAnimationListenerTest.kt @@ -0,0 +1,44 @@ +package com.faceclaw.app + +import java.util.Collections +import kotlin.test.* + +/** + * The phone mirror pulls previews only when told to, while the compositor's player advances an + * animated display list on its own timer. It needs a notification after every redraw, the + * settled one included, or it keeps a mid-slide frame. Host-only: it runs the real redraw thread. + */ +class PreviewAnimationListenerTest { + // FrameDisplayListTest's menu golden (a 2x2 highlight sliding in to rest at (3, 2)), + // restarted at 0 of its 240 ms so the slide is still in flight at the first pull. + private val menu = "077f0000000300020002000200022a00000096000000010002000200ffffffff0200080000ff24017e30020000000001812c8001812c1611010017328001812c16103001812c3023414031ff24017c30020000000001812c8001812c1611010017328001812c16103001812c30234140310200020001000103100400000000000000001f" + .chunked(2).map { it.toInt(16).toByte() }.toByteArray().also { it.fill(0, 18, 22) } + + private fun settledPreview(): ByteArray { + val reader = ArrayByteReader(menu) + assertEquals(DrawRecordKind.DISPLAY_LIST, reader.get().toInt()) + val list = FrameDisplayList.read(reader, drawAnimationTimeMs() - 10_000) + return ShellScene(emptyList(), listOf(list)).preview(ByteArray(64), 8, 8) + } + + @Test fun mirrorIsNotifiedOfEachRedrawThroughTheSettledFrame() { + val c = SurfaceCompositor() + c.configureScreen(8, 8) + c.configureSurface("app", 0, 0, 8, 8, 0, 0) + val pulled = Collections.synchronizedList(ArrayList()) + c.setPreviewAnimationListener { pulled.add(assertNotNull(c.previewComposite()).gray) } + c.submitSurface("app", ArrayByteReader(ByteArray(64)), 0, 0, 8, 8, "menu", ArrayByteReader(menu)) + val settled = settledPreview() + assertFalse(assertNotNull(c.previewComposite()).gray.contentEquals(settled), "the first pull is mid-slide") + // Redraws are 45 ms apart; 150 ms without one means the player has stopped. + val deadline = System.nanoTime() + 2_000_000_000L + var count = -1 + while (System.nanoTime() < deadline && pulled.size != count) { + count = pulled.size + Thread.sleep(150) + } + assertEquals(count, pulled.size, "the player stops once the slide's timeline ends") + assertTrue(pulled.size >= 2, "one notification per redraw") + assertContentEquals(settled, pulled.last(), "the last notification's pull is the settled frame") + } +}