From 38da8e6168924f43aedac80021426dc7cd88736c Mon Sep 17 00:00:00 2001 From: James Babcock Date: Mon, 5 Oct 2026 01:36:40 -0700 Subject: [PATCH] Fix several bugs that could lead to reconnect-looping Fix misinterpretation of EvenHub event type 5. Notification modals stacked, in a way that's O(n) in device resource memory; replace with a queue. --- app/g2/dashboard-controller.ts | 1 - app/ui/layers.ts | 13 +++ app/ui/shell/modal-layer.ts | 6 +- app/ui/shell/notification-modal-queue.ts | 91 +++++++++++++++ app/ui/shell/shell.ts | 49 ++++++--- .../faceclaw/app/g2protocol/ScenePlanner.kt | 60 ++++++++-- .../g2protocol/session/GlassesSessionCore.kt | 15 ++- .../g2protocol/session/GlassesSessionSend.kt | 22 +++- .../app/graphics/DisplayListRenderer.kt | 9 +- .../com/faceclaw/app/graphics/ShellScene.kt | 42 ++++--- tests/ai-chat.test.cjs | 3 +- tests/brightness-picker.test.cjs | 1 + .../com/faceclaw/app/DisplayListTest.kt | 31 ++++++ tests/notification-modal-queue.test.cjs | 104 ++++++++++++++++++ tests/tsconfig.json | 1 + 15 files changed, 396 insertions(+), 52 deletions(-) create mode 100644 app/ui/shell/notification-modal-queue.ts create mode 100644 tests/notification-modal-queue.test.cjs diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index c5cbfb5c..bfde6f13 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -2291,7 +2291,6 @@ class DashboardController { this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`; this.appendLog(`sys-event ${this.lastSys}`); if ( - event.eventType === OsEventTypeList.FOREGROUND_EXIT_EVENT || event.eventType === OsEventTypeList.ABNORMAL_EXIT_EVENT || event.eventType === OsEventTypeList.SYSTEM_EXIT_EVENT ) { diff --git a/app/ui/layers.ts b/app/ui/layers.ts index 25272ca5..4c2cb796 100644 --- a/app/ui/layers.ts +++ b/app/ui/layers.ts @@ -198,6 +198,19 @@ export class LayerStack { return true; } + /** + * Remove one stacked layer wherever it sits, leaving the layers above it in + * place (e.g. a notification modal that closes under a menu opened over it). + * Returns false if the layer isn't stacked (the base layer can never be removed). + */ + remove(layer: Layer): boolean { + const index = this.layers.indexOf(layer); + if (index <= 0) return false; + this.layers.splice(index, 1); + notifyRemoved(layer); + return true; + } + isAtBase(): boolean { return this.layers.length === 1; } diff --git a/app/ui/shell/modal-layer.ts b/app/ui/shell/modal-layer.ts index 9001d0ae..1ca89df6 100644 --- a/app/ui/shell/modal-layer.ts +++ b/app/ui/shell/modal-layer.ts @@ -40,10 +40,14 @@ export function modalRect(): { x: number; y: number; width: number; height: numb export class ShellModalLayer implements Layer { private readonly stack: LayerStack; - constructor(baseLayer: Layer, actions: LayerActions) { + constructor(baseLayer: Layer, actions: LayerActions, private readonly removed?: () => void) { this.stack = new LayerStack(baseLayer, actions, modalInterior()); } + onRemoved(): void { + this.removed?.(); + } + paint(_ctx: LayerContext, paintBelow: PaintBelow): GrayImage { const image = paintBelow(); // Track a display-mode switch while the modal is up, so the inner stack diff --git a/app/ui/shell/notification-modal-queue.ts b/app/ui/shell/notification-modal-queue.ts new file mode 100644 index 00000000..bf6f8eff --- /dev/null +++ b/app/ui/shell/notification-modal-queue.ts @@ -0,0 +1,91 @@ +/** + * New-notification modals, one at a time. Each modal is a near-64 KiB + * on-glasses surface, and stacking one per posted notification overflowed the + * glasses' resource cache (#42). Notifications that arrive while a modal is + * open wait here and open in turn as each one closes. + * + * `M` is the host's modal handle; the host creates and stacks modals, and + * reports back how each one left the stack. + */ +export interface NotificationModalHost { + /** Create and stack a modal showing this notification. */ + open(notificationKey: string): M; + /** Whether a queued notification is still worth opening (not dismissed, popups still on). */ + isAvailable(notificationKey: string): boolean; + /** Put the screen back to sleep. */ + sleep(): void; +} + +interface Entry { + readonly notificationKey: string; + /** Whether this notification woke the screen, so closing the last modal sleeps again. */ + readonly wokeScreen: boolean; +} + +export class NotificationModalQueue { + private current: (Entry & { readonly modal: M }) | null = null; + private readonly queued: Entry[] = []; + + constructor(private readonly host: NotificationModalHost) {} + + /** + * A notification was posted. Opens it now, or queues it behind the open + * modal. A repost of the open notification needs nothing: its modal repaints + * from the live notification. + */ + post(notificationKey: string, wokeScreen: boolean): void { + if (this.current) { + if (this.current.notificationKey !== notificationKey + && !this.queued.some((entry) => entry.notificationKey === notificationKey)) { + this.queued.push({ notificationKey, wokeScreen }); + } + return; + } + this.current = { modal: this.host.open(notificationKey), notificationKey, wokeScreen }; + } + + /** + * The user closed `modal`. Opens the next queued notification; once none + * are left, goes back to sleep if any of them woke the screen. Call before + * unstacking the modal, so the removal it reports is already accounted for. + */ + closed(modal: M): void { + const current = this.current; + if (current?.modal !== modal) return; + this.current = null; + if (!this.openNext(current.wokeScreen) && current.wokeScreen) { + this.host.sleep(); + } + } + + /** + * `modal` left the stack some other way. Going to sleep clears popups (the + * notifications stay in the Notifications app), so a screen that is off + * drops the queue; otherwise the next notification opens. + */ + removed(modal: M, screenOn: boolean): void { + const current = this.current; + if (current?.modal !== modal) return; + this.current = null; + if (!screenOn) { + this.queued.length = 0; + return; + } + this.openNext(current.wokeScreen); + } + + /** Notifications waiting behind the open modal (for diagnostics and tests). */ + queuedKeys(): string[] { + return this.queued.map((entry) => entry.notificationKey); + } + + private openNext(wokeScreen: boolean): boolean { + while (this.queued.length > 0) { + const next = this.queued.shift()!; + if (!this.host.isAvailable(next.notificationKey)) continue; + this.post(next.notificationKey, wokeScreen || next.wokeScreen); + return true; + } + return false; + } +} diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 87a6c72a..1ad07d0b 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -50,6 +50,9 @@ import { import { onAmbientCardsChanged } from "./ambient-cards"; import { ShellChromeLayer, sidebarContentSpan, type ShellChromeState, type ShellChromeWindow } from "./chrome-layer"; import { ShellModalLayer } from "./modal-layer"; +import { NotificationModalQueue } from "./notification-modal-queue"; +import { ALL_NOTIFICATIONS, readActiveNotifications } from "../../native/notification-icons"; +import { shouldShowNotificationOnGlasses } from "../../native/notification-sources"; import { ToolDebugMenuLayer } from "./tool-debug-layer"; import type { InProcessWindow } from "./in-process-window"; import { BrightnessPickerLayer } from "./brightness-picker-layer"; @@ -399,6 +402,14 @@ class Shell { private readonly trayIcons = new Map(); private activeVoiceLayer: VoiceInputLayer | null = null; private activeKeyboardLayer: KeyboardInputLayer | null = null; + private readonly notificationModals = new NotificationModalQueue({ + open: (notificationKey) => this.pushNotificationModal(notificationKey), + isAvailable: (notificationKey) => { + const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey); + return !!notification && shouldShowNotificationOnGlasses(notification.packageName); + }, + sleep: () => this.sleep(), + }); private conversations: AssistantConversations | null = null; private get assistantSession(): AssistantSession | null { return this.conversations?.current().session ?? null; @@ -762,28 +773,36 @@ class Shell { } /** - * Show a new notification in a shell modal over the app viewport. If the - * notification woke the screen, closing the modal goes back to sleep + * Show a new notification in a shell modal over the app viewport, or queue + * it behind the one already open (see NotificationModalQueue). If + * notifications woke the screen, closing the last modal goes back to sleep * (matching the old sleep-popup behavior). */ openNotificationModal(notificationKey: string, wokeScreen: boolean): void { if (!this.screenOn) return; - const modal: ShellModalLayer = new ShellModalLayer( - new SingleNotificationLayer(notificationKey, { - origin: "new-notification-modal", - onClose: () => this.closeNotificationModal(modal, wokeScreen), - }), - this.config.actions, - ); - this.stack.push(modal); + this.notificationModals.post(notificationKey, wokeScreen); this.config.requestShellRender(); } - private closeNotificationModal(modal: ShellModalLayer, wokeScreen: boolean): void { - this.stack.popIfTop((layer) => layer === modal); - if (wokeScreen) { - this.sleep(); - } + private pushNotificationModal(notificationKey: string): ShellModalLayer { + const modal: ShellModalLayer = new ShellModalLayer( + new SingleNotificationLayer(notificationKey, { + origin: "new-notification-modal", + onClose: () => this.closeNotificationModal(modal), + }), + this.config.actions, + () => this.notificationModals.removed(modal, this.screenOn), + ); + this.stack.push(modal); + return modal; + } + + private closeNotificationModal(modal: ShellModalLayer): void { + // The queue first: it opens the next notification or puts the screen back + // to sleep, and then ignores the removal below. remove() rather than a + // pop, so a menu opened over the modal doesn't leave it stuck. + this.notificationModals.closed(modal); + this.stack.remove(modal); this.config.requestShellRender(); } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt index fe9bf23b..d9a4bf8e 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/ScenePlanner.kt @@ -2,9 +2,13 @@ package com.faceclaw.app import kotlin.jvm.JvmOverloads -/** The delta base contains only app pixels. Shell resources are composed at presentation. */ +/** + * The delta base contains only app pixels, except in frames that bake shell content (see [plan]). + * Shell resources are composed at presentation. + */ class ScenePlanner(private val cache: ResourceCacheState) { - class Plan(val commands: List, val nextFid: Int) + /** [bakedLayers] and [bakedSelections] say what a frame over the resource budget baked into its pixels. */ + class Plan(val commands: List, val nextFid: Int, val bakedLayers: Int = 0, val bakedSelections: Boolean = false) private val owned = HashMap() private val pixels = HashMap() private var previous: ByteArray? = null @@ -12,6 +16,14 @@ class ScenePlanner(private val cache: ResourceCacheState) { private var screenHeight = 0 private var epoch = -1 private var root = -1 + + /** + * When the shell's surfaces and retained drawings don't all fit the resource budget, the frame + * bakes the bottom of the scene into its screen pixels until the rest fits: first the scene's + * own retained drawings, then layers from the bottom (the chrome) up, so the top overlay stays + * a live surface longest. A frame that can't be planned even fully flattened throws, leaving + * the cache and this planner as they were. + */ @JvmOverloads fun plan(screen: ByteArray, width: Int, height: Int, draws: Array?, scene: ShellScene, firstId: Int, textures: Boolean = true, incremental: Boolean = true, multiRect: Boolean = true, maxRects: Int = 6): Plan { require(width in 1..640 && height in 1..480) @@ -19,42 +31,66 @@ class ScenePlanner(private val cache: ResourceCacheState) { require(screen.size == ((width + 1) / 2) * height) if (width != screenWidth || height != screenHeight) previous = null screenWidth = width; screenHeight = height + val rollback = cache.checkpoint() + try { + planScene(screen, width, height, draws, scene, firstId, textures, incremental, multiRect, maxRects)?.let { return it } + for (layerCount in 0..scene.layers.size) { + if (layerCount == 0 && scene.selections.isEmpty()) continue + rollback() + val kept = ShellScene(scene.layers.drop(layerCount), screenDepth = scene.screenDepth) + planScene(screen, width, height, draws, kept, firstId, textures, incremental, multiRect, maxRects) { + scene.flatten(screen, width, height, layerCount) + }?.let { return Plan(it.commands, it.nextFid, layerCount, scene.selections.isNotEmpty()) } + } + error("Shell scene exceeds the resource budget even fully flattened") + } catch (t: Throwable) { + rollback() + throw t + } + } + + /** + * One attempt at [plan], or null if [scene] doesn't fit the budget. [baked] supplies the + * screen pixels once the shell fits. Planner state changes only when the attempt succeeds. + */ + private fun planScene(screen: ByteArray, width: Int, height: Int, draws: Array?, scene: ShellScene, firstId: Int, textures: Boolean, incremental: Boolean, multiRect: Boolean, maxRects: Int, baked: (() -> ByteArray)? = null): Plan? { val commands = ArrayList() val keys = scene.layers.map { "shell:${it.key}:${it.width}x${it.height}" } - val surfaces = scene.layers.mapIndexed { i, layer -> owned.getOrPut(keys[i]) { - CachedResource(DrawProtocol.rawImage(layer.width, layer.height), keys[i]) - } } - owned.keys.retainAll(keys.toSet()); pixels.keys.retainAll(keys.toSet()) + val surfaces = scene.layers.mapIndexed { i, layer -> + owned[keys[i]] ?: CachedResource(DrawProtocol.rawImage(layer.width, layer.height), keys[i]) + } val selectedResources = scene.retainedResources cache.pin(surfaces + selectedResources) val absent = surfaces.map { cache.resourceId(it) < 0 } val prepared = cache.prepare(surfaces + selectedResources) + if (prepared.any { it < 0 }) return null val ids = prepared.copyOfRange(0, surfaces.size) val selectedIds = prepared.copyOfRange(surfaces.size, prepared.size) - check(prepared.all { it >= 0 }) { "Shell surfaces exceed the resource budget" } + val frame = baked?.invoke() ?: screen commands.addAll(cache.drainCommands(3600)) val rootCalls = scene.calls(width, height, ids, selectedIds) for ((i, layer) in scene.layers.withIndex()) { val old = if (absent[i]) null else pixels[keys[i]] commands.addAll(DrawProtocol.messages(pixelCalls(old, layer.packed, layer.width, layer.height, ids[i]))) - pixels[keys[i]] = layer.packed } val base = if (incremental) previous else null - val texture = if (textures) TexturePlanner.plan(base, screen, width, height, draws, cache, firstId, multiRect, maxRects) else null + val texture = if (textures) TexturePlanner.plan(base, frame, width, height, draws, cache, firstId, multiRect, maxRects) else null if (texture != null) { commands.addAll(texture.resourceCommands) commands.addAll(DrawProtocol.messages(DrawProtocol.fromOptimized(texture.payload, width, height))) - } else commands.addAll(DrawProtocol.messages(pixelCalls(base, screen, width, height, null))) + } else commands.addAll(DrawProtocol.messages(pixelCalls(base, frame, width, height, null))) val list = CachedResource(DrawProtocol.displayList(rootCalls)) val rootId = cache.prepare(listOf(list))[0] - check(rootId >= 0) { "No space for root display list" } + if (rootId < 0) return null commands.addAll(cache.drainCommands(3600)) cache.pin(surfaces + selectedResources + list) val evicted = commands.any { it[0].toInt() == ResourceCacheState.EVICT_MODE } if (evicted) commands.add(0, DrawProtocol.root(DrawProtocol.SCREEN)) if (root != rootId || evicted) commands.add(DrawProtocol.root(rootId)) commands.add(byteArrayOf(DrawProtocol.PRESENT.toByte())) - root = rootId; previous = screen.copyOf() + owned.keys.retainAll(keys.toSet()); pixels.keys.retainAll(keys.toSet()) + for ((i, key) in keys.withIndex()) { owned[key] = surfaces[i]; pixels[key] = scene.layers[i].packed } + root = rootId; previous = frame.copyOf() return Plan(commands, texture?.nextFid ?: firstId) } private fun pixelCalls(old: ByteArray?, next: ByteArray, width: Int, height: Int, target: Int?): List { diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt index 2381369b..8a41081d 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionCore.kt @@ -1580,13 +1580,26 @@ class GlassesSessionCore( updateSilentModeLocked(false) } if ("sys-event" == decoded.kind) { - if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT || decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { + if (decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) { if (shutdownRequested) { lastShutdownExitAtMs = now() } fixedLayoutCreated = false displayedFingerprint = "" clearAllMessagesLocked("firmware exit event") + } else if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_ENTER) { + logLine("firmware overlay opened over the EvenHub page") + } else if (decoded.eventType == BleProtocol.EVENT_FOREGROUND_EXIT) { + // Despite its name this does not end the page: stock + // evenhub_page_event_handler sends it when a firmware + // overlay announced by FOREGROUND_ENTER closes, and + // restores the page and its keepalive. Teardown is + // always SYSTEM_EXIT/ABNORMAL_EXIT. Re-creating the + // layout here sent a create the firmware ignores while + // the page exists, so the session timed out (#42). + // Re-present over whatever the overlay left instead. + logLine("firmware overlay closed; re-presenting") + lastEnqueuedFingerprint = "" } } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt index 6930eb5d..3d246f02 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/session/GlassesSessionSend.kt @@ -795,9 +795,25 @@ internal fun GlassesSessionCore.enqueueDesiredImageLocked() { // Visibility belongs to this immutable composite, not the latest UI request. enqueueBrightnessLocked(!fingerprint.startsWith("blanked:")) if (customFirmwareDetected && packedFrame.size > 0) { - val rendered = scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId, - connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, - connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS) + val rendered = try { + scenePlanner.plan(packedFrame, width, height, draws, scene, nextImageFrameId, + connectionOptions.TEXTURE_CACHE_FRAMES, connectionOptions.INCREMENTAL_FRAMES, + connectionOptions.MULTI_RECT_FRAMES, ConnectionOptions.MULTI_RECT_MAX_RECTS) + } catch (e: Exception) { + // Planning depends only on this frame, so reconnecting would fail + // the same way (#42). Drop the frame (the planner and resource + // cache are as they were) and wait for the scene to change. + lastEnqueuedFingerprint = fingerprint + logLine("frame plan failed; frame dropped: " + safeMessage(e)) + finishFrame(frameId, "discarded: frame plan failed") + return + } + if (rendered.bakedLayers > 0 || rendered.bakedSelections) { + val baked = "shell over resource budget: baked " + rendered.bakedLayers + "/" + scene.layers.size + + " layers" + (if (rendered.bakedSelections) " and retained drawings" else "") + " into the frame" + frameTimings.log(frameId, baked) + logLine("$baked fingerprint=$fingerprint") + } nextImageFrameId = rendered.nextFid val commands = rendered.commands for (i in 0 until commands.size - 1) enqueueResourceCommandLocked(commands[i]) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/DisplayListRenderer.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/DisplayListRenderer.kt index 441e8c06..9723ec7f 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/DisplayListRenderer.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/DisplayListRenderer.kt @@ -1,10 +1,14 @@ package com.faceclaw.app -/** Reference interpreter. It uses the firmware's packed rows, clipping, LUT and call grammar. */ +/** + * Reference interpreter. It uses the firmware's packed rows, clipping, LUT and call grammar. + * [flat] ignores stereo depth, for pixels both lenses share (see ShellScene.flatten). + */ class DisplayListRenderer( private val resources: Map, private val builtin: ((Int, Int) -> BuiltinGlyph)? = null, private val rightLens: Boolean = false, + private val flat: Boolean = false, ) { class BuiltinGlyph(val image: ByteArray, val advance: Int, val x: Int = 0, val y: Int = 0) @@ -239,7 +243,8 @@ class DisplayListRenderer( target = target(id).shifted(inherited.shiftX) } if (flags and DRAW_FLAG_DEPTH != 0) { - target = target.shifted(DrawProtocol.depthOffset(reader.readS8(), rightLens)) + val depth = reader.readS8() + if (!flat) target = target.shifted(DrawProtocol.depthOffset(depth, rightLens)) } if (flags and DRAW_FLAG_CLIP != 0) { target = target.clipped(reader.readS16(), reader.readS16(), reader.readU16(), reader.readU16()) diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt index 62544f8a..7818e75d 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt @@ -45,15 +45,34 @@ class ShellScene(val layers: List, val selections: List fun preview(screenGray: ByteArray, width: Int, height: Int, rightLens: Boolean = false): ByteArray { val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height) - val output = ByteArray(screen.size); val resources = HashMap() + val output = ByteArray(screen.size) + DisplayListRenderer(rendererResources(width, height), rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height)) + val target = DisplayListRenderer.Target(output, width, height) + return ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() } + } + + /** + * [screen] (packed 4bpp) with this scene's own retained drawings and its bottom [layerCount] + * layers drawn in, for a frame whose shell exceeds the resource budget (see ScenePlanner.plan). + * Both lenses share the result, so stereo depth is dropped, and animations are drawn settled. + */ + fun flatten(screen: ByteArray, width: Int, height: Int, layerCount: Int): ByteArray { + val baked = ShellScene(layers.take(layerCount), selections) + val output = ByteArray(screen.size) + DisplayListRenderer(baked.rendererResources(width, height), flat = true).render(511, + DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height), SETTLED_MS) + return output + } + + /** Layer surfaces, then retained resources, by index, and the root list at 511. */ + private fun rendererResources(width: Int, height: Int): HashMap { + val resources = HashMap() val surfaces = IntArray(layers.size) { id -> val layer = layers[id]; resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed); id } val rows = IntArray(retainedResources.size) { id -> resources[id + layers.size] = retainedResources[id].bytes; id + layers.size } resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows)) - DisplayListRenderer(resources, rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height)) - val target = DisplayListRenderer.Target(output, width, height) - return ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() } + return resources } internal class AnimatedPreview(val player: DisplayListPlayer, var pixels: ByteArray) @@ -63,20 +82,9 @@ class ShellScene(val layers: List, val selections: List ): AnimatedPreview { val screen = BmpUtil.pack4bppFromGray8(screenGray, width, height) val output = ByteArray(screen.size) - val resources = HashMap() - val surfaces = IntArray(layers.size) { id -> - val layer = layers[id] - resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed) - id - } - val rows = IntArray(retainedResources.size) { id -> - resources[id + layers.size] = retainedResources[id].bytes - id + layers.size - } - resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows)) val target = DisplayListRenderer.Target(output, width, height) lateinit var preview: AnimatedPreview - val player = DisplayListPlayer(DisplayListRenderer(resources), + val player = DisplayListPlayer(DisplayListRenderer(rendererResources(width, height)), DisplayListRenderer.Target(screen, width, height), target, clock = { drawAnimationTimeMs() }, schedule = schedule, displayed = { preview.pixels = ByteArray(width * height) { (target.get(it % width, it / width) * 16).toByte() } }) @@ -87,6 +95,8 @@ class ShellScene(val layers: List, val selections: List companion object { val EMPTY = ShellScene(emptyList()) + /** Past every animation's end: TIME counts from PRESENT and clamps to the animation's length. */ + private const val SETTLED_MS = 3_600_000L fun decode(reader: ByteReader): ShellScene { val count = reader.getShort().toInt() and 65535; require(count <= 64) val layers = ArrayList() diff --git a/tests/ai-chat.test.cjs b/tests/ai-chat.test.cjs index ddc6b05b..7bd82e94 100644 --- a/tests/ai-chat.test.cjs +++ b/tests/ai-chat.test.cjs @@ -260,7 +260,8 @@ function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) '../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {}, '../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {}, './chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {}, - './brightness-picker-layer': {}, + './brightness-picker-layer': {}, './notification-modal-queue': load('app/ui/shell/notification-modal-queue.ts', {}), + '../../native/notification-icons': {}, '../../native/notification-sources': {}, '../../assistant/tool-registry': {}, './geometry': { sidebarWidth: () => 64, minWindowTop: () => 96, switcherPosition: () => 'left', uiDepth: () => 0, appViewportRect: () => ({ x: 64, y: 124, width: 576, height: 260 }), diff --git a/tests/brightness-picker.test.cjs b/tests/brightness-picker.test.cjs index c7df4740..bc212b57 100644 --- a/tests/brightness-picker.test.cjs +++ b/tests/brightness-picker.test.cjs @@ -114,6 +114,7 @@ test('system menu hides brightness in Auto and opens its picker between Voice in '../../graphics/image': graphics, '../layers': layers, '../menu': menu, '../gestures': gestures, '../dashboard-settings': f.settings, './geometry': f.geometry, './brightness-picker-layer': f.picker, './chrome-layer': { ShellChromeLayer: class {} }, + './notification-modal-queue': load('app/ui/shell/notification-modal-queue.ts'), }); shell.registerWindow({ windowId: 'test', appId: 'test', closeable: true, handleInput() {} }); shell.openSystemMenu('test'); diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt index 92419258..bed1f4c0 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt @@ -341,6 +341,37 @@ class DisplayListTest { c.setSurfaceDepth("glance", -16) assertEquals(-16, c.composite().shellScene.screenDepth) } + /** #42: stacked modal-sized layers overflow the cache, so the frame bakes the bottom ones instead of throwing. */ + @Test fun shellOverTheResourceBudgetBakesBottomLayersAndRecovers() { + val cache = ResourceCacheState(); val planner = ScenePlanner(cache) + val gray = ByteArray(640 * 480) { 48 }; val app = BmpUtil.pack4bppFromGray8(gray, 640, 480) + fun layer(key: Int, y: Int, h: Int, nibble: Int, depth: Int = 0) = + ShellScene.Layer(key, 14, y, 612, h, 256, ByteArray(306 * h) { (nibble * 17).toByte() }, depth = depth) + // Each 612x214 layer is just under the 64 KiB resource cap; three exceed the arena on their own. + fun chrome(depth: Int) = layer(1, 444, 36, 7, depth) + val modals = listOf(layer(2, 10, 214, 9), layer(3, 120, 214, 11), layer(4, 230, 214, 13, depth = 4)) + val overflow = ShellScene(listOf(chrome(6)) + modals) + val plan = planner.plan(app, 640, 480, null, overflow, 1) + assertEquals(2, plan.bakedLayers); assertFalse(plan.bakedSelections) + assertTrue(plan.commands.all { it.size <= 65535 }) + assertTrue(cache.usedBytes() <= ResourceCacheState.CACHE_SIZE) + // Baked layers lose their stereo depth; the live ones keep it. + val expected = ShellScene(listOf(chrome(0)) + modals) + val lenses = listOf(Glasses(), Glasses(true)) + for (glasses in lenses) { + glasses.apply(plan.commands) + assertContentEquals(BmpUtil.pack4bppFromGray8(expected.preview(gray, 640, 480, glasses.right), 640, 480), glasses.composition) + } + // Once the shell fits again, the next frame is fully live and the screen is the app's again. + val fits = ShellScene(listOf(chrome(0), modals[2])) + val next = planner.plan(app, 640, 480, null, fits, plan.nextFid) + assertEquals(0, next.bakedLayers) + for (glasses in lenses) { + glasses.apply(next.commands) + assertContentEquals(app, glasses.screen) + assertContentEquals(BmpUtil.pack4bppFromGray8(fits.preview(gray, 640, 480, glasses.right), 640, 480), glasses.composition) + } + } @Test fun noisyFramesRemainBoundedAndResourceBudgetIs192KiB() { val cache=ResourceCacheState();val planner=ScenePlanner(cache);val glasses=Glasses() val app=ByteArray(640*480/2){(it*37).toByte()} diff --git a/tests/notification-modal-queue.test.cjs b/tests/notification-modal-queue.test.cjs new file mode 100644 index 00000000..2edad5e3 --- /dev/null +++ b/tests/notification-modal-queue.test.cjs @@ -0,0 +1,104 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { NotificationModalQueue } = require("../.test-build/app/ui/shell/notification-modal-queue.js"); + +/** A host that records opened modals (one handle object each) and sleeps. */ +function makeHost(unavailable = new Set()) { + const host = { + opened: [], + sleeps: 0, + open(notificationKey) { + const modal = { notificationKey }; + host.opened.push(modal); + return modal; + }, + isAvailable: (notificationKey) => !unavailable.has(notificationKey), + sleep() { host.sleeps++; }, + }; + return host; +} + +test("only one modal is open; later notifications wait and open as each closes", () => { + const host = makeHost(); + const queue = new NotificationModalQueue(host); + queue.post("a", false); + queue.post("b", false); + queue.post("c", false); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a"]); + assert.deepEqual(queue.queuedKeys(), ["b", "c"]); + queue.closed(host.opened[0]); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a", "b"]); + queue.closed(host.opened[1]); + queue.closed(host.opened[2]); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a", "b", "c"]); + assert.equal(host.sleeps, 0); + // With nothing open, the next notification opens at once. + queue.post("d", false); + assert.equal(host.opened.length, 4); +}); + +test("reposts of the open or a queued notification are not duplicated", () => { + const host = makeHost(); + const queue = new NotificationModalQueue(host); + queue.post("a", false); + queue.post("a", false); + queue.post("b", false); + queue.post("b", false); + assert.equal(host.opened.length, 1); + assert.deepEqual(queue.queuedKeys(), ["b"]); +}); + +test("queued notifications that went away are skipped", () => { + const unavailable = new Set(["b"]); + const host = makeHost(unavailable); + const queue = new NotificationModalQueue(host); + queue.post("a", false); + queue.post("b", false); + queue.post("c", false); + queue.closed(host.opened[0]); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a", "c"]); +}); + +test("a screen woken by a notification sleeps again once the last modal closes", () => { + const host = makeHost(); + const queue = new NotificationModalQueue(host); + queue.post("a", true); + queue.post("b", false); + queue.closed(host.opened[0]); + assert.equal(host.sleeps, 0, "the next notification opens instead of sleeping"); + queue.closed(host.opened[1]); + assert.equal(host.sleeps, 1); + // A modal opened with the screen already on doesn't sleep when closed. + queue.post("c", false); + queue.closed(host.opened[2]); + assert.equal(host.sleeps, 1); +}); + +test("sleep drops the queue; another removal opens the next without sleeping", () => { + const host = makeHost(); + const queue = new NotificationModalQueue(host); + queue.post("a", true); + queue.post("b", false); + queue.removed(host.opened[0], false); + assert.deepEqual(queue.queuedKeys(), []); + assert.equal(host.opened.length, 1); + queue.post("c", false); + queue.post("d", false); + queue.removed(host.opened[1], true); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a", "c", "d"]); + assert.equal(host.sleeps, 0); +}); + +test("the removal reported after a close is ignored, as are stale modals", () => { + const host = makeHost(); + const queue = new NotificationModalQueue(host); + queue.post("a", false); + queue.post("b", false); + const first = host.opened[0]; + queue.closed(first); + queue.removed(first, true); + queue.closed(first); + assert.deepEqual(host.opened.map((modal) => modal.notificationKey), ["a", "b"]); + assert.deepEqual(queue.queuedKeys(), []); +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 73bcf527..cc9c2fc3 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -34,6 +34,7 @@ "../app/g2/glasses-artwork.ts", "../app/g2/glasses-display-state.ts", "../app/g2/glance-state.ts", + "../app/ui/shell/notification-modal-queue.ts", "../app/apps/glanceboard/layout.ts", "../app/apps/files/video-layout.ts", "../app/g2/pairing-candidates.ts",