From 440674b261c808d848fb0e375eb3a2127cc6bcac Mon Sep 17 00:00:00 2001 From: James Babcock Date: Wed, 23 Sep 2026 22:05:06 -0700 Subject: [PATCH] Animations are mainly on the typescript side, not kotlin --- app/graphics/display-list.md | 84 ++++++++++ app/graphics/display-list.ts | 133 +++++++++++++++ app/graphics/draw-expression.ts | 152 ++++++++++++++++++ app/graphics/glyph-wire.ts | 5 +- app/graphics/image.ts | 26 ++- app/graphics/menu-selection-list.ts | 18 +++ app/graphics/presentation-wire.ts | 50 +++--- app/graphics/surface-compositor.ts | 2 +- .../faceclaw/app/g2protocol/ScenePlanner.kt | 2 +- .../faceclaw/app/graphics/DrawRecordKind.kt | 4 +- .../faceclaw/app/graphics/FrameDisplayList.kt | 133 +++++++++++++++ .../faceclaw/app/graphics/MenuSelection.kt | 31 ++-- .../com/faceclaw/app/graphics/ShellScene.kt | 24 +-- .../app/graphics/SurfaceCompositor.kt | 6 +- tests/ai-chat.test.cjs | 2 +- tests/brightness-picker.test.cjs | 2 +- tests/compass-debug.test.cjs | 2 +- tests/compass-texture-cache.test.cjs | 4 +- tests/display-list-authoring.test.cjs | 83 ++++++++++ tests/input-events.test.cjs | 2 +- .../com/faceclaw/app/DrawExpressionTest.kt | 42 ----- .../com/faceclaw/app/FrameDisplayListTest.kt | 90 +++++++++++ tests/menu-display-list.test.cjs | 2 +- tests/menu-highlight-motion.test.cjs | 8 +- tests/music-settings.test.cjs | 2 +- tests/notification-filter.test.cjs | 2 +- 26 files changed, 783 insertions(+), 128 deletions(-) create mode 100644 app/graphics/display-list.md create mode 100644 app/graphics/display-list.ts create mode 100644 app/graphics/draw-expression.ts create mode 100644 app/graphics/menu-selection-list.ts create mode 100644 native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FrameDisplayList.kt create mode 100644 tests/display-list-authoring.test.cjs create mode 100644 tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameDisplayListTest.kt diff --git a/app/graphics/display-list.md b/app/graphics/display-list.md new file mode 100644 index 00000000..38462c02 --- /dev/null +++ b/app/graphics/display-list.md @@ -0,0 +1,84 @@ +# TypeScript display lists + +`GrayImage.drawDisplayList(list, x, y, width, height, depth)` retains a list alongside +the image's ordinary pixels. Existing frame submission (`prepareFrameDraws`) and +shell scene submission carry it to Kotlin. Bounds describe its footprint for plane +occlusion; draw coordinates are relative to its placement. Keep the list, its image +resources, and their pixel buffers immutable after submission. + +```ts +import { DrawExpression as E } from './draw-expression'; +import { DrawOp, type DisplayList } from './display-list'; + +const list: DisplayList = { + resources: [], + timeline: { token: animationId, startedAt: Date.now() }, + calls: [{ + op: DrawOp.ROUNDED_RECT, + x: E.progress(300).ease().lerp(E.f32(0), E.f32(100)).toInt(), + y: 0, + width: 40, height: 20, radius: 6, + background: 4, border: 15, + }], +}; +frame.drawDisplayList(list, 20, 30, 140, 20, 0); +``` + +Supported calls in this first pass: + +- `ROUNDED_RECT`: literal or expression `x`/`y`, literal dimensions/radius, and + 4-bit colors (0–15). Border 16 means no border. Fill takes the maximum of the + background color and existing pixels. +- `IMAGE`: a local resource index, integer `x`/`y`, and `transparent`. Transparent + images skip pixels quantized to zero. Resources have `width`, `height`, and + gray8 `pixels`; a baked `GrayImage` can be used directly. +- `RECT_COPY`: a local resource index, integer source rectangle and destination + `dx`/`dy`. `SCREEN` reads the uncomposed frame; its source coordinates are also + relative to this list's placement. Each call may add its own `depth` to the + list's depth. A screen restore normally uses the negative of the list depth. + +Kotlin allocates/cache-pins all image resources, resolves the local indices, applies +surface placement, and includes the calls in the root firmware display list. The +phone's existing display-list renderer consumes the same compiled calls. This uses +the current firmware protocol; no firmware update is needed for this bridge change. + +## Expressions and time + +`E.i32` and `E.f32` construct typed constants. Builders support arithmetic, min/max, +integer remainder, negation, explicit conversions, lerp, and the easing functions +in `ExprOp`. `ease()` defaults to smoothstep. Integers wrap at 32 bits; float +operations round to f32. The firmware's existing stack/error semantics apply. +Only rounded-rectangle x/y accept expressions in this first pass. + +`maximum.time()` returns `min(milliseconds since PRESENT, maximum)`. The existing +45 ms repaint timer remains active while this is below maximum. `E.elapsed()` is +a bridge binding: Kotlin replaces it with the timeline's elapsed milliseconds when +building a new root list. It is a constant within that PRESENT, not a second clock. +`E.progress(durationMs)` combines the two and clamps to 0..1 so intervening frames +and PRESENTs continue an animation instead of restarting it. + +Keep `timeline.token` and `timeline.startedAt` stable for an animation; use a new +token when restarting it. JS sends an elapsed duration, so JS and native clocks +need not share an epoch. Fingerprints include the token and expression code but +exclude the changing elapsed duration. Without a timeline, raw `time()` expressions +start at each PRESENT. Native compilation strips the bridge-only elapsed opcode. + +`menu-selection-list.ts` authors both the rounded highlight and the transparent row +image using this API. Navigation eligibility, duration, and easing remain in +TypeScript. There is no animated-menu tag or menu animation policy in Kotlin. + +## Bridge framing + +Tag 7 is followed by a little-endian u32 byte length and a bounded body: +placement (i16 x/y, u16 width/height, i8 depth), timeline (u32 token/elapsed), +u16 resource count, gray8 images (u16 width/height followed by pixels), u16 call +count, then calls. Calls start with the firmware opcode and an i16 relative depth; +operands otherwise use their fixed-width fields, except rounded x/y which use +extended varints. Expressions use the firmware bytecode plus bridge-only +`ELAPSED = 128`. Resource indices are local to this record, never firmware IDs. +Rect-copy source x/y are signed here to permit relative SCREEN coordinates. + +Limits: 256 images per record, 64 KiB per packed image, 4 MiB per bridge record, +4096 calls, and 900 input bytes per expression. Native validates the bound program +against the firmware's 1024-byte limit. The combined scene must also fit the +existing resource cache, root-list size, and VM stack limits. diff --git a/app/graphics/display-list.ts b/app/graphics/display-list.ts new file mode 100644 index 00000000..a7eb5c36 --- /dev/null +++ b/app/graphics/display-list.ts @@ -0,0 +1,133 @@ +import { DrawBytes, encodeValue, evaluate, integer, readValue, type DrawValue } from './draw-expression'; + +/** Local resources are assigned firmware cache IDs by Kotlin. SCREEN reads the uncomposed frame. */ +export const SCREEN = 65535; +export const DrawOp = { RECT_COPY: 2, IMAGE: 4, ROUNDED_RECT: 8 } as const; +export const DISPLAY_LIST_RECORD = 7; +export type ListImage = { readonly width: number; readonly height: number; readonly pixels: Uint8Array }; +export type ListCall = ( + | { op: typeof DrawOp.ROUNDED_RECT; x: DrawValue; y: DrawValue; width: number; height: number; radius: number; background: number; border: number } + | { op: typeof DrawOp.IMAGE; resource: number; x: number; y: number; transparent: boolean } + | { op: typeof DrawOp.RECT_COPY; resource: number; x: number; y: number; width: number; height: number; dx: number; dy: number } +) & { depth?: number }; +export type DisplayList = { + readonly resources: readonly ListImage[]; + readonly calls: readonly ListCall[]; + /** Stable identity plus JS clock origin. Native binds ELAPSED again for each PRESENT. */ + /** Set by the decoder: the local PRESENT origin for software replay. */ + readonly presentedAt?: number; + readonly timeline?: { readonly token: number; readonly startedAt: number }; +}; +export type PlacedDisplayList = { displayList: DisplayList; x: number; y: number; width: number; height: number; depth: number }; + +class Writer { + bytes: number[] = []; + u8(n: number): void { this.bytes.push(integer(n, 0, 255)); } + i8(n: number): void { this.u8(integer(n, -128, 127) & 255); } + u16(n: number): void { integer(n, 0, 65535); this.bytes.push(n & 255, n >>> 8); } + i16(n: number): void { this.u16(integer(n, -32768, 32767) & 65535); } + u32(n: number): void { integer(n, 0, 4294967295); this.u16(n & 65535); this.u16(n >>> 16); } + raw(bytes: ArrayLike): void { for (let i = 0; i < bytes.length; i++) this.bytes.push(bytes[i]); } +} + +/** Length-delimited bridge envelope; call operands use fixed widths except extended x/y. */ +export function encodeDisplayList(placed: PlacedDisplayList, now = Date.now()): Uint8Array { + const { displayList: list } = placed, out = new Writer(); + integer(placed.width, 1, 640); integer(placed.height, 1, 480); + out.i16(placed.x); out.i16(placed.y); out.u16(placed.width); out.u16(placed.height); out.i8(placed.depth); + out.u32(list.timeline?.token ?? 0); + out.u32(list.timeline ? Math.min(2147483647, Math.max(0, Math.trunc(now - list.timeline.startedAt))) : 0); + integer(list.resources.length, 0, 256); out.u16(list.resources.length); + for (const image of list.resources) { + integer(image.width, 1, 640); integer(image.height, 1, 480); + if (image.pixels.length !== image.width * image.height || 5 + Math.ceil(image.width / 2) * image.height > 65536) throw new Error('Invalid display-list image'); + out.u16(image.width); out.u16(image.height); out.raw(image.pixels); + } + integer(list.calls.length, 0, 4096); out.u16(list.calls.length); + for (const call of list.calls) { + out.u8(call.op); out.i16(call.depth ?? 0); + integer(placed.depth + (call.depth ?? 0), -128, 127); + if (call.op === DrawOp.ROUNDED_RECT) { + out.raw(encodeValue(call.x)); out.raw(encodeValue(call.y)); + out.u16(integer(call.width, 1, 640)); out.u16(integer(call.height, 1, 480)); out.u16(call.radius); + out.u8(integer(call.background, 0, 15)); out.u8(integer(call.border, 0, 16)); + } else { + if (call.resource !== SCREEN) integer(call.resource, 0, list.resources.length - 1); + if (call.op === DrawOp.RECT_COPY && call.resource !== SCREEN && (call.x < 0 || call.y < 0)) throw new Error('Negative image source coordinate'); + if (call.op === DrawOp.IMAGE && call.resource === SCREEN) throw new Error('Use rectCopy to read SCREEN'); + out.u16(call.resource); + if (call.op === DrawOp.IMAGE) { out.i16(call.x); out.i16(call.y); out.u8(call.transparent ? 31 : 15); } + else { + out.i16(call.x); out.i16(call.y); out.u16(integer(call.width, 1, 640)); out.u16(integer(call.height, 1, 480)); out.i16(call.dx); out.i16(call.dy); + } + } + } + if (out.bytes.length > 4 * 1024 * 1024) throw new Error('Display list exceeds bridge limit'); + const header = new Writer(); header.u8(DISPLAY_LIST_RECORD); header.u32(out.bytes.length); header.raw(out.bytes); + return Uint8Array.from(header.bytes); +} + +export function readDisplayList(bytes: Uint8Array, offset: number, now = Date.now()): { placed: PlacedDisplayList; end: number } { + const envelope = new DrawBytes(bytes.subarray(offset)); + if (envelope.u8() !== DISPLAY_LIST_RECORD) throw new Error('Invalid display-list record'); + const length = envelope.u32(); integer(length, 0, 4 * 1024 * 1024); + const r = new DrawBytes(envelope.take(length)); + const x = r.i16(), y = r.i16(), width = r.u16(), height = r.u16(), depth = r.u8() << 24 >> 24; + const token = r.u32(), elapsed = r.u32(), count = r.u16(); integer(count, 0, 256); + const resources: ListImage[] = []; + for (let i = 0; i < count; i++) { const width = r.u16(), height = r.u16(); resources.push({ width, height, pixels: r.take(width * height) }); } + const calls: ListCall[] = [], callCount = r.u16(); integer(callCount, 0, 4096); + for (let i = 0; i < callCount; i++) { + const op = r.u8(), depth = r.i16(); + if (op === DrawOp.ROUNDED_RECT) calls.push({ op, depth, x: readValue(r), y: readValue(r), width: r.u16(), height: r.u16(), radius: r.u16(), background: r.u8(), border: r.u8() }); + else if (op === DrawOp.IMAGE) calls.push({ op, depth, resource: r.u16(), x: r.i16(), y: r.i16(), transparent: (r.u8() & 16) !== 0 }); + else if (op === DrawOp.RECT_COPY) calls.push({ op, depth, resource: r.u16(), x: r.i16(), y: r.i16(), width: r.u16(), height: r.u16(), dx: r.i16(), dy: r.i16() }); + else throw new Error('Unsupported display-list call'); + } + if (r.offset !== length) throw new Error('Trailing display-list bytes'); + const placed = { x, y, width, height, depth, displayList: { resources, calls, presentedAt: now, timeline: { token, startedAt: now - elapsed } } }; + // The writer is also the shared structural validator for software preview records. + encodeDisplayList(placed, now); + return { placed, end: offset + envelope.offset }; +} + +export function dimDisplayList(list: DisplayList, factor: number): DisplayList { + const dim = (v: number) => v === 0 ? 0 : Math.max(1, Math.round(v * factor)); + return { ...list, resources: list.resources.map(r => ({ ...r, pixels: r.pixels.map(dim) })), + calls: list.calls.map(c => c.op === DrawOp.ROUNDED_RECT ? { ...c, background: Math.min(15, Math.round(c.background * factor)), border: c.border === 16 ? 16 : Math.min(15, Math.round(c.border * factor)) } : c) }; +} + +/** Fallback compositor; native preview and glasses replay the compiled list with their existing timers. */ +export function paintDisplayList(output: Uint8Array, screen: Uint8Array, width: number, height: number, placed: PlacedDisplayList, right = false, now = Date.now()): void { + const q = (n: number) => Math.min(15, (n + 8) >> 4) * 16; + const list = placed.displayList; + const elapsed = list.timeline ? Math.max(0, (list.presentedAt ?? now) - list.timeline.startedAt) : 0; + const presentTime = list.presentedAt === undefined ? 0 : Math.max(0, now - list.presentedAt); + const inside = (x: number, y: number, w: number, h: number, radius: number) => { + if (x < 0 || y < 0 || x >= w || y >= h) return false; + const r = Math.min(radius, Math.floor(w / 2), Math.floor(h / 2)); + const dx = Math.max(0, 2 * r - (2 * Math.min(x, w - 1 - x) + 1)), dy = Math.max(0, 2 * r - (2 * Math.min(y, h - 1 - y) + 1)); + return dx * dx + dy * dy <= 4 * r * r; + }; + for (const c of placed.displayList.calls) { + const depth = placed.depth + (c.depth ?? 0), shift = right ? -Math.floor((depth + 1) / 2) : Math.floor(depth / 2); + if (c.op === DrawOp.ROUNDED_RECT) { + const x = placed.x + evaluate(c.x, elapsed, presentTime).value + shift, y = placed.y + evaluate(c.y, elapsed, presentTime).value; + for (let yy = 0; yy < c.height; yy++) for (let xx = 0; xx < c.width; xx++) { + if (x + xx < 0 || y + yy < 0 || x + xx >= width || y + yy >= height || !inside(xx, yy, c.width, c.height, c.radius)) continue; + const p = (y + yy) * width + x + xx; + output[p] = c.border !== 16 && !inside(xx - 1, yy - 1, c.width - 2, c.height - 2, Math.max(0, c.radius - 1)) ? c.border * 16 : Math.max(output[p], c.background * 16); + } + } else { + const source = c.resource === SCREEN ? { width, height, pixels: screen } : placed.displayList.resources[c.resource]; + const w = c.op === DrawOp.IMAGE ? source.width : c.width, h = c.op === DrawOp.IMAGE ? source.height : c.height; + const sx = c.op === DrawOp.IMAGE ? 0 : c.x + (c.resource === SCREEN ? placed.x : 0), sy = c.op === DrawOp.IMAGE ? 0 : c.y + (c.resource === SCREEN ? placed.y : 0); + const x = placed.x + (c.op === DrawOp.IMAGE ? c.x : c.dx) + shift, y = placed.y + (c.op === DrawOp.IMAGE ? c.y : c.dy); + for (let yy = 0; yy < h; yy++) for (let xx = 0; xx < w; xx++) { + if (x + xx < 0 || y + yy < 0 || x + xx >= width || y + yy >= height || sx + xx < 0 || sy + yy < 0 || sx + xx >= source.width || sy + yy >= source.height) continue; + const v = q(source.pixels[(sy + yy) * source.width + sx + xx]); + if (c.op !== DrawOp.IMAGE || !c.transparent || v !== 0) output[(y + yy) * width + x + xx] = v; + } + } + } +} diff --git a/app/graphics/draw-expression.ts b/app/graphics/draw-expression.ts new file mode 100644 index 00000000..869ea641 --- /dev/null +++ b/app/graphics/draw-expression.ts @@ -0,0 +1,152 @@ +/** Revision 27 expression bytecode. ELAPSED is a bridge binding, never sent to firmware. */ +export const ExprOp = { + I32: 1, F32: 2, DUP: 3, DROP: 4, SWAP: 5, + IADD: 16, ISUB: 17, IMUL: 18, IDIV: 19, IMOD: 20, INEG: 21, IMIN: 22, IMAX: 23, + FADD: 32, FSUB: 33, FMUL: 34, FDIV: 35, FNEG: 36, FMIN: 37, FMAX: 38, + I2F: 48, F2I: 49, TIME: 50, LERP: 64, SMOOTHSTEP: 65, + EASE_IN_QUAD: 66, EASE_OUT_QUAD: 67, EASE_IN_OUT_QUAD: 68, + EASE_IN_CUBIC: 69, EASE_OUT_CUBIC: 70, EASE_IN_OUT_CUBIC: 71, + ELAPSED: 128, +} as const; + +export function integer(value: number, min: number, max: number): number { + if (!Number.isInteger(value) || value < min || value > max) throw new Error('Draw integer out of range'); + return value; +} +export function varint(value: number, signed = true): number[] { + integer(value, signed ? -2147483648 : 0, signed ? 2147483647 : 4294967295); + const widths = [7, 14, 21, 28, 32], prefixes = [0, 128, 192, 224, 240]; + const size = widths.findIndex(bits => bits === 32 || (signed + ? value >= -(2 ** (bits - 1)) && value < 2 ** (bits - 1) : value < 2 ** bits)) + 1; + const bytes = [size === 5 ? 240 : prefixes[size - 1] | ((value >>> (8 * (size - 1))) & ((1 << (8 - size)) - 1))]; + for (let i = 1; i < size; i++) bytes.push((value >>> (8 * (size - 1 - i))) & 255); + return bytes; +} + +export class DrawBytes { + offset = 0; + constructor(readonly bytes: Uint8Array) {} + take(size: number): Uint8Array { + integer(size, 0, this.bytes.length - this.offset); + const result = this.bytes.slice(this.offset, this.offset + size); this.offset += size; return result; + } + u8(): number { return this.take(1)[0]; } + u16(): number { return this.u8() | this.u8() << 8; } + i16(): number { return this.u16() << 16 >> 16; } + u32(): number { return (this.u16() | this.u16() << 16) >>> 0; } + variable(signed = true): number { + const first = this.u8(); + const size = first < 128 ? 1 : first < 192 ? 2 : first < 224 ? 3 : first < 240 ? 4 : first === 240 ? 5 : 0; + if (!size) throw new Error('Reserved extended-varint prefix'); + let value = size === 5 ? 0 : first & ((1 << (8 - size)) - 1); + for (let i = 1; i < size; i++) value = (value << 8) | this.u8(); + const bits = [7, 14, 21, 28, 32][size - 1]; + return signed ? bits < 32 ? value << (32 - bits) >> (32 - bits) : value : value >>> 0; + } +} + +/** Immutable, typed expression builder; operations use firmware i32/f32 semantics. */ +export class DrawExpression { + private constructor(readonly type: T, readonly code: readonly number[]) { + // Leave room for native elapsed bindings and surface translation. + if (code.length > 900) throw new Error('Draw expression too large'); + } + static i32(value: number): DrawExpression<'i32'> { return new DrawExpression('i32', [ExprOp.I32, ...varint(value)]); } + static f32(value: number): DrawExpression<'f32'> { + if (!Number.isFinite(Math.fround(value))) throw new Error('Invalid draw float'); + const bytes = new Uint8Array(4); new DataView(bytes.buffer).setFloat32(0, value, true); + return new DrawExpression('f32', [ExprOp.F32, ...bytes]); + } + /** Time already elapsed on this list's timeline when the next PRESENT happens. */ + static elapsed(): DrawExpression<'i32'> { return new DrawExpression('i32', [ExprOp.ELAPSED]); } + /** A 0..1 timeline fraction that survives subsequent PRESENT clock resets. */ + static progress(durationMs: number): DrawExpression<'f32'> { + const duration = DrawExpression.i32(integer(durationMs, 1, 2147483647)); + const elapsed = DrawExpression.elapsed().min(duration); + return duration.sub(elapsed).max(DrawExpression.i32(0)).time() + .add(elapsed).toFloat().div(duration.toFloat()); + } + static decode(code: Uint8Array): DrawExpression<'i32'> { return new DrawExpression('i32', Array.from(code)); } + private binary(other: DrawExpression, i: number, f: number): DrawExpression { + if (other.type !== this.type) throw new Error('Draw expression type mismatch'); + return new DrawExpression(this.type, [...this.code, ...other.code, this.type === 'i32' ? i : f]); + } + add(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.IADD, ExprOp.FADD); } + sub(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.ISUB, ExprOp.FSUB); } + mul(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.IMUL, ExprOp.FMUL); } + div(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.IDIV, ExprOp.FDIV); } + min(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.IMIN, ExprOp.FMIN); } + max(b: DrawExpression): DrawExpression { return this.binary(b, ExprOp.IMAX, ExprOp.FMAX); } + mod(this: DrawExpression<'i32'>, b: DrawExpression<'i32'>): DrawExpression<'i32'> { return this.binary(b, ExprOp.IMOD, ExprOp.IMOD); } + neg(): DrawExpression { return new DrawExpression(this.type, [...this.code, this.type === 'i32' ? ExprOp.INEG : ExprOp.FNEG]); } + toFloat(this: DrawExpression<'i32'>): DrawExpression<'f32'> { return new DrawExpression('f32', [...this.code, ExprOp.I2F]); } + toInt(this: DrawExpression<'f32'>): DrawExpression<'i32'> { return new DrawExpression('i32', [...this.code, ExprOp.F2I]); } + time(this: DrawExpression<'i32'>): DrawExpression<'i32'> { return new DrawExpression('i32', [...this.code, ExprOp.TIME]); } + ease(this: DrawExpression<'f32'>, op: number = ExprOp.SMOOTHSTEP): DrawExpression<'f32'> { + integer(op, ExprOp.SMOOTHSTEP, ExprOp.EASE_IN_OUT_CUBIC); + return new DrawExpression('f32', [...this.code, op]); + } + lerp(this: DrawExpression<'f32'>, from: DrawExpression<'f32'>, to: DrawExpression<'f32'>): DrawExpression<'f32'> { + return new DrawExpression('f32', [...from.code, ...to.code, ...this.code, ExprOp.LERP]); + } +} +export type DrawValue = number | DrawExpression<'i32'>; +export function encodeValue(value: DrawValue): number[] { + return typeof value === 'number' ? varint(value) : [255, ...varint(value.code.length, false), ...value.code]; +} +export function readValue(reader: DrawBytes): DrawValue { + if (reader.bytes[reader.offset] !== 255) return reader.variable(); + reader.u8(); return DrawExpression.decode(reader.take(reader.variable(false))); +} + +/** Software preview of the same VM; invalid programs return zero and discard pending animation. */ +export function evaluate(value: DrawValue, elapsed: number, presentTime = 0): { value: number; pending: boolean } { + if (typeof value === 'number') return { value, pending: false }; + const reader = new DrawBytes(Uint8Array.from(value.code)), stack: { value: number; float: boolean }[] = []; + let pending = false; + const push = (value: number, float = false) => { + value = float ? Math.fround(value) : value | 0; + if (!Number.isFinite(value) || stack.length === 32) throw new Error('Invalid expression stack'); + stack.push({ value, float }); + }; + const pop = (float: boolean) => { const v = stack.pop(); if (!v || v.float !== float) throw new Error('Expression type mismatch'); return v.value; }; + const toInt = (v: number) => { if (v < -2147483648 || v >= 2147483648) throw new Error('Invalid integer conversion'); return Math.trunc(v); }; + try { + while (reader.offset < reader.bytes.length) { + const op = reader.u8(); + if (op === ExprOp.I32) push(reader.variable()); + else if (op === ExprOp.F32) { const b = reader.take(4); push(new DataView(b.buffer).getFloat32(0, true), true); } + else if (op === ExprOp.ELAPSED) push(Math.min(2147483647, Math.max(0, elapsed))); + else if (op === ExprOp.I2F) push(pop(false), true); + else if (op === ExprOp.F2I) push(toInt(pop(true))); + else if (op === ExprOp.TIME) { const max = pop(false); if (max < 0) throw new Error('Invalid time'); const t = Math.min(Math.max(0, presentTime), max); pending ||= t !== max; push(t); } + else if (op === ExprOp.INEG || op === ExprOp.FNEG) push(-pop(op === ExprOp.FNEG), op === ExprOp.FNEG); + else if (op >= ExprOp.IADD && op <= ExprOp.IMAX || op >= ExprOp.FADD && op <= ExprOp.FMAX) { + const float = op >= ExprOp.FADD, b = pop(float), a = pop(float); + const key = float ? op - ExprOp.FADD : op - ExprOp.IADD; + if ((key === 3 || !float && key === 4) && b === 0) throw new Error('Division by zero'); + const result = key === 0 ? a + b : key === 1 ? a - b : key === 2 ? float ? a * b : Math.imul(a, b) : key === 3 ? a / b : !float && key === 4 ? a % b : key === (float ? 5 : 6) ? Math.min(a, b) : Math.max(a, b); + push(result, float); + } else if (op === ExprOp.LERP) { const t = pop(true), b = pop(true), a = pop(true); push(a + Math.fround(Math.fround(b - a) * t), true); } + else if (op >= ExprOp.SMOOTHSTEP && op <= ExprOp.EASE_IN_OUT_CUBIC) { + const t = Math.max(0, Math.min(1, pop(true))), u = Math.fround(1 - t), f = Math.fround; + let result: number; + switch (op) { + case ExprOp.SMOOTHSTEP: result = f(t * t) * f(3 - f(2 * t)); break; + case ExprOp.EASE_IN_QUAD: result = t * t; break; + case ExprOp.EASE_OUT_QUAD: result = 1 - f(u * u); break; + case ExprOp.EASE_IN_OUT_QUAD: result = t < .5 ? f(2 * t) * t : 1 - f(f(2 * u) * u); break; + case ExprOp.EASE_IN_CUBIC: result = f(t * t) * t; break; + case ExprOp.EASE_OUT_CUBIC: result = 1 - f(f(u * u) * u); break; + default: result = t < .5 ? f(f(4 * t) * t) * t : 1 - f(f(f(4 * u) * u) * u); + } + push(result, true); + } else if (op === ExprOp.DUP) { const a = stack[stack.length - 1]; if (!a) throw new Error('Underflow'); push(a.value, a.float); } + else if (op === ExprOp.DROP) { if (!stack.pop()) throw new Error('Underflow'); } + else if (op === ExprOp.SWAP) { if (stack.length < 2) throw new Error('Underflow'); const a = stack.pop()!, b = stack.pop()!; stack.push(a, b); } + else throw new Error('Unknown expression opcode'); + } + const top = stack.pop(); if (!top) throw new Error('Empty expression'); + return { value: top.float ? toInt(top.value) : top.value, pending }; + } catch { return { value: 0, pending: false }; } +} diff --git a/app/graphics/glyph-wire.ts b/app/graphics/glyph-wire.ts index 624137ec..cd15f0e2 100644 --- a/app/graphics/glyph-wire.ts +++ b/app/graphics/glyph-wire.ts @@ -159,6 +159,7 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | // Pass 1: resolve ids, register unseen rasters, size the buffer. let registration: Map | null = null; const fwRunOk = new Map(); + const presentations = new Map(); let bytes = 0; for (const placed of draws) { if (placed.kind === "glyph") { @@ -177,7 +178,7 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | (placed.glyph.coverage ? group.aaGlyphs : group.glyphs).push(placed.glyph); } else if (placed.kind === "image") { if (!inRange16(placed.x) || !inRange16(placed.y)) continue; - if (placed.presentation) bytes += encodePresentation(placed).length; + if (placed.presentation) { const record = encodePresentation(placed); presentations.set(placed, record); bytes += record.length; } else if (imageId(placed) !== null) bytes += IMAGE_RECORD_BYTES; } else { const ok = prepareFwTextRun(placed); @@ -210,7 +211,7 @@ export function prepareFrameDraws(draws: readonly DeferredDraw[]): ArrayBuffer | offset += GLYPH_RECORD_BYTES; } else if (placed.kind === "image") { if (!inRange16(placed.x) || !inRange16(placed.y)) continue; - if (placed.presentation) { const bytes = encodePresentation(placed); new Uint8Array(out.buffer).set(bytes, offset); offset += bytes.length; continue; } + if (placed.presentation) { const bytes = presentations.get(placed)!; new Uint8Array(out.buffer).set(bytes, offset); offset += bytes.length; continue; } const id = imageId(placed); if (id === null) continue; out.setUint8(offset, DrawRecordKind.TEXTURE_IMAGE); diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 8541ab7c..ac339a3d 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -1,3 +1,5 @@ +import { menuSelectionList } from "./menu-selection-list"; +import { encodeDisplayList, dimDisplayList, paintDisplayList, type DisplayList } from "./display-list"; import type { MenuHighlightAnimation } from "../ui/menu-highlight-motion"; import { Glyph } from "./bdffont"; import { wrapText } from "./textwrap"; @@ -74,7 +76,7 @@ export type PlacedGlyph = { * from the moment it is placed. */ export type PlacedImage = { - presentation?: { animation?: MenuHighlightAnimation; mode?: "image" | "masked-image"; radius: number; background: number; border: number; depth: number; occlusions?: readonly { x: number; y: number; width: number; height: number }[] }; + presentation?: { displayList?: DisplayList; mode?: "image" | "masked-image"; radius: number; background: number; border: number; depth: number; occlusions?: readonly { x: number; y: number; width: number; height: number }[] }; kind: "image"; source: GrayImage; x: number; @@ -395,7 +397,9 @@ export class GrayImage { hash = mixInt(hash, placed.source.width); hash = mixInt(hash, placed.source.height); hash = mixInt(hash, placed.source.sourceContentHash32()); - if (placed.presentation) { const p = placed.presentation; for (const value of [p.radius, p.background, p.border, p.depth, p.mode === "image" ? 1 : p.mode === "masked-image" ? 2 : 0, p.animation?.token ?? 0, p.animation?.dx ?? 0, p.animation?.dy ?? 0, p.animation?.durationMs ?? 0]) hash = mixInt(hash, value); } + if (placed.presentation) { const p = placed.presentation; for (const value of [p.radius, p.background, p.border, p.depth, p.mode === "image" ? 1 : p.mode === "masked-image" ? 2 : 0]) hash = mixInt(hash, value); + if (p.displayList) for (const byte of encodeDisplayList({ displayList: p.displayList, x: 0, y: 0, width: placed.source.width, height: placed.source.height, depth: p.depth }, p.displayList.timeline?.startedAt ?? 0)) hash = mixInt(hash, byte); + } } else { hash = mixInt(hash, 0xf17e); hash = mixInt(hash, placed.font.atlasTag); @@ -508,8 +512,15 @@ export class GrayImage { /** Retain the selected row separately from the menu surface for glasses-side composition. */ drawMenuSelection(source: GrayImage, x: number, y: number, background: number, border: number, radius = 8, depth = 2, animation?: MenuHighlightAnimation): void { - this.drawList.push({ kind: "image", source: source.withDrawsBaked(), x, y, - presentation: { radius, background, border, depth, animation } }); + const baked = source.withDrawsBaked(); + this.drawList.push({ kind: "image", source: baked, x, y, + presentation: { radius, background, border, depth, displayList: menuSelectionList(baked, background, border, radius, animation) } }); + } + + /** Submit a retained list alongside this frame; coordinates are relative to x/y. */ + drawDisplayList(displayList: DisplayList, x: number, y: number, width: number, height: number, depth = 0): void { + this.drawList.push({ kind: "image", source: new GrayImage(width, height), x, y, + presentation: { radius: 0, background: 0, border: 0, depth, displayList } }); } /** Replay an image at stereo depth. Masked images preserve intentional gray8 black (1). */ @@ -570,7 +581,7 @@ export class GrayImage { for (const placed of this.drawList) { if (placed.kind === "image" && placed.presentation) { const p = placed.presentation; - copy.drawList.push({ ...placed, source: placed.source.dimmed(factor), presentation: { ...p, background: dimValue(p.background, scale), border: dimValue(p.border, scale) } }); + copy.drawList.push({ ...placed, source: placed.source.dimmed(factor), presentation: { ...p, background: dimValue(p.background, scale), border: dimValue(p.border, scale), displayList: p.displayList && dimDisplayList(p.displayList, factor) } }); } else if (placed.kind !== "image") { copy.drawList.push({ ...placed, value: dimValue(placed.value, scale) }); } @@ -613,6 +624,11 @@ export class GrayImage { } else if (placed.kind === "image") { if (placed.presentation) { if (!presentations) continue; + if (placed.presentation.displayList) { + paintDisplayList(target.pixels, target.pixels.slice(), target.width, target.height, + { displayList: placed.presentation.displayList, x: placed.x + dx, y: placed.y + dy, width: placed.source.width, height: placed.source.height, depth: placed.presentation.depth }); + continue; + } if (placed.presentation.mode) { target.bitBlt(placed.source, placed.x + dx, placed.y + dy, { transparentZero: true }); continue; diff --git a/app/graphics/menu-selection-list.ts b/app/graphics/menu-selection-list.ts new file mode 100644 index 00000000..6a334039 --- /dev/null +++ b/app/graphics/menu-selection-list.ts @@ -0,0 +1,18 @@ +import type { MenuHighlightAnimation } from '../ui/menu-highlight-motion'; +import { DrawExpression as E } from './draw-expression'; +import { DrawOp, type DisplayList, type ListImage } from './display-list'; + +/** Menu policy and easing live in TypeScript; the bridge carries only generic calls/expressions. */ +export function menuSelectionList(image: ListImage, background: number, border: number, radius: number, animation?: MenuHighlightAnimation): DisplayList { + const coordinate = (delta: number) => { + if (!animation || !delta) return 0; + const progress = E.progress(animation.durationMs).ease(); + return progress.lerp(E.i32(delta).toFloat(), E.f32(0)).toInt(); + }; + const q = (n: number) => Math.min(15, (n + 8) >> 4); + return { resources: [image], timeline: animation, + calls: [ + { op: DrawOp.ROUNDED_RECT, x: coordinate(animation?.dx ?? 0), y: coordinate(animation?.dy ?? 0), width: image.width, height: image.height, radius, background: q(background), border: q(border) }, + { op: DrawOp.IMAGE, resource: 0, x: 0, y: 0, transparent: true }, + ] }; +} diff --git a/app/graphics/presentation-wire.ts b/app/graphics/presentation-wire.ts index 4dca3be1..0e8c361d 100644 --- a/app/graphics/presentation-wire.ts +++ b/app/graphics/presentation-wire.ts @@ -1,5 +1,5 @@ import type { PlacedImage } from './image' -import type { MenuHighlightAnimation } from '../ui/menu-highlight-motion' +import { DISPLAY_LIST_RECORD, DrawOp, SCREEN, encodeDisplayList, readDisplayList, paintDisplayList, type DisplayList } from './display-list' /** TS/native bridge tags, not firmware draw opcodes. Keep in sync with DrawRecordKind.kt. */ export const DrawRecordKind = { @@ -9,38 +9,36 @@ export const DrawRecordKind = { MENU_SELECTION: 3, TRANSPARENT_IMAGE: 4, MASKED_IMAGE: 5, - ANIMATED_MENU_SELECTION: 6, + DISPLAY_LIST: DISPLAY_LIST_RECORD, } as const export function isPresentationKind(kind: number | undefined): boolean { return kind === DrawRecordKind.MENU_SELECTION || kind === DrawRecordKind.TRANSPARENT_IMAGE || - kind === DrawRecordKind.MASKED_IMAGE || kind === DrawRecordKind.ANIMATED_MENU_SELECTION + kind === DrawRecordKind.MASKED_IMAGE || kind === DrawRecordKind.DISPLAY_LIST } -/** Presentation record: tag, signed xy, u16 wh/radius, gray fill/stroke, signed depth, restore-count, optional animation, gray pixels, restore rectangles. */ +/** Presentation record: tag, signed xy, u16 wh/radius, gray fill/stroke, signed depth, restore-count, gray pixels, restore rectangles. */ export function encodePresentation(draw: PlacedImage): Uint8Array { const p = draw.presentation! const { width, height, pixels } = draw.source - if (width < 1 || height < 1 || width > 640 || height > 480 || 5 + Math.ceil(width / 2) * height > 65536) throw new Error('Presentation image exceeds the resource size limit') + if (width < 1 || height < 1 || width > 640 || height > 480 || !p.displayList && 5 + Math.ceil(width / 2) * height > 65536) throw new Error('Presentation image exceeds the resource size limit') if (!Number.isInteger(p.depth) || p.depth < -128 || p.depth > 127) throw new Error('Invalid menu depth') if ((p.occlusions?.length ?? 0) > 2048) throw new Error('Too many menu occlusion rectangles') - const animation = p.animation; - if (animation && (!Number.isInteger(animation.durationMs) || animation.durationMs < 1 || animation.durationMs > 65535)) throw new Error('Invalid menu animation duration') - const header = animation ? 28 : 16; + if (p.displayList) { + const calls = [...p.displayList.calls, ...(p.occlusions ?? []).map(r => ({ + op: DrawOp.RECT_COPY, resource: SCREEN, x: r.x - draw.x, y: r.y - draw.y, + width: r.width, height: r.height, dx: r.x - draw.x, dy: r.y - draw.y, depth: -p.depth, + }))]; + return encodeDisplayList({ displayList: { ...p.displayList, calls }, x: draw.x, y: draw.y, width, height, depth: p.depth }); + } + const header = 16; const bytes = new Uint8Array(header + width * height + (p.occlusions?.length ?? 0) * 8), view = new DataView(bytes.buffer) - bytes[0] = animation ? DrawRecordKind.ANIMATED_MENU_SELECTION - : p.mode === "image" ? DrawRecordKind.TRANSPARENT_IMAGE + bytes[0] = p.mode === "image" ? DrawRecordKind.TRANSPARENT_IMAGE : p.mode === "masked-image" ? DrawRecordKind.MASKED_IMAGE : DrawRecordKind.MENU_SELECTION view.setInt16(1, draw.x, true); view.setInt16(3, draw.y, true) view.setUint16(5, width, true); view.setUint16(7, height, true); view.setUint16(9, p.radius, true) bytes[11] = p.background; bytes[12] = p.border; view.setInt8(13, p.depth) view.setUint16(14, p.occlusions?.length ?? 0, true) - if (animation) { - view.setInt16(16, animation.dx, true); view.setInt16(18, animation.dy, true); - view.setUint16(20, Math.max(0, Math.min(animation.durationMs, Date.now() - animation.startedAt)), true); - view.setUint32(22, animation.token, true); - view.setUint16(26, animation.durationMs, true); - } bytes.set(pixels, header) let offset = header + pixels.length for (const rect of p.occlusions ?? []) { @@ -50,20 +48,18 @@ export function encodePresentation(draw: PlacedImage): Uint8Array { return bytes } -export type Selection = { animation?: MenuHighlightAnimation; mode?: "image" | "masked-image"; x: number; y: number; width: number; height: number; radius: number; background: number; border: number; depth: number; pixels: Uint8Array; occlusions: { x: number; y: number; width: number; height: number }[] } +export type Selection = { displayList?: DisplayList; mode?: "image" | "masked-image"; x: number; y: number; width: number; height: number; radius: number; background: number; border: number; depth: number; pixels: Uint8Array; occlusions: { x: number; y: number; width: number; height: number }[] } export function readPresentation(bytes: Uint8Array, offset: number): { selection: Selection; end: number } { + if (bytes[offset] === DrawRecordKind.DISPLAY_LIST) { + const { placed, end } = readDisplayList(bytes, offset); + return { selection: { ...placed, radius: 0, background: 0, border: 0, pixels: new Uint8Array(0), occlusions: [] }, end }; + } const v = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength), w = v.getUint16(offset + 5, true), h = v.getUint16(offset + 7, true) const kind = bytes[offset] - const header = kind === DrawRecordKind.ANIMATED_MENU_SELECTION ? 28 : 16 + const header = 16 const selection: Selection = { mode: kind === DrawRecordKind.TRANSPARENT_IMAGE ? "image" : kind === DrawRecordKind.MASKED_IMAGE ? "masked-image" : undefined, x: v.getInt16(offset+1,true), y: v.getInt16(offset+3,true), width:w, height:h, radius:v.getUint16(offset+9,true), background:bytes[offset+11]!, border:bytes[offset+12]!, depth:v.getInt8(offset+13), pixels:bytes.slice(offset+header,offset+header+w*h), occlusions:[] } - if (kind === DrawRecordKind.ANIMATED_MENU_SELECTION) { - const durationMs = v.getUint16(offset+26,true), elapsed = v.getUint16(offset+20,true) - if (!durationMs || elapsed > durationMs) throw new Error('Invalid menu animation timing') - selection.animation = { dx:v.getInt16(offset+16,true), dy:v.getInt16(offset+18,true), - startedAt:Date.now()-elapsed, token:v.getUint32(offset+22,true), durationMs } - } let end=offset+header+w*h for(let i=0;iMath.min(15,(n+8)>>4), shift=right?-Math.floor((row.depth+1)/2):Math.floor(row.depth/2) const inside=(x:number,y:number,w:number,h:number,r:number)=>{ if(x<0||y<0||x>=w||y>=h) return false @@ -88,10 +85,7 @@ export function paintPresentation(output: Uint8Array, screen: Uint8Array, width: const dx=Math.max(0,2*r-(2*Math.min(x,w-1-x)+1)),dy=Math.max(0,2*r-(2*Math.min(y,h-1-y)+1)) return dx*dx+dy*dy<=4*r*r } - const t = row.animation ? Math.max(0, Math.min(1, (Date.now() - row.animation.startedAt) / row.animation.durationMs)) : 1 - const remaining = 1 - t * t * (3 - 2 * t) - const highlightX = Math.trunc(row.x + (row.animation?.dx ?? 0) * remaining) - const highlightY = Math.trunc(row.y + (row.animation?.dy ?? 0) * remaining) + const highlightX = row.x, highlightY = row.y if (!row.mode) for(let yy=0;yy=width||y>=height || !inside(xx,yy,row.width,row.height,row.radius)) continue diff --git a/app/graphics/surface-compositor.ts b/app/graphics/surface-compositor.ts index 9d0fa772..e110bac9 100644 --- a/app/graphics/surface-compositor.ts +++ b/app/graphics/surface-compositor.ts @@ -93,7 +93,7 @@ export class SurfaceCompositor { if(x+xx+shift>=0 && y+yy>=0 && x+xx+shift true + MENU_SELECTION, TRANSPARENT_IMAGE, MASKED_IMAGE, DISPLAY_LIST -> true else -> false } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FrameDisplayList.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FrameDisplayList.kt new file mode 100644 index 00000000..0d7c467f --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/FrameDisplayList.kt @@ -0,0 +1,133 @@ +package com.faceclaw.app + +/** A retained draw submitted with frame pixels, before cache IDs have been assigned. */ +interface RetainedDrawing { + val resources: List + val fingerprint: String + fun translated(dx: Int, dy: Int): RetainedDrawing + fun calls(ids: IntArray, nowMs: Long): List +} + +internal fun readRetainedDrawing(reader: ByteReader, kind: Int): RetainedDrawing = + if (kind == DrawRecordKind.DISPLAY_LIST) FrameDisplayList.read(reader) else MenuSelection.read(reader, kind) + +/** Generic TS-authored calls. No menu or easing policy lives in this bridge. */ +class FrameDisplayList private constructor( + private val x: Int, private val y: Int, private val depth: Int, + override val resources: List, private val commands: List, + internal val startedAt: Long, private val identity: String, +) : RetainedDrawing { + override val fingerprint: String get() = "$x,$y,$depth,$identity" + override fun translated(dx: Int, dy: Int) = FrameDisplayList(x + dx, y + dy, depth, resources, commands, startedAt, identity) + + private class Command(val op: Int, val depth: Int, val values: IntArray, + val x: DrawValue = DrawValue.Integer(0), val y: DrawValue = DrawValue.Integer(0)) + + override fun calls(ids: IntArray, nowMs: Long): List { + require(ids.size == resources.size) + val elapsed = (nowMs - startedAt).coerceIn(0, Int.MAX_VALUE.toLong()).toInt() + fun resource(index: Int) = if (index == DrawProtocol.SCREEN) index else ids[index] + return commands.map { c -> + val v = c.values + when (c.op) { + DRAW_OP_ROUNDED_RECT -> DrawProtocol.roundedRect(bind(c.x, elapsed, x), bind(c.y, elapsed, y), + v[0], v[1], v[2], v[3], v[4], depth + c.depth) + DRAW_OP_IMAGE -> DrawProtocol.image(resource(v[0]), x + v[1], y + v[2], v[3], depth = depth + c.depth) + DRAW_OP_RECT_COPY -> DrawProtocol.rectCopy(resource(v[0]), + v[1] + if (v[0] == DrawProtocol.SCREEN) x else 0, + v[2] + if (v[0] == DrawProtocol.SCREEN) y else 0, + v[3], v[4], x + v[5], y + v[6], depth = depth + c.depth) + else -> error("Unsupported frame display-list opcode") + } + } + } + + companion object { + // Bridge-only constant binding, removed before a program reaches either renderer. + private const val ELAPSED = 128 + private const val MAX_RECORD_BYTES = 4 * 1024 * 1024 + + private fun bind(value: DrawValue, elapsed: Int, offset: Int): DrawValue { + if (value is DrawValue.Integer) return DrawValue.Integer(value.value + offset) + val reader = DrawReader((value as DrawValue.Expression).program) + val out = ByteSink() + while (reader.remaining > 0) { + when (val op = reader.readU8()) { + ELAPSED -> { out.write(DrawExpression.PUSH_I32); out.write(ExtendedVarint.signed(elapsed)) } + DrawExpression.PUSH_I32 -> { out.write(op); out.write(ExtendedVarint.signed(ExtendedVarint.read(reader, true))) } + DrawExpression.PUSH_F32 -> { out.write(op); out.write(reader.readBytes(4)) } + else -> out.write(op) + } + } + if (offset != 0) { + out.write(DrawExpression.PUSH_I32); out.write(ExtendedVarint.signed(offset)); out.write(DrawExpression.IADD) + } + val program = out.toByteArray() + require(program.size <= DrawExpression.MAX_BYTES) + return DrawValue.Expression(program) + } + + fun read(input: ByteReader, nowMs: Long = drawAnimationTimeMs()): FrameDisplayList { + val length = input.getInt() + require(length in 21..MAX_RECORD_BYTES && length <= input.remaining()) + val bytes = ByteArray(length).also { input.get(it) } + val r = DrawReader(bytes) + val x = r.readS16(); val y = r.readS16() + val width = r.readU16(); val height = r.readU16() + require(width in 1..640 && height in 1..480) + val depth = r.readS8() + r.readBytes(4) // Stable timeline token is included in the identity below. + val elapsed = r.readU16() or (r.readU16() shl 16) + require(elapsed >= 0) + val resourceCount = r.readU16(); require(resourceCount <= 256) + val resources = List(resourceCount) { + val w = r.readU16(); val h = r.readU16() + require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536) + val gray = r.readBytes(w * h) + CachedResource(DrawProtocol.rawImage(w, h, BmpUtil.pack4bppFromGray8(gray, w, h))) + } + fun value(): DrawValue { + val first = r.readU8() + if (first != 255) return DrawValue.Integer(ExtendedVarint.read(r, true, first)) + val size = ExtendedVarint.read(r, false).toUInt() + require(size <= 900u) + return DrawValue.Expression(r.readBytes(size.toInt())).also { + // Validate framing and worst-case binding growth before retaining the frame. + bind(it, Int.MAX_VALUE, Int.MAX_VALUE) + } + } + fun resource(): Int = r.readU16().also { require(it == DrawProtocol.SCREEN || it in resources.indices) } + val count = r.readU16(); require(count <= 4096) + val calls = List(count) { + val op = r.readU8(); val d = r.readS16(); require(depth + d in -128..127) + when (op) { + DRAW_OP_ROUNDED_RECT -> { + val xx = value(); val yy = value() + val w = r.readU16(); val h = r.readU16(); val radius = r.readU16() + val background = r.readU8(); val border = r.readU8() + require(w in 1..640 && h in 1..480 && background in 0..15 && border in 0..16) + Command(op, d, intArrayOf(w, h, radius, background, border), xx, yy) + } + DRAW_OP_IMAGE -> { + val id = resource(); require(id != DrawProtocol.SCREEN) + val xx = r.readS16(); val yy = r.readS16(); val options = r.readU8() + require(options == 15 || options == 31) + Command(op, d, intArrayOf(id, xx, yy, options)) + } + DRAW_OP_RECT_COPY -> { + val id = resource(); val sx = r.readS16(); val sy = r.readS16() + val w = r.readU16(); val h = r.readU16(); val dx = r.readS16(); val dy = r.readS16() + require(w in 1..640 && h in 1..480) + require(id == DrawProtocol.SCREEN || sx >= 0 && sy >= 0) + Command(op, d, intArrayOf(id, sx, sy, w, h, dx, dy)) + } + else -> error("Unsupported frame display-list opcode") + } + } + r.requireDone() + // Elapsed changes while the same timeline is resubmitted; it is not content identity. + bytes.fill(0, 13, 17) + return FrameDisplayList(x, y, depth, resources, calls, nowMs - elapsed, CachedResource(bytes).hash.toString()) + } + } +} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt index c386a6e4..fda4e8e6 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/MenuSelection.kt @@ -2,17 +2,15 @@ package com.faceclaw.app /** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int, - val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List = emptyList(), val animation: Animation? = null) { - class Animation(val dx: Int, val dy: Int, val startedAt: Long, val token: Int, val durationMs: Int) + val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List = emptyList()) : RetainedDrawing { val resource = CachedResource(DrawProtocol.rawImage(width, height, packed)) - val fingerprint = "${animation?.token},${animation?.dx},${animation?.dy},${animation?.durationMs},$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}" - fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask, animation) - fun calls(id: Int, nowMs: Long = drawAnimationTimeMs()): List { - val elapsed = animation?.let { (nowMs - it.startedAt).coerceIn(0, it.durationMs.toLong()).toInt() } ?: 0 - val highlightX = animation?.let { DrawValue.animate(x + it.dx, x, it.durationMs, elapsed) } ?: DrawValue.Integer(x) - val highlightY = animation?.let { DrawValue.animate(y + it.dy, y, it.durationMs, elapsed) } ?: DrawValue.Integer(y) + override val resources = listOf(resource) + override val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}" + override fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask) + override fun calls(ids: IntArray, nowMs: Long) = calls(ids.single()) + fun calls(id: Int): List { val image = when (kind) { - DrawRecordKind.MENU_SELECTION -> listOf(DrawProtocol.roundedRect(highlightX, highlightY, width, height, radius, background, border, depth), + DrawRecordKind.MENU_SELECTION -> listOf(DrawProtocol.roundedRect(x, y, width, height, radius, background, border, depth), DrawProtocol.image(id, x, y, CFW_TEXTURE_OPT_BRIGHTNESS_MASK or CFW_TEXTURE_OPT_TRANSPARENT, depth = depth)) DrawRecordKind.TRANSPARENT_IMAGE -> listOf(DrawProtocol.image(id, x, y, CFW_TEXTURE_OPT_BRIGHTNESS_MASK or CFW_TEXTURE_OPT_TRANSPARENT, depth = depth)) // Gray8 zero is transparent but gray8 one is opaque black. Rect copies preserve @@ -31,7 +29,7 @@ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val companion object { /** The caller has consumed a presentation tag from [DrawRecordKind]. Colors and pixels cross in gray8. */ fun read(reader: ByteReader, kind: Int = DrawRecordKind.MENU_SELECTION): MenuSelection { - require(DrawRecordKind.isPresentation(kind)) + require(kind in listOf(DrawRecordKind.MENU_SELECTION, DrawRecordKind.TRANSPARENT_IMAGE, DrawRecordKind.MASKED_IMAGE)) require(reader.remaining() >= 15) val x = reader.getShort().toInt(); val y = reader.getShort().toInt() val w = reader.getShort().toInt() and 65535; val h = reader.getShort().toInt() and 65535 @@ -42,16 +40,6 @@ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val val count = reader.getShort().toInt() and 65535 require(count <= 2048) require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536 && reader.remaining() >= w * h) - val animation = if (kind == DrawRecordKind.ANIMATED_MENU_SELECTION) { - require(reader.remaining() >= 12) - val dx = reader.getShort().toInt() - val dy = reader.getShort().toInt() - val elapsed = reader.getShort().toInt() and 65535 - val token = reader.getInt() - val duration = reader.getShort().toInt() and 65535 - require(duration > 0 && elapsed <= duration) - Animation(dx, dy, drawAnimationTimeMs() - elapsed, token, duration) - } else null val gray = ByteArray(w * h); reader.get(gray) require(reader.remaining() >= count * 8) val occlusions = List(count) { IntArray(4) { reader.getShort().toInt() and 65535 } } @@ -68,8 +56,7 @@ class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val } } } - val normalizedKind = if (kind == DrawRecordKind.ANIMATED_MENU_SELECTION) DrawRecordKind.MENU_SELECTION else kind - return MenuSelection(x, y, w, h, radius, background, border, depth, BmpUtil.pack4bppFromGray8(gray, w, h), occlusions, normalizedKind, mask, animation) + return MenuSelection(x, y, w, h, radius, background, border, depth, BmpUtil.pack4bppFromGray8(gray, w, h), occlusions, kind, mask) } } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt index 17195151..c48a81c2 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ShellScene.kt @@ -1,18 +1,24 @@ package com.faceclaw.app /** Immutable shell snapshot. Keys identify writable surfaces across repaints, not their pixels. */ -class ShellScene(val layers: List, val selections: List = emptyList()) { - class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray, val selections: List = emptyList(), val depth: Int = 0) +class ShellScene(val layers: List, val selections: List = emptyList()) { + class Layer(val key: Int, val x: Int, val y: Int, val width: Int, val height: Int, val dim: Int, val packed: ByteArray, val selections: List = emptyList(), val depth: Int = 0) val allSelections = selections + layers.flatMap { it.selections } + val retainedResources = allSelections.flatMap { it.resources } + init { require(retainedResources.size + layers.size < 511) } val fingerprint: String = layers.joinToString(";") { "${it.key},${it.x},${it.y},${it.width},${it.height},${it.dim},${it.depth},${CachedResource(it.packed).hash},${it.selections.joinToString { row -> row.fingerprint }}" } + selections.joinToString { it.fingerprint } fun calls(width: Int, height: Int, surfaces: IntArray, selected: IntArray): List { val calls = ArrayList(); var rowId = 0 val now = drawAnimationTimeMs() - for (row in selections) calls.addAll(row.calls(selected[rowId++], now)) + fun add(row: RetainedDrawing) { + calls.addAll(row.calls(selected.copyOfRange(rowId, rowId + row.resources.size), now)) + rowId += row.resources.size + } + for (row in selections) add(row) for ((index, layer) in layers.withIndex()) { if (layer.dim < 256) calls.add(DrawProtocol.lut(width, height, layer.dim)) calls.add(DrawProtocol.image(surfaces[index], layer.x, layer.y, depth = layer.depth)) - for (row in layer.selections) calls.addAll(row.calls(selected[rowId++], now)) + for (row in layer.selections) add(row) } return calls } @@ -22,7 +28,7 @@ class ShellScene(val layers: List, val selections: List = val surfaces = IntArray(layers.size) { id -> val layer = layers[id]; resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed); id } - val rows = IntArray(allSelections.size) { id -> resources[id + layers.size] = allSelections[id].resource.bytes; id + layers.size } + val rows = IntArray(retainedResources.size) { id -> resources[id + layers.size] = retainedResources[id].bytes; id + layers.size } resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows)) DisplayListRenderer(resources, rightLens = rightLens).render(511, DisplayListRenderer.Target(screen, width, height), DisplayListRenderer.Target(output, width, height)) val target = DisplayListRenderer.Target(output, width, height) @@ -42,8 +48,8 @@ class ShellScene(val layers: List, val selections: List = resources[id] = DrawProtocol.rawImage(layer.width, layer.height, layer.packed) id } - val rows = IntArray(allSelections.size) { id -> - resources[id + layers.size] = allSelections[id].resource.bytes + val rows = IntArray(retainedResources.size) { id -> + resources[id + layers.size] = retainedResources[id].bytes id + layers.size } resources[511] = DrawProtocol.displayList(calls(width, height, surfaces, rows)) @@ -73,11 +79,11 @@ class ShellScene(val layers: List, val selections: List = val depth = reader.getShort().toInt(); require(depth in -128..127) require(w in 1..640 && h in 1..480 && 5 + (w + 1) / 2 * h <= 65536 && reader.remaining() >= w * h) val gray = ByteArray(w * h); reader.get(gray) - val selections = List(selectionCount) { MenuSelection.read(reader, reader.get().toInt()) } + val selections = List(selectionCount) { readRetainedDrawing(reader, reader.get().toInt()) } layers.add(Layer(key, x, y, w, h, dim, BmpUtil.pack4bppFromGray8(gray, w, h), selections, depth)) } require(reader.remaining() == 0) - require(layers.sumOf { it.selections.size } + layers.size < 511) + require(layers.sumOf { layer -> layer.selections.sumOf { it.resources.size } } + layers.size < 511) require(layers.map { it.key }.toSet().size == layers.size) return ShellScene(layers) } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt index c862d47d..0a52c70e 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt @@ -51,7 +51,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr var out: MutableList = ArrayList() while ((cursor.remaining() >= 1)) { var kind: Int = (cursor.get() and 0xff) - if (DrawRecordKind.isPresentation(kind)) { out.add(ScreenDraw.image(0, 0, 0).also { it.selection = MenuSelection.read(cursor, kind) }); continue } + if (DrawRecordKind.isPresentation(kind)) { out.add(ScreenDraw.image(0, 0, 0).also { it.selection = readRetainedDrawing(cursor, kind) }); continue } if (((kind == ScreenDraw.KIND_GLYPH) && (cursor.remaining() >= 11))) { var fontId: Int = (cursor.getShort().toInt() and 0xffff) var encoding: Int = cursor.getInt() @@ -108,7 +108,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr * raster comes from ImageAtlas under imageId. */ class ScreenDraw { - @JvmField var selection: MenuSelection? = null + @JvmField var selection: RetainedDrawing? = null companion object { const val KIND_GLYPH: Int = DrawRecordKind.GLYPH @@ -631,7 +631,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr var fingerprint: StringBuilder = StringBuilder() fingerprint.append(screenWidth).append('x').append(screenHeight) var draws: MutableList = ArrayList() - val selections = ArrayList() + val selections = ArrayList() for (surface in ordered) { if (!surface.visible || (shellScene != null && surface.id == "shell")) { continue diff --git a/tests/ai-chat.test.cjs b/tests/ai-chat.test.cjs index 8367b291..bfbcf03b 100644 --- a/tests/ai-chat.test.cjs +++ b/tests/ai-chat.test.cjs @@ -355,7 +355,7 @@ function chatLayerEnv() { const env = conversations(); const textwrap = load('app/graphics/textwrap.ts', {}); const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); - const graphics = load('app/graphics/image.ts', { './textwrap': textwrap }); + const graphics = require('../.test-build/app/graphics/image.js'); const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8')); let windowOptions; const { createAiChatWindow } = load('app/apps/ai-chat/ai-chat-app.ts', { diff --git a/tests/brightness-picker.test.cjs b/tests/brightness-picker.test.cjs index 59569452..037d45c5 100644 --- a/tests/brightness-picker.test.cjs +++ b/tests/brightness-picker.test.cjs @@ -15,7 +15,7 @@ function load(file, modules = {}) { } const textwrap = load('app/graphics/textwrap.ts'); -const graphics = load('app/graphics/image.ts', { './textwrap': textwrap }); +const graphics = require('../.test-build/app/graphics/image.js'); const plane = load('app/graphics/plane.ts', { './image': graphics }); const gestures = load('app/ui/gestures.ts'); const numeric = load('app/util/numeric-util.ts'); diff --git a/tests/compass-debug.test.cjs b/tests/compass-debug.test.cjs index d9057409..215dcc88 100644 --- a/tests/compass-debug.test.cjs +++ b/tests/compass-debug.test.cjs @@ -17,7 +17,7 @@ function load(file, modules = {}, globals = {}) { return context.exports; } const textwrap = load('app/graphics/textwrap.ts'); -const graphics = load('app/graphics/image.ts', { './textwrap': textwrap }); +const graphics = require('../.test-build/app/graphics/image.js'); const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); const font = (size) => BdfFont.parse(fs.readFileSync(path.join(__dirname, `../app/fonts/terminus/ter-u${size}n.bdf`), 'utf8')); const small = font(18), large = font(32); diff --git a/tests/compass-texture-cache.test.cjs b/tests/compass-texture-cache.test.cjs index 08e36e12..8cc9670f 100644 --- a/tests/compass-texture-cache.test.cjs +++ b/tests/compass-texture-cache.test.cjs @@ -19,7 +19,7 @@ function load(file, modules = {}, extra = '', globals = {}) { const textwrap = load('app/graphics/textwrap.ts'); const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); -const graphics = load('app/graphics/image.ts', { './textwrap': textwrap }); +const graphics = require('../.test-build/app/graphics/image.js'); const { GrayImage } = graphics; const font = (size) => BdfFont.parse(fs.readFileSync( path.join(__dirname, `../app/fonts/terminus/ter-u${size}n.bdf`), 'utf8')); @@ -156,7 +156,7 @@ test('the background registers once and uses a nine-byte image reference on late return 1; } } } } }, }); - const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', { '../native/texture-atlas': atlas, './presentation-wire': load('app/graphics/presentation-wire.ts') }); + const { prepareFrameDraws } = load('app/graphics/glyph-wire.ts', { '../native/texture-atlas': atlas, './presentation-wire': require('../.test-build/app/graphics/presentation-wire.js') }); const background = compass.createCompassBackground(576, 260, 256, 193.25, 98, 140, 1); for (const heading of [0, 1, 2, 45, 90]) { const image = background.clone(); diff --git a/tests/display-list-authoring.test.cjs b/tests/display-list-authoring.test.cjs new file mode 100644 index 00000000..d3f7cc61 --- /dev/null +++ b/tests/display-list-authoring.test.cjs @@ -0,0 +1,83 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { DrawExpression: E, ExprOp, varint, DrawBytes, evaluate } = require('../.test-build/app/graphics/draw-expression.js'); +const { DrawOp, SCREEN, encodeDisplayList, readDisplayList, paintDisplayList } = require('../.test-build/app/graphics/display-list.js'); +const { GrayImage } = require('../.test-build/app/graphics/image.js'); +const { encodePresentation } = require('../.test-build/app/graphics/presentation-wire.js'); +// Also consumed by FrameDisplayListTest.kt: bridge bytes, not independently rebuilt native calls. +const MENU = '077e0000000300020002000200022a00000096000000010002000200ffffffff0200080000ff24017e30020000000001812c8001812c1611010017328001812c16103001812c3023414031ff24017c30020000000001812c8001812c1611010017328001812c16103001812c302341403102000200010001030400000000000000001f'; +const MULTI = '074c000000030002000400010002000000000000000002000200010000ff010001006003000400000000000000001f020000010000000000010001000200000002feffffffffff000001000100ffff0000'; + +test('extended integers agree with firmware widths and sign extension', () => { + const vectors = [[63,'3f'],[-64,'40'],[64,'8040'],[-65,'bfbf'],[8191,'9fff'],[-8192,'a000'],[8192,'c02000'],[-8193,'dfdfff'],[1048575,'cfffff'],[-1048576,'d00000'],[1048576,'e0100000'],[-1048577,'efefffff'],[134217727,'e7ffffff'],[-134217728,'e8000000'],[134217728,'f008000000'],[2147483647,'f07fffffff'],[-2147483648,'f080000000']]; + for (const [value, hex] of vectors) { + assert.equal(Buffer.from(varint(value)).toString('hex'), hex); + assert.equal(new DrawBytes(Buffer.from(hex, 'hex')).variable(), value); + } + for (const n of [0,127,128,16383,16384,2097151,2097152,268435455,268435456,4294967295]) assert.equal(new DrawBytes(Uint8Array.from(varint(n,false))).variable(false), n); + assert.throws(() => E.i32(2147483648)); + assert.throws(() => E.f32(Infinity)); +}); + +test('typed expressions follow i32/f32 arithmetic, easing, time and error semantics', () => { + assert.equal(evaluate(E.i32(2147483647).add(E.i32(1)),0).value, -2147483648); + assert.equal(evaluate(E.i32(7).neg().div(E.i32(2)),0).value, -3); + assert.equal(evaluate(E.i32(-2147483648).div(E.i32(-1)),0).value, -2147483648); + assert.equal(evaluate(E.f32(1.5).mul(E.f32(2)).toInt(),0).value, 3); + for (let op = ExprOp.SMOOTHSTEP; op <= ExprOp.EASE_IN_OUT_CUBIC; op++) { + assert.equal(evaluate(E.f32(0).ease(op),0).value, 0); + assert.equal(evaluate(E.f32(1).ease(op),0).value, 1); + } + assert.deepEqual(evaluate(E.i32(300).time(),0,150), {value:150,pending:true}); + assert.deepEqual(evaluate(E.i32(300).time(),0,300), {value:300,pending:false}); + assert.deepEqual(evaluate(E.i32(300).time().div(E.i32(0)),0,150), {value:0,pending:false}); + assert.deepEqual(evaluate(E.decode(Uint8Array.of(ExprOp.IADD)),0), {value:0,pending:false}); +}); + +test('menu authors the golden generic list and stable timeline identity', () => { + const original = Date.now; Date.now = () => 1150; + try { + const image = new GrayImage(8,4); + image.drawMenuSelection(new GrayImage(2,2,255),3,2,16,48,1,2,{dx:-2,dy:-4,startedAt:1000,token:42,durationMs:300}); + assert.equal(Buffer.from(encodePresentation(image.draws[0])).toString('hex'), MENU); + const before = image.fingerprint(); Date.now = () => 1250; + assert.equal(image.fingerprint(), before, 'wall time does not change frame identity'); + const decoded = readDisplayList(Buffer.from(MENU,'hex'),0,1150).placed; + const y = decoded.displayList.calls[0].y; + assert.deepEqual(evaluate(y,150,0), {value:-2,pending:true}); + assert.deepEqual(evaluate(y,150,150), {value:0,pending:false}); + assert.deepEqual(evaluate(y,300,0), {value:0,pending:false}); + } finally { Date.now = original; } +}); + +test('local resources, relative placement, stereo and screen restoration round-trip', () => { + const list = { resources:[{width:2,height:1,pixels:Uint8Array.of(0,255)},{width:1,height:1,pixels:Uint8Array.of(96)}], calls:[ + {op:DrawOp.IMAGE,resource:0,x:0,y:0,transparent:true}, + {op:DrawOp.RECT_COPY,resource:1,x:0,y:0,width:1,height:1,dx:2,dy:0}, + {op:DrawOp.RECT_COPY,resource:SCREEN,x:-1,y:0,width:1,height:1,dx:-1,dy:0,depth:-2}, + ] }; + const bytes = encodeDisplayList({displayList:list,x:3,y:2,width:4,height:1,depth:2}); + assert.equal(Buffer.from(bytes).toString('hex'), MULTI); + const {placed,end} = readDisplayList(bytes,0); + assert.equal(end,bytes.length); + for (const right of [false,true]) { + const screen = new Uint8Array(64).fill(32), output = screen.slice(); + paintDisplayList(output,screen,8,8,placed,right); + assert.equal(output[2*8+(right?3:5)],240); + assert.equal(output[2*8+(right?4:6)],96); + assert.equal(output[2*8+2],32); + } + for (let i=0;i readDisplayList(bytes.slice(0,i),0)); + const invalid = {...list, calls:[{op:DrawOp.IMAGE,resource:2,x:0,y:0,transparent:true}]}; + assert.throws(() => encodeDisplayList({displayList:invalid,x:0,y:0,width:4,height:1,depth:0})); +}); + +test('frame submission supports resource-free lists and plain PRESENT-relative time', () => { + const image = new GrayImage(8,8); + image.drawDisplayList({resources:[],calls:[{op:DrawOp.ROUNDED_RECT,x:E.i32(4).time(),y:0,width:2,height:2,radius:0,background:15,border:16}]},0,0,8,8); + const bytes = encodePresentation(image.draws[0]); + const {placed} = readDisplayList(bytes,0,1000); + const output = new Uint8Array(64), screen = output.slice(); + paintDisplayList(output,screen,8,8,placed,false,1002); + assert.equal(output[2],240); assert.equal(output[0],0); +}); diff --git a/tests/input-events.test.cjs b/tests/input-events.test.cjs index cf20fa8c..41735ccb 100644 --- a/tests/input-events.test.cjs +++ b/tests/input-events.test.cjs @@ -83,7 +83,7 @@ test('bounded history preserves equal-time events, snapshots and arrival gaps; p test('Input events captures while visible, pauses for history, and unsubscribes on removal', () => { const textwrap = load('app/graphics/textwrap.ts'); - const graphics = load('app/graphics/image.ts', { './textwrap': textwrap }); + const graphics = require('../.test-build/app/graphics/image.js'); const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); const font = BdfFont.parse(read('app/fonts/terminus/ter-u18n.bdf')); class RecordingImage extends graphics.GrayImage { diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DrawExpressionTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DrawExpressionTest.kt index dae59f65..11fd7001 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DrawExpressionTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DrawExpressionTest.kt @@ -72,48 +72,6 @@ class DrawExpressionTest { assertEquals(110 to false, evaluate(continued.program, 250)) } - @Test fun animatedMenuBridgePreservesElapsedTimeAndTranslation() { - // Tag 6 record emitted by the TypeScript bridge: elapsed=200, delta=(-2,-4), duration=300. - val wire = hex("06030002000200020001001030020000fefffcffc8002a0000002c01ffffffff") - val input = ArrayByteReader(wire) - assertEquals(6, input.get().toInt()) - val row = MenuSelection.read(input, DrawRecordKind.ANIMATED_MENU_SELECTION).translated(10, 10) - assertEquals(0, input.remaining()) - val motion = assertNotNull(row.animation) - assertEquals(42, motion.token) - assertEquals(300, motion.durationMs) - assertEquals(-2, motion.dx) - assertEquals(-4, motion.dy) - val call = row.calls(7, motion.startedAt + 200).first() - for ((elapsed, expected) in listOf(0L to (12 to 10), 100L to (13 to 12))) { - val reader = DrawReader(call) - assertEquals(8, reader.readU8()) - assertEquals(DRAW_FLAG_DEPTH, reader.readU8()) - assertEquals(2, reader.readS8()) - val frame = DrawEvaluation(elapsed) - assertEquals(expected.first, ExtendedVarint.evaluate(reader, frame)) - assertEquals(expected.second, ExtendedVarint.evaluate(reader, frame)) - assertEquals(elapsed < 100, frame.animationPending) - } - for (end in 1 until wire.size) assertFails { - MenuSelection.read(ArrayByteReader(wire.copyOf(end), 1), DrawRecordKind.ANIMATED_MENU_SELECTION) - } - } - - @Test fun menuCoordinatesUseTheSuppliedDuration() { - val row = MenuSelection(0, 4, 2, 2, 0, 15, 16, 0, ByteArray(2), - animation = MenuSelection.Animation(0, -4, 1000, 42, 120)) - val call = row.calls(7, 1000).first() - for ((elapsed, y) in listOf(0L to 0, 60L to 2, 120L to 4)) { - val reader = DrawReader(call) - reader.readBytes(3) // opcode, flags, depth - val frame = DrawEvaluation(elapsed) - assertEquals(0, ExtendedVarint.evaluate(reader, frame)) - assertEquals(y, ExtendedVarint.evaluate(reader, frame)) - assertEquals(elapsed < 120, frame.animationPending) - } - } - @Test fun playerSchedulesOnlyUnfinishedFramesAndCancelsStaleCallbacks() { val call = DrawProtocol.roundedRect(DrawValue.Integer(0), DrawValue.animate(0, 4, 300), 2, 2, 0, 15) val renderer = DisplayListRenderer(mapOf(1 to DrawProtocol.displayList(listOf(call)))) diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameDisplayListTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameDisplayListTest.kt new file mode 100644 index 00000000..29975bc8 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/FrameDisplayListTest.kt @@ -0,0 +1,90 @@ +package com.faceclaw.app + +import kotlin.test.* + +class FrameDisplayListTest { + // Golden records emitted by display-list-authoring.test.cjs. + private val menu = "077e0000000300020002000200022a00000096000000010002000200ffffffff0200080000ff24017e30020000000001812c8001812c1611010017328001812c16103001812c3023414031ff24017c30020000000001812c8001812c1611010017328001812c16103001812c302341403102000200010001030400000000000000001f" + private val multi = "074c000000030002000400010002000000000000000002000200010000ff010001006003000400000000000000001f020000010000000000010001000200000002feffffffffff000001000100ffff0000" + private fun hex(s: String) = s.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + private fun read(bytes: ByteArray, now: Long = 1150): FrameDisplayList { + val reader = ArrayByteReader(bytes) + assertEquals(DrawRecordKind.DISPLAY_LIST, reader.get().toInt()) + return FrameDisplayList.read(reader, now).also { assertEquals(0, reader.remaining()) } + } + + @Test fun authoredExpressionsRebindOnPresentAndTranslateWithTheSurface() { + val row = read(hex(menu)).translated(10, 10) + assertEquals(1, row.resources.size) + fun coordinates(now: Long, elapsed: Long): Triple { + val call = DrawReader(row.calls(intArrayOf(7), now).first()) + assertEquals(DRAW_OP_ROUNDED_RECT, call.readU8()) + assertEquals(DRAW_FLAG_DEPTH, call.readU8()) + assertEquals(2, call.readS8()) + val frame = DrawEvaluation(elapsed) + val x = ExtendedVarint.evaluate(call, frame) + val y = ExtendedVarint.evaluate(call, frame) + return Triple(x, y, frame.animationPending) + } + assertEquals(Triple(12, 10, true), coordinates(1150, 0)) + assertEquals(Triple(13, 12, false), coordinates(1150, 150)) + assertEquals(coordinates(1150, 75), coordinates(1225, 0), "another PRESENT preserves progress") + assertEquals(Triple(13, 12, false), coordinates(1400, 0)) + val later = hex(menu).also { it[18] = 200.toByte() } // envelope 5 + elapsed offset 13 + assertEquals(read(hex(menu)).fingerprint, read(later).fingerprint) + val other = hex(menu).also { it[14] = 43 } // timeline token + assertNotEquals(read(hex(menu)).fingerprint, read(other).fingerprint) + } + + @Test fun localResourceIdsAreResolvedAndScreenCopiesStayAtScreenDepth() { + val list = read(hex(multi)).translated(10, 10) + val calls = list.calls(intArrayOf(41, 42), 1150) + assertContentEquals(DrawProtocol.image(41, 13, 12, 31, depth = 2), calls[0]) + assertContentEquals(DrawProtocol.rectCopy(42, 0, 0, 1, 1, 15, 12, depth = 2), calls[1]) + assertContentEquals(DrawProtocol.rectCopy(DrawProtocol.SCREEN, 12, 12, 1, 1, 12, 12, depth = 0), calls[2]) + val scene = ShellScene(emptyList(), listOf(read(hex(multi)))) + assertEquals(2, scene.retainedResources.size) + for (right in listOf(false, true)) { + val pixels = scene.preview(ByteArray(64) { 32 }, 8, 8, right) + assertEquals(240, pixels[16 + if (right) 3 else 5].toInt() and 255) + assertEquals(96, pixels[16 + if (right) 4 else 6].toInt() and 255) + assertEquals(32, pixels[18].toInt() and 255) + } + val cache = ResourceCacheState() + val planner = ScenePlanner(cache) + val screen = ByteArray(32) { 0x22 } + planner.plan(screen, 8, 8, null, scene, 1) + val ids = scene.retainedResources.map { cache.resourceId(it) } + assertTrue(ids.all { it >= 0 }); assertNotEquals(ids[0], ids[1]) + planner.plan(screen, 8, 8, null, scene, 1) + assertEquals(ids, scene.retainedResources.map { cache.resourceId(it) }) + } + + @Test fun frameAndShellSubmissionUseTheSameGenericDecoder() { + val c = SurfaceCompositor() + c.configureScreen(8, 8) + c.configureSurface("app", 0, 0, 8, 8, 0, 0) + c.submitSurface("app", ArrayByteReader(ByteArray(64) { 32 }), 0, 0, 8, 8, "list", + ArrayByteReader(hex(multi))) + val frame = c.composite() + assertEquals(2, frame.shellScene.retainedResources.size) + assertEquals(240, frame.gray[21].toInt() and 255) + assertTrue(frame.screenGray.all { it.toInt() == 32 }) + // One opaque 1x1 shell layer, with one generic list after its pixels. + val layer = DrawProtocol.word(1) + DrawProtocol.word(1) + DrawProtocol.word(0) + DrawProtocol.word(0) + + DrawProtocol.word(1) + DrawProtocol.word(1) + DrawProtocol.word(256) + DrawProtocol.word(1) + + DrawProtocol.word(0) + byteArrayOf(32) + hex(multi) + val shell = ShellScene.decode(ArrayByteReader(layer)) + assertEquals(2, shell.retainedResources.size) + assertEquals(240, shell.preview(ByteArray(64) { 32 }, 8, 8)[21].toInt() and 255) + } + + @Test fun rejectsTruncationBadResourcesAndTrailingData() { + val bytes = hex(multi) + for (end in 1 until bytes.size) assertFails { read(bytes.copyOf(end)) } + assertFails { read(bytes.copyOf().also { it[40] = 9 }) } // first image's local resource + val trailing = bytes + byteArrayOf(0) + trailing[1] = (trailing[1] + 1).toByte() + assertFails { read(trailing) } + } +} diff --git a/tests/menu-display-list.test.cjs b/tests/menu-display-list.test.cjs index 6d6167e3..2709219b 100644 --- a/tests/menu-display-list.test.cjs +++ b/tests/menu-display-list.test.cjs @@ -48,7 +48,7 @@ test('shell menu selection survives serialization and is not baked into its surf test('later opaque planes occlude a replayed selection without erasing its visible part',()=>{ const image=menu().paint(),row=image.draws.find(d=>d.presentation),cover=new graphics.GrayImage(10,8,1); const flat=planes.flattenPlanesWithDraws([{image,x:0,y:0},{image:cover,x:row.x+10,y:row.y+2}]); - const records=wire.presentationRecords(prepareFrameDraws(flat.draws));assert.ok(records[0].occlusions.length); + const records=wire.presentationRecords(prepareFrameDraws(flat.draws));assert.ok(records[0].displayList.calls.some(c => c.op === 2 && c.resource === 65535)); const c=new SurfaceCompositor(80,80);c.configureSurface('app',{x:0,y:0,width:80,height:80,zOrder:0,transparency:'opaque'}); c.submitSurfaceFrame('app',flat.image.pixels,{x:0,y:0,width:80,height:80},prepareFrameDraws(flat.draws)); const preview=c.composite(); diff --git a/tests/menu-highlight-motion.test.cjs b/tests/menu-highlight-motion.test.cjs index f74279b9..ca561f4b 100644 --- a/tests/menu-highlight-motion.test.cjs +++ b/tests/menu-highlight-motion.test.cjs @@ -35,7 +35,7 @@ test('bridge and fallback preview use the supplied animation duration', () => { image.drawMenuSelection(new GrayImage(2, 2), 5, 0, 0, 255, 0, 0, { dx: -4, dy: 0, startedAt: 1000, token: 1, durationMs: 100 }); const record = readPresentation(encodePresentation(image.draws[0]), 0); - assert.equal(record.selection.animation.durationMs, 100); + assert.ok(record.selection.displayList); const screen = new Uint8Array(16), output = new Uint8Array(16); paintPresentation(output, screen, 8, 2, record.selection); assert.equal(output[3], 240, 'halfway at 50 ms'); @@ -56,11 +56,11 @@ test('animated selection bridge carries relative start, elapsed time and stable image.drawMenuSelection(new GrayImage(2, 2, 255), 3, 2, 16, 48, 1, 2, { dx: -2, dy: -4, startedAt: 1000, token: 42, durationMs: MENU_HIGHLIGHT_DURATION_MS }); const bytes = encodePresentation(image.draws[0]); - assert.equal(bytes[0], 6); - assert.equal(bytes.length, 32); + assert.equal(bytes[0], 7); const record = readPresentation(bytes, 0); assert.equal(record.end, bytes.length); - assert.deepEqual(record.selection.animation, { dx: -2, dy: -4, startedAt: 1000, token: 42, durationMs: MENU_HIGHLIGHT_DURATION_MS }); + assert.deepEqual(record.selection.displayList.timeline, { startedAt: 1000, token: 42 }); + assert.equal(record.selection.displayList.calls[0].op, 8); assert.equal(record.selection.x, 3); assert.equal(record.selection.y, 2); } finally { Date.now = now; } diff --git a/tests/music-settings.test.cjs b/tests/music-settings.test.cjs index bffedd4d..248b8caf 100644 --- a/tests/music-settings.test.cjs +++ b/tests/music-settings.test.cjs @@ -137,7 +137,7 @@ test('Music settings keeps toggles and selection when a new app arrives and supp const { prefs } = store(); let refreshes = 0, closes = 0; const textwrap = load('app/graphics/textwrap.ts'); - const graphics = load('app/graphics/image.ts', () => textwrap); + const graphics = require('../.test-build/app/graphics/image.js'); const { BdfFont } = load('app/graphics/bdffont.ts', () => ({})); const font = BdfFont.parse(source('app/fonts/terminus/ter-u20n.bdf')); class RecordingImage extends graphics.GrayImage { diff --git a/tests/notification-filter.test.cjs b/tests/notification-filter.test.cjs index 81811ca5..07d2c57b 100644 --- a/tests/notification-filter.test.cjs +++ b/tests/notification-filter.test.cjs @@ -53,7 +53,7 @@ function ui(fontSize = 12) { let dismissals = 0; let closes = 0; const textwrap = load('app/graphics/textwrap.ts'); - const graphics = load('app/graphics/image.ts', () => textwrap); + const graphics = require('../.test-build/app/graphics/image.js'); const { BdfFont } = load('app/graphics/bdffont.ts', () => ({})); const font = BdfFont.parse(source(`app/fonts/terminus/ter-u${fontSize}n.bdf`)); class RecordingImage extends graphics.GrayImage {