diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt index 1df754f9..5b0167b2 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt @@ -108,6 +108,64 @@ class ImageFileLoader private constructor() { } } + /** Longest side decodeRgbaFromBytes decodes at; larger images are subsampled. */ + private const val MAX_RGBA_SIDE = 4096 + + /** + * Decode encoded image bytes (JPEG, PNG, WebP, GIF's first frame, BMP) + * to [widthLo, widthHi, heightLo, heightHi, R, G, B, A, ...] at native + * size, not premultiplied, so a transparent pixel keeps its colour; + * empty when undecodable. The EvenHub host converts these to grey the + * way the Even app does (app/apps/evenhub/image-pipeline.ts). + */ + @JvmStatic + fun decodeRgbaFromBytes(buffer: java.nio.ByteBuffer?): ByteArray { + if (buffer == null) return ByteArray(0) + try { + val source = buffer.duplicate() + source.rewind() + val data = ByteArray(source.remaining()) + source.get(data) + if (data.isEmpty()) return ByteArray(0) + val bounds = BitmapFactory.Options().apply { inJustDecodeBounds = true } + BitmapFactory.decodeByteArray(data, 0, data.size, bounds) + if (bounds.outWidth <= 0 || bounds.outHeight <= 0) return ByteArray(0) + var sample = 1 + while (maxOf(bounds.outWidth, bounds.outHeight) / sample > MAX_RGBA_SIDE) sample *= 2 + val options = BitmapFactory.Options().apply { + inPreferredConfig = Bitmap.Config.ARGB_8888 + inPremultiplied = false + inSampleSize = sample + } + val bitmap = BitmapFactory.decodeByteArray(data, 0, data.size, options) ?: return ByteArray(0) + val width = bitmap.width + val height = bitmap.height + val argb = IntArray(width * height) + bitmap.getPixels(argb, 0, width, 0, 0, width, height) + bitmap.recycle() + val packet = ByteArray(4 + argb.size * 4) + packet[0] = (width and 0xff).toByte() + packet[1] = ((width shr 8) and 0xff).toByte() + packet[2] = (height and 0xff).toByte() + packet[3] = ((height shr 8) and 0xff).toByte() + for (i in argb.indices) { + val pixel = argb[i] + val o = 4 + i * 4 + packet[o] = (pixel shr 16).toByte() + packet[o + 1] = (pixel shr 8).toByte() + packet[o + 2] = pixel.toByte() + packet[o + 3] = (pixel ushr 24).toByte() + } + return packet + } catch (e: Exception) { + Log.w(TAG, "RGBA image decode failed", e) + return ByteArray(0) + } catch (e: OutOfMemoryError) { + Log.w(TAG, "RGBA image decode failed", e) + return ByteArray(0) + } + } + /** * Scale a bitmap to fit within maxWidth x maxHeight (preserving aspect, * never upscaling) and convert it to the grayscale packet format. diff --git a/App_Resources/iOS/src/FaceclawGraphics.h b/App_Resources/iOS/src/FaceclawGraphics.h index 2f71ddcd..f948c1e3 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.h +++ b/App_Resources/iOS/src/FaceclawGraphics.h @@ -14,4 +14,10 @@ + (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize; /** bgraPacket of encoded image bytes (JPEG, PNG, ...); nil when undecodable. */ + (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize; +/** + * [widthLo, widthHi, heightLo, heightHi, RGBA...] of encoded image bytes at native size + * (at most 4096 px a side), first frame only; nil when undecodable. CoreGraphics + * premultiplies, so transparent pixels come out black. For the EvenHub image pipeline. + */ ++ (NSData *)rgbaPacketFromImageData:(NSData *)data; @end diff --git a/App_Resources/iOS/src/FaceclawGraphics.m b/App_Resources/iOS/src/FaceclawGraphics.m index cd8f3f16..fbc7fb9e 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.m +++ b/App_Resources/iOS/src/FaceclawGraphics.m @@ -158,4 +158,26 @@ CGImageRelease(image); return packet; } ++ (NSData *)rgbaPacketFromImageData:(NSData *)data { + if (!data.length) return nil; + CGImageSourceRef source = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL); + if (!source) return nil; + CGImageRef image = CGImageSourceCreateImageAtIndex(source, 0, NULL); + CFRelease(source); + if (!image) return nil; + size_t width = CGImageGetWidth(image), height = CGImageGetHeight(image); + if (width == 0 || height == 0 || width > 4096 || height > 4096) { CGImageRelease(image); return nil; } + uint8_t dimensions[] = { width & 255, (width >> 8) & 255, height & 255, (height >> 8) & 255 }; + NSMutableData *packet = [NSMutableData dataWithBytes:dimensions length:4]; + [packet increaseLengthBy:width * height * 4]; + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate((uint8_t *)packet.mutableBytes + 4, width, height, 8, width * 4, space, + kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast); + CGColorSpaceRelease(space); + if (!context) { CGImageRelease(image); return nil; } + CGContextDrawImage(context, CGRectMake(0, 0, width, height), image); + CGContextRelease(context); + CGImageRelease(image); + return packet; +} @end diff --git a/app/apps/evenhub/compositor.ts b/app/apps/evenhub/compositor.ts index 0d86f8e0..7886b5b2 100644 --- a/app/apps/evenhub/compositor.ts +++ b/app/apps/evenhub/compositor.ts @@ -7,15 +7,22 @@ * match what apps expect. Extended layouts may pick other fonts per container * (fonts.ts), which wrap with Faceclaw's own text wrapping. * + * Containers stack in stock's order (paintOrder): without zOrderIndex, lists, + * then texts, then images, so an image container covers text beneath it. Text + * containers are transparent windows that clip their text and scroll it. + * * Known deviations from stock, acceptable for now: * - List rendering is a plain vertical list. A non-zero itemWidth sets the * selection outline's width and truncates long items to fit, as on stock; * whether stock also lays narrow items out side by side is unknown. * - Lists scroll with our menus' animation and bounce (see list-menu.ts). + * - Text scrolls in jumps; stock eases each 28px step over 200ms and + * rubber-bands at the ends. */ import { GrayImage } from "../../graphics/image"; import { containerFace } from "./fonts"; import { + borderGrey, paintOrder, type EvenHubImageContainer, type EvenHubListContainer, @@ -30,35 +37,122 @@ export const EVENHUB_SCREEN_HEIGHT = 288; const TEXT_WHITE = 255; /** - * SDK 0.0.14 textColor levels 0..4 as grey values. Level 4 is the default and - * the only one apps got before 0.0.14, so it must stay exactly TEXT_WHITE. - * Level 0 is the firmware's "fully dim" step, which over the black page - * background means the text is effectively not shown. + * SDK 0.0.14 textColor levels 0..4 as the greys stock draws them (measured + * in the firmware emulator): level 0 is dim, not off. A missing or + * out-of-range level draws white, the only colour before 0.0.14. */ -const TEXT_BRIGHTNESS_GREYS = [0, 64, 128, 191, 255]; +const TEXT_BRIGHTNESS_GREYS = [51, 102, 153, 204, 255]; function textGrey(level: number | undefined): number { return level === undefined ? TEXT_WHITE : (TEXT_BRIGHTNESS_GREYS[level] ?? TEXT_WHITE); } -/** Chars the font lacks are skipped silently on stock (no tofu); EvenHubFont - * skips unknown glyphs, matching that. */ -function paintTextContainer(image: GrayImage, container: EvenHubTextContainer): void { - // Only the text dims: textColor is a text-brightness level, and stock draws - // the container's border at full brightness regardless. - paintBorder(image, container); +/** + * The scrollbar stock draws on any text container whose text overflows + * (LV_SCROLLBAR_MODE_AUTO), whether or not it captures input: 4px wide, 10px + * in from the right edge, in a track 6px in from top and bottom, with its + * corner pixels cut. Measured in the firmware emulator. + */ +const SCROLLBAR_GREY = 187; +const SCROLLBAR_WIDTH = 4; +const SCROLLBAR_RIGHT_INSET = 10; +const SCROLLBAR_TRACK_INSET = 6; + +type Rect = { x: number; y: number; width: number; height: number }; + +/** A text container's wrapped lines and the geometry its scrolling works in. */ +export type TextLayout = { + lines: string[]; + lineHeight: number; + /** Left edge and unscrolled top of the text. */ + contentX: number; + contentY: number; + /** Height of the window the text scrolls in (inside border and padding). */ + viewportHeight: number; + /** Largest scrollY: how far the text overflows that window. */ + maxScroll: number; +}; + +/** + * Stock lays text out at x + border + padding and wraps it at width - 2 * + * padding: the label is sized ignoring the border, so a line can run into + * the right border, where it is clipped. + */ +export function layoutText(container: EvenHubTextContainer): TextLayout { const inset = container.borderWidth + container.paddingLength; const face = containerFace(container.font); - const lines = face.wrap(container.content, Math.max(1, container.width - 2 * inset)); - for (let i = 0; i < lines.length; i++) { - face.drawText(image, container.x + inset, container.y + inset + i * face.lineHeight, lines[i]!, textGrey(container.textColor)); + const lines = face.wrap(container.content, Math.max(1, container.width - 2 * container.paddingLength)); + const viewportHeight = Math.max(0, container.height - 2 * inset); + return { + lines, + lineHeight: face.lineHeight, + contentX: container.x + inset, + contentY: container.y + inset, + viewportHeight, + maxScroll: Math.max(0, lines.length * face.lineHeight - viewportHeight), + }; +} + +/** + * Text is clipped at the border's inner edge (at the container's edge when it + * has no border). Lines wholly inside stay deferred firmware-text draws; a + * line the edge cuts is rasterized and blitted clipped. Chars the font lacks + * are skipped silently, as on stock (no tofu). + */ +function paintTextContainer(image: GrayImage, container: EvenHubTextContainer): void { + // Only the text dims: textColor is a text-brightness level; the border has its own color. + paintBorder(image, container); + const layout = layoutText(container); + const face = containerFace(container.font); + const grey = textGrey(container.textColor); + const border = container.borderWidth; + const clip: Rect = { + x: container.x + border, + y: container.y + border, + width: Math.max(0, container.width - 2 * border), + height: Math.max(0, container.height - 2 * border), + }; + for (let i = 0; i < layout.lines.length; i++) { + const y = layout.contentY + i * layout.lineHeight - container.scrollY; + const bottom = y + layout.lineHeight; + if (bottom <= clip.y || y >= clip.y + clip.height) continue; + const line = layout.lines[i]!; + if (y >= clip.y && bottom <= clip.y + clip.height && layout.contentX + face.measureLine(line) <= clip.x + clip.width) { + face.drawText(image, layout.contentX, y, line, grey); + continue; + } + const strip = new GrayImage(container.width, layout.lineHeight, 0); + face.drawText(strip, layout.contentX - container.x, 0, line, grey); + const visibleTop = Math.max(y, clip.y); + image.bitBlt(strip.withDrawsBaked(), clip.x, visibleTop, { + sx: clip.x - container.x, + sy: visibleTop - y, + width: clip.width, + height: Math.min(bottom, clip.y + clip.height) - visibleTop, + transparentZero: true, + }); + } + if (layout.maxScroll > 0) paintScrollbar(image, container, layout); +} + +function paintScrollbar(image: GrayImage, container: EvenHubTextContainer, layout: TextLayout): void { + const height = container.height; + const track = height - 2 * SCROLLBAR_TRACK_INSET; + const length = Math.floor((track * height) / (height + layout.maxScroll)); + if (track <= 0 || length <= 0) return; + const top = container.y + SCROLLBAR_TRACK_INSET + Math.floor(((track - length) * container.scrollY) / layout.maxScroll); + const left = container.x + container.width - SCROLLBAR_RIGHT_INSET; + for (let row = 0; row < length; row++) { + const end = row === 0 || row === length - 1; + image.fillRect(end ? left + 1 : left, top + row, end ? SCROLLBAR_WIDTH - 2 : SCROLLBAR_WIDTH, 1, SCROLLBAR_GREY); } } function paintBorder( image: GrayImage, - container: { x: number; y: number; width: number; height: number; borderWidth: number; borderRadius: number }, + container: { x: number; y: number; width: number; height: number; borderWidth: number; borderColor: number; borderRadius: number }, ): void { + const grey = borderGrey(container.borderColor); for (let i = 0; i < container.borderWidth; i++) { if (container.borderRadius > 0) { image.drawRoundedRect( @@ -66,28 +160,28 @@ function paintBorder( container.y + i, container.width - 2 * i, container.height - 2 * i, - TEXT_WHITE, + grey, Math.max(0, container.borderRadius - i), ); } else { - image.drawRect(container.x + i, container.y + i, container.width - 2 * i, container.height - 2 * i, TEXT_WHITE); + image.drawRect(container.x + i, container.y + i, container.width - 2 * i, container.height - 2 * i, grey); } } } +/** + * An opaque bitmap (black pixels cover what's beneath). Updates are converted + * to exactly the container's size (image-pipeline.ts); pixels an extended + * layout carried over from a differently sized container are clipped. + */ function paintImageContainer(image: GrayImage, container: EvenHubImageContainer): void { if (!container.pixels || container.pixelsWidth <= 0 || container.pixelsHeight <= 0) return; const source = new GrayImage(container.pixelsWidth, container.pixelsHeight, 0); source.pixels.set(container.pixels); - // Stock quirk: raw data smaller than the container tiles/repeats to fill it. - for (let ty = 0; ty < container.height; ty += container.pixelsHeight) { - for (let tx = 0; tx < container.width; tx += container.pixelsWidth) { - image.bitBlt(source, container.x + tx, container.y + ty, { - width: Math.min(container.pixelsWidth, container.width - tx), - height: Math.min(container.pixelsHeight, container.height - ty), - }); - } - } + image.bitBlt(source, container.x, container.y, { + width: Math.min(container.pixelsWidth, container.width), + height: Math.min(container.pixelsHeight, container.height), + }); } function paintListContainer(image: GrayImage, container: EvenHubListContainer, focused: boolean): void { @@ -101,14 +195,17 @@ export function compositePage(page: EvenHubPage | null, size: { width: number; h if (!page) return image; for (const container of paintOrder(page.containers)) { switch (container.kind) { - case "image": + case "image": { + if (!container.pixels) break; // Nothing is drawn before the first update. // Text painted so far is deferred draws (firmware-text runs), which - // would otherwise render above this container's raster. Stock puts a - // later image container OVER earlier text, so bake the pending draws - // first; text painted after this container stays deferred (and above). - image.bakeDeferredDrawsInPlace(); + // would otherwise render above this container's raster. The image is + // opaque and covers what's under it, so bake the draws it overlaps + // first; the others, and text painted later, stay deferred. + const box: Rect = { x: container.x, y: container.y, width: container.width, height: container.height }; + image.bakeDeferredDrawsWithin(box); paintImageContainer(image, container); break; + } case "text": paintTextContainer(image, container); break; diff --git a/app/apps/evenhub/containers.ts b/app/apps/evenhub/containers.ts index 4096391a..a2bb79d7 100644 --- a/app/apps/evenhub/containers.ts +++ b/app/apps/evenhub/containers.ts @@ -6,6 +6,7 @@ * reads go through loose key matching (the SDK's pickLoose behavior). */ import { mergeFontSpecs, parseFontSpec, type EvenHubFontSpec } from "./font-spec"; +import { inStockImageRange } from "./image-pipeline"; export type EvenHubTextContainer = { kind: "text"; @@ -16,6 +17,8 @@ export type EvenHubTextContainer = { width: number; height: number; borderWidth: number; + /** Firmware border color index 0..16; see borderGrey. */ + borderColor: number; borderRadius: number; paddingLength: number; isEventCapture: boolean; @@ -27,6 +30,11 @@ export type EvenHubTextContainer = { textColor: number | undefined; /** Extended-layout font (merged with the layout's default); undefined is the stock font. */ font: EvenHubFontSpec | undefined; + /** + * How far the text is scrolled up, in pixels. Host-local like a list's + * selection: stock scrolls an overflowing capture container itself. + */ + scrollY: number; }; export type EvenHubImageContainer = { @@ -54,6 +62,8 @@ export type EvenHubListContainer = { width: number; height: number; borderWidth: number; + /** Firmware border color index 0..16; see borderGrey. */ + borderColor: number; borderRadius: number; paddingLength: number; isEventCapture: boolean; @@ -80,7 +90,7 @@ export type EvenHubMenuItem = { }; export type EvenHubPage = { - /** In declaration order (lists, images, texts); z-sorted at paint time. */ + /** Lists, then texts, then images, each in declaration order (stock's container list order); z-sorted at paint time. */ containers: EvenHubContainer[]; /** * The page's contextual-menu entries (SDK 0.0.14 `menuObject`). Empty when @@ -92,6 +102,26 @@ export type EvenHubPage = { /** Stock draws at most a 5px border; wider requests are clamped. */ const MAX_BORDER_WIDTH = 5; +/** + * Stock firmware limits for list and text containers (common_text_create / + * common_list_create clamp to these), from the evenRealities-openCFW + * reconstruction. Extended layouts are exempt: they have their own canvas. + */ +const STOCK_SCREEN_WIDTH = 576; +const STOCK_SCREEN_HEIGHT = 288; +const STOCK_MAX_BORDER_COLOR = 16; +const STOCK_MAX_BORDER_RADIUS = 10; +const STOCK_MAX_PADDING = 32; +const STOCK_MAX_LIST_ITEMS = 20; + +/** + * The border color index's grey: indices 0..15 step by 0x11 (0 is black, so a + * border without a color is invisible) and 16 is white. + */ +export function borderGrey(index: number): number { + return index >= 0 && index < 16 ? index * 0x11 : 0xff; +} + /** Firmware brightness levels for text; 4 is the device default. */ const MIN_TEXT_BRIGHTNESS = 0; const MAX_TEXT_BRIGHTNESS = 4; @@ -154,6 +184,11 @@ function readBorderWidth(json: Record): number { return Math.max(0, Math.min(MAX_BORDER_WIDTH, readNumber(json, "borderWidth", 0))); } +/** A protobuf field, so an omitted borderColor is 0 (black). */ +function readBorderColor(json: Record): number { + return Math.max(0, readNumber(json, "borderColor", 0)); +} + /** A container's font: its own `font` over the layout-wide default. */ function readFont(json: Record, layoutFont: EvenHubFontSpec | undefined): EvenHubFontSpec | undefined { return mergeFontSpecs(layoutFont, parseFontSpec(pickLoose(json, "font"))); @@ -169,6 +204,7 @@ function parseTextContainer(json: Record, layoutFont: EvenHubFo width: readNumber(json, "width", 0), height: readNumber(json, "height", 0), borderWidth: readBorderWidth(json), + borderColor: readBorderColor(json), borderRadius: readNumber(json, "borderRadius", 0), paddingLength: readNumber(json, "paddingLength", 0), isEventCapture: readFlag(json, "isEventCapture"), @@ -177,6 +213,7 @@ function parseTextContainer(json: Record, layoutFont: EvenHubFo content: readString(json, "content", ""), textColor: readTextBrightness(json), font: readFont(json, layoutFont), + scrollY: 0, }; } @@ -262,6 +299,7 @@ function parseListContainer(json: Record, layoutFont: EvenHubFo width: readNumber(json, "width", 0), height: readNumber(json, "height", 0), borderWidth: readBorderWidth(json), + borderColor: readBorderColor(json), borderRadius: readNumber(json, "borderRadius", 0), paddingLength: readNumber(json, "paddingLength", 0), isEventCapture: readFlag(json, "isEventCapture"), @@ -277,9 +315,8 @@ function parseListContainer(json: Record, layoutFont: EvenHubFo /** * Parse a CreateStartUpPageContainer / RebuildPageContainer payload into a - * page. Containers keep declaration order within and across the three arrays - * (lists, then images, then texts — texts last so info panels and capture - * containers sit above image tiles when no zOrderIndex is given). + * page. Containers are kept in stock's container-list order: lists, then + * texts, then images, each in declaration order (evenhub_build_container_list). * * The whole page is replaced on rebuild, which is also what gives an omitted * menuObject its documented meaning: the previous contextual menu goes away. @@ -292,42 +329,100 @@ export function parsePage(data: Record): EvenHubPage { if (Array.isArray(lists)) { for (const item of lists) containers.push(parseListContainer(asRecord(item), layoutFont)); } - const images = pickLoose(data, "imageObject"); - if (Array.isArray(images)) { - for (const item of images) containers.push(parseImageContainer(asRecord(item))); - } const texts = pickLoose(data, "textObject"); if (Array.isArray(texts)) { for (const item of texts) containers.push(parseTextContainer(asRecord(item), layoutFont)); } + const images = pickLoose(data, "imageObject"); + if (Array.isArray(images)) { + for (const item of images) containers.push(parseImageContainer(asRecord(item))); + } return { containers, menuItems: parseMenuItems(data) }; } /** - * Bottom-to-top paint order for a page's containers. - * - * With zOrderIndex, larger values render closer to the front (painted later). - * Stock honors it only when every container has one (the SDK enforces that - * app-side); we sort whenever any container has one, treating a missing index - * as 0. - * - * Without it, image containers sit below lists and text, keeping declaration - * order within each group. That is our choice, and it may not match stock: - * the evenRealities-openCFW firmware reconstruction builds the container list - * as lists, then texts, then images, which would put images on top. Not yet - * checked on hardware; layout-debug.ts logs where the two orders differ. + * Clamp a stock page's containers the way the firmware does when it creates + * them: positions and sizes to the 576x288 screen (images keep their size, + * which a separate check rejects instead), border color, radius and padding to + * their maxima, and lists to 20 items. Not for extended layouts. */ -export function paintOrder(containers: readonly EvenHubContainer[]): EvenHubContainer[] { - const anyZ = containers.some((c) => c.zOrderIndex !== undefined); - if (anyZ) return [...containers].sort((a, b) => (a.zOrderIndex ?? 0) - (b.zOrderIndex ?? 0)); - const images = containers.filter((c) => c.kind === "image"); - const rest = containers.filter((c) => c.kind !== "image"); - return [...images, ...rest]; +export function clampToStockLimits(page: EvenHubPage): void { + for (const c of page.containers) { + c.x = Math.min(c.x, STOCK_SCREEN_WIDTH); + c.y = Math.min(c.y, STOCK_SCREEN_HEIGHT); + if (c.kind === "image") continue; + c.width = Math.min(c.width, STOCK_SCREEN_WIDTH); + c.height = Math.min(c.height, STOCK_SCREEN_HEIGHT); + c.borderColor = Math.min(c.borderColor, STOCK_MAX_BORDER_COLOR); + c.borderRadius = Math.min(c.borderRadius, STOCK_MAX_BORDER_RADIUS); + c.paddingLength = Math.min(c.paddingLength, STOCK_MAX_PADDING); + if (c.kind === "list" && c.itemNames.length > STOCK_MAX_LIST_ITEMS) { + c.itemNames = c.itemNames.slice(0, STOCK_MAX_LIST_ITEMS); + } + } } -/** The container that owns gesture events (exactly one per valid page). */ -export function eventCaptureContainer(page: EvenHubPage): EvenHubContainer | undefined { - return page.containers.find( - (container) => container.kind !== "image" && container.isEventCapture, - ); +/** The firmware's fixed protobuf arrays: more containers than this fail to decode. */ +const STOCK_MAX_LISTS = 4; +const STOCK_MAX_TEXTS = 8; +const STOCK_MAX_IMAGES = 4; + +/** EhStartUpPageCreateResult values for a page the firmware can't build. */ +export const PAGE_CREATE_INVALID = 1; +export const PAGE_CREATE_OUT_OF_MEMORY = 3; + +/** + * Why the stock firmware would fail to build this page, with the create result + * the app gets, or null when it builds. Too many containers overflow the + * firmware's protobuf arrays, so decoding fails (it replies with the code the + * phone app maps to outOfMemory). An image container outside 20..288 x 20..144 + * fails to create (invalid container). + */ +export function stockPageFailure(page: EvenHubPage): { code: number; reason: string } | null { + const count = (kind: EvenHubContainer["kind"]) => page.containers.filter((c) => c.kind === kind).length; + const [lists, texts, images] = [count("list"), count("text"), count("image")]; + if (lists > STOCK_MAX_LISTS || texts > STOCK_MAX_TEXTS || images > STOCK_MAX_IMAGES) { + return { + code: PAGE_CREATE_OUT_OF_MEMORY, + reason: `${lists} list, ${texts} text and ${images} image containers (at most ${STOCK_MAX_LISTS}, ${STOCK_MAX_TEXTS} and ${STOCK_MAX_IMAGES})`, + }; + } + for (const c of page.containers) { + if (c.kind === "image" && !inStockImageRange(c.width, c.height)) { + return { code: PAGE_CREATE_INVALID, reason: `image container ${JSON.stringify(c.name)} is ${c.width}x${c.height}, outside 20..288 x 20..144` }; + } + } + return null; +} + +/** + * Bottom-to-top paint order, as stock stacks its containers: the container + * list order (lists, texts, images), stably sorted by zOrderIndex (larger is + * closer to the front) only when every container has one and there are at + * least two. So without z-order, images are drawn over text. + */ +export function paintOrder(containers: readonly EvenHubContainer[]): EvenHubContainer[] { + const allHaveZ = containers.every((c) => c.zOrderIndex !== undefined); + if (!allHaveZ || containers.length < 2) return [...containers]; + return [...containers].sort((a, b) => (a.zOrderIndex ?? 0) - (b.zOrderIndex ?? 0)); +} + +/** + * The container that owns gesture events. Stock binds the first list or text + * container with isEventCapture in paint order and ignores any later ones. + */ +export function eventCaptureContainer(page: EvenHubPage): EvenHubTextContainer | EvenHubListContainer | undefined { + for (const c of paintOrder(page.containers)) { + if (c.kind !== "image" && c.isEventCapture) return c; + } + return undefined; +} + +/** + * The container stock resolves an id to: the first one with that id in paint + * order, whatever its kind, so an update aimed at a later container that + * shares the id never reaches it. + */ +export function containerById(page: EvenHubPage | null, id: number): EvenHubContainer | undefined { + return page ? paintOrder(page.containers).find((c) => c.id === id) : undefined; } diff --git a/app/apps/evenhub/image-pipeline.ts b/app/apps/evenhub/image-pipeline.ts new file mode 100644 index 00000000..3f5345f8 --- /dev/null +++ b/app/apps/evenhub/image-pipeline.ts @@ -0,0 +1,247 @@ +/** + * updateImageRawData the way stock turns an app's imageData into the 4-bit + * bitmap the glasses show. The Even phone app does the conversion + * (EvenHubAppService.updateImageRawData → _prepare / _prepareLegacyRaw, read + * from its blutter decompilation); the glasses firmware then shows the result + * only if it is exactly the container's size (common_image_container_set_data). + * + * - imageData must be a JSON array of integers; a base64 string or an object + * (a typed array that skipped the SDK's toJson) fails as sendFailed. + * - PNG, JPEG, WebP, GIF and BMP are decoded, scaled to cover the container + * (aspect-fill, centre crop, nearest neighbour) and converted to grey with + * alpha ignored. + * - Anything else is raw pixels, accepted only with compressMode 2 (which the + * SDK sends since 0.0.12) and exactly the container's size: one byte per + * pixel (values up to 15 are already 4-bit levels) or 4-bit packed rows. + * + * Native-free so it runs under Node; JPEG/WebP/GIF decoding is injected. + */ +const UPNG = require("upng-js"); + +/** EhImageUpdateResult, as the SDK reads the host's integer. */ +export const IMAGE_SUCCESS = 0; +export const IMAGE_EXCEPTION = 1; +export const IMAGE_SIZE_INVALID = 2; +export const IMAGE_TO_GRAY4_FAILED = 3; +export const IMAGE_SEND_FAILED = 4; + +/** Stock image containers are 20..288 wide and 20..144 high. */ +export const STOCK_IMAGE_MIN_SIZE = 20; +export const STOCK_IMAGE_MAX_WIDTH = 288; +export const STOCK_IMAGE_MAX_HEIGHT = 144; + +/** The compressMode the SDK sends, and the phone app requires for raw pixels. */ +const RAW_COMPRESS_MODE = 2; + +export function inStockImageRange(width: number, height: number): boolean { + return width >= STOCK_IMAGE_MIN_SIZE && width <= STOCK_IMAGE_MAX_WIDTH && + height >= STOCK_IMAGE_MIN_SIZE && height <= STOCK_IMAGE_MAX_HEIGHT; +} + +export type ImageFormat = "png" | "jpeg" | "webp" | "gif" | "bmp" | "raw"; + +/** Format by magic bytes, as the phone app's EvenHubImageFormatDetector. */ +export function detectImageFormat(b: Uint8Array): ImageFormat { + const at = (i: number, ...values: number[]) => values.every((v, j) => b[i + j] === v); + if (at(0, 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a)) return "png"; + if (at(0, 0xff, 0xd8, 0xff)) return "jpeg"; + if (at(0, 0x52, 0x49, 0x46, 0x46) && at(8, 0x57, 0x45, 0x42, 0x50)) return "webp"; + if (at(0, 0x47, 0x49, 0x46, 0x38, 0x37, 0x61) || at(0, 0x47, 0x49, 0x46, 0x38, 0x39, 0x61)) return "gif"; + if (at(0, 0x42, 0x4d)) return "bmp"; + return "raw"; +} + +/** Decoded pixels, RGBA, not premultiplied. */ +export type RgbaImage = { width: number; height: number; rgba: Uint8Array }; + +export type NativeImageDecoder = (bytes: Uint8Array) => RgbaImage | null; + +/** 8-bit grey from RGB (alpha ignored), then the nearest of 16 levels. */ +export function grayLevel(r: number, g: number, b: number): number { + const g8 = Math.floor((299 * r + 587 * g + 114 * b + 500) / 1000); + return Math.min(15, Math.max(0, Math.floor((g8 * 15 + 127) / 255))); +} + +export function rgbaToLevels(image: RgbaImage): Uint8Array { + const levels = new Uint8Array(image.width * image.height); + for (let i = 0; i < levels.length; i++) { + const o = i * 4; + levels[i] = grayLevel(image.rgba[o]!, image.rgba[o + 1]!, image.rgba[o + 2]!); + } + return levels; +} + +/** + * Scale to exactly width x height covering it (aspect-fill), cropping the + * overflow equally on both sides, sampling the nearest source pixel centre. + */ +export function resizeRgbaCover(image: RgbaImage, width: number, height: number): RgbaImage { + const scale = Math.max(width / image.width, height / image.height); + const offX = (image.width - width / scale) / 2; + const offY = (image.height - height / scale) / 2; + const rgba = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) { + const sy = Math.min(image.height - 1, Math.max(0, Math.floor(offY + (y + 0.5) / scale))); + for (let x = 0; x < width; x++) { + const sx = Math.min(image.width - 1, Math.max(0, Math.floor(offX + (x + 0.5) / scale))); + const from = (sy * image.width + sx) * 4; + rgba.set(image.rgba.subarray(from, from + 4), (y * width + x) * 4); + } + } + return { width, height, rgba }; +} + +function decodePng(bytes: Uint8Array): RgbaImage | null { + try { + const decoded = UPNG.decode(bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength)); + const frames = UPNG.toRGBA8(decoded) as ArrayBuffer[]; + if (!frames.length) return null; + return { width: decoded.width, height: decoded.height, rgba: new Uint8Array(frames[0]!) }; + } catch { + return null; + } +} + +/** + * Uncompressed Windows BMP ('BM', BI_RGB): 1/4/8-bit paletted and 24/32-bit, + * top-down or bottom-up. Null for other variants (the native decoder may still + * read them). EvenChess sends 1-bit BMPs for full-board refreshes. + */ +export function decodeBmp(bytes: Uint8Array): RgbaImage | null { + if (bytes.length < 54 || bytes[0] !== 0x42 || bytes[1] !== 0x4d) return null; + const u16 = (o: number) => bytes[o]! | (bytes[o + 1]! << 8); + const u32 = (o: number) => (bytes[o]! | (bytes[o + 1]! << 8) | (bytes[o + 2]! << 16) | (bytes[o + 3]! << 24)) >>> 0; + const i32 = (o: number) => bytes[o]! | (bytes[o + 1]! << 8) | (bytes[o + 2]! << 16) | (bytes[o + 3]! << 24); + const pixelOffset = u32(10); + const dibSize = u32(14); + const width = i32(18); + const heightRaw = i32(22); + const bitCount = u16(28); + if (u32(30) !== 0) return null; // only BI_RGB + if (width <= 0 || width > 4096) return null; + const topDown = heightRaw < 0; + const height = Math.abs(heightRaw); + if (height === 0 || height > 4096) return null; + if (![1, 4, 8, 24, 32].includes(bitCount)) return null; + + // Palette entries are 4 bytes, BGRx. + let palette: Uint8Array | null = null; + if (bitCount <= 8) { + let count = u32(46); + if (count === 0 || count > 1 << bitCount) count = 1 << bitCount; + palette = new Uint8Array(count * 3); + for (let i = 0; i < count; i++) { + const o = 14 + dibSize + i * 4; + if (o + 2 >= bytes.length) break; + palette.set([bytes[o + 2]!, bytes[o + 1]!, bytes[o]!], i * 3); + } + } + const stride = ((width * bitCount + 31) >> 5) << 2; + const rgba = new Uint8Array(width * height * 4); + for (let y = 0; y < height; y++) { + const row = pixelOffset + (topDown ? y : height - 1 - y) * stride; + if (row >= bytes.length) break; + for (let x = 0; x < width; x++) { + let r: number, g: number, b: number; + if (palette) { + let index: number; + if (bitCount === 1) index = ((bytes[row + (x >> 3)] ?? 0) >> (7 - (x & 7))) & 1; + else if (bitCount === 4) index = (x & 1) === 0 ? (bytes[row + (x >> 1)] ?? 0) >> 4 : (bytes[row + (x >> 1)] ?? 0) & 0x0f; + else index = bytes[row + x] ?? 0; + if (index * 3 >= palette.length) index = 0; + [r, g, b] = [palette[index * 3]!, palette[index * 3 + 1]!, palette[index * 3 + 2]!]; + } else { + const o = row + x * (bitCount / 8); + [r, g, b] = [bytes[o + 2] ?? 0, bytes[o + 1] ?? 0, bytes[o] ?? 0]; + } + rgba.set([r, g, b, 255], (y * width + x) * 4); + } + } + return { width, height, rgba }; +} + +/** imageData as bytes: a JSON array of integers only (Uint8List.fromList keeps the low 8 bits). */ +function imageDataBytes(raw: unknown): Uint8Array | null { + if (raw === undefined || raw === null) return new Uint8Array(0); + if (!Array.isArray(raw)) return null; + const out = new Uint8Array(raw.length); + for (let i = 0; i < raw.length; i++) { + const value: unknown = raw[i]; + if (typeof value !== "number" || !Number.isInteger(value)) return null; + out[i] = value & 0xff; + } + return out; +} + +export type PreparedImage = { + result: number; + /** 4-bit levels (0..15), row-major, when result is success. */ + levels: Uint8Array | null; + width: number; + height: number; + format: ImageFormat | null; + /** Why it failed, for the debug log. */ + reason: string; +}; + +export type ImageRequest = { + imageData: unknown; + compressMode: number; + /** width/height sent with the update; when both are positive they replace the container's. */ + requestWidth: number; + requestHeight: number; + /** The image container's size, looked up by id or name; null when none matched. */ + geometry: { width: number; height: number } | null; + decodeNative: NativeImageDecoder; + /** Stock pages limit image sizes; Faceclaw's extended layouts don't. */ + stockLimits: boolean; +}; + +/** Convert an update's imageData to 4-bit levels the way the phone app does. */ +export function prepareImageUpdate(request: ImageRequest): PreparedImage { + const fail = (result: number, reason: string, format: ImageFormat | null = null): PreparedImage => + ({ result, levels: null, width: 0, height: 0, format, reason }); + const bytes = imageDataBytes(request.imageData); + if (!bytes) return fail(IMAGE_SEND_FAILED, "imageData is not an array of integers"); + const inRange = (w: number, h: number) => (request.stockLimits ? inStockImageRange(w, h) : w > 0 && h > 0); + const [width, height] = request.requestWidth > 0 && request.requestHeight > 0 + ? [request.requestWidth, request.requestHeight] + : [request.geometry?.width ?? 0, request.geometry?.height ?? 0]; + + const format = detectImageFormat(bytes); + if (format === "raw") { + if (request.stockLimits && request.compressMode !== RAW_COMPRESS_MODE) { + return fail(IMAGE_EXCEPTION, `raw pixels need compressMode ${RAW_COMPRESS_MODE}`, format); + } + if (!inRange(width, height)) return fail(IMAGE_SIZE_INVALID, `raw size ${width}x${height} out of range`, format); + const levels = new Uint8Array(width * height); + const packedStride = Math.ceil(width / 2); + if (bytes.length === width * height) { + for (let i = 0; i < levels.length; i++) { + const v = bytes[i]!; + levels[i] = v <= 15 ? v : Math.floor((v * 15 + 127) / 255); + } + } else if (bytes.length === packedStride * height) { + for (let y = 0; y < height; y++) { + for (let x = 0; x < width; x++) { + const byte = bytes[y * packedStride + (x >> 1)]!; + levels[y * width + x] = (x & 1) === 0 ? byte >> 4 : byte & 0x0f; + } + } + } else { + return fail(IMAGE_EXCEPTION, `raw length ${bytes.length} fits neither ${width}x${height} 8-bit nor 4-bit`, format); + } + return { result: IMAGE_SUCCESS, levels, width, height, format, reason: "" }; + } + + let decoded = format === "png" ? decodePng(bytes) : format === "bmp" ? decodeBmp(bytes) : null; + decoded ??= request.decodeNative(bytes); + if (!decoded || decoded.width <= 0 || decoded.height <= 0) return fail(IMAGE_EXCEPTION, `${format} decode failed`, format); + if (inRange(width, height) && (decoded.width !== width || decoded.height !== height)) { + decoded = resizeRgbaCover(decoded, width, height); + } + if (!inRange(decoded.width, decoded.height)) { + return fail(IMAGE_SIZE_INVALID, `${decoded.width}x${decoded.height} out of range`, format); + } + return { result: IMAGE_SUCCESS, levels: rgbaToLevels(decoded), width: decoded.width, height: decoded.height, format, reason: "" }; +} diff --git a/app/apps/evenhub/layout-debug.ts b/app/apps/evenhub/layout-debug.ts index 5c228780..2653b394 100644 --- a/app/apps/evenhub/layout-debug.ts +++ b/app/apps/evenhub/layout-debug.ts @@ -2,50 +2,24 @@ * Debug-log descriptions of EvenHub page layouts and container updates. * session.ts sends them to logcat at debug level (tag FaceclawEvenHubLayout). * - * Lines starting "stock:" flag places where our emulation may draw something - * stock would not, or the reverse. The stock rules come from the - * evenRealities-openCFW reconstruction of firmware 2.2.6.10 - * (evenhub_ui_containers.c, evenhub_ui_page.c, common_image_container.c) and - * the community simulator's hardware notes. Nobody has checked them against - * captures of our own, so they are leads, not established facts. They also - * describe the glasses firmware only: the Even phone app sits between the SDK - * and the firmware and may validate or convert payloads first. + * "note:" lines point out page quirks that change what stock (and so we) do + * with it: shared ids, missing or extra capture containers, a wrong + * containerTotalNum. */ import { paintOrder, readOptionalNumber, type EvenHubContainer, - type EvenHubImageContainer, type EvenHubListContainer, type EvenHubPage, type EvenHubTextContainer, } from "./containers"; - -/** Stock image-container size limits; outside them the container, and so the whole page, fails to create. */ -const STOCK_IMAGE_MIN_SIZE = 20; -const STOCK_IMAGE_MAX_WIDTH = 288; -const STOCK_IMAGE_MAX_HEIGHT = 144; +import type { PreparedImage } from "./image-pipeline"; const CONTENT_PREVIEW_CHARS = 40; const RAW_STRING_CHARS = 120; const RAW_ARRAY_ITEMS = 32; -/** - * Stock's bottom-to-top container order: lists, then texts, then images (each - * in declaration order). The firmware sorts by zOrderIndex (stably) only when - * every container has one and there are at least two. - */ -export function stockPaintOrder(containers: readonly EvenHubContainer[]): EvenHubContainer[] { - const legacy = [ - ...containers.filter((c) => c.kind === "list"), - ...containers.filter((c) => c.kind === "text"), - ...containers.filter((c) => c.kind === "image"), - ]; - const allHaveZ = legacy.every((c) => c.zOrderIndex !== undefined); - if (!allHaveZ || legacy.length < 2) return legacy; - return legacy.sort((a, b) => (a.zOrderIndex ?? 0) - (b.zOrderIndex ?? 0)); -} - function label(c: EvenHubContainer): string { return `${c.kind}#${c.id} ${JSON.stringify(c.name)}`; } @@ -56,7 +30,7 @@ function preview(text: string): string { function decoration(c: EvenHubTextContainer | EvenHubListContainer): string { let out = ""; - if (c.borderWidth > 0) out += ` border=${c.borderWidth}${c.borderRadius > 0 ? `r${c.borderRadius}` : ""}`; + if (c.borderWidth > 0) out += ` border=${c.borderWidth}c${c.borderColor}${c.borderRadius > 0 ? `r${c.borderRadius}` : ""}`; if (c.paddingLength > 0) out += ` pad=${c.paddingLength}`; if (c.isEventCapture) out += " capture"; if (c.font) out += " font"; @@ -78,66 +52,22 @@ function describeContainer(c: EvenHubContainer): string { } } -function overlaps(a: EvenHubContainer, b: EvenHubContainer): boolean { - if (a.width <= 0 || a.height <= 0 || b.width <= 0 || b.height <= 0) return false; - return a.x < b.x + b.width && b.x < a.x + a.width && a.y < b.y + b.height && b.y < a.y + a.height; -} - -/** Ways a page would lay out or behave differently on stock firmware. */ -export function stockDifferences(page: EvenHubPage, declaredTotal?: number): string[] { +/** Quirks of a page that change how stock treats it. */ +export function pageNotes(page: EvenHubPage, declaredTotal?: number): string[] { const notes: string[] = []; - const containers = page.containers; - const ours = paintOrder(containers); - const stock = stockPaintOrder(containers); - - const withZ = containers.filter((c) => c.zOrderIndex !== undefined).length; - if (withZ > 0 && withZ < containers.length) { - notes.push(`zOrderIndex is set on ${withZ} of ${containers.length} containers: we sort by it (missing = 0), stock ignores it`); - } - if (ours.some((c, i) => c !== stock[i])) { - notes.push(`paint order differs; stock bottom to top is ${stock.map(label).join(", ")}`); - for (let i = 0; i < ours.length; i++) { - for (let j = i + 1; j < ours.length; j++) { - const below = ours[i]!; - const above = ours[j]!; - if (overlaps(below, above) && stock.indexOf(below) > stock.indexOf(above)) { - notes.push(`${label(below)} overlaps ${label(above)}: we draw ${label(above)} on top, stock draws ${label(below)} on top`); - } - } - } - } - - for (const c of containers) { - if (c.kind !== "image") continue; - if (c.width < STOCK_IMAGE_MIN_SIZE || c.height < STOCK_IMAGE_MIN_SIZE || c.width > STOCK_IMAGE_MAX_WIDTH || c.height > STOCK_IMAGE_MAX_HEIGHT) { - notes.push( - `${label(c)} is ${c.width}x${c.height}; stock fails the whole page for an image container outside ` + - `${STOCK_IMAGE_MIN_SIZE}-${STOCK_IMAGE_MAX_WIDTH} x ${STOCK_IMAGE_MIN_SIZE}-${STOCK_IMAGE_MAX_HEIGHT}`, - ); - } - } - - // The firmware finds a container by id with a first-match walk of its - // (stock-ordered) list, for image updates and events alike. + const ordered = paintOrder(page.containers); const byId = new Map(); - for (const c of stock) byId.set(c.id, [...(byId.get(c.id) ?? []), c]); + for (const c of ordered) byId.set(c.id, [...(byId.get(c.id) ?? []), c]); byId.forEach((shared, id) => { if (shared.length > 1) { - notes.push(`id ${id} is shared by ${shared.map(label).join(", ")}; stock resolves it to ${label(shared[0]!)} only`); + notes.push(`id ${id} is shared by ${shared.map(label).join(", ")}; updates and events resolve to ${label(shared[0]!)} only`); } }); - - const captures = stock.filter((c) => c.kind !== "image" && c.isEventCapture); - if (captures.length !== 1) { - notes.push( - captures.length === 0 - ? "no event-capture container (stock requires exactly one)" - : `${captures.length} event-capture containers; stock binds only the first, ${label(captures[0]!)}`, - ); - } - - if (declaredTotal !== undefined && declaredTotal !== containers.length) { - notes.push(`containerTotalNum is ${declaredTotal} but ${containers.length} containers were sent`); + const captures = ordered.filter((c) => c.kind !== "image" && c.isEventCapture); + if (captures.length === 0) notes.push("no event-capture container, so the app gets no input"); + else if (captures.length > 1) notes.push(`${captures.length} event-capture containers; only the first, ${label(captures[0]!)}, gets input`); + if (declaredTotal !== undefined && declaredTotal !== page.containers.length) { + notes.push(`containerTotalNum is ${declaredTotal} but ${page.containers.length} containers were sent (nothing checks it)`); } return notes; } @@ -159,7 +89,7 @@ function rawJson(data: Record): string { } } -/** A create/rebuild/extension layout: one line per container in our paint order, then stock differences, then the raw payload. */ +/** A create/rebuild/extension layout: one line per container in paint order, then notes, then the raw payload. */ export function describePage(method: string, page: EvenHubPage, data: Record): string[] { const containers = page.containers; const count = (kind: EvenHubContainer["kind"]) => containers.filter((c) => c.kind === kind).length; @@ -168,10 +98,10 @@ export function describePage(method: string, page: EvenHubPage, data: Record 0 ? `, ${page.menuItems.length} menu items` : ""; const lines = [ `${method}: ${containers.length} containers (${count("list")} list, ${count("text")} text, ${count("image")} image)` + - `${total}${menu}; we paint bottom to top:`, + `${total}${menu}; painted bottom to top:`, ]; paintOrder(containers).forEach((c, i) => lines.push(` ${i}: ${describeContainer(c)}`)); - for (const note of stockDifferences(page, declaredTotal)) lines.push(` stock: ${note}`); + for (const note of pageNotes(page, declaredTotal)) lines.push(` note: ${note}`); lines.push(` raw: ${rawJson(data)}`); return lines; } @@ -184,75 +114,55 @@ function payloadKind(raw: unknown): string { return typeof raw; } -function imageFormat(bytes: Uint8Array, container: EvenHubImageContainer): string { - if (bytes.length > 4 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) return "PNG"; - // BITMAPINFOHEADER biBitCount: 14-byte file header + 14. - if (bytes.length > 30 && bytes[0] === 0x42 && bytes[1] === 0x4d) return `BMP ${bytes[28]! | (bytes[29]! << 8)}bpp`; - const area = container.width * container.height; - if (bytes.length === area) return "raw8"; - if (bytes.length === Math.ceil(area / 2)) return "raw4"; - const head = Array.from(bytes.subarray(0, 4), (b) => b.toString(16).padStart(2, "0")).join(" "); - return `unrecognized (${bytes.length} bytes, starts ${head})`; -} - -/** Lit-pixel count and bounding box, so a blank "hide this" image is recognizable. */ -function litSummary(decoded: { pixels: Uint8Array; width: number; height: number }): string { +/** Lit-pixel count and bounding box of 4-bit levels, so a blank "hide this" image is recognizable. */ +function litSummary(levels: Uint8Array, width: number): string { let lit = 0; let max = 0; - let x0 = decoded.width; - let y0 = decoded.height; + let x0 = width; + let y0 = Infinity; let x1 = -1; let y1 = -1; - for (let i = 0; i < decoded.pixels.length; i++) { - const value = decoded.pixels[i]!; - if (value === 0) continue; + for (let i = 0; i < levels.length; i++) { + const level = levels[i]!; + if (level === 0) continue; lit++; - if (value > max) max = value; - const x = i % decoded.width; - const y = Math.floor(i / decoded.width); + if (level > max) max = level; + const x = i % width; + const y = Math.floor(i / width); if (x < x0) x0 = x; if (x > x1) x1 = x; if (y < y0) y0 = y; if (y > y1) y1 = y; } - return lit === 0 ? "blank" : `lit=${lit}px max=${max} bbox=${x0},${y0}-${x1},${y1}`; + return lit === 0 ? "blank" : `lit=${lit}px max=${max}/15 bbox=${x0},${y0}-${x1},${y1}`; } export type ImageUpdateDebug = { - page: EvenHubPage | null; sentId: number; sentName: string; - container: EvenHubImageContainer | null; raw: unknown; - bytes: Uint8Array | null; - decoded: { pixels: Uint8Array; width: number; height: number } | null; - /** BleEhSendImageResult: 0 success, 1 imageException, 2 sizeInvalid, 3 sendFailed. */ + /** The container the id resolves to. */ + target: EvenHubContainer | undefined; + prepared: PreparedImage; + /** Whether the container now shows the update. */ + shown: boolean; + /** EhImageUpdateResult returned to the app. */ result: number; }; export function describeImageUpdate(u: ImageUpdateDebug): string[] { - const parts = [`updateImageRawData ${u.container ? label(u.container) : `id=${u.sentId} ${JSON.stringify(u.sentName)}`}`]; - if (u.container && u.sentName !== "" && u.sentName !== u.container.name) parts.push(`(sent name ${JSON.stringify(u.sentName)})`); + const parts = [`updateImageRawData id=${u.sentId} ${JSON.stringify(u.sentName)}`]; + parts.push(`-> ${u.target ? label(u.target) : "no container"}`); parts.push(`payload=${payloadKind(u.raw)}`); - if (u.bytes && u.container) parts.push(`format=${imageFormat(u.bytes, u.container)}`); - if (u.decoded) parts.push(`decoded=${u.decoded.width}x${u.decoded.height} ${litSummary(u.decoded)}`); - parts.push(`result=${u.result}`); + if (u.prepared.format) parts.push(`format=${u.prepared.format}`); + if (u.prepared.levels) parts.push(`${u.prepared.width}x${u.prepared.height} ${litSummary(u.prepared.levels, u.prepared.width)}`); + parts.push(`result=${u.result}`, u.shown ? "shown" : "not shown"); const lines = [parts.join(" ")]; - - if (!u.container) lines.push(` no image container with id ${u.sentId} on this page`); - else if (u.result !== 0) lines.push(" rejected; the container keeps its previous pixels"); - if (typeof u.raw === "string") { - lines.push(" stock: real hardware reportedly ignores a base64-string imageData (it needs bytes); we decode and draw it"); - } - if (u.container && u.decoded && (u.decoded.width !== u.container.width || u.decoded.height !== u.container.height)) { - lines.push( - ` stock: the firmware only accepts a bitmap exactly the container's size (${u.container.width}x${u.container.height}); ` + - "we tile or crop it to fit", - ); - } - const stockTarget = u.page ? stockPaintOrder(u.page.containers).find((c) => c.id === u.sentId) : undefined; - if (stockTarget && stockTarget.kind !== "image") { - lines.push(` stock: id ${u.sentId} resolves to ${label(stockTarget)} first, so stock drops this update`); + if (u.prepared.reason) lines.push(` ${u.prepared.reason}`); + if (u.prepared.result === 0 && u.target?.kind !== "image") { + lines.push(` the id ${u.target ? `resolves to ${label(u.target)}` : "matches no container"}, so the glasses refuse the transfer`); + } else if (u.prepared.result === 0 && !u.shown && u.target) { + lines.push(` ${u.prepared.width}x${u.prepared.height} isn't the container's ${u.target.width}x${u.target.height}, so the glasses drop it`); } return lines; } @@ -264,7 +174,7 @@ export function describeTextUpgrade( offset: number | undefined, sentLength: number, ): string { - if (!container) return `textContainerUpgrade id=${sentId} ${JSON.stringify(sentName)}: no matching text container`; + if (!container) return `textContainerUpgrade id=${sentId} ${JSON.stringify(sentName)}: no container with that id`; const at = offset === undefined ? "" : ` offset=${offset}`; const color = container.textColor === undefined ? "" : ` color=${container.textColor}`; return `textContainerUpgrade ${label(container)}${at} sent=${sentLength} chars -> len=${container.content.length}${color} ${preview(container.content)}`; diff --git a/app/apps/evenhub/payload-validation.ts b/app/apps/evenhub/payload-validation.ts new file mode 100644 index 00000000..4d001b91 --- /dev/null +++ b/app/apps/evenhub/payload-validation.ts @@ -0,0 +1,141 @@ +/** + * The checks the Even phone app applies to createStartUpPageContainer, + * rebuildPageContainer and textContainerUpgrade payloads before it sends + * anything to the glasses (read from its blutter decompilation: + * EvenHubProtoJsonMapper.validateTextBrightness, mapProtoJsonKeys and + * protobuf.dart's tag-keyed JSON parsing): + * + * - every key must be one the message defines (a numeric key passes as a raw + * protobuf tag, which we then ignore); + * - numeric fields take an integer or a numeric string — not a boolean, a + * fraction or null; string fields take a string; repeated fields an array; + * - any key named textColor, anywhere, must be an integer from 0 to 4. + * + * A failure means the call fails without reaching the glasses: create + * returns invalid (1), rebuild and textContainerUpgrade return false. + * Faceclaw's extended layouts are not checked; they carry their own keys. + */ + +type FieldType = "uint" | "string" | "any" | { list: FieldType } | { message: Schema }; +type Schema = Readonly>; + +const BOX: Schema = { + xPosition: "uint", + yPosition: "uint", + width: "uint", + height: "uint", + containerID: "uint", + containerName: "string", + zOrderIndex: "uint", +}; + +const DECORATED: Schema = { + ...BOX, + borderWidth: "uint", + borderColor: "uint", + borderRadius: "uint", + paddingLength: "uint", + isEventCapture: "uint", +}; + +const TEXT: Schema = { ...DECORATED, content: "string", textColor: "uint" }; + +const ITEM_CONTAINER: Schema = { + itemCount: "uint", + itemWidth: "uint", + isItemSelectBorderEn: "uint", + itemName: { list: "string" }, +}; + +const LIST: Schema = { ...DECORATED, itemContainer: { message: ITEM_CONTAINER } }; + +// The app knows position, subItems and checkedIndex (second-level menus, not +// yet in the SDK); their value types are unconfirmed, so they aren't checked. +const MENU_ITEM: Schema = { itemName: "string", itemID: "uint", position: "any", subItems: "any", checkedIndex: "any" }; + +const MENU: Schema = { menuItems: { list: { message: MENU_ITEM } } }; + +const REBUILD: Schema = { + containerTotalNum: "uint", + listObject: { list: { message: LIST } }, + textObject: { list: { message: TEXT } }, + imageObject: { list: { message: BOX } }, + menuObject: { message: MENU }, +}; + +const CREATE: Schema = { ...REBUILD, widgetId: "uint" }; + +const TEXT_UPGRADE: Schema = { + containerID: "uint", + containerName: "string", + contentOffset: "uint", + contentLength: "uint", + content: "string", + textColor: "uint", +}; + +const SCHEMAS = { create: CREATE, rebuild: REBUILD, textUpgrade: TEXT_UPGRADE } as const; + +/** What Dart's int.parse accepts. */ +const INT_STRING = /^\s*[+-]?(?:\d+|0[xX][0-9a-fA-F]+)\s*$/; + +function isRecord(value: unknown): value is Record { + return !!value && typeof value === "object" && !Array.isArray(value); +} + +function checkValue(type: FieldType, value: unknown, path: string): string | null { + if (type === "any") return null; + if (value === null || value === undefined) return `${path} is null`; + if (type === "uint") { + if (typeof value === "number" ? Number.isInteger(value) : typeof value === "string" && INT_STRING.test(value)) return null; + return `${path} must be an integer, not ${JSON.stringify(value)}`; + } + if (type === "string") return typeof value === "string" ? null : `${path} must be a string, not ${JSON.stringify(value)}`; + if ("list" in type) { + if (!Array.isArray(value)) return `${path} must be an array`; + for (let i = 0; i < value.length; i++) { + const error = checkValue(type.list, value[i], `${path}[${i}]`); + if (error) return error; + } + return null; + } + return isRecord(value) ? checkMessage(type.message, value, path) : `${path} must be an object`; +} + +function checkMessage(schema: Schema, message: Record, path: string): string | null { + for (const [key, value] of Object.entries(message)) { + const field = schema[key]; + if (!field) { + if (INT_STRING.test(key)) continue; + return `unknown key ${path ? `${path}.` : ""}${key}`; + } + const error = checkValue(field, value, path ? `${path}.${key}` : key); + if (error) return error; + } + return null; +} + +/** validateTextBrightness: every textColor key, at any depth, holds an integer 0..4. */ +function checkTextColors(value: unknown, path: string): string | null { + if (Array.isArray(value)) { + for (let i = 0; i < value.length; i++) { + const error = checkTextColors(value[i], `${path}[${i}]`); + if (error) return error; + } + } else if (isRecord(value)) { + for (const [key, child] of Object.entries(value)) { + const at = path ? `${path}.${key}` : key; + if (key === "textColor" && !(typeof child === "number" && Number.isInteger(child) && child >= 0 && child <= 4)) { + return `${at} must be an integer from 0 to 4, not ${JSON.stringify(child)}`; + } + const error = checkTextColors(child, at); + if (error) return error; + } + } + return null; +} + +/** Why the Even app would refuse this payload without sending it, or null if it would send it. */ +export function stockPayloadError(kind: keyof typeof SCHEMAS, data: Record): string | null { + return checkTextColors(data, "") ?? checkMessage(SCHEMAS[kind], data, ""); +} diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 71191a6b..18ba982e 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -38,7 +38,11 @@ import { import { EvenHubApiKeyDialogLayer } from "./api-key-dialog"; import { asRecord, + clampToStockLimits, + containerById, declaredMenuItems, + PAGE_CREATE_INVALID, + stockPageFailure, eventCaptureContainer, parsePage, readNumber, @@ -50,9 +54,13 @@ import { type EvenHubContainer, type EvenHubMenuItem, type EvenHubPage, + type EvenHubTextContainer, } from "./containers"; -import { compositePage } from "./compositor"; +import { compositePage, layoutText } from "./compositor"; import { describeImageUpdate, describePage, describeTextUpgrade } from "./layout-debug"; +import { IMAGE_SEND_FAILED, IMAGE_SUCCESS, prepareImageUpdate } from "./image-pipeline"; +import { stockPayloadError } from "./payload-validation"; +import { decodeImageRgba } from "../../native/image-rgba"; import { stepListSelection } from "./list-menu"; import { type EvenHubManifest } from "./ehpk"; import { permissionsIncludeMicrophone, permissionsInclude } from "./permissions"; @@ -75,7 +83,6 @@ import { parseWindowIcon, WindowIconRenderer } from "./window-icon"; import { availableFontFamilies, containerFace, measureContainerText } from "./fonts"; import { parseFontSpec } from "./font-spec"; -const UPNG = require("upng-js"); /** * Page layouts and container updates go to logcat at debug level (see @@ -93,12 +100,20 @@ const SCROLL_BOTTOM_EVENT = 2; const DOUBLE_CLICK_EVENT = 3; const FOREGROUND_ENTER_EVENT = 4; const FOREGROUND_EXIT_EVENT = 5; +const ABNORMAL_EXIT_EVENT = 6; const SYSTEM_EXIT_EVENT = 7; const IMU_DATA_REPORT_EVENT = 8; /** SDK 0.0.14: the press gesture, delivered like any other list/text/sys event. */ const LONG_PRESS_EVENT = 9; const LONG_PRESS_RELEASE_EVENT = 10; +/** After a failed rebuild, how long the app gets to hear ABNORMAL_EXIT before it is closed. */ +const ABNORMAL_EXIT_CLOSE_DELAY_MS = 200; + +/** Stock text scrolling: one step per swipe, and "at an end" within this tolerance. */ +const TEXT_SCROLL_STEP_PX = 28; +const TEXT_SCROLL_EDGE_PX = 3; + /** * Injected into every served HTML document at document start (before any app * JS): (1) the promise-returning flutter_inappwebview.callHandler the EvenHub @@ -426,6 +441,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH private page: EvenHubPage | null = null; private pageCreated = false; + /** The page came from Faceclaw's extended layout API, which isn't held to stock limits. */ + private extendedLayout = false; private webViewHandle: EvenHubWebViewHandle | null = null; private windowHooks: EvenHubWindowHooks | null = null; private closed = false; @@ -606,6 +623,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH const eventType = event.type === "scroll-up" ? SCROLL_TOP_EVENT : SCROLL_BOTTOM_EVENT; if (capture?.kind === "list") { this.scrollList(capture, eventType, inputSource); + } else if (capture?.kind === "text") { + this.scrollText(capture, eventType, inputSource); } else { this.emitSysEvent(eventType, 0, inputSource); } @@ -630,6 +649,25 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.pushMessage("evenHubEvent", { type: "menuItemClickEvent", jsonData: { itemID } }); } + /** + * Stock scrolls a capture text container that overflows by itself, a 28px + * step per swipe, and tells the app only at the ends (a textEvent naming the + * container). Text that fits reports every swipe as reaching an end. + */ + private scrollText(text: EvenHubTextContainer, eventType: number, inputSource?: ExtensionInputSource): void { + const { maxScroll } = layoutText(text); + const forward = eventType === SCROLL_BOTTOM_EVENT; + const atEnd = forward ? maxScroll - text.scrollY < TEXT_SCROLL_EDGE_PX : text.scrollY < TEXT_SCROLL_EDGE_PX; + if (maxScroll === 0 || atEnd) { + this.emitTextEvent(text, eventType, inputSource); + return; + } + text.scrollY = forward + ? Math.min(maxScroll, text.scrollY + TEXT_SCROLL_STEP_PX) + : Math.max(0, text.scrollY - TEXT_SCROLL_STEP_PX); + this.windowHooks?.requestRender(); + } + private scrollList(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void { // Swipe up selects the previous item, swipe down the next (stock direction). const delta = eventType === SCROLL_TOP_EVENT ? -1 : 1; @@ -818,7 +856,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH deliverImu(reading: ImuReading): void { this.pushEvenHubEvent("sysEvent", { eventType: IMU_DATA_REPORT_EVENT, - imuData: { x: reading.x, y: reading.y, z: reading.z }, + // The Even app's field name (protobuf IMU_Data); the SDK reads it as imuData. + iMUData: { x: reading.x, y: reading.y, z: reading.z }, }); } @@ -868,7 +907,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH const first = args[0]; const message = asRecord(typeof first === "string" ? JSON.parse(first) : first); const method = readString(message, "method", ""); - const data = asRecord(message.data); + // As the Even app: `data`, else `payload`. + const data = asRecord(message.data ?? message.payload); result = await this.dispatch(method, data); } catch (error) { this.log(`evenhub bridge call failed: ${error}`); @@ -930,11 +970,25 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH await delay(DUPLICATE_CREATE_DELAY_MS); return 1; } + // The Even app refuses some payloads without sending them; then the + // firmware may fail to build the page. + const payloadError = stockPayloadError("create", data); + if (payloadError) { + this.layoutLog(() => [`createStartUpPageContainer refused, as the Even app would: ${payloadError}`]); + return PAGE_CREATE_INVALID; + } const page = parsePage(data); - this.warnOnInvalidPage(page, data); + const failure = stockPageFailure(page); + if (failure) { + this.layoutLog(() => [`createStartUpPageContainer failed, as the firmware would: ${failure.reason}`]); + return failure.code; + } + clampToStockLimits(page); + this.warnOnDroppedMenuItems(page, data); this.layoutLog(() => describePage("createStartUpPageContainer", page, data)); this.page = page; this.pageCreated = true; + this.extendedLayout = false; // The launch focus arrived before the page existed; deliver the ENTER now. if (this.foreground && !this.launchEnterSent) { this.launchEnterSent = true; @@ -945,17 +999,39 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH } private rebuildPage(data: Record): boolean { - // Also serves as create for apps that fall back to rebuild-on-error. + // Only createStartUpPageContainer starts an EvenHub page on the glasses. + if (!this.pageCreated) { + this.layoutLog(() => ["rebuildPageContainer before createStartUpPageContainer: no page to rebuild"]); + return false; + } + const payloadError = stockPayloadError("rebuild", data); + if (payloadError) { + this.layoutLog(() => [`rebuildPageContainer refused, as the Even app would: ${payloadError}`]); + return false; + } const page = parsePage(data); - this.warnOnInvalidPage(page, data); + const failure = stockPageFailure(page); + if (failure) { + // The firmware has already torn the old page down; it reports an + // abnormal exit and the Even app closes the app. + this.layoutLog(() => [`rebuildPageContainer failed, as the firmware would: ${failure.reason}; closing the app`]); + this.page = { containers: [], menuItems: [] }; + this.windowHooks?.requestRender(); + this.emitSysEvent(ABNORMAL_EXIT_EVENT, 0); + this.systemExitSent = true; + setTimeout(() => this.close(), ABNORMAL_EXIT_CLOSE_DELAY_MS); + return false; + } + clampToStockLimits(page); + this.warnOnDroppedMenuItems(page, data); this.layoutLog(() => describePage("rebuildPageContainer", page, data)); this.page = page; - this.pageCreated = true; + this.extendedLayout = false; this.windowHooks?.requestRender(); return true; } - private warnOnInvalidPage(page: EvenHubPage, data: Record): void { + private warnOnDroppedMenuItems(page: EvenHubPage, data: Record): void { const declaredMenuCount = declaredMenuItems(data).length; if (declaredMenuCount > page.menuItems.length) { this.log( @@ -963,77 +1039,86 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH "items (max 10, itemID must be non-zero and unique, itemName at most 32 UTF-8 bytes)", ); } - // Stock rejects these outright; we accept-and-log until rejection is - // proven necessary for compatibility. - const captures = page.containers.filter((c) => c.kind !== "image" && c.isEventCapture).length; - if (captures !== 1) this.log(`evenhub: page has ${captures} event-capture containers (stock requires exactly 1)`); - const texts = page.containers.filter((c) => c.kind === "text").length; - const images = page.containers.filter((c) => c.kind === "image").length; - if (texts > 8 || images > 4) this.log(`evenhub: page exceeds stock limits (${texts} text, ${images} image)`); } private textUpgrade(data: Record): boolean { - const id = readNumber(data, "containerID", -1); - // containerID is the identity; containerName is optional in the wild (e.g. - // level-even-g2 omits it). Match by id, enforcing the name only when the app - // actually supplies a non-empty one. (updateImage already matches by id.) + const id = readNumber(data, "containerID", 0); const name = readString(data, "containerName", ""); - const container = this.page?.containers.find( - (c) => c.kind === "text" && c.id === id && (name === "" || c.name === name), - ); const content = readString(data, "content", ""); const offset = readOptionalNumber(data, "contentOffset"); - if (!container || container.kind !== "text") { + const payloadError = stockPayloadError("textUpgrade", data); + if (payloadError) { + this.layoutLog(() => [`textContainerUpgrade refused, as the Even app would: ${payloadError}`]); + return false; + } + // The firmware finds the container by id alone (its first container with + // that id, of any kind); the name isn't checked anywhere. + const target = containerById(this.page, id); + if (!target) { this.layoutLog(() => [describeTextUpgrade(id, name, null, offset, content.length)]); return false; } - // (contentLength is intentionally ignored — see below.) - // Stock semantics are "write `content` at contentOffset and truncate the - // rest" (a null-terminating strcpy(buf+offset, content)), NOT a splice that - // keeps the old tail after offset+contentLength. Keeping the tail left stray - // trailing text when an app (e.g. g2oom) shrank a string with an - // under-sized contentLength; dropping it matches stock and still supports - // suffix rewrites (offset>0) and full replaces (offset 0 / omitted). - if (offset !== undefined && offset > 0) { - container.content = container.content.slice(0, offset) + content; - } else { - container.content = content; + // Aimed at a list or image, the firmware writes into it as if it were + // text and reports success; nothing we could show. + if (target.kind !== "text") return true; + // The content replaces the whole text: stock ignores contentOffset and + // contentLength (firmware 2.2.9.22 in the emulator showed "XY" alone for + // every offset/length tried). The firmware re-sets the label, scrolled + // back to the top, only for non-empty text, so emptying a container + // leaves its old text showing (apps send " " to clear one). + if (content !== "") { + target.content = content; + target.scrollY = 0; } // SDK 0.0.14: textColor on an incremental update sets the container's // brightness; omitting it keeps whatever level the container has. const brightness = readTextBrightness(data); - if (brightness !== undefined) container.textColor = brightness; - this.layoutLog(() => [describeTextUpgrade(id, name, container, offset, content.length)]); + if (brightness !== undefined) target.textColor = brightness; + this.layoutLog(() => [describeTextUpgrade(id, name, target, offset, content.length)]); this.windowHooks?.requestRender(); return true; } - /** BleEhSendImageResult: 0 success, 1 imageException, 2 sizeInvalid, 3 sendFailed. */ + /** EhImageUpdateResult: 0 success, 1 imageException, 2 sizeInvalid, 3 toGray4Failed, 4 sendFailed. */ private updateImage(data: Record): number { - const id = readNumber(data, "containerID", -1); - const found = this.page?.containers.find((c) => c.kind === "image" && c.id === id); - const container = found?.kind === "image" ? found : null; - const raw = data.imageData ?? data.rawData ?? data.mapRawData; - const bytes = container ? imagePayloadBytes(raw) : null; - const decoded = container && bytes && bytes.length > 0 ? decodeImageBytes(bytes, container) : null; - const result = !container ? 3 : !decoded ? 1 : 0; - this.layoutLog(() => - describeImageUpdate({ - page: this.page, - sentId: id, - sentName: readString(data, "containerName", ""), - container, - raw, - bytes, - decoded, - result, - }), - ); - if (!container || !decoded) return result; - container.pixels = decoded.pixels; - container.pixelsWidth = decoded.width; - container.pixelsHeight = decoded.height; - this.windowHooks?.requestRender(); + const id = readNumber(data, "containerID", 0); + const name = readString(data, "containerName", ""); + const raw = data.imageData; + // Before the page exists (and after shutdown) the Even app fails it outright. + if (!this.pageCreated || !this.page) { + this.layoutLog(() => [`updateImageRawData id=${id}: no page yet, sendFailed`]); + return IMAGE_SEND_FAILED; + } + // The Even app sizes the bitmap from the image container it finds by id, + // then by trimmed name. + const images = this.page.containers.filter((c): c is EvenHubImageContainer => c.kind === "image"); + const geometry = (id > 0 ? images.find((c) => c.id === id) : undefined) ?? + (name.trim() ? images.find((c) => c.name.trim() === name.trim()) : undefined); + const prepared = prepareImageUpdate({ + imageData: raw, + compressMode: readNumber(data, "compressMode", 0), + requestWidth: readNumber(data, "width", 0), + requestHeight: readNumber(data, "height", 0), + geometry: geometry ?? null, + decodeNative: decodeImageRgba, + stockLimits: !this.extendedLayout, + }); + // The firmware takes the transfer only for an id whose first container is + // an image, and then shows the bitmap only if it's exactly that size. + const target = containerById(this.page, id); + let result = prepared.result; + let shown = false; + if (result === IMAGE_SUCCESS && target?.kind !== "image") { + result = IMAGE_SEND_FAILED; + } else if (result === IMAGE_SUCCESS && target?.kind === "image" && prepared.levels && + prepared.width === target.width && prepared.height === target.height) { + target.pixels = prepared.levels.map((level) => level * 17); + target.pixelsWidth = prepared.width; + target.pixelsHeight = prepared.height; + shown = true; + this.windowHooks?.requestRender(); + } + this.layoutLog(() => describeImageUpdate({ sentId: id, sentName: name, raw, target, prepared, shown, result })); return result; } @@ -1121,6 +1206,15 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.pushEvenHubEvent("sysEvent", { eventType, eventSource, [INPUT_SOURCE_FIELD]: inputSource }); } + private emitTextEvent(text: EvenHubTextContainer, eventType: number, inputSource?: ExtensionInputSource): void { + this.pushEvenHubEvent("textEvent", { + containerID: text.id, + containerName: text.name, + eventType, + [INPUT_SOURCE_FIELD]: inputSource, + }); + } + private emitListEvent(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void { this.pushEvenHubEvent("listEvent", { containerID: list.id, @@ -1301,6 +1395,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.layoutLog(() => describePage("extension layout", page, data)); this.page = page; this.pageCreated = true; + this.extendedLayout = true; // The extended layout uses the taller app area. this.windowHooks?.setTallCanvas(true); if (this.foreground && !this.launchEnterSent) { @@ -1606,82 +1701,6 @@ function normalizeBuzzerSteps(raw: unknown[]): Step[] { return out; } -/** - * Decode an uncompressed Windows BMP ('BM', BI_RGB) to 8bpp grayscale. Handles - * 1/4/8-bit paletted and 24/32-bit BGR(A), top-down or bottom-up. Returns null - * for compressed or unsupported variants. Apps like EvenChess send 1-bit BMPs - * (black/white palette) for full-board refreshes. - */ -function decodeBmp(bytes: Uint8Array): { pixels: Uint8Array; width: number; height: number } | null { - if (bytes.length < 54 || bytes[0] !== 0x42 || bytes[1] !== 0x4d) return null; - const u16 = (o: number) => bytes[o]! | (bytes[o + 1]! << 8); - const u32 = (o: number) => (bytes[o]! | (bytes[o + 1]! << 8) | (bytes[o + 2]! << 16) | (bytes[o + 3]! << 24)) >>> 0; - const i32 = (o: number) => bytes[o]! | (bytes[o + 1]! << 8) | (bytes[o + 2]! << 16) | (bytes[o + 3]! << 24); - - const pixelOffset = u32(10); - const dibSize = u32(14); - const width = i32(18); - const heightRaw = i32(22); - const bitCount = u16(28); - const compression = u32(30); - if (compression !== 0) return null; // only BI_RGB (uncompressed) - if (width <= 0 || width > 4096) return null; - const topDown = heightRaw < 0; - const height = Math.abs(heightRaw); - if (height === 0 || height > 4096) return null; - - // Palette (grayscale per index) for <=8bpp; each entry is 4 bytes BGRA. - let palette: Uint8Array | null = null; - if (bitCount <= 8) { - let numColors = u32(46); - if (numColors === 0 || numColors > (1 << bitCount)) numColors = 1 << bitCount; - const palOffset = 14 + dibSize; - palette = new Uint8Array(numColors); - for (let i = 0; i < numColors; i++) { - const o = palOffset + i * 4; - if (o + 2 >= bytes.length) break; - palette[i] = luminance(bytes[o + 2]!, bytes[o + 1]!, bytes[o]!); - } - } - - const stride = (((width * bitCount + 31) >> 5) << 2); // rows are 4-byte aligned - const pixels = new Uint8Array(width * height); - for (let y = 0; y < height; y++) { - const srcY = topDown ? y : height - 1 - y; - const rowStart = pixelOffset + srcY * stride; - if (rowStart >= bytes.length) break; - const dstRow = y * width; - for (let x = 0; x < width; x++) { - let gray: number; - if (bitCount === 1) { - const bit = ((bytes[rowStart + (x >> 3)] ?? 0) >> (7 - (x & 7))) & 1; - gray = palette?.[bit] ?? (bit ? 255 : 0); - } else if (bitCount === 4) { - const byte = bytes[rowStart + (x >> 1)] ?? 0; - const nib = (x & 1) === 0 ? byte >> 4 : byte & 0x0f; - gray = palette?.[nib] ?? nib * 17; - } else if (bitCount === 8) { - const idx = bytes[rowStart + x] ?? 0; - gray = palette?.[idx] ?? idx; - } else if (bitCount === 24) { - const o = rowStart + x * 3; - gray = luminance(bytes[o + 2] ?? 0, bytes[o + 1] ?? 0, bytes[o] ?? 0); - } else if (bitCount === 32) { - const o = rowStart + x * 4; - gray = luminance(bytes[o + 2] ?? 0, bytes[o + 1] ?? 0, bytes[o] ?? 0); - } else { - return null; - } - pixels[dstRow + x] = gray; - } - } - return { pixels, width, height }; -} - -function luminance(r: number, g: number, b: number): number { - return Math.round(0.2126 * r + 0.7152 * g + 0.0722 * b); -} - /** * Input-device names in the extension API: the touchDown event's source, and * the `faceclawInputSource` field added to stock gesture events. @@ -1733,97 +1752,3 @@ function elideZeroFields(obj: Record): Record return out; } -/** Accept the payload forms seen on the wire: number[], JSON-ified typed - * array ({"0": 1, ...}), or base64 string. */ -function imagePayloadBytes(raw: unknown): Uint8Array | null { - if (Array.isArray(raw)) { - const out = new Uint8Array(raw.length); - for (let i = 0; i < raw.length; i++) out[i] = Number(raw[i]) & 0xff; - return out; - } - if (typeof raw === "string") return base64Decode(raw); - if (raw && typeof raw === "object") { - const record = raw as Record; - const keys = Object.keys(record); - const out = new Uint8Array(keys.length); - for (let i = 0; i < keys.length; i++) { - const value = record[String(i)]; - if (typeof value !== "number") return null; - out[i] = value & 0xff; - } - return out; - } - return null; -} - -const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; - -function base64Decode(text: string): Uint8Array | null { - const clean = text.replace(/[\s=]+$/g, "").replace(/\s+/g, ""); - const out = new Uint8Array(Math.floor((clean.length * 3) / 4)); - let bits = 0; - let bitCount = 0; - let outIndex = 0; - for (const char of clean) { - const value = BASE64_ALPHABET.indexOf(char); - if (value < 0) return null; - bits = (bits << 6) | value; - bitCount += 6; - if (bitCount >= 8) { - bitCount -= 8; - out[outIndex++] = (bits >> bitCount) & 0xff; - } - } - return out.subarray(0, outIndex); -} - -/** - * Decode an image payload: PNG or BMP (both are sent by real apps — EvenChess - * force-refreshes as 1-bit BMP but streams live moves as PNG, so a host that - * only did PNG dropped the initial board), or raw grayscale sized to the - * container (8bpp, or 4bpp packed two pixels per byte, high nibble first). - */ -function decodeImageBytes( - bytes: Uint8Array, - container: EvenHubImageContainer, -): { pixels: Uint8Array; width: number; height: number } | null { - if (bytes.length > 2 && bytes[0] === 0x42 && bytes[1] === 0x4d) { - const bmp = decodeBmp(bytes); - if (bmp) return bmp; - } - if (bytes.length > 4 && bytes[0] === 0x89 && bytes[1] === 0x50 && bytes[2] === 0x4e && bytes[3] === 0x47) { - try { - const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); - const decoded = UPNG.decode(buffer); - const frames = UPNG.toRGBA8(decoded) as ArrayBuffer[]; - if (!frames.length) return null; - const rgba = new Uint8Array(frames[0]!); - const pixels = new Uint8Array(decoded.width * decoded.height); - for (let i = 0; i < pixels.length; i++) { - const offset = i * 4; - const alpha = (rgba[offset + 3] ?? 0) / 255; - pixels[i] = Math.round( - (0.2126 * (rgba[offset] ?? 0) + 0.7152 * (rgba[offset + 1] ?? 0) + 0.0722 * (rgba[offset + 2] ?? 0)) * alpha, - ); - } - return { pixels, width: decoded.width, height: decoded.height }; - } catch (error) { - console.warn(`evenhub: PNG decode failed: ${error}`); - return null; - } - } - const area = container.width * container.height; - if (bytes.length === area) { - return { pixels: Uint8Array.from(bytes), width: container.width, height: container.height }; - } - if (bytes.length === Math.ceil(area / 2)) { - const pixels = new Uint8Array(area); - for (let i = 0; i < area; i++) { - const byte = bytes[i >> 1]!; - const nibble = i % 2 === 0 ? byte >> 4 : byte & 0x0f; - pixels[i] = nibble * 17; - } - return { pixels, width: container.width, height: container.height }; - } - return null; -} diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 0792454e..ea4c69a2 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -596,6 +596,25 @@ export class GrayImage { } } + /** + * bakeDeferredDrawsInPlace, but only for the draws whose ink may fall inside + * `rect`; the rest stay deferred. For covering part of the image with later + * raster (the EvenHub compositor painting an image container over text) + * without giving up the deferred wire path for everything else. + */ + bakeDeferredDrawsWithin(rect: { x: number; y: number; width: number; height: number }): void { + if (!this.drawList.length) return; + const covered = (b: { x: number; y: number; width: number; height: number } | null) => + !b || (b.x < rect.x + rect.width && rect.x < b.x + b.width && b.y < rect.y + rect.height && rect.y < b.y + b.height); + const bake: DeferredDraw[] = []; + const keep: DeferredDraw[] = []; + for (const placed of this.drawList) (covered(placedDrawBounds(placed)) ? bake : keep).push(placed); + if (!bake.length) return; + this.drawList.splice(0, this.drawList.length, ...bake); + this.bakeDeferredDrawsInPlace(); + this.drawList.push(...keep); + } + /** * A copy of this image with its brightness scaled by `factor` (0..1): * raster and glyph / firmware-text draws alike, keeping the draws deferred @@ -705,6 +724,35 @@ export function grayToNibble(value: number): number { /** Where rasterizeGlyph writes: an image, or a clipped buffer (paintReplayDraws). */ type PixelSink = { setPixel(x: number, y: number, value: number): void }; +/** A deferred draw's ink rectangle, or null when it can't be bounded (treat as anywhere). */ +function placedDrawBounds(placed: DeferredDraw): { x: number; y: number; width: number; height: number } | null { + if (placed.kind === "glyph") { + const { glyph } = placed; + return { + x: placed.x + glyph.bbxX, + y: placed.y + placed.font.ascent - (glyph.bbxHeight + glyph.bbxY), + width: glyph.bbxWidth, + height: glyph.bbxHeight, + }; + } + if (placed.kind === "image") { + return placed.presentation ? null : { x: placed.x, y: placed.y, width: placed.source.width, height: placed.source.height }; + } + let left = Infinity, top = Infinity, right = -Infinity, bottom = -Infinity; + for (const member of placed.glyphs) { + if (!member.ink) continue; + const wire = placed.font.fwGlyphWireData(member.cp); + if (!wire) return null; + const x = placed.x + member.dx + wire.ofsX; + const y = placed.y + wire.inkTop; + left = Math.min(left, x); + top = Math.min(top, y); + right = Math.max(right, x + wire.boxW); + bottom = Math.max(bottom, y + wire.boxH); + } + return right < left ? { x: placed.x, y: placed.y, width: 0, height: 0 } : { x: left, y: top, width: right - left, height: bottom - top }; +} + /** Bake one glyph into an image's pixel buffer (y is the top of the line). */ function rasterizeGlyph( target: PixelSink, diff --git a/app/graphics/lvgl-wrap.ts b/app/graphics/lvgl-wrap.ts index 3ad87ec3..b4626dde 100644 --- a/app/graphics/lvgl-wrap.ts +++ b/app/graphics/lvgl-wrap.ts @@ -1,19 +1,21 @@ /** - * Word wrap for EvenHub text containers, after LVGL's lv_text_get_next_line - * as ported by @evenrealities/pretext's measureTextWrap, so soft breaks land - * where apps that measure with pretext expect them. The loop mirrors - * measureTextWrap's, including rescanning from the break point when it wraps - * at an earlier opportunity. + * Word wrap for EvenHub text containers, matching the stock firmware's LVGL + * labels as measured by running firmware 2.2.9.22 in the g2-firmware-emulator: * - * One deliberate difference from pretext: pretext skips leading spaces at the - * start of every line, but stock keeps them at the start of the text and after - * a '\n'; only a soft-wrapped continuation line loses them. Apps indent with - * spaces to clear image containers placed beside their text: Iris Pro indents a - * line 11 spaces (44px) past a 40px glyph. Stripping that indent slid the text - * under the icon. + * - Spaces are kept wherever they fall: at the start of the text, after a + * '\n', and on soft-wrapped lines. A space that doesn't fit at the end of a + * line starts the next one, so "HHHHHHHH EEEE" in 100px (8 H = 96px, + * 5px spaces) wraps with all four spaces indenting the second line. + * - A line otherwise breaks after its last space, hyphen or CJK ideograph, + * or, with none, before the character that overflows. + * + * @evenrealities/pretext's measureTextWrap, which apps measure with, differs + * in one way: it drops spaces at the start of every line. Apps rely on stock's + * behavior, though: Iris Pro indents a line 11 spaces past a 40px glyph placed + * beside it, and stripping the indent slid the text under the icon. */ -/** Wrap opportunities: space, hyphen, and between CJK ideographs (pretext parity). */ +/** Wrap opportunities: space, hyphen, and between CJK ideographs. */ export function isLvglBreakable(cp: number): boolean { if (cp === 32 || cp === 45) return true; return (cp >= 0x2e80 && cp <= 0x9fff) || (cp >= 0xf900 && cp <= 0xfaff) || (cp >= 0xac00 && cp <= 0xd7af); @@ -30,55 +32,43 @@ export function wrapLvglText(text: string, maxWidth: number, advanceOf: (cp: num let line: number[] = []; let lineWidth = 0; // The last break opportunity on this line, as an index into cps, and how - // many of `line`'s codepoints a wrap there keeps (a breaking space is dropped). + // many of `line`'s codepoints end at it. let lastBreak = -1; let lastBreakKeep = 0; - // Whether `line` continues a soft-wrapped line, whose leading spaces are dropped. - let continuation = false; - const endLine = (keep: number[], next: number[], nextWidth: number) => { + const endLine = (keep: number[]) => { lines.push(String.fromCodePoint(...keep)); - line = next; - lineWidth = nextWidth; + line = []; + lineWidth = 0; lastBreak = -1; - continuation = true; }; let i = 0; while (i < cps.length) { const cp = cps[i]!; if (cp === 10) { - endLine(line, [], 0); - continuation = false; - i++; - continue; - } - if (continuation && line.length === 0 && cp === 32) { + endLine(line); i++; continue; } const adv = advanceOf(cp, cps[i + 1] ?? 0); - if (lineWidth + adv <= maxWidth) { + if (lineWidth + adv <= maxWidth || line.length === 0) { + // A character wider than the whole line still takes one to itself. line.push(cp); lineWidth += adv; if (isLvglBreakable(cp)) { lastBreak = i; - lastBreakKeep = cp === 32 ? line.length - 1 : line.length; + lastBreakKeep = line.length; } i++; - } else if (cp === 32) { - // The overflowing char is a space: break here and discard it. - endLine(line, [], 0); - i++; - } else if (lastBreak !== -1) { - // Wrap at the last break opportunity and rescan from just after it. + } else if (cp !== 32 && lastBreak !== -1) { + // Wrap after the last break opportunity and rescan from there. const restart = lastBreak + 1; - endLine(line.slice(0, lastBreakKeep), [], 0); + endLine(line.slice(0, lastBreakKeep)); i = restart; } else { - // No break opportunity: hard break before the overflowing char. - endLine(line, [cp], adv); - i++; + // An overflowing space, or no break opportunity: this character starts the next line. + endLine(line); } } lines.push(String.fromCodePoint(...line)); diff --git a/app/native/image-rgba.ios.ts b/app/native/image-rgba.ios.ts new file mode 100644 index 00000000..7ef09dba --- /dev/null +++ b/app/native/image-rgba.ios.ts @@ -0,0 +1,9 @@ +import { toData, fromData } from './kotlin-data' +import { rgbaFromPacket } from './image-rgba' +declare const FaceclawGraphics: any + +/** Decoded RGBA of encoded image bytes, or null when undecodable (transparent pixels come out black). */ +export function decodeImageRgba(bytes: Uint8Array): { width: number; height: number; rgba: Uint8Array } | null { + const data = FaceclawGraphics.rgbaPacketFromImageData(toData(bytes)) + return data ? rgbaFromPacket(fromData(data)) : null +} diff --git a/app/native/image-rgba.ts b/app/native/image-rgba.ts new file mode 100644 index 00000000..28c8d7b1 --- /dev/null +++ b/app/native/image-rgba.ts @@ -0,0 +1,22 @@ +import { toUint8Array } from '../util/array-util' +import { copyToJavaByteBuffer } from './java-direct-buffer' +declare const com: any + +/** Decoded RGBA (not premultiplied) of encoded image bytes, or null when undecodable. */ +export function decodeImageRgba(bytes: Uint8Array): { width: number; height: number; rgba: Uint8Array } | null { + try { + return rgbaFromPacket(toUint8Array(com.faceclaw.app.ImageFileLoader.decodeRgbaFromBytes(copyToJavaByteBuffer(bytes)))) + } catch (error) { + console.warn(`decodeImageRgba failed: ${error}`) + return null + } +} + +/** [widthLo, widthHi, heightLo, heightHi, RGBA...] as an image, or null when short. */ +export function rgbaFromPacket(packet: Uint8Array): { width: number; height: number; rgba: Uint8Array } | null { + if (packet.length < 4) return null + const width = packet[0]! | (packet[1]! << 8) + const height = packet[2]! | (packet[3]! << 8) + if (width <= 0 || height <= 0 || packet.length < 4 + width * height * 4) return null + return { width, height, rgba: packet.slice(4, 4 + width * height * 4) } +} diff --git a/tests/evenhub-containers.test.cjs b/tests/evenhub-containers.test.cjs index 7527df66..98ffa986 100644 --- a/tests/evenhub-containers.test.cjs +++ b/tests/evenhub-containers.test.cjs @@ -130,9 +130,9 @@ test("a layout font is the default for text and list containers, which override page.containers.map((c) => [c.name, c.font]), [ ["l", { family: "Inter", size: 24, weight: 700 }], - ["i", undefined], ["t", { family: "Inter", size: 24 }], ["u", { family: "Terminus", size: 24 }], + ["i", undefined], ], ); }); diff --git a/tests/evenhub-image-pipeline.test.cjs b/tests/evenhub-image-pipeline.test.cjs new file mode 100644 index 00000000..b0fca69f --- /dev/null +++ b/tests/evenhub-image-pipeline.test.cjs @@ -0,0 +1,126 @@ +// updateImageRawData conversion (app/apps/evenhub/image-pipeline.ts), checked +// against what the Even phone app's decompiled _prepare / _prepareLegacyRaw do. +const test = require("node:test"); +const assert = require("node:assert/strict"); +const UPNG = require("upng-js"); + +const { + IMAGE_EXCEPTION, + IMAGE_SEND_FAILED, + IMAGE_SIZE_INVALID, + IMAGE_SUCCESS, + detectImageFormat, + grayLevel, + prepareImageUpdate, + resizeRgbaCover, +} = require("../.test-build/app/apps/evenhub/image-pipeline.js"); + +const noNative = () => null; +const request = (over) => ({ + imageData: [], + compressMode: 2, + requestWidth: 0, + requestHeight: 0, + geometry: { width: 20, height: 20 }, + decodeNative: noNative, + stockLimits: true, + ...over, +}); + +/** A PNG of the given RGBA pixels. */ +function png(width, height, rgba) { + return Array.from(new Uint8Array(UPNG.encode([Uint8Array.from(rgba).buffer], width, height, 0))); +} + +test("imageData must be an array of integers", () => { + assert.equal(prepareImageUpdate(request({ imageData: "AAAA" })).result, IMAGE_SEND_FAILED); + assert.equal(prepareImageUpdate(request({ imageData: { 0: 1, 1: 2 } })).result, IMAGE_SEND_FAILED); + assert.equal(prepareImageUpdate(request({ imageData: [1, 2.5] })).result, IMAGE_SEND_FAILED); +}); + +test("formats are recognized by their magic bytes", () => { + assert.equal(detectImageFormat(Uint8Array.from(png(1, 1, [0, 0, 0, 255]))), "png"); + assert.equal(detectImageFormat(Uint8Array.from([0xff, 0xd8, 0xff, 0xe0])), "jpeg"); + assert.equal(detectImageFormat(Uint8Array.from([0x52, 0x49, 0x46, 0x46, 0, 0, 0, 0, 0x57, 0x45, 0x42, 0x50])), "webp"); + assert.equal(detectImageFormat(Uint8Array.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61])), "gif"); + assert.equal(detectImageFormat(Uint8Array.from([0x42, 0x4d, 0, 0])), "bmp"); + assert.equal(detectImageFormat(Uint8Array.from([1, 2, 3])), "raw"); +}); + +test("grey uses 299/587/114 weights rounded to 16 levels, ignoring alpha", () => { + assert.equal(grayLevel(255, 0, 0), 4); + assert.equal(grayLevel(255, 255, 255), 15); + assert.equal(grayLevel(0, 0, 0), 0); + // A transparent white pixel is white: the phone app never looks at alpha. + const result = prepareImageUpdate(request({ imageData: png(20, 20, new Array(20 * 20).fill([255, 255, 255, 0]).flat()) })); + assert.equal(result.result, IMAGE_SUCCESS); + assert.ok(result.levels.every((level) => level === 15)); +}); + +test("an encoded image is scaled to cover the container and centre-cropped", () => { + // 4x2: columns 0..3 lit 0, 85, 170, 255. Covering 20x20 scales by 10 to + // 40x20 and crops 10px a side, so only columns 1 and 2 survive. + const row = [0, 85, 170, 255].flatMap((v) => [v, v, v, 255]); + const result = prepareImageUpdate(request({ imageData: png(4, 2, [...row, ...row]) })); + assert.equal(result.result, IMAGE_SUCCESS); + assert.deepEqual([result.width, result.height], [20, 20]); + assert.deepEqual(Array.from(result.levels.slice(0, 20)), [...new Array(10).fill(grayLevel(85, 85, 85)), ...new Array(10).fill(grayLevel(170, 170, 170))]); +}); + +test("the cover resize samples source pixel centres", () => { + const source = { width: 3, height: 1, rgba: Uint8Array.from([10, 0, 0, 0, 20, 0, 0, 0, 30, 0, 0, 0]) }; + const wide = resizeRgbaCover(source, 6, 2); + assert.deepEqual(Array.from(wide.rgba.filter((_, i) => i % 4 === 0)), [10, 10, 20, 20, 30, 30, 10, 10, 20, 20, 30, 30]); +}); + +test("width and height sent with the update replace the container's", () => { + const black = new Array(30 * 30).fill([0, 0, 0, 255]).flat(); + const result = prepareImageUpdate(request({ imageData: png(30, 30, black), requestWidth: 40, requestHeight: 25 })); + assert.deepEqual([result.width, result.height], [40, 25]); +}); + +test("without a usable container size, the decoded size must itself be in range", () => { + const tiny = new Array(10 * 10).fill([0, 0, 0, 255]).flat(); + assert.equal(prepareImageUpdate(request({ imageData: png(10, 10, tiny), geometry: null })).result, IMAGE_SIZE_INVALID); +}); + +test("JPEG, WebP and GIF go to the native decoder; a failed decode is an exception", () => { + const jpeg = [0xff, 0xd8, 0xff, 0xe0]; + const decoded = prepareImageUpdate(request({ + imageData: jpeg, + decodeNative: () => ({ width: 20, height: 20, rgba: new Uint8Array(20 * 20 * 4).fill(255) }), + })); + assert.equal(decoded.result, IMAGE_SUCCESS); + assert.equal(decoded.format, "jpeg"); + assert.equal(prepareImageUpdate(request({ imageData: jpeg })).result, IMAGE_EXCEPTION); +}); + +test("raw pixels need compressMode 2 and exactly the container's size", () => { + const raw = new Array(400).fill(0); + assert.equal(prepareImageUpdate(request({ imageData: raw })).result, IMAGE_SUCCESS); + assert.equal(prepareImageUpdate(request({ imageData: raw, compressMode: 0 })).result, IMAGE_EXCEPTION); + assert.equal(prepareImageUpdate(request({ imageData: raw.slice(1) })).result, IMAGE_EXCEPTION); + assert.equal(prepareImageUpdate(request({ imageData: raw, geometry: { width: 400, height: 1 } })).result, IMAGE_SIZE_INVALID); +}); + +test("raw bytes up to 15 are already 4-bit levels; larger ones are 8-bit grey", () => { + const raw = new Array(400).fill(0); + raw.splice(0, 4, 15, 16, 128, 255); + const { levels } = prepareImageUpdate(request({ imageData: raw })); + assert.deepEqual(Array.from(levels.slice(0, 4)), [15, 1, 8, 15]); +}); + +test("4-bit raw rows are padded to whole bytes", () => { + // 21 wide: 11 bytes a row, high nibble first, the last low nibble unused. + const rows = []; + for (let y = 0; y < 20; y++) rows.push(0xf0, ...new Array(9).fill(0), 0x0f << 4); + const { result, levels } = prepareImageUpdate(request({ imageData: rows, geometry: { width: 21, height: 20 } })); + assert.equal(result, IMAGE_SUCCESS); + assert.deepEqual([levels[0], levels[1], levels[20], levels[21]], [15, 0, 15, 15]); +}); + +test("extended layouts skip the stock size and compressMode rules", () => { + const big = new Array(400 * 200).fill(0); + const result = prepareImageUpdate(request({ imageData: big, compressMode: 0, geometry: { width: 400, height: 200 }, stockLimits: false })); + assert.equal(result.result, IMAGE_SUCCESS); +}); diff --git a/tests/evenhub-layout-debug.test.cjs b/tests/evenhub-layout-debug.test.cjs index 7e134ae7..17257d6f 100644 --- a/tests/evenhub-layout-debug.test.cjs +++ b/tests/evenhub-layout-debug.test.cjs @@ -1,18 +1,10 @@ -// The EvenHub layout debug log: the stock container-order rules it compares -// our paint order against (from the evenRealities-openCFW reconstruction of -// evenhub_ui_containers.c), and the stock notes it attaches to updates. +// The EvenHub layout debug log (app/apps/evenhub/layout-debug.ts): page notes +// and the per-update lines it writes to logcat. const test = require("node:test"); const assert = require("node:assert/strict"); -const { parsePage, paintOrder } = require("../.test-build/app/apps/evenhub/containers.js"); -const { - describeImageUpdate, - describePage, - stockDifferences, - stockPaintOrder, -} = require("../.test-build/app/apps/evenhub/layout-debug.js"); - -const names = (containers) => containers.map((c) => c.name); +const { parsePage } = require("../.test-build/app/apps/evenhub/containers.js"); +const { describeImageUpdate, describePage, pageNotes } = require("../.test-build/app/apps/evenhub/layout-debug.js"); const iconOverBody = (extra = {}) => parsePage({ @@ -21,102 +13,51 @@ const iconOverBody = (extra = {}) => imageObject: [{ containerID: 2, containerName: "icon", xPosition: 8, yPosition: 8, width: 24, height: 24, ...extra.image }], }); -test("without zOrderIndex, stock orders lists, then texts, then images", () => { - const page = parsePage({ - imageObject: [{ containerID: 1, containerName: "img" }], - textObject: [{ containerID: 2, containerName: "txt" }], - listObject: [{ containerID: 3, containerName: "lst" }], - }); - assert.deepEqual(names(stockPaintOrder(page.containers)), ["lst", "txt", "img"]); - // ...where we put images at the bottom. - assert.deepEqual(names(paintOrder(page.containers)), ["img", "lst", "txt"]); -}); - -test("stock sorts by zOrderIndex only when every container has one", () => { - const all = iconOverBody({ text: { zOrderIndex: 2 }, image: { zOrderIndex: 1 } }); - assert.deepEqual(names(stockPaintOrder(all.containers)), ["icon", "body"]); - assert.deepEqual(names(paintOrder(all.containers)), ["icon", "body"]); - - const some = iconOverBody({ text: { zOrderIndex: 2 } }); - assert.deepEqual(names(stockPaintOrder(some.containers)), ["body", "icon"]); - assert.deepEqual(names(paintOrder(some.containers)), ["icon", "body"]); - assert.ok(stockDifferences(some).some((note) => note.includes("zOrderIndex is set on 1 of 2"))); -}); - -test("an image overlapping text is flagged when the two orders disagree", () => { - const notes = stockDifferences(iconOverBody()); - assert.ok(notes.some((note) => note.startsWith("paint order differs"))); - assert.ok(notes.includes('image#2 "icon" overlaps text#1 "body": we draw text#1 "body" on top, stock draws image#2 "icon" on top')); -}); - -test("non-overlapping containers in a different order raise no overlap note", () => { - const notes = stockDifferences(iconOverBody({ text: { yPosition: 100, height: 100 } })); - assert.ok(!notes.some((note) => note.includes("overlaps"))); -}); - -test("image containers outside stock's size limits are flagged", () => { - const notes = stockDifferences(iconOverBody({ image: { width: 16, height: 16 } })); - assert.ok(notes.some((note) => note.includes("16x16") && note.includes("fails the whole page"))); -}); - -test("a shared container id resolves to the first container in stock order", () => { - const page = iconOverBody({ image: { containerID: 1 } }); - assert.ok(stockDifferences(page).includes('id 1 is shared by text#1 "body", image#1 "icon"; stock resolves it to text#1 "body" only')); - const lines = describeImageUpdate({ - page, - sentId: 1, - sentName: "", - container: page.containers.find((c) => c.kind === "image"), - raw: [], - bytes: null, - decoded: null, - result: 1, - }); - assert.ok(lines.some((line) => line.includes("stock drops this update"))); -}); - -test("image updates note base64 payloads, size mismatches and blank images", () => { - const page = iconOverBody(); - const icon = page.containers.find((c) => c.kind === "image"); - const lines = describeImageUpdate({ - page, - sentId: 2, - sentName: "icon", - container: icon, - raw: "AAAA", - bytes: new Uint8Array(3), - decoded: { pixels: new Uint8Array(4), width: 2, height: 2 }, - result: 0, - }); - assert.match(lines[0], /payload=string\(4 chars\)/); - assert.match(lines[0], /decoded=2x2 blank/); - assert.ok(lines.some((line) => line.includes("base64-string"))); - assert.ok(lines.some((line) => line.includes("exactly the container's size (24x24)"))); -}); - -test("lit pixels are summarized with a bounding box", () => { - const page = iconOverBody(); - const icon = page.containers.find((c) => c.kind === "image"); - const pixels = new Uint8Array(24 * 24); - pixels[24 * 3 + 5] = 200; - pixels[24 * 10 + 7] = 255; - const [first] = describeImageUpdate({ - page, - sentId: 2, - sentName: "", - container: icon, - raw: Array.from(pixels), - bytes: pixels, - decoded: { pixels, width: 24, height: 24 }, - result: 0, - }); - assert.match(first, /format=raw8 decoded=24x24 lit=2px max=255 bbox=5,3-7,10 result=0$/); -}); - test("describePage lists containers bottom to top and ends with the raw payload", () => { const lines = describePage("rebuildPageContainer", iconOverBody(), { containerTotalNum: 2 }); assert.match(lines[0], /^rebuildPageContainer: 2 containers \(0 list, 1 text, 1 image\), containerTotalNum=2/); - assert.match(lines[1], /^ {2}0: image#2 "icon" @8,8 24x24 no pixels yet$/); - assert.match(lines[2], /^ {2}1: text#1 "body" @0,0 576x288 capture len=0 ""$/); + assert.match(lines[1], /^ {2}0: text#1 "body" @0,0 576x288 capture len=0 ""$/); + assert.match(lines[2], /^ {2}1: image#2 "icon" @8,8 24x24 no pixels yet$/); assert.equal(lines.at(-1), ' raw: {"containerTotalNum":2}'); }); + +test("a shared container id is noted with the container it resolves to", () => { + assert.ok(pageNotes(iconOverBody({ image: { containerID: 1 } })).includes( + 'id 1 is shared by text#1 "body", image#1 "icon"; updates and events resolve to text#1 "body" only', + )); +}); + +test("capture-container problems and a wrong containerTotalNum are noted", () => { + assert.ok(pageNotes(iconOverBody({ text: { isEventCapture: 0 } })).some((n) => n.startsWith("no event-capture container"))); + assert.ok(pageNotes(iconOverBody(), 3).some((n) => n.startsWith("containerTotalNum is 3 but 2"))); + assert.deepEqual(pageNotes(iconOverBody(), 2), []); +}); + +const prepared = (over = {}) => ({ result: 0, levels: null, width: 0, height: 0, format: "raw", reason: "", ...over }); + +test("an update lists payload, size, lit pixels and outcome", () => { + const page = iconOverBody(); + const levels = new Uint8Array(24 * 24); + levels[24 * 3 + 5] = 9; + levels[24 * 10 + 7] = 15; + const [line] = describeImageUpdate({ + sentId: 2, sentName: "icon", raw: Array.from(levels), target: page.containers[1], + prepared: prepared({ levels, width: 24, height: 24 }), shown: true, result: 0, + }); + assert.equal(line, 'updateImageRawData id=2 "icon" -> image#2 "icon" payload=number[576] format=raw 24x24 lit=2px max=15/15 bbox=5,3-7,10 result=0 shown'); +}); + +test("an update says why it wasn't shown", () => { + const page = iconOverBody({ image: { containerID: 1 } }); + const toText = describeImageUpdate({ + sentId: 1, sentName: "", raw: [], target: page.containers[0], + prepared: prepared({ levels: new Uint8Array(1), width: 1, height: 1 }), shown: false, result: 4, + }); + assert.ok(toText.some((line) => line.includes('resolves to text#1 "body", so the glasses refuse'))); + const failed = describeImageUpdate({ + sentId: 2, sentName: "", raw: "AAAA", target: undefined, + prepared: prepared({ result: 4, format: null, reason: "imageData is not an array of integers" }), shown: false, result: 4, + }); + assert.match(failed[0], /payload=string\(4 chars\) result=4 not shown$/); + assert.equal(failed[1], " imageData is not an array of integers"); +}); diff --git a/tests/evenhub-payload-validation.test.cjs b/tests/evenhub-payload-validation.test.cjs new file mode 100644 index 00000000..05a25686 --- /dev/null +++ b/tests/evenhub-payload-validation.test.cjs @@ -0,0 +1,61 @@ +// The Even phone app's checks before it sends a page or text upgrade to the +// glasses (app/apps/evenhub/payload-validation.ts). +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { stockPayloadError } = require("../.test-build/app/apps/evenhub/payload-validation.js"); + +// What SDK 0.0.16 sends for a small page (captured with a stub bridge). +const sdkPage = () => ({ + containerTotalNum: 2, + textObject: [{ containerID: 1, containerName: "t", xPosition: 0, yPosition: 0, width: 200, height: 100, content: "hi", isEventCapture: 1, borderWidth: 2 }], + imageObject: [{ containerID: 2, containerName: "i", xPosition: 0, yPosition: 0, width: 40, height: 40 }], + listObject: [{ containerID: 3, containerName: "l", itemContainer: { itemCount: 2, itemName: ["a", "b"], isItemSelectBorderEn: 1 } }], + menuObject: { menuItems: [{ itemName: "Pause", itemID: 1 }] }, +}); + +test("SDK-shaped payloads pass", () => { + assert.equal(stockPayloadError("create", sdkPage()), null); + assert.equal(stockPayloadError("rebuild", sdkPage()), null); + assert.equal(stockPayloadError("textUpgrade", { containerID: 1, containerName: "t", contentOffset: 2, contentLength: 3, content: "abc" }), null); +}); + +test("unknown keys fail, numeric tag keys pass", () => { + const page = sdkPage(); + page.textObject[0].Container_ID = 1; + assert.match(stockPayloadError("create", page), /unknown key textObject\[0\]\.Container_ID/); + assert.match(stockPayloadError("rebuild", { ...sdkPage(), widgetId: 1 }), /unknown key widgetId/); + assert.equal(stockPayloadError("create", { ...sdkPage(), 7: 1 }), null); +}); + +test("numeric fields take integers or numeric strings only", () => { + const withField = (value) => { + const page = sdkPage(); + page.textObject[0].xPosition = value; + return stockPayloadError("create", page); + }; + assert.equal(withField(-10), null, "negative values wrap rather than fail"); + assert.equal(withField("12"), null); + assert.match(withField(10.5), /must be an integer/); + assert.match(withField(true), /must be an integer/); + assert.match(withField(null), /is null/); + assert.match(withField("left"), /must be an integer/); +}); + +test("string and repeated fields are type-checked", () => { + const page = sdkPage(); + page.textObject[0].content = 5; + assert.match(stockPayloadError("create", page), /content must be a string/); + const list = sdkPage(); + list.listObject[0].itemContainer.itemName = "a"; + assert.match(stockPayloadError("create", list), /itemName must be an array/); +}); + +test("textColor must be an integer 0..4 wherever it appears", () => { + const page = sdkPage(); + page.textObject[0].textColor = 4; + assert.equal(stockPayloadError("create", page), null); + page.textObject[0].textColor = 15; + assert.match(stockPayloadError("create", page), /textColor must be an integer from 0 to 4/); + assert.match(stockPayloadError("textUpgrade", { containerID: 1, content: "x", textColor: "3" }), /textColor/); +}); diff --git a/tests/evenhub-session-stock.test.cjs b/tests/evenhub-session-stock.test.cjs new file mode 100644 index 00000000..39e1191b --- /dev/null +++ b/tests/evenhub-session-stock.test.cjs @@ -0,0 +1,161 @@ +// EvenHubSession behaviors that follow stock (the Even phone app plus +// firmware 2.2.9.22, as decompiled and run in the firmware emulator): text +// scrolling and its boundary events, text upgrades, image routing, and what a +// page the firmware can't build does. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const vm = require('node:vm'); +const { loader } = require('./helpers/load-typescript.cjs'); + +// A face with 10px characters and 27px lines, wrapping only at '\n'. +const face = { + lineHeight: 27, + measureLine: text => text.length * 10, + wrap: text => text.split('\n'), + drawText() {}, + drawTextWrapped() {}, +}; + +function harness() { + const timers = []; + const load = loader({ global: { isIOS: true }, setTimeout: fn => { timers.push(fn); return timers.length; }, clearTimeout() {}, Promise }, { + '@nativescript/core': { ApplicationSettings: { getString: (_k, d) => d, setString() {} } }, + 'upng-js': require('upng-js'), + '../../graphics/evenhub-font': { EvenHubFont: { get: () => face } }, + './fonts': { containerFace: () => face, availableFontFamilies: () => [], measureContainerText: () => ({ width: 0, lineHeight: 27 }) }, + '../../ui/dashboard-settings': {}, '../../ui/sound-effects': {}, './api-key-dialog': {}, + './mic-router': { evenHubMicRouter: {} }, './imu-router': { evenHubImuRouter: {} }, + './compass-router': { evenHubCompassRouter: {} }, '../../assistant/tool-registry': {}, + '../../native/location': {}, '../../native/location-tracker': {}, '../../native/location-permissions': {}, + '../../native/image-rgba': { decodeImageRgba: () => null }, + }); + const api = load('app/apps/evenhub/session.ts'); + const session = new api.EvenHubSession({ name: 'Probe', packageId: 'test.stock', permissions: [] }, '/test/dist', () => {}, ''); + let windowClosed = false; + session.attachWindow({ requestRender() {}, windowId: 'probe', closeWindow: () => { windowClosed = true; } }); + const web = vm.createContext({ Promise, Map, Date, console }); + web.window = web; + web.__faceclawEvenHub = { postMessage: (name, args, id) => session.handleBridgeCall(name, args, id) }; + session.attachWebView({ evaluateJs: script => vm.runInContext(script, web), destroy() {} }); + vm.runInContext(api.EVENHUB_BRIDGE_INJECT_SCRIPT, web); + const events = []; + web._listenEvenAppMessage = value => { + const message = typeof value === 'string' ? JSON.parse(value) : value; + if (message.method === 'evenHubEvent') events.push(JSON.parse(JSON.stringify(message.data))); + }; + const call = (method, data = {}) => web.flutter_inappwebview.callHandler('evenAppMessage', JSON.stringify({ method, data })); + const runTimers = () => { for (const fn of timers.splice(0)) fn(); }; + return { session, call, events, runTimers, windowClosed: () => windowClosed }; +} + +const lines = n => Array.from({ length: n }, (_, i) => `line ${i}`).join('\n'); +const textPage = (content, extra = {}) => ({ + containerTotalNum: 1, + textObject: [{ containerID: 1, containerName: 'body', xPosition: 0, yPosition: 0, width: 300, height: 100, content, isEventCapture: 1, ...extra }], +}); +const swipe = (h, direction) => h.session.handleGesture({ type: direction === 'down' ? 'scroll-down' : 'scroll-up' }); + +test('an overflowing capture text container scrolls itself and reports only at its ends', async () => { + const h = harness(); + // 6 lines = 162px in a 100px window: 62px to scroll, three 28px swipes. + assert.equal(await h.call('createStartUpPageContainer', textPage(lines(6))), 0); + h.events.length = 0; + swipe(h, 'down'); swipe(h, 'down'); swipe(h, 'down'); + assert.deepEqual(h.events, [], 'no events while scrolling, nor for the swipe that reaches the end'); + swipe(h, 'down'); + assert.deepEqual(h.events, [{ type: 'textEvent', jsonData: { containerID: 1, containerName: 'body', eventType: 2 } }]); + h.events.length = 0; + swipe(h, 'up'); swipe(h, 'up'); swipe(h, 'up'); + assert.deepEqual(h.events, []); + swipe(h, 'up'); + assert.deepEqual(h.events, [{ type: 'textEvent', jsonData: { containerID: 1, containerName: 'body', eventType: 1 } }]); +}); + +test('text that fits reports every swipe as reaching an end', async () => { + const h = harness(); + await h.call('createStartUpPageContainer', textPage('short')); + h.events.length = 0; + swipe(h, 'down'); swipe(h, 'down'); + assert.equal(h.events.filter(e => e.type === 'textEvent' && e.jsonData.eventType === 2).length, 2); +}); + +test('text upgrades replace the whole text by id, ignore offsets, and keep old text when emptied', async () => { + const h = harness(); + await h.call('createStartUpPageContainer', textPage('Hello world')); + const body = () => h.session.page.containers[0]; + assert.equal(await h.call('textContainerUpgrade', { containerID: 1, containerName: 'other name', contentOffset: 3, contentLength: 2, content: 'XY' }), true); + assert.equal(body().content, 'XY'); + assert.equal(await h.call('textContainerUpgrade', { containerID: 1, content: '' }), true); + assert.equal(body().content, 'XY', 'an empty upgrade leaves the old text showing'); + assert.equal(await h.call('textContainerUpgrade', { containerID: 9, content: 'nope' }), false); + assert.equal(await h.call('textContainerUpgrade', { containerID: 1, content: 'x', junk: 1 }), false, 'unknown keys are refused'); +}); + +test('an upgrade scrolls the text back to the top', async () => { + const h = harness(); + await h.call('createStartUpPageContainer', textPage(lines(6))); + swipe(h, 'down'); + assert.equal(h.session.page.containers[0].scrollY, 28); + await h.call('textContainerUpgrade', { containerID: 1, content: lines(7) }); + assert.equal(h.session.page.containers[0].scrollY, 0); +}); + +test('image updates need a page, an image at that id, and a usable payload', async () => { + const h = harness(); + const raw = new Array(400).fill(0); + assert.equal(await h.call('updateImageRawData', { containerID: 2, imageData: raw, compressMode: 2 }), 4, 'no page yet'); + await h.call('createStartUpPageContainer', { + containerTotalNum: 3, + textObject: [{ containerID: 1, containerName: 'body', width: 300, height: 100, content: 'x', isEventCapture: 1 }, + { containerID: 3, containerName: 'label', width: 50, height: 30, content: 'y' }], + imageObject: [{ containerID: 2, containerName: 'icon', width: 20, height: 20 }, { containerID: 3, containerName: 'shadowed', width: 20, height: 20 }], + }); + assert.equal(await h.call('updateImageRawData', { containerID: 2, imageData: raw, compressMode: 2 }), 0); + assert.equal(await h.call('updateImageRawData', { containerID: 2, imageData: 'AAAA', compressMode: 2 }), 4, 'base64 is refused'); + assert.equal(await h.call('updateImageRawData', { containerID: 3, imageData: raw, compressMode: 2 }), 4, 'id 3 resolves to the text container'); + // With no image at that id the Even app has no size for raw pixels: 0x0 is out of range. + assert.equal(await h.call('updateImageRawData', { containerID: 7, imageData: raw, compressMode: 2 }), 2); +}); + +test('a page the firmware cannot build fails create, and a failed rebuild closes the app', async () => { + const h = harness(); + const tinyImage = { ...textPage('x'), imageObject: [{ containerID: 2, containerName: 'dot', width: 10, height: 10 }] }; + assert.equal(await h.call('createStartUpPageContainer', tinyImage), 1); + assert.equal(await h.call('rebuildPageContainer', textPage('x')), false, 'nothing to rebuild before a page exists'); + assert.equal(await h.call('createStartUpPageContainer', textPage('x')), 0); + h.events.length = 0; + assert.equal(await h.call('rebuildPageContainer', tinyImage), false); + assert.equal(h.session.page.containers.length, 0); + assert.ok(h.events.some(e => e.type === 'sysEvent' && e.jsonData.eventType === 6), 'ABNORMAL_EXIT'); + h.runTimers(); + assert.equal(h.windowClosed(), true); +}); + +test('the Even app refuses malformed pages before they reach the glasses', async () => { + const h = harness(); + assert.equal(await h.call('createStartUpPageContainer', textPage('x', { isEventCapture: true })), 1); + assert.equal(await h.call('createStartUpPageContainer', textPage('x', { textColor: 7 })), 1); + assert.equal(await h.call('createStartUpPageContainer', textPage('x')), 0); + assert.equal(await h.call('rebuildPageContainer', { ...textPage('y'), Container_Total_Num: 1 }), false); + assert.equal(h.session.page.containers[0].content, 'x', 'a refused rebuild leaves the page alone'); +}); + +test('borders use the colour index, overflowing text gets a scrollbar, and images cover text', async () => { + const h = harness(); + // 300x100 with a 2px border of colour 3; 6 lines overflow the 96px window by 66px. + await h.call('createStartUpPageContainer', { + containerTotalNum: 2, + textObject: [{ containerID: 1, containerName: 'body', xPosition: 0, yPosition: 0, width: 300, height: 100, content: lines(6), isEventCapture: 1, borderWidth: 2, borderColor: 3 }], + imageObject: [{ containerID: 2, containerName: 'icon', xPosition: 0, yPosition: 40, width: 20, height: 20 }], + }); + await h.call('updateImageRawData', { containerID: 2, imageData: new Array(400).fill(0), compressMode: 2 }); + const image = h.session.paint({ width: 576, height: 288 }, true); + const at = (x, y) => image.pixels[y * 576 + x]; + assert.equal(at(150, 0), 3 * 17, 'border grey'); + assert.equal(at(0, 50), 0, 'the black image covers the border under it'); + // Track 6..94 (88px); length floor(88 * 100 / 166) = 53; x 290..293. + assert.deepEqual([at(290, 7), at(291, 7), at(293, 7), at(289, 7), at(294, 7)], [187, 187, 187, 0, 0]); + assert.deepEqual([at(290, 6), at(291, 6), at(292, 6), at(293, 6)], [0, 187, 187, 0], 'chamfered top row'); + assert.equal(at(291, 58), 187); + assert.equal(at(291, 59), 0, 'bar ends after 53 rows'); +}); diff --git a/tests/evenhub-stock-layout.test.cjs b/tests/evenhub-stock-layout.test.cjs new file mode 100644 index 00000000..f8bc218f --- /dev/null +++ b/tests/evenhub-stock-layout.test.cjs @@ -0,0 +1,81 @@ +// Stock container-model rules the EvenHub emulation follows: stacking order, +// which container captures input, id resolution, firmware clamps and border +// greys. Source: the evenRealities-openCFW reconstruction of firmware 2.2.6.10 +// (evenhub_ui_containers.c, evenhub_ui_page.c, common_text/list_container.c). +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { + borderGrey, + clampToStockLimits, + containerById, + eventCaptureContainer, + paintOrder, + parsePage, +} = require("../.test-build/app/apps/evenhub/containers.js"); + +const names = (containers) => containers.map((c) => c.name); + +test("without zOrderIndex containers stack lists, then texts, then images", () => { + const page = parsePage({ + imageObject: [{ containerID: 1, containerName: "img" }], + textObject: [{ containerID: 2, containerName: "txt" }], + listObject: [{ containerID: 3, containerName: "lst" }], + }); + assert.deepEqual(names(paintOrder(page.containers)), ["lst", "txt", "img"]); +}); + +test("zOrderIndex sorts only when every container has one", () => { + const all = parsePage({ + textObject: [{ containerID: 1, containerName: "body", zOrderIndex: 2 }], + imageObject: [{ containerID: 2, containerName: "icon", zOrderIndex: 1 }], + }); + assert.deepEqual(names(paintOrder(all.containers)), ["icon", "body"]); + const some = parsePage({ + textObject: [{ containerID: 1, containerName: "body", zOrderIndex: 2 }], + imageObject: [{ containerID: 2, containerName: "icon" }], + }); + assert.deepEqual(names(paintOrder(some.containers)), ["body", "icon"]); +}); + +test("the first capture container in stacking order owns input", () => { + const page = parsePage({ + textObject: [ + { containerID: 1, containerName: "a", isEventCapture: 1, zOrderIndex: 5 }, + { containerID: 2, containerName: "b", isEventCapture: 1, zOrderIndex: 3 }, + ], + }); + assert.equal(eventCaptureContainer(page).name, "b"); +}); + +test("an id resolves to its first container in stacking order, whatever the kind", () => { + const page = parsePage({ + textObject: [{ containerID: 7, containerName: "label" }], + imageObject: [{ containerID: 7, containerName: "icon" }], + }); + assert.equal(containerById(page, 7).kind, "text"); + assert.equal(containerById(page, 8), undefined); +}); + +test("stock clamps text and list geometry but leaves image sizes alone", () => { + const page = parsePage({ + textObject: [{ containerID: 1, containerName: "t", xPosition: 900, yPosition: 400, width: 800, height: 400, borderColor: 40, borderRadius: 30, paddingLength: 50 }], + listObject: [{ containerID: 2, containerName: "l", itemContainer: { itemName: Array.from({ length: 25 }, (_, i) => `item ${i}`) } }], + imageObject: [{ containerID: 3, containerName: "i", xPosition: 600, width: 400, height: 200 }], + }); + clampToStockLimits(page); + const [list, text, image] = page.containers; + assert.deepEqual([text.x, text.y, text.width, text.height], [576, 288, 576, 288]); + assert.deepEqual([text.borderColor, text.borderRadius, text.paddingLength], [16, 10, 32]); + assert.equal(list.itemNames.length, 20); + assert.deepEqual([image.x, image.width, image.height], [576, 400, 200]); +}); + +test("border color indices are 16 grey steps, with 16 meaning white", () => { + assert.equal(borderGrey(0), 0); + assert.equal(borderGrey(3), 0x33); + assert.equal(borderGrey(15), 0xff); + assert.equal(borderGrey(16), 0xff); + const page = parsePage({ textObject: [{ containerID: 1, containerName: "t", borderWidth: 2 }] }); + assert.equal(page.containers[0].borderColor, 0, "an omitted borderColor is the protobuf default, black"); +}); diff --git a/tests/ios-evenhub.test.cjs b/tests/ios-evenhub.test.cjs index 7e9921f3..3447a9cc 100644 --- a/tests/ios-evenhub.test.cjs +++ b/tests/ios-evenhub.test.cjs @@ -87,7 +87,8 @@ function sessionHarness(overrides = {}, playBuzzer = () => {}, { packageId = 'te '../../ui/dashboard-settings': {}, '../../ui/sound-effects': {}, './api-key-dialog': {}, './mic-router': { evenHubMicRouter: {} }, './imu-router': { evenHubImuRouter: {} }, './compass-router': { evenHubCompassRouter: {} }, '../../assistant/tool-registry': {}, - '../../native/location': {}, '../../native/location-tracker': {}, '../../native/location-permissions': {}, ...overrides, + '../../native/location': {}, '../../native/location-tracker': {}, '../../native/location-permissions': {}, + '../../native/image-rgba': { decodeImageRgba: () => null }, ...overrides, }); const api = load('app/apps/evenhub/session.ts'); const session = new api.EvenHubSession({ name: 'Probe', packageId, permissions: [] }, '/test/dist', line => logs.push(line), '', playBuzzer); @@ -105,10 +106,12 @@ function sessionHarness(overrides = {}, playBuzzer = () => {}, { packageId = 'te test('injected EvenHub RPC creates, updates and composites an image page with no Android globals', async () => { const h = sessionHarness(); assert.equal(await h.call('createStartUpPageContainer', { containerTotalNum: 1, - imageObject: [{ containerID: 1, containerName: 'board', xPosition: 0, yPosition: 0, width: 2, height: 2 }] }), 0); - assert.equal(await h.call('updateImageRawData', { containerID: 1, containerName: 'board', imageData: [0, 80, 160, 255] }), 0); + imageObject: [{ containerID: 1, containerName: 'board', xPosition: 0, yPosition: 0, width: 20, height: 20 }] }), 0); + // Raw 8-bit grey as the SDK sends it (compressMode 2), quantized to 4-bit levels as the Even app does. + const raw = new Array(400).fill(0); raw.splice(0, 4, 0, 80, 160, 255); + assert.equal(await h.call('updateImageRawData', { containerID: 1, containerName: 'board', imageData: raw, compressMode: 2 }), 0); const painted = h.session.paint({ width: 576, height: 288 }, true); - assert.deepEqual([painted.pixels[0], painted.pixels[1], painted.pixels[576], painted.pixels[577]], [0, 80, 160, 255]); + assert.deepEqual(Array.from(painted.pixels.slice(0, 4)), [0, 5 * 17, 9 * 17, 255]); assert.equal(await h.call('setLocalStorage', { key: 'score', value: '42' }), true); assert.equal(await h.call('getLocalStorage', { key: 'score' }), '42'); const events = []; diff --git a/tests/lvgl-wrap.test.cjs b/tests/lvgl-wrap.test.cjs index 09821955..3c5642ca 100644 --- a/tests/lvgl-wrap.test.cjs +++ b/tests/lvgl-wrap.test.cjs @@ -1,45 +1,35 @@ -// EvenHub text-container wrapping (app/graphics/lvgl-wrap.ts): pretext's -// breaks, except that leading spaces survive at the start of the text and -// after '\n' — apps indent with them to clear icons placed beside the text. +// EvenHub text-container wrapping (app/graphics/lvgl-wrap.ts), against what +// stock firmware 2.2.9.22 drew in the g2-firmware-emulator: spaces are kept +// everywhere, including ones that overflow onto a soft-wrapped line. const test = require("node:test"); const assert = require("node:assert/strict"); const { wrapLvglText } = require("../.test-build/app/graphics/lvgl-wrap.js"); -// Spaces are 4px like the stock 20px font; everything else 10px. -const fixed = (cp) => (cp === 32 ? 4 : 10); +// The stock 20px font's advances for these: H 12px, E 11px, space 5px. +const ADVANCES = { H: 12, E: 11, " ": 5 }; +const advance = (cp) => ADVANCES[String.fromCodePoint(cp)] ?? 10; -test("indentation after a newline is kept (Iris Pro's icon-clearing indent)", () => { - const content = " \n R E C O R D\n Conversation"; - const lines = wrapLvglText(content, 248, fixed); - assert.deepEqual(lines, [" ", " R E C O R D", " Conversation"]); +test("indentation after a newline and at the start is kept (Iris Pro's icon-clearing indent)", () => { + assert.deepEqual(wrapLvglText(" \n R E C O R D", 248, advance), [" ", " R E C O R D"]); + assert.deepEqual(wrapLvglText("AB\n CD", 248, advance), ["AB", " CD"]); + assert.deepEqual(wrapLvglText(" EF", 248, advance), [" EF"]); }); -test("indentation at the very start of the text is kept", () => { - assert.deepEqual(wrapLvglText(" 55°", 248, fixed), [" 55°"]); +test("spaces that overflow a line start the next one (emulator Q3)", () => { + const text = "HHHHHHHH EEEE"; + assert.deepEqual(wrapLvglText(text, 100, advance), ["HHHHHHHH", " EEEE"]); + assert.deepEqual(wrapLvglText(text, 110, advance), ["HHHHHHHH ", " EEEE"]); + assert.deepEqual(wrapLvglText(text, 120, advance), ["HHHHHHHH ", "EEEE"]); }); -test("a soft-wrapped continuation line drops its leading spaces", () => { - // "aaaa" (40px) fits; the space after it overflows at width 40 and is - // discarded, and the next space would lead the continuation line. - assert.deepEqual(wrapLvglText("aaaa bbbb", 40, fixed), ["aaaa", "bbbb"]); - // Wrapping at an earlier break opportunity. - assert.deepEqual(wrapLvglText("aa bb cc", 54, fixed), ["aa bb", "cc"]); +test("a word that overflows wraps after the last space or hyphen", () => { + assert.deepEqual(wrapLvglText("HH EEEEEEEE", 100, advance), ["HH ", "EEEEEEEE"]); + assert.deepEqual(wrapLvglText("HH-EEEEEEEE", 100, advance), ["HH-", "EEEEEEEE"]); }); -test("unindented text breaks exactly where pretext's measureTextWrap says", async () => { - const pretext = await import("@evenrealities/pretext"); - const advance = (a, b) => (b ? pretext.getTextWidth(String.fromCodePoint(a, b)) - pretext.getTextWidth(String.fromCodePoint(b)) : pretext.getTextWidth(String.fromCodePoint(a))); - const samples = [ - "Market moving from feature flex to everyday integration", - "No one owns human, calm intelligence\nConversation likely becomes next differentiation layer", - "Supercalifragilisticexpialidocious-and-then-some words that will not fit on one line at all", - "short\n\nlines\nhere", - ]; - for (const text of samples) { - for (let width = 40; width <= 560; width += 7) { - const ours = wrapLvglText(text, width, advance); - assert.equal(ours.length, pretext.measureTextWrap(text, width).lineCount, `${JSON.stringify(text)} at ${width}px`); - } - } +test("a run with no break opportunity breaks before the overflowing character", () => { + assert.deepEqual(wrapLvglText("HHHHHHHHHH", 50, advance), ["HHHH", "HHHH", "HH"]); + // A character wider than the line still gets one to itself. + assert.deepEqual(wrapLvglText("HH", 5, advance), ["H", "H"]); }); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 1de19ea5..f3bd5c75 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -57,6 +57,8 @@ "../app/apps/microphones/name-guess.ts", "../app/apps/evenhub/containers.ts", "../app/apps/evenhub/layout-debug.ts", + "../app/apps/evenhub/image-pipeline.ts", + "../app/apps/evenhub/payload-validation.ts", "../app/apps/evenhub/list-menu.ts", "../app/apps/evenhub/device-status.ts", "../app/apps/evenhub/font-spec.ts",