Optimize phone CPU usage

This commit is contained in:
James Babcock committed 2026-09-27 23:16:41 -07:00
1 parent 449cb0bf73
commit 45d2ece0d2
19 files changed
+485 -103

No files matched your search

@@ -349,6 +349,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
contentFingerprint, paintMs, frameId, if (glyphs == null) null else AndroidByteReader(glyphs))
}
/** Whether submitting this full-surface content would change nothing (see [GlassesSessionCore.isSurfaceCurrent]). */
fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean = core.isSurfaceCurrent(surfaceId, fingerprint)
/** Play a tone sequence via CFW load_image_z mode 5 kind 4 (payload built on the TS side). */
fun playBuzzerSequence(payload: java.nio.ByteBuffer?) {
val bytes = ByteArray(if (payload == null) 0 else payload.remaining())
@@ -23,6 +23,8 @@ import java.nio.charset.StandardCharsets
import java.util.ArrayDeque
import java.util.ArrayList
import java.util.Collections
import java.util.concurrent.ExecutorService
import java.util.concurrent.Executors
/**
* Phone side of the Wear OS integration: the single place that talks to the
@@ -114,6 +116,16 @@ class FaceclawWearBridge private constructor(context: Context) {
private val pending = ArrayDeque<PendingMessage>()
private val watchNodes: MutableList<Node> = ArrayList()
private var lastPublishedState: String? = null
/** Latest state waiting for [publishExecutor]; null when none is queued. Guarded by [lock]. */
private var queuedState: String? = null
/**
* Serializing a DataMap and handing it to Play services costs ~10-25ms per
* publish, and the switcher publishes on every foreground change, so it
* runs here instead of on the main thread.
*/
private val publishExecutor: ExecutorService = Executors.newSingleThreadExecutor { runnable ->
Thread(runnable, "FaceclawWearPublish").apply { isDaemon = true }
}
private class PendingMessage(
val path: String,
@@ -210,12 +222,28 @@ class FaceclawWearBridge private constructor(context: Context) {
publishState(json, false)
}
/** As [publishState]; `force` re-sends an unchanged state. */
/**
* As [publishState]; `force` re-sends an unchanged state. The put happens
* on [publishExecutor]; states that arrive while one is still waiting
* replace it, since only the latest matters to the watch.
*/
fun publishState(json: String?, force: Boolean) {
if (!available || json == null) return
synchronized(lock) {
if (!force && json == lastPublishedState) return
lastPublishedState = json
val alreadyQueued = queuedState != null
queuedState = json
if (alreadyQueued) return
}
publishExecutor.execute { publishQueuedState() }
}
private fun publishQueuedState() {
val json: String
synchronized(lock) {
json = queuedState ?: return
queuedState = null
}
try {
val request = PutDataMapRequest.create(PATH_STATE)
+10
View File
@@ -51,9 +51,19 @@ export function installedEvenHubPackageId(appId: string): string | null {
}
}
// The parsed index for the last stored value: the launcher and the watch
// state both list installed apps on every refresh, and the sort's
// localeCompare dominated the cost.
let parsedIndex: { raw: string; apps: InstalledEvenHubApp[] } | null = null;
export function getInstalledEvenHubApps(): InstalledEvenHubApp[] {
const raw = getStringSetting(STORAGE_KEY, "");
if (!raw) return [];
if (parsedIndex?.raw !== raw) parsedIndex = { raw, apps: parseInstalledIndex(raw) };
return parsedIndex.apps.map((app) => ({ ...app }));
}
function parseInstalledIndex(raw: string): InstalledEvenHubApp[] {
try {
const parsed = JSON.parse(raw);
if (!Array.isArray(parsed)) return [];
+7
View File
@@ -2515,6 +2515,13 @@ class DashboardController {
return;
}
const fingerprint = frameTimings.span(frameId, "fingerprint", () => planesFingerprint(planes));
// An identical repaint (e.g. every in-process window re-renders when the
// switcher brings it to the foreground) would flatten, encode and
// composite only to be deduped before sending.
if (display.isSurfaceCurrent(surfaceId, fingerprint)) {
frameTimings.finishFrame(frameId, "discarded: surface unchanged");
return;
}
const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes));
const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer());
const preparedDraws = frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws));
+27 -7
View File
@@ -36,14 +36,34 @@ export type DisplayList = {
};
export type PlacedDisplayList = { displayList: DisplayList; x: number; y: number; width: number; height: number; depth: number };
/**
* Growable byte buffer. Lists are encoded on every frame that carries one, and
* twice more for its fingerprint and the preview; resources (whole menu rows,
* scroll strips) are tens of KB, so they go in with one set() rather than a
* push per byte.
*/
class Writer {
bytes: number[] = [];
u8(n: number): void { this.bytes.push(integer(n, 0, 255)); }
private buffer = new Uint8Array(256);
length = 0;
private reserve(count: number): void {
const needed = this.length + count;
if (needed <= this.buffer.length) return;
let size = this.buffer.length * 2;
while (size < needed) size *= 2;
const grown = new Uint8Array(size);
grown.set(this.buffer.subarray(0, this.length));
this.buffer = grown;
}
u8(n: number): void { integer(n, 0, 255); this.reserve(1); this.buffer[this.length++] = n; }
i8(n: number): void { this.u8(integer(n, -128, 127) & 255); }
u16(n: number): void { integer(n, 0, 65535); this.bytes.push(n & 255, n >>> 8); }
u16(n: number): void { integer(n, 0, 65535); this.reserve(2); this.buffer[this.length++] = n & 255; this.buffer[this.length++] = n >>> 8; }
i16(n: number): void { this.u16(integer(n, -32768, 32767) & 65535); }
u32(n: number): void { integer(n, 0, 4294967295); this.u16(n & 65535); this.u16(n >>> 16); }
raw(bytes: ArrayLike<number>): void { for (let i = 0; i < bytes.length; i++) this.bytes.push(bytes[i]); }
raw(bytes: ArrayLike<number>): void { this.reserve(bytes.length); this.buffer.set(bytes, this.length); this.length += bytes.length; }
/** The written bytes, in a buffer of their own. */
bytes(): Uint8Array { return this.buffer.slice(0, this.length); }
/** A view of the written bytes; valid until the next write. */
view(): Uint8Array { return this.buffer.subarray(0, this.length); }
}
/** Length-delimited bridge envelope; call operands use fixed widths except extended x/y. */
@@ -85,9 +105,9 @@ export function encodeDisplayList(placed: PlacedDisplayList, now = Date.now()):
}
}
}
if (out.bytes.length > 4 * 1024 * 1024) throw new Error('Display list exceeds bridge limit');
const header = new Writer(); header.u8(DISPLAY_LIST_RECORD); header.u32(out.bytes.length); header.raw(out.bytes);
return Uint8Array.from(header.bytes);
if (out.length > 4 * 1024 * 1024) throw new Error('Display list exceeds bridge limit');
const header = new Writer(); header.u8(DISPLAY_LIST_RECORD); header.u32(out.length); header.raw(out.view());
return header.bytes();
}
/** Replay-record sizes by tag, as in glyph-wire's frame draw buffer. */
+47 -32
View File
@@ -277,17 +277,28 @@ export class GrayImage {
const destX = dx | 0;
const destY = dy | 0;
for (let row = 0; row < copyHeight; row++) {
const sy = srcY + row;
const ty = destY + row;
if (sy < 0 || sy >= source.height || ty < 0 || ty >= this.height) continue;
for (let col = 0; col < copyWidth; col++) {
const sx = srcX + col;
const tx = destX + col;
if (sx < 0 || sx >= source.width || tx < 0 || tx >= this.width) continue;
const value = source.pixels[sy * source.width + sx] ?? 0;
// Clip once, then copy row spans: flattening a frame blits every plane,
// and per-pixel bounds checks made that a sizeable share of a paint.
const colStart = Math.max(0, -destX);
const colEnd = Math.min(copyWidth, source.width - srcX, this.width - destX);
const rowStart = Math.max(0, -destY);
const rowEnd = Math.min(copyHeight, source.height - srcY, this.height - destY);
const span = colEnd - colStart;
const src = source.pixels;
const dst = this.pixels;
// A blit within one image copies pixel by pixel, forwards, as it always has.
const bulk = !opts.transparentZero && source !== this;
for (let row = rowStart; span > 0 && row < rowEnd; row++) {
const from = (srcY + row) * source.width + srcX + colStart;
const to = (destY + row) * this.width + destX + colStart;
if (bulk) {
dst.set(src.subarray(from, from + span), to);
continue;
}
for (let i = 0; i < span; i++) {
const value = src[from + i]!;
if (opts.transparentZero && value === 0) continue;
this.pixels[ty * this.width + tx] = value;
dst[to + i] = value;
}
}
@@ -398,7 +409,7 @@ export class GrayImage {
hash = mixInt(hash, placed.source.height);
hash = mixInt(hash, placed.source.sourceContentHash32());
if (placed.presentation) { const p = placed.presentation; for (const value of [p.radius, p.background, p.border, p.depth, p.mode === "image" ? 1 : p.mode === "masked-image" ? 2 : 0]) hash = mixInt(hash, value);
if (p.displayList) for (const byte of encodeDisplayList({ displayList: p.displayList, x: 0, y: 0, width: placed.source.width, height: placed.source.height, depth: p.depth }, p.displayList.timeline?.startedAt ?? 0)) hash = mixInt(hash, byte);
if (p.displayList) hash = mixInt(hash, bytesHash32(encodeDisplayList({ displayList: p.displayList, x: 0, y: 0, width: placed.source.width, height: placed.source.height, depth: p.depth }, p.displayList.timeline?.startedAt ?? 0)));
}
} else {
hash = mixInt(hash, 0xf17e);
@@ -415,28 +426,9 @@ export class GrayImage {
return hash >>> 0;
}
/**
* FNV-1a over the raster pixels. Four bytes at a time where alignment
* allows: this runs on every frame's fingerprint, over the whole screen, on
* the thread that is about to hand the frame to the transport.
*/
/** Hash of the raster pixels; see bytesHash32. */
private pixelsHash32(): number {
const pixels = this.pixels;
let hash = 2166136261;
let i = 0;
if ((pixels.byteOffset & 3) === 0) {
const words = new Uint32Array(pixels.buffer, pixels.byteOffset, pixels.length >>> 2);
for (let w = 0; w < words.length; w++) {
hash ^= words[w]!;
hash = Math.imul(hash, 16777619);
}
i = words.length << 2;
}
for (; i < pixels.length; i++) {
hash ^= pixels[i]!;
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
return bytesHash32(this.pixels);
}
/**
@@ -744,6 +736,29 @@ function clampByte(value: number): number {
}
/** Fold a 32-bit int into an FNV-style running hash. */
/**
* FNV-1a over bytes. Four bytes at a time where alignment allows: this runs on
* every frame's fingerprint, over the whole screen, on the thread that is
* about to hand the frame to the transport.
*/
function bytesHash32(bytes: Uint8Array): number {
let hash = 2166136261;
let i = 0;
if ((bytes.byteOffset & 3) === 0) {
const words = new Uint32Array(bytes.buffer, bytes.byteOffset, bytes.length >>> 2);
for (let w = 0; w < words.length; w++) {
hash ^= words[w]!;
hash = Math.imul(hash, 16777619);
}
i = words.length << 2;
}
for (; i < bytes.length; i++) {
hash ^= bytes[i]!;
hash = Math.imul(hash, 16777619);
}
return hash >>> 0;
}
function mixInt(hash: number, value: number): number {
hash ^= value | 0;
return Math.imul(hash, 16777619);
+8
View File
@@ -536,6 +536,14 @@ export class FaceclawCommunicatorBridge {
await this.enqueueNativeCall(() => { this.communicator.setScreenBlankedBlanked(Boolean(blanked)); });
}
/**
* See the Android bridge. Not wired to the shared compositor's check on iOS
* yet, so every repaint is submitted as before.
*/
isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean {
return false;
}
async submitSurfaceFrame(
surfaceId: string,
pixels8bpp: Uint8Array,
+23 -2
View File
@@ -133,6 +133,8 @@ export class FaceclawCommunicatorBridge {
private javaCallQueue: Promise<void> = Promise.resolve();
/** Calls waiting on javaCallQueue; 0 means enqueueJavaCall's fast path is safe. */
private queuedJavaCalls = 0;
/** Queued calls that may change what a surface holds; see isSurfaceCurrent. */
private queuedSurfaceChanges = 0;
// Reused Java-side buffers for byte payloads; passing a JS ArrayBuffer to
// Java leaks it (see java-direct-buffer.ts). Loaded inside the queued call,
// immediately before the synchronous Java call that consumes them.
@@ -267,8 +269,12 @@ export class FaceclawCommunicatorBridge {
* the frame timings), which is pure latency on a call the caller is already
* awaiting. Off by default: a call that might block for a while should keep
* yielding to the main looper first.
*
* preservesSurfaces marks a call that neither changes a surface's retained
* content nor leaves a change waiting for a later composite (it composites
* itself), so having it queued does not make isSurfaceCurrent answer false.
*/
private enqueueJavaCall<T>(operation: () => T, inlineWhenIdle = false): Promise<T> {
private enqueueJavaCall<T>(operation: () => T, inlineWhenIdle = false, preservesSurfaces = false): Promise<T> {
if (inlineWhenIdle && this.queuedJavaCalls === 0) {
try {
return Promise.resolve(operation());
@@ -277,10 +283,12 @@ export class FaceclawCommunicatorBridge {
}
}
this.queuedJavaCalls++;
if (!preservesSurfaces) this.queuedSurfaceChanges++;
const run = () =>
new Promise<T>((resolve, reject) => {
setTimeout(() => {
this.queuedJavaCalls--;
if (!preservesSurfaces) this.queuedSurfaceChanges--;
try {
resolve(operation());
} catch (error) {
@@ -549,6 +557,7 @@ export class FaceclawCommunicatorBridge {
await this.enqueueJavaCall(
() => this.communicator.submitShellScene(this.shellSceneBuffer.load(snapshot), paintMs, frameId),
true,
true,
);
}
@@ -562,7 +571,19 @@ export class FaceclawCommunicatorBridge {
async setSurfaceVisible(id: string, visible: boolean): Promise<void> {
await this.enqueueJavaCall(() => {
this.communicator.setSurfaceVisible(id, Boolean(visible));
});
}, false, true);
}
/**
* Whether submitting a full-surface frame with this fingerprint would
* change nothing: Java already holds that content for the surface, and no
* queued or not-yet-composited change is waiting on a composite. Lets a
* repaint that came out identical (a window re-rendered when it comes to
* the foreground, say) skip flattening and submitting its frame.
*/
isSurfaceCurrent(surfaceId: string, fingerprint: string): boolean {
if (this.queuedSurfaceChanges > 0) return false;
return Boolean(this.communicator.isSurfaceCurrent(surfaceId, fingerprint));
}
/**
+18
View File
@@ -7,7 +7,25 @@ export type PhoneBatteryState = {
charging: boolean | null;
};
// The Android read is a sticky-broadcast binder round trip, and the top bar
// asks on every shell paint; the level moves far slower than that.
const ANDROID_CACHE_MS = 5000;
let cachedState: PhoneBatteryState | null = null;
let cachedAtMs = 0;
export function readPhoneBatteryState(): PhoneBatteryState {
if (global.isAndroid) {
const now = Date.now();
if (cachedState === null || now - cachedAtMs >= ANDROID_CACHE_MS || now < cachedAtMs) {
cachedState = readPhoneBatteryStateUncached();
cachedAtMs = now;
}
return { ...cachedState };
}
return readPhoneBatteryStateUncached();
}
function readPhoneBatteryStateUncached(): PhoneBatteryState {
if (global.isIOS) {
const device = UIDevice.currentDevice
const level = device.batteryLevel
+6
View File
@@ -18,6 +18,7 @@ export type DisplayTarget = Pick<
| "setUnderlayDim"
| "setScreenBlanked"
| "submitSurfaceFrame"
| "isSurfaceCurrent"
| "submitShellScene"
| "waitForFrameFinished"
| "getCompositePreview"
@@ -109,6 +110,11 @@ export class PreviewDisplayTarget implements DisplayTarget {
this.composited();
}
/** Never skips: preview frames are cheap and nothing is transmitted. */
isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean {
return false;
}
async submitShellScene(bytes: Uint8Array, _paintMs = 0, _frameId = 0): Promise<void> {
this.compositor.setShellScene(bytes);
this.composited();
+6
View File
@@ -24,6 +24,7 @@ export type DisplayTarget = Pick<
| "setUnderlayDim"
| "setScreenBlanked"
| "submitSurfaceFrame"
| "isSurfaceCurrent"
| "submitShellScene"
| "waitForFrameFinished"
| "getCompositePreview"
@@ -109,6 +110,11 @@ export class PreviewDisplayTarget implements DisplayTarget {
this.compositor.setSurfaceDepth(id, Math.round(depth));
}
/** Never skips: preview frames are cheap and nothing is transmitted. */
isSurfaceCurrent(_surfaceId: string, _fingerprint: string): boolean {
return false;
}
async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise<void> {
this.compositor.submitShellScene(this.shellSceneBuffer.load(bytes), paintMs, frameId);
}
+81 -16
View File
@@ -8,17 +8,35 @@
import { Utils } from "@nativescript/core";
declare const com: any;
declare const android: any;
let javaInstance: any = null;
// The Java-side listener proxy must stay referenced or it gets GC'd.
let retainedListenerProxy: any = null;
const changeListeners = new Set<(key: string) => void>();
// Values already read through JNI, which cost a bridge call each and are read
// many times per paint (display mode, fonts, clock format). Only used while
// this isolate's Java listener is registered on a thread with a Looper: every
// write, from any isolate, notifies that listener, which drops the key before
// any other change listener runs. A write from another isolate is visible here
// once its notification has been dispatched (one message-loop tick), the same
// point at which change listeners hear about it.
// Strings are cached without a default (null = unset). Booleans are cached per
// default, since the Java getter cannot report "unset", except after a write
// from this isolate (defaultValue null: stored, so any default reads it).
const stringCache = new Map<string, string | null>();
const booleanCache = new Map<string, { defaultValue: boolean | null; value: boolean }>();
let cachingEnabled: boolean | null = null;
/** Release this isolate's native observer before its worker is terminated. */
export function disposeSettingsStore(): void {
changeListeners.clear();
if (retainedListenerProxy !== null) getJava().unregisterListener(retainedListenerProxy);
retainedListenerProxy = null;
cachingEnabled = false;
stringCache.clear();
booleanCache.clear();
}
function getJava(): any {
@@ -31,20 +49,80 @@ function getJava(): any {
return javaInstance;
}
function canCache(): boolean {
if (cachingEnabled === null) {
cachingEnabled = false;
try {
// FaceclawSettings dispatches notifications through the registering
// thread's Looper; without one this isolate would never hear of writes.
if (android.os.Looper.myLooper() !== null) {
ensureJavaListener();
cachingEnabled = true;
}
} catch {
cachingEnabled = false;
}
}
return cachingEnabled;
}
export function getStringSetting(key: string, defaultValue: string): string {
return String(getJava().getString(key, defaultValue));
if (!canCache()) return String(getJava().getString(key, defaultValue));
let value = stringCache.get(key);
if (value === undefined) {
const raw = getJava().getString(key, null);
value = raw === null || raw === undefined ? null : String(raw);
stringCache.set(key, value);
}
return value ?? defaultValue;
}
export function setStringSetting(key: string, value: string): void {
getJava().setString(key, value);
if (cachingEnabled) stringCache.set(key, value === null || value === undefined ? null : String(value));
}
export function getBooleanSetting(key: string, defaultValue: boolean): boolean {
return Boolean(getJava().getBoolean(key, defaultValue));
if (!canCache()) return Boolean(getJava().getBoolean(key, defaultValue));
const cached = booleanCache.get(key);
if (cached !== undefined && (cached.defaultValue === null || cached.defaultValue === defaultValue)) {
return cached.value;
}
const value = Boolean(getJava().getBoolean(key, defaultValue));
booleanCache.set(key, { defaultValue, value });
return value;
}
export function setBooleanSetting(key: string, value: boolean): void {
getJava().setBoolean(key, value);
if (cachingEnabled) booleanCache.set(key, { defaultValue: null, value: Boolean(value) });
}
function forgetCachedSetting(key: string | null | undefined): void {
if (key === null || key === undefined) {
stringCache.clear();
booleanCache.clear();
return;
}
stringCache.delete(key);
booleanCache.delete(key);
}
function ensureJavaListener(): void {
if (retainedListenerProxy !== null) return;
retainedListenerProxy = new com.faceclaw.app.FaceclawSettingsListener({
onSettingChanged: (key: string) => {
forgetCachedSetting(key === null || key === undefined ? key : String(key));
for (const registered of Array.from(changeListeners)) {
try {
registered(String(key));
} catch (error) {
console.warn("settings change listener failed", error);
}
}
},
});
getJava().registerListener(retainedListenerProxy);
}
/**
@@ -53,20 +131,7 @@ export function setBooleanSetting(key: string, value: boolean): void {
* change.
*/
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
if (retainedListenerProxy === null) {
retainedListenerProxy = new com.faceclaw.app.FaceclawSettingsListener({
onSettingChanged: (key: string) => {
for (const registered of Array.from(changeListeners)) {
try {
registered(String(key));
} catch (error) {
console.warn("settings change listener failed", error);
}
}
},
});
getJava().registerListener(retainedListenerProxy);
}
ensureJavaListener();
changeListeners.add(listener);
return () => {
changeListeners.delete(listener);
+7 -3
View File
@@ -14,8 +14,9 @@ export type PersistedOpenApps = {
const STATE_VERSION = 1;
const FILE_NAME = "open-apps.json";
// Coalesces bursts (restore, sidebar scrolling) into one write; short enough
// that a build-reinstall kill right after a change rarely loses it.
// Coalesces bursts (restore, sidebar scrolling) into one write once they
// settle; short enough that a build-reinstall kill right after a change
// rarely loses it.
const WRITE_DELAY_MS = 1000;
function openAppsFilePath(): string {
@@ -50,7 +51,10 @@ let pendingState: PersistedOpenApps | null = null;
/** Save the open-app state (debounced; the latest state wins). */
export function savePersistedOpenApps(state: PersistedOpenApps): void {
pendingState = state;
if (writeTimer !== null) return;
// Restarted on every change: scrolling through the switcher changes the
// foreground app a few times a second, and each synchronous write costs
// several milliseconds of main thread.
if (writeTimer !== null) clearTimeout(writeTimer);
writeTimer = setTimeout(() => {
writeTimer = null;
const state = pendingState;
@@ -63,7 +63,7 @@ class ScenePlanner(private val cache: ResourceCacheState) {
val calls = ArrayList<ByteArray>()
for (top in 0 until height step 32) {
val rows = minOf(32, height - top)
if (old != null && (top * stride until (top + rows) * stride).all { old[it] == next[it] }) continue
if (old != null && sameBytes(old, next, top * stride, (top + rows) * stride)) continue
var left = width; var right = -1
for (y in top until top + rows) for (x in 0 until width) {
val p = y * stride + x / 2; val shift = if (x % 2 == 0) 4 else 0
@@ -75,4 +75,12 @@ class ScenePlanner(private val cache: ResourceCacheState) {
}
return calls
}
private fun sameBytes(a: ByteArray, b: ByteArray, from: Int, to: Int): Boolean {
var i = from
while (i < to) {
if (a[i] != b[i]) return false
i++
}
return true
}
}
@@ -1017,6 +1017,10 @@ class GlassesSessionCore(
storeDesiredComposite(composite, paintMs, frameId)
}
/** See [SurfaceCompositor.isSurfaceCurrent]. */
fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean =
compositor.isSurfaceCurrent(surfaceId, fingerprint)
/** Store a composite as the desired frame unless a newer one won the race. */
internal fun storeDesiredComposite(composite: SurfaceCompositor.Composite, paintMs: Int, frameId: Int) {
var supersededFrameId = 0
@@ -2,11 +2,14 @@ package com.faceclaw.app
/** A retained selection or image, replayed at stereo depth over the screen and shell surfaces. */
class MenuSelection(val x: Int, val y: Int, val width: Int, val height: Int, val radius: Int,
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList()) : RetainedDrawing {
val resource = CachedResource(DrawProtocol.rawImage(width, height, packed))
val background: Int, val border: Int, val depth: Int, val packed: ByteArray, val occlusions: List<IntArray> = emptyList(), val kind: Int = DrawRecordKind.MENU_SELECTION, val mask: List<IntArray> = emptyList(),
/** [packed] as a raw-image resource; translated copies share it (and its hash) instead of rebuilding it. */
val resource: CachedResource = CachedResource(DrawProtocol.rawImage(width, height, packed))) : RetainedDrawing {
override val resources = listOf(resource)
override val fingerprint = "$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
override fun translated(dx: Int, dy: Int) = MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask)
override val fingerprint: String by lazy {
"$kind,${mask.joinToString { it.joinToString() }},$x,$y,$width,$height,$radius,$background,$border,$depth,${resource.hash},${occlusions.joinToString { it.joinToString() }}"
}
override fun translated(dx: Int, dy: Int) = if (dx == 0 && dy == 0) this else MenuSelection(x + dx, y + dy, width, height, radius, background, border, depth, packed, occlusions.map { intArrayOf(it[0] + dx, it[1] + dy, it[2], it[3]) }, kind, mask, resource)
override fun calls(ids: IntArray, nowMs: Long) = calls(ids.single())
fun calls(id: Int): List<ByteArray> {
val image = when (kind) {
@@ -43,11 +43,19 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return maxOf(1, (((value * dim) + 128) shr 8))
}
/**
* The rest of [reader] as a heap-array reader. Parsing reads field by field, and through a
* platform buffer (Android's direct ByteBuffer) each byte was its own native peek; one bulk
* copy first is far cheaper.
*/
private fun heapReader(reader: ByteReader): ByteReader =
if (reader is ArrayByteReader) reader else ArrayByteReader(ByteArray(reader.remaining()).also { reader.get(it) })
private fun parseDraws(draws: ByteReader?): Array<ScreenDraw> {
if (((draws == null) || (draws.remaining() < 1))) {
return NO_DRAWS
}
val cursor: ByteReader = draws
val cursor: ByteReader = heapReader(draws)
var out: MutableList<ScreenDraw> = ArrayList()
while ((cursor.remaining() >= 1)) {
var kind: Int = (cursor.get() and 0xff)
@@ -253,8 +261,11 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var shellScene: ShellScene? = null
fun setShellScene(reader: ByteReader) {
val scene = ShellScene.decode(reader)
lock.withLock { shellScene = scene }
val scene = ShellScene.decode(heapReader(reader))
lock.withLock {
shellScene = scene
changedSinceComposite = true
}
}
private class Surface {
@@ -284,6 +295,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
/** Surface-local deferred draws of the retained content (already baked into pixels). */
@JvmField var draws: Array<ScreenDraw> = NO_DRAWS
/**
* [draws]' retained selections in screen coordinates, as built for [placedFor] at
* ([placedX], [placedY]). Reused by every composite until either changes, so the shell
* scene built from them (and its fingerprint) can be reused too.
*/
@JvmField var placedSelections: List<RetainedDrawing> = emptyList()
@JvmField var placedFor: Array<ScreenDraw>? = null
@JvmField var placedX: Int = 0
@JvmField var placedY: Int = 0
constructor(id: String) {
this.id = id
}
@@ -291,6 +312,8 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private val lock = protocolPlatform().createLock()
private var animatedPreview: ShellScene.AnimatedPreview? = null
/** The last composite's scene; reused while its inputs are unchanged (see compositeLocked). */
private var lastScene: ShellScene? = null
private var previewKey: String? = null
private fun previewScene(scene: ShellScene, gray: ByteArray, key: String): ByteArray {
@@ -318,6 +341,13 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
private var nextCompositeSeq: Long = 1
/**
* Set by changes that take effect at the next composite (geometry, dim, depth, blanking,
* visibility, a new shell scene, content retained without compositing) and cleared by a
* composite that is kept (not a preview); see [isSurfaceCurrent].
*/
private var changedSinceComposite: Boolean = false
/** Set the output frame size. Must be called before any surface work. */
fun configureScreen(width: Int, height: Int): Unit {
if (((width <= 0) || (height <= 0))) {
@@ -376,12 +406,27 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
surface.height = height
surface.zOrder = zOrder
surface.transparency = transparency
changedSinceComposite = true
}
}
fun removeSurface(id: String): Unit {
lock.withLock {
surfaces.remove(id)
changedSinceComposite = true
}
}
/**
* Whether a full-surface update carrying [fingerprint] would change nothing: the surface
* already retains that content and every other change has reached a kept composite. A
* submitter can then drop an identical repaint without flattening or sending it.
*/
fun isSurfaceCurrent(id: String, fingerprint: String): Boolean {
lock.withLock {
if (changedSinceComposite) return false
val surface = surfaces.get(id) ?: return false
return surface.fingerprint.isNotEmpty() && surface.fingerprint == fingerprint
}
}
@@ -398,6 +443,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
underlayDimBelowZOrder = belowZOrder
underlayDim = maxOf(0, minOf(256, factor256))
changedSinceComposite = true
}
}
@@ -419,6 +465,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
throw IllegalArgumentException(("unknown surface " + id))
}
surface.visible = visible
changedSinceComposite = true
}
}
@@ -432,6 +479,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
lock.withLock {
val surface = surfaces.get(id) ?: throw IllegalArgumentException(("unknown surface " + id))
surface.depth = depth
changedSinceComposite = true
}
}
@@ -450,6 +498,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
fun setBlanked(blanked: Boolean): Unit {
lock.withLock {
this.blanked = blanked
changedSinceComposite = true
}
}
@@ -596,6 +645,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
surface.fingerprint = (if ((contentFingerprint == null)) "" else contentFingerprint)
surface.draws = parsed
if (!composeAfter) changedSinceComposite = true
return if (composeAfter) compositeLocked() else null
}
}
@@ -618,8 +668,10 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
return null
}
val seq = nextCompositeSeq
val changed = changedSinceComposite
val result = compositeLocked(includePreview = true)
nextCompositeSeq = seq
changedSinceComposite = changed
return result
}
}
@@ -640,6 +692,7 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
private fun compositeLocked(includePreview: Boolean = includePreviewInFrames): Composite {
changedSinceComposite = false
var gray: ByteArray = ByteArray((screenWidth * screenHeight))
if (blanked) {
animatedPreview?.player?.stop()
@@ -666,9 +719,9 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
blendLocked(gray, surface)
var dim: Int = dimForLocked(surface)
selections.addAll(placedSelectionsLocked(surface))
for (draw in surface.draws) {
val selected = draw.selection
if (selected != null) { selections.add(selected.translated(surface.x, surface.y)); continue }
if (draw.selection != null) continue
if (((dim < 256) && (draw.kind == ScreenDraw.KIND_IMAGE))) {
continue
}
@@ -710,8 +763,16 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
}
}
val screenDepth = screenDepthLocked(ordered)
val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene(emptyList(), screenDepth = screenDepth)
else ShellScene(shellScene?.layers ?: emptyList(), selections, screenDepth)
val overlaid = surfaces.values.any { it.visible && it.zOrder > 1 }
val sceneLayers = if (overlaid) emptyList() else shellScene?.layers ?: emptyList()
val sceneSelections: List<RetainedDrawing> = if (overlaid) emptyList() else selections
// Built per composite, a scene rebuilt its fingerprint and resource lists every time even
// though its inputs (the decoded shell layers, cached placed selections) rarely change.
val previous = lastScene
val scene = if (previous != null && previous.layers === sceneLayers && previous.screenDepth == screenDepth &&
sameElements(previous.selections, sceneSelections)) previous
else ShellScene(sceneLayers, sceneSelections, screenDepth)
lastScene = scene
val sceneKey = fingerprint.append("|shell:").append(scene.fingerprint).toString()
val preview = if (includePreview && (shellScene != null || selections.isNotEmpty())) {
previewScene(scene, gray, sceneKey)
@@ -727,6 +788,22 @@ class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFr
nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene }
}
private fun placedSelectionsLocked(surface: Surface): List<RetainedDrawing> {
if (surface.placedFor !== surface.draws || surface.placedX != surface.x || surface.placedY != surface.y) {
surface.placedSelections = surface.draws.mapNotNull { it.selection?.translated(surface.x, surface.y) }
surface.placedFor = surface.draws
surface.placedX = surface.x
surface.placedY = surface.y
}
return surface.placedSelections
}
private fun sameElements(a: List<RetainedDrawing>, b: List<RetainedDrawing>): Boolean {
if (a.size != b.size) return false
for (i in a.indices) if (a[i] !== b[i]) return false
return true
}
private fun blendLocked(gray: ByteArray, surface: Surface): Unit {
var srcX: Int = maxOf(0, -surface.x)
var srcY: Int = maxOf(0, -surface.y)
@@ -268,13 +268,16 @@ class FrameTimingsCore(
get() = lock.withLock { exportDirty }
/** The full export text (does not clear the dirty flag). */
fun buildExport(): String = lock.withLock { buildExportLocked() }
fun buildExport(): String = formatExport(lock.withLock { exportSnapshotLocked() })
/** The export text if anything changed since the last take, else null; clears the dirty flag. */
fun takeExport(): String? = lock.withLock {
if (!exportDirty) return null
exportDirty = false
buildExportLocked()
fun takeExport(): String? {
val snapshot = lock.withLock {
if (!exportDirty) return null
exportDirty = false
exportSnapshotLocked()
}
return formatExport(snapshot)
}
// ---------------------------------------------------------------------
@@ -360,30 +363,72 @@ class FrameTimingsCore(
)
}
private fun buildExportLocked(): String {
/**
* Frame state as of one moment, formatted into the export text outside the lock. Formatting
* the full export takes tens of milliseconds, and holding the lock for it stalled every
* thread recording a span meanwhile, the main thread included.
*/
private class FrameSnapshot(
val id: Int,
val reason: String,
val startedAtMs: Long,
val startedWallClockMs: Long,
val durationMs: Long,
val open: Boolean,
val outcome: String?,
val subtreeEndAtMs: Long,
val subtreeHasOpenFrame: Boolean,
val lines: List<Line>,
val children: List<FrameSnapshot>,
)
private class ExportSnapshot(
val wallClockMs: Long,
val statsSummary: String,
val slowest: List<FrameSnapshot>,
val recent: List<FrameSnapshot>,
)
private fun snapshotLocked(frame: Frame, now: Long): FrameSnapshot = FrameSnapshot(
frame.id, frame.reason, frame.startedAtMs, frame.startedWallClockMs, frame.durationMs(), frame.isOpen(),
frame.outcome, frame.subtreeEndAtMs(now), frame.subtreeHasOpenFrame(), frame.lines.toList(),
frame.children.map { snapshotLocked(it, now) },
)
private fun exportSnapshotLocked(): ExportSnapshot {
val now = platform.elapsedRealtimeMs()
// Stable sort, descending by whole-tree duration (matches Collections.sort).
val treeDurations = HashMap<Frame, Long>()
for (frame in slowestRoots) treeDurations[frame] = frame.subtreeEndAtMs(now) - frame.startedAtMs
slowestRoots.sortWith(compareByDescending { treeDurations.getValue(it) })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
return ExportSnapshot(
wallClockMs(),
statsSummaryLocked(),
slowestRoots.map { snapshotLocked(it, now) },
recentRoots.map { snapshotLocked(it, now) },
)
}
private fun formatExport(snapshot: ExportSnapshot): String {
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append("FrameTimings export at ").append(formatWallClock(snapshot.wallClockMs)).append('\n')
out.append(snapshot.statsSummary).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
// Stable sort, descending by whole-tree duration (matches Collections.sort).
slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.slowest) {
appendTree(out, frame, frame, 0)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, now)
for (frame in snapshot.recent) {
appendTree(out, frame, frame, 0)
}
return out.toString()
}
@@ -416,28 +461,22 @@ class FrameTimingsCore(
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
private fun appendTree(out: StringBuilder, frame: FrameSnapshot, root: FrameSnapshot, depth: Int) {
val indent = " ".repeat(depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(formatWallClock(frame.startedWallClockMs))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
val totalMs = frame.subtreeEndAtMs - frame.startedAtMs
out.append(" duration ").append(frame.durationMs).append("ms")
if (frame.children.isNotEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
.append(if (frame.subtreeHasOpenFrame) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
.append(" duration ").append(if (frame.open) "open" else (frame.durationMs.toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
@@ -445,7 +484,7 @@ class FrameTimingsCore(
.append("ms [").append(line.thread).append("] ").append(line.message).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, now)
appendTree(out, child, root, depth + 1)
}
}
}
@@ -97,6 +97,46 @@ class SharedGraphicsTest {
assertContentEquals(byteArrayOf(-56, -56, 0, -56, -56, 0), c.composite().gray)
}
@Test
fun surfaceIsCurrentOnlyAfterItsContentAndEveryChangeWasComposited() {
val c = SurfaceCompositor()
c.configureScreen(3, 2)
c.configureSurface("app", 0, 0, 3, 2, 0, 0)
assertFalse(c.isSurfaceCurrent("app", ""))
c.applyAndComposite("app", ArrayByteReader(ByteArray(6) { 200.toByte() }), 0, 0, 3, 2, "a")
assertTrue(c.isSurfaceCurrent("app", "a"))
assertFalse(c.isSurfaceCurrent("app", "b"))
assertFalse(c.isSurfaceCurrent("missing", "a"))
// A preview composite is not kept, so it does not apply pending changes.
c.setUnderlayDim(1, 128)
assertFalse(c.isSurfaceCurrent("app", "a"))
c.previewComposite()
assertFalse(c.isSurfaceCurrent("app", "a"))
c.composite()
assertTrue(c.isSurfaceCurrent("app", "a"))
for (change in listOf<() -> Unit>(
{ c.configureSurface("app", 1, 0, 3, 2, 0, 0) },
{ c.setSurfaceVisible("app", false) },
{ c.setSurfaceDepth("app", 2) },
{ c.setBlanked(false) },
{ c.removeSurface("other") },
)) {
change()
assertFalse(c.isSurfaceCurrent("app", "a"))
c.composite()
assertTrue(c.isSurfaceCurrent("app", "a"))
}
// Submitting without compositing retains the content but leaves it unapplied.
c.submitSurface("app", ArrayByteReader(ByteArray(6)), 0, 0, 3, 2, "b")
assertFalse(c.isSurfaceCurrent("app", "b"))
c.composite()
assertTrue(c.isSurfaceCurrent("app", "b"))
// A resize discards the retained content.
c.configureSurface("app", 0, 0, 2, 2, 0, 0)
c.composite()
assertFalse(c.isSurfaceCurrent("app", "b"))
}
@Test
fun pngChunksAndPixels() {
val png = SharedScreenshots.encode4BitGrayPng(byteArrayOf(0, 16, -1, 32, 48, 64), 3, 2)