diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java index 05d7bfc0..cbed7e39 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.java @@ -258,7 +258,9 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { // included in the newer composite). private long lastStoredCompositeSeq; - private final SurfaceCompositor compositor = new SurfaceCompositor(); + // Wire submissions need screenGray + the shell scene, never preview pixels. + // Preview/screenshot/recording callers render those explicitly on demand. + private final SurfaceCompositor compositor = new SurfaceCompositor(false); // Phone-side model of the CFW's 192 KiB resource cache. // Reset whenever the image pipeline / EvenHub session is torn down: the @@ -4025,17 +4027,6 @@ public class FaceclawBleCommunicator implements FaceclawBleListener, Runnable { private void logLine(String line) { Log.i(TAG, line); - final FaceclawBleCommunicatorListener current = listener; - if (current == null) { - return; - } - mainHandler.post(() -> { - try { - current.onLog(line); - } catch (Throwable t) { - Log.w(TAG, "listener onLog failed", t); - } - }); } private static String timestamp(long elapsedMs) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java index b0304992..e0351588 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.java @@ -111,7 +111,7 @@ public final class FaceclawPreviewCompositor { int frameId, java.nio.ByteBuffer glyphs ) { - compositor.applyAndComposite( + compositor.submitSurface( surfaceId, new AndroidByteReader(pixels8bpp), rectX, rectY, rectWidth, rectHeight, contentFingerprint, glyphs == null ? null : new AndroidByteReader(glyphs)); FrameTimings.getInstance().finishFrame(frameId, "composited (preview-only, no glasses)"); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java index 1b602978..a44f47c9 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java @@ -1,5 +1,7 @@ package com.faceclaw.app; +import android.os.Handler; +import android.os.Looper; import java.io.ByteArrayOutputStream; import java.net.InetAddress; import java.net.NetworkInterface; @@ -13,7 +15,7 @@ import java.security.MessageDigest; import java.security.SecureRandom; import org.json.JSONObject; -/** Explicit-address, bounded, one-request-per-connection input port. JS polls on its own thread. */ +/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */ public final class FaceclawRemoteInput { private static final FaceclawRemoteInput INSTANCE = new FaceclawRemoteInput(); public static FaceclawRemoteInput getInstance() { return INSTANCE; } @@ -22,6 +24,22 @@ public final class FaceclawRemoteInput { private String pending, reply; private long nextId, requestId; private long deadline; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + private Runnable requestListener; + + public synchronized void setRequestListener(Runnable listener) { requestListener = listener; } + + private void notifyRequestReady(long id) { + // Called with this monitor held; posting avoids entering JS on the socket thread. + mainHandler.post(() -> { + Runnable callback; + synchronized (this) { + if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return; + callback = requestListener; + } + if (callback != null) callback.run(); + }); + } public synchronized String start(int port) { return startAddress(port, "127.0.0.1"); } public synchronized String startAddress(int port, String address) { @@ -81,6 +99,7 @@ public final class FaceclawRemoteInput { reply = null; pending = new JSONObject().put("id", requestId).put("expiresAt", deadline) .put("body", text).toString(); + notifyRequestReady(requestId); while (reply == null && server == listener && System.currentTimeMillis() < deadline) { wait(Math.max(1, deadline - System.currentTimeMillis())); } diff --git a/App_Resources/iOS/src/FaceclawRemoteInput.h b/App_Resources/iOS/src/FaceclawRemoteInput.h index dbbce5e0..4a5708f4 100644 --- a/App_Resources/iOS/src/FaceclawRemoteInput.h +++ b/App_Resources/iOS/src/FaceclawRemoteInput.h @@ -5,6 +5,7 @@ - (NSString *)start:(int)port; - (NSString *)start:(int)port address:(NSString *)address; - (void)stop; +- (void)setRequestListener:(void (^)(void))listener; - (NSString *)nextRequest; - (void)complete:(long long)identifier response:(NSString *)response; - (NSString *)randomSecret; diff --git a/App_Resources/iOS/src/FaceclawRemoteInput.m b/App_Resources/iOS/src/FaceclawRemoteInput.m index 0c1c46af..b7e73446 100644 --- a/App_Resources/iOS/src/FaceclawRemoteInput.m +++ b/App_Resources/iOS/src/FaceclawRemoteInput.m @@ -17,6 +17,7 @@ long long _nextId, _requestId; NSTimeInterval _deadline; NSString *_pending, *_reply; + void (^_requestListener)(void); } + (instancetype)shared { static FaceclawRemoteInput *instance; static dispatch_once_t once; @@ -65,6 +66,20 @@ NSString *result = NSDate.date.timeIntervalSince1970 < _deadline ? _pending : nil; _pending = nil; [_condition unlock]; return result; } +- (void)setRequestListener:(void (^)(void))listener { + [_condition lock]; _requestListener = [listener copy]; [_condition unlock]; +} +- (void)notifyRequestReady:(long long)identifier { + // Dispatch from under the condition lock; invoke JS only on the main thread, + // outside the lock, and discard notifications invalidated by stop/timeout. + dispatch_async(dispatch_get_main_queue(), ^{ + [self->_condition lock]; + void (^callback)(void) = self->_requestId == identifier && self->_pending && + NSDate.date.timeIntervalSince1970 < self->_deadline ? self->_requestListener : nil; + [self->_condition unlock]; + if (callback) callback(); + }); +} - (void)complete:(long long)identifier response:(NSString *)response { [_condition lock]; if (_requestId == identifier && NSDate.date.timeIntervalSince1970 < _deadline) { _reply = response; [_condition signal]; } @@ -104,6 +119,7 @@ _requestId = ++_nextId; _deadline = NSDate.date.timeIntervalSince1970 + 5; _reply = nil; NSData *json = [NSJSONSerialization dataWithJSONObject:@{@"id":@(_requestId), @"expiresAt":@(_deadline * 1000), @"body":text} options:0 error:nil]; _pending = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]; + [self notifyRequestReady:_requestId]; while (!_reply && generation == _generation && NSDate.date.timeIntervalSince1970 < _deadline) [_condition waitUntilDate:[NSDate dateWithTimeIntervalSince1970:_deadline]]; NSString *reply = _reply ?: @"{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"; diff --git a/app/apps/flappy/flappy-app.worker.ts b/app/apps/flappy/flappy-app.worker.ts index d3cf5cce..c0e91451 100644 --- a/app/apps/flappy/flappy-app.worker.ts +++ b/app/apps/flappy/flappy-app.worker.ts @@ -761,17 +761,19 @@ function renderAndSubmit(window: FlappyWindow, inputFrameId: number): void { } const { image, draws } = frameTimings.span(frameId, "flatten", () => flattenPlanesWithDraws(planes)); const buffer = frameTimings.span(frameId, "to8bpp", () => image.to8bppBuffer()); - communicator.submitSurfaceFrame( - buffer.buffer, - window.surfaceId, - 0, - 0, - image.width, - image.height, - fingerprint, - paintMs, - frameId, - frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), + frameTimings.span(frameId, "submitSurfaceFrame", () => + communicator.submitSurfaceFrame( + buffer.buffer, + window.surfaceId, + 0, + 0, + image.width, + image.height, + fingerprint, + paintMs, + frameId, + frameTimings.span(frameId, "prepareFrameDraws", () => prepareFrameDraws(draws)), + ) ); window.lastSubmittedFingerprint = fingerprint; } catch (error) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 9c431239..3db547e3 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -268,7 +268,6 @@ class DashboardController { private lockSurfaceConfigured = false; private lastLockScreenEnabled = lockScreenEnabledSetting.get(); private offState: (() => void) | null = null; - private offLog: (() => void) | null = null; private offRing: (() => void) | null = null; private offBattery: (() => void) | null = null; private offSilentMode: (() => void) | null = null; @@ -1327,9 +1326,6 @@ class DashboardController { ring: ringAddress, }); this.communicator = communicator; - this.offLog = communicator.onLog((line) => { - this.appendLog(line); - }); this.offState = communicator.onStateChange((state) => { if (state.phase !== "connected") resetRingInputFilter(); if (state.phase === "unpaired") { @@ -1542,8 +1538,6 @@ class DashboardController { const message = this.formatError(error); this.offState?.(); this.offState = null; - this.offLog?.(); - this.offLog = null; this.offRing?.(); this.offRing = null; this.offBattery?.(); @@ -1682,8 +1676,6 @@ class DashboardController { this.clearDashboardTimer(); this.offState?.(); this.offState = null; - this.offLog?.(); - this.offLog = null; this.offRing?.(); this.offRing = null; this.offBattery?.(); @@ -2593,7 +2585,7 @@ class DashboardController { } private appendLog(line: string): void { - console.log(`[${formatTimestamp(new Date())}] ${line}`); + //console.log(`[${formatTimestamp(new Date())}] ${line}`); } private setDisplayPreview(preview: ImageSource | null): void { @@ -2610,6 +2602,22 @@ class DashboardController { */ private previewTrailingTimer: ReturnType | null = null; private lastRecordCaptureAtMs = 0; + private phonePreviewVisible: (() => boolean) | null = null; + + /** The main page owns the mirror; other phone pages must not keep it rendering. */ + attachPhonePreview(isVisible: () => boolean): () => void { + this.phonePreviewVisible = isVisible; + this.lastConnectedPreviewUpdateAtMs = 0; + this.updateCompositePreview(); + return () => { + if (this.phonePreviewVisible !== isVisible) return; + this.phonePreviewVisible = null; + if (this.previewTrailingTimer) { + clearTimeout(this.previewTrailingTimer); + this.previewTrailingTimer = null; + } + }; + } /** * Refresh the preview now if the floor allows, otherwise once the floor @@ -2617,6 +2625,7 @@ class DashboardController { * of waiting for the safety-net poll. */ private schedulePreviewUpdate(): void { + if (!this.phonePreviewVisible && !this.screenRecordingActive) return; if (this.previewTrailingTimer) return; const wait = this.lastConnectedPreviewUpdateAtMs + CONNECTED_PREVIEW_MIN_UPDATE_MS - Date.now(); if (wait <= 0) { @@ -2644,6 +2653,13 @@ class DashboardController { return; } this.lastConnectedPreviewUpdateAtMs = now; + // A user-requested GIF is an independent consumer, including while the + // phone is asleep or another page/app is foregrounded. + if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) { + this.lastRecordCaptureAtMs = now; + display.recordScreenFrame(); + } + if (!this.phonePreviewVisible?.()) return; // The connected foreground service intentionally keeps this controller // alive after the phone UI is backgrounded. Do not keep constructing // 640x480 Android Bitmaps for a window that cannot display them: besides @@ -2652,10 +2668,8 @@ class DashboardController { if (global.isAndroid) { const activity = Application.android.foregroundActivity; if (!activity || !activity.hasWindowFocus()) return; - } - if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) { - this.lastRecordCaptureAtMs = now; - display.recordScreenFrame(); + const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager; + if (!power?.isInteractive()) return; } const preview = display.getCompositePreview(previewColorSetting.get() === "green"); if (preview) { diff --git a/app/graphics/image.ts b/app/graphics/image.ts index bb725293..05f7125d 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -149,7 +149,9 @@ export class GrayImage { this.width = width; this.height = height; this.pixels = new Uint8Array(width * height); - this.clear(fill); + if (fill !== 0) { + this.clear(fill); + } } get draws(): readonly DeferredDraw[] { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index bd4f2bcc..5d36a3f7 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -137,7 +137,6 @@ export class FaceclawCommunicatorBridge { // does not race against finishes that land before the wait starts. private readonly finishedFrameOutcomes = new Map(); private readonly frameFinishedWaiters = new Map void>>(); - private readonly logListeners = new Set<(line: string) => void>(); private readonly stateListeners = new Set<(state: CommunicatorState) => void>(); private readonly ringListeners = new Set<(event: RawInputEvent) => void>(); private readonly batteryListeners = new Set<(state: HeadsetBatteryState) => void>(); @@ -161,9 +160,6 @@ export class FaceclawCommunicatorBridge { addresses.ring ?? "", ); this.listenerProxy = new com.faceclaw.app.FaceclawBleCommunicatorListener({ - onLog: (line: string) => { - this.emitAsync(this.logListeners, String(line)); - }, onStateChange: (phase: string, status: string) => { const state = { phase: String(phase) as CommunicatorPhase, @@ -293,11 +289,6 @@ export class FaceclawCommunicatorBridge { return result; } - onLog(listener: (line: string) => void): () => void { - this.logListeners.add(listener); - return () => this.logListeners.delete(listener); - } - onStateChange(listener: (state: CommunicatorState) => void): () => void { this.stateListeners.add(listener); return () => this.stateListeners.delete(listener); diff --git a/app/native/remote-input.ios.ts b/app/native/remote-input.ios.ts index d20e9e5c..511cd41c 100644 --- a/app/native/remote-input.ios.ts +++ b/app/native/remote-input.ios.ts @@ -1,9 +1,10 @@ import { createListenerPool, type InterfaceAddress } from '../remote/listeners'; declare const FaceclawRemoteInput: any; -export function remoteNative() { +export function remoteNative(onRequestReady: () => void) { return createListenerPool(() => { const native = FaceclawRemoteInput.alloc().init(); - return { start: (port, address) => String(native.startAddress(port, address)), stop: () => native.stop(), + native.setRequestListener(onRequestReady); + return { start: (port, address) => String(native.startAddress(port, address)), stop: () => { native.setRequestListener(null); native.stop(); }, nextRequest: () => native.nextRequest(), complete: (id, response) => native.completeResponse(id, response) }; }); } diff --git a/app/native/remote-input.ts b/app/native/remote-input.ts index 8650ef07..0be486be 100644 --- a/app/native/remote-input.ts +++ b/app/native/remote-input.ts @@ -1,10 +1,12 @@ import { createListenerPool, type InterfaceAddress } from '../remote/listeners'; import { Utils } from '@nativescript/core'; declare const com: any; -export function remoteNative() { +export function remoteNative(onRequestReady: () => void) { return createListenerPool(() => { const native = new com.faceclaw.app.FaceclawRemoteInput(); - return { start: (port, address) => String(native.startAddress(port, address)), stop: () => native.stop(), + const ready = new java.lang.Runnable({ run: onRequestReady }); + native.setRequestListener(ready); + return { start: (port, address) => String(native.startAddress(port, address)), stop: () => { native.setRequestListener(null); native.stop(); }, nextRequest: () => native.nextRequest(), complete: (id, response) => native.complete(id, response) }; }); } diff --git a/app/phone-ui/main-page.ts b/app/phone-ui/main-page.ts index 244b6d05..b22a42d5 100644 --- a/app/phone-ui/main-page.ts +++ b/app/phone-ui/main-page.ts @@ -15,6 +15,7 @@ type MainPageState = { model: MainViewModel propertyChangeHandler: (args: EventData & { propertyName?: string }) => void orientationHandler: () => void + detachPreview: () => void } function getPageState(page: Page): MainPageState | undefined { @@ -32,6 +33,7 @@ function cleanupPage(page: Page): void { return } state.model.off(Observable.propertyChangeEvent, state.propertyChangeHandler) + state.detachPreview() Application.off(Application.orientationChangedEvent, state.orientationHandler) // navigatingTo builds a fresh model each visit; drop the old one's // controller/settings subscriptions or every navigation leaks a listener. @@ -130,6 +132,8 @@ export function loaded(args: EventData) { const state: MainPageState = { model, + detachPreview: dashboardController.attachPhonePreview(() => + page.isLoaded && !model.displayPreviewMessage && (!isAndroid || !!page.android?.isShown())), orientationHandler: () => { setTimeout(() => { model.refreshLayoutMetrics() diff --git a/app/remote/service.ts b/app/remote/service.ts index 02e64c5b..2c324229 100644 --- a/app/remote/service.ts +++ b/app/remote/service.ts @@ -10,17 +10,33 @@ export function remoteInputStatus(): string { return status; } export function remoteInterfaceSelection(): string { return getStringSetting(INTERFACE_STORAGE_KEY, 'tailscale'); } export function selectRemoteInterface(value: string): void { setStringSetting(INTERFACE_STORAGE_KEY, value); } -/** Owned by the main controller. Native socket work never enters a JS isolate. */ +/** Owned by the main controller. Native sockets notify JS only for complete requests. */ export function startRemoteInput(host: RemoteHost): () => void { - const native = remoteNative(); - let timer: ReturnType | null = null; + const native = remoteNative(() => { void drain(); }); let busy = false; + let stopped = false; let networkTimer: ReturnType | null = null; + // A notification while handling another request is covered by the next drain + // iteration. No timer or native bridge calls are needed while sockets are idle. + const drain = async () => { + if (busy || stopped) return; + busy = true; + try { + while (!stopped) { + const raw = native.nextRequest(); + if (!raw) break; + const request = JSON.parse(String(raw)); + if (Date.now() >= request.expiresAt) continue; + let reply; + try { reply = await handleRequest(request.body, remoteTokens, host); } + catch { reply = { ok: false, error: 'failed', message: 'Request failed.' }; } + if (!stopped) native.complete(request.id, JSON.stringify(reply)); + } + } finally { busy = false; } + }; const sync = () => { if (!remoteTokens.list().length) { native.stop(); - if (timer !== null) clearInterval(timer); - timer = null; if (networkTimer !== null) clearInterval(networkTimer); networkTimer = null; status = 'No tokens; listener stopped.'; @@ -34,21 +50,8 @@ export function startRemoteInput(host: RemoteHost): () => void { if (failure) status += `\n${failure}`; // Reconcile after VPN reconnects/address changes; unchanged sockets stay open. if (networkTimer === null) networkTimer = setInterval(sync, 3000); - if (timer !== null) return; - timer = setInterval(() => { - if (busy) return; - const raw = native.nextRequest(); - if (!raw) return; - const request = JSON.parse(String(raw)); - if (Date.now() >= request.expiresAt) return; - busy = true; - void handleRequest(request.body, remoteTokens, host) - .then(reply => native.complete(request.id, JSON.stringify(reply))) - .catch(() => native.complete(request.id, JSON.stringify({ ok: false, error: 'failed', message: 'Request failed.' }))) - .finally(() => { busy = false; }); - }, 15); }; sync(); const off = onSettingsStoreChanged(key => { if (key === TOKEN_STORAGE_KEY || key === INTERFACE_STORAGE_KEY) sync(); }); - return () => { off(); if (timer !== null) clearInterval(timer); if (networkTimer !== null) clearInterval(networkTimer); native.stop(); }; + return () => { stopped = true; off(); if (networkTimer !== null) clearInterval(networkTimer); native.stop(); }; } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt index b972112e..106b41ee 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/callbacks/FaceclawBleCommunicatorListener.kt @@ -1,8 +1,6 @@ package com.faceclaw.app interface FaceclawBleCommunicatorListener { - fun onLog(line: String?): Unit - fun onStateChange(phase: String?, status: String?): Unit fun onRingEvent( diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt index 0e06f6c8..2eeb86b3 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/BleImageOptimizer.kt @@ -574,16 +574,20 @@ class BleImageOptimizer { */ @JvmStatic fun rleEncode(pix: ByteArray): ByteArray { - var n: Int = (pix.size * 2) - var out: ByteArray = ByteArray(n) + val n = pix.size * 2 + val out = ByteArray(n) var o: Int = 0 var i: Int = 0 while ((i < n)) { - var v: Int = nibbleAt(pix, i) - var j: Int = (i + 1) - while (((j < n) && (nibbleAt(pix, j) == v))) { - j++ - } + val first = pix[i ushr 1].toInt() and 255 + val v = if (i and 1 == 0) first ushr 4 else first and 15 + // Start at a byte boundary, consuming the known low nibble + // first if the previous run ended halfway through a byte. + var j = i + (i and 1) + val pair = ((v shl 4) or v).toByte() + while (j < n && pix[j ushr 1] == pair) j += 2 + // A final high nibble can match even when its low nibble doesn't. + if (j < n && (pix[j ushr 1].toInt() and 240) == v shl 4) j++ var run: Int = (j - i) while ((run > 0)) { var c: Int = minOf(run, 0xffff) @@ -607,10 +611,6 @@ class BleImageOptimizer { return out.copyOf(o) } - private fun nibbleAt(pix: ByteArray, i: Int): Int { - var b: Int = (pix[(i shr 1)] and 0xff) - return (if (((i and 1) != 0)) (b and 0x0f) else (b shr 4)) - } } constructor() {} diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt index 48057a2b..fb1b2cd1 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/g2protocol/TexturePlanner.kt @@ -591,35 +591,26 @@ class TexturePlanner { left: Int, top: Int, ): Boolean { - run { - var row: Int = 0 - while ((row < atlas.height)) { - var y: Int = (top + row) - if (((y < 0) || (y >= height))) { - row++ - continue + val pixels = atlas.nibbles + val imageWidth = atlas.width + val startX = maxOf(0, left) + val endX = minOf(width, left + imageWidth) + val endY = minOf(height, top + atlas.height) + var y = maxOf(0, top) + while (y < endY) { + var src = (y - top) * imageWidth + startX - left + val dstRow = y * stride + var x = startX + while (x < endX) { + val source = pixels[src++].toInt() and 255 + if (source != 0) { + val byte = packed[dstRow + (x ushr 1)].toInt() and 255 + val actual = if (x and 1 == 0) byte ushr 4 else byte and 15 + if (actual != source) return false } - run { - var col: Int = 0 - while ((col < atlas.width)) { - var source: Int = atlas.nibbleAt(col, row) - if ((source == 0)) { - col++ - continue - } - var x: Int = (left + col) - if (((x < 0) || (x >= width))) { - col++ - continue - } - if ((nibbleAt(packed, stride, x, y) != source)) { - return false - } - col++ - } - } - row++ + x++ } + y++ } return true } @@ -661,29 +652,39 @@ class TexturePlanner { } } } else { - var atlas: ImageAtlas.Entry = sel.imageAtlas!! - run { - var row: Int = 0 - while ((row < atlas.height)) { - var y: Int = (sel.draw.y + row) - if (((y < 0) || (y >= height))) { - row++ - continue - } - run { - var col: Int = 0 - while ((col < atlas.width)) { - if ((atlas.nibbleAt(col, row) == 0)) { - col++ - continue - } - punchPixel(packed, stride, width, (sel.gx + col), y) - col++ - } - } - row++ + punchImage(packed, stride, width, height, sel.imageAtlas!!, sel.gx, sel.draw.y) + } + } + + /** Clear only visible nonzero image pixels, preserving the neighboring nibble. */ + private fun punchImage( + packed: ByteArray, + stride: Int, + width: Int, + height: Int, + atlas: ImageAtlas.Entry, + left: Int, + top: Int, + ) { + val pixels = atlas.nibbles + val imageWidth = atlas.width + val startX = maxOf(0, left) + val endX = minOf(width, left + imageWidth) + val endY = minOf(height, top + atlas.height) + var y = maxOf(0, top) + while (y < endY) { + var src = (y - top) * imageWidth + startX - left + val dstRow = y * stride + var x = startX + while (x < endX) { + if (pixels[src++].toInt() != 0) { + val index = dstRow + (x ushr 1) + val mask = if (x and 1 == 0) 15 else 240 + packed[index] = (packed[index].toInt() and mask).toByte() } + x++ } + y++ } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt index 2a8b21e7..8acc7d00 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/ImageAtlas.kt @@ -93,7 +93,7 @@ class ImageAtlas { /** The 4bpp value the mode-13 draw would write at (col, row); 0 = skipped. */ fun nibbleAt(col: Int, row: Int): Int { - return (nibbles[((row * width) + col)] and 0xff) + return nibbles[row * width + col].toInt() and 255 } private fun encodeCachedImage(): ByteArray { diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt index bc70d8d7..21f937ee 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/SurfaceCompositor.kt @@ -2,6 +2,7 @@ package com.faceclaw.app import kotlin.jvm.JvmField import kotlin.jvm.JvmStatic +import kotlin.jvm.JvmOverloads /** * Retained-surface compositor: the phone-side model of what is on the glasses screen. @@ -30,7 +31,7 @@ import kotlin.jvm.JvmStatic * atomically under an internal lock, and each composite carries a monotonic sequence number so * callers can detect when a composite was superseded by a concurrent one before being acted on. */ -class SurfaceCompositor { +class SurfaceCompositor @JvmOverloads constructor(private val includePreviewInFrames: Boolean = true) { companion object { const val TRANSPARENCY_OPAQUE: Int = 0 @@ -206,7 +207,11 @@ class SurfaceCompositor { class Composite { @JvmField var screenGray: ByteArray @JvmField var shellScene: ShellScene = ShellScene.EMPTY - /** Full-screen 8bpp grayscale pixels, screenWidth*screenHeight bytes. */ + /** + * Full-screen preview pixels. When includePreviewInFrames is false, + * submitted composites alias screenGray here; use previewComposite() + * for the shell/selection-rendered preview, screenshots or recording. + */ @JvmField val gray: ByteArray @JvmField val width: Int @@ -573,7 +578,7 @@ class SurfaceCompositor { return null } val seq = nextCompositeSeq - val result = compositeLocked() + val result = compositeLocked(includePreview = true) nextCompositeSeq = seq return result } @@ -594,7 +599,7 @@ class SurfaceCompositor { return gray } - private fun compositeLocked(): Composite { + private fun compositeLocked(includePreview: Boolean = includePreviewInFrames): Composite { var gray: ByteArray = ByteArray((screenWidth * screenHeight)) if (blanked) { return Composite( @@ -662,7 +667,7 @@ class SurfaceCompositor { } } val scene = if (surfaces.values.any { it.visible && it.zOrder > 1 }) ShellScene.EMPTY else ShellScene(shellScene?.layers ?: emptyList(), selections) - val preview = if (shellScene != null || selections.isNotEmpty()) scene.preview(gray, screenWidth, screenHeight) else gray + val preview = if (includePreview && (shellScene != null || selections.isNotEmpty())) scene.preview(gray, screenWidth, screenHeight) else gray return Composite(preview, screenWidth, screenHeight, fingerprint.append("|shell:").append(scene.fingerprint).toString(), nextCompositeSeq++, draws.toTypedArray()).also { it.screenGray = gray; it.shellScene = scene } } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt index 1e3636db..b4815df1 100644 --- a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/BmpUtil.kt @@ -13,7 +13,7 @@ class BmpUtil { } private val GRAY_TO_NIBBLE = - ByteArray(256) { v -> if (v == 0) 0 else minOf(15, (v + 8) shr 4).toByte() } + IntArray(256) { v -> minOf(15, (v + 8) shr 4) } /** * The 4bpp level an 8-bit gray value packs to. Public so the resource-cache planner computes @@ -22,7 +22,7 @@ class BmpUtil { */ @JvmStatic fun nibbleForGray(gray: Int): Int { - return (GRAY_TO_NIBBLE[(gray and 0xff)] and 0xff) + return GRAY_TO_NIBBLE[gray and 0xff] } /** @@ -37,32 +37,34 @@ class BmpUtil { if (((width <= 0) || (height <= 0))) { return ByteArray(0) } - var stride: Int = ((width + 1) shr 1) - var out: ByteArray = ByteArray((stride * height)) + val stride = (width + 1) ushr 1 + val out = ByteArray(stride * height) if (((gray8 == null) || (gray8.size < (width * height)))) { return out } - var pairs: Int = (width shr 1) - var src: Int = 0 - var dst: Int = 0 - run { - var y: Int = 0 - while ((y < height)) { - run { - var i: Int = 0 - while ((i < pairs)) { - out[dst++] = - (((GRAY_TO_NIBBLE[(gray8[src] and 0xff)] shl 4) or - (GRAY_TO_NIBBLE[(gray8[(src + 1)] and 0xff)] and 0xff))) - .toByte() - src += 2 - i++ - } + // Work with Ints: Byte.and / Byte.shl in ByteSink.kt are ordinary + // function calls, not Kotlin intrinsics. Avoid four calls per pair + // even in an interpreted/debug ART build, and load the LUT once. + val levels = GRAY_TO_NIBBLE + var src = 0 + var dst = 0 + if (width and 1 == 0) { + // No row padding: the common 640x480 frame is one linear loop. + while (dst < out.size) { + out[dst++] = ((levels[gray8[src].toInt() and 255] shl 4) or + levels[gray8[src + 1].toInt() and 255]).toByte() + src += 2 + } + } else { + repeat(height) { + val end = src + width - 1 + while (src < end) { + out[dst++] = ((levels[gray8[src].toInt() and 255] shl 4) or + levels[gray8[src + 1].toInt() and 255]).toByte() + src += 2 } - if (((width and 1) != 0)) { - out[dst++] = ((GRAY_TO_NIBBLE[(gray8[src++] and 0xff)] shl 4)).toByte() - } - y++ + // Pad each odd-width row with a zero low nibble. + out[dst++] = (levels[gray8[src++].toInt() and 255] shl 4).toByte() } } return out diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt index 86bf698e..e5177b95 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/DisplayListTest.kt @@ -1,9 +1,47 @@ package com.faceclaw.app import kotlin.test.* +import kotlin.random.Random class DisplayListTest { private fun hex(s: String) = s.chunked(2).map { it.toInt(16).toByte() }.toByteArray() + @Test fun screenCopyPreservesOffsetsClippingPaddingAndAliasing() { + val random = Random(42) + val renderer = DisplayListRenderer(emptyMap()) + for (sourceWidth in listOf(1, 2, 3, 7, 8, 640)) for (destinationWidth in listOf(1, 2, 3, 7, 8, 640)) { + for (shared in listOf(false, true)) for (shift in listOf(-3, 0, 2)) { + val sourceHeight = random.nextInt(1, 5); val destinationHeight = random.nextInt(1, 5) + val sourceOffset = random.nextInt(4); val destinationOffset = random.nextInt(4) + val size = maxOf(sourceOffset + (sourceWidth + 1) / 2 * sourceHeight, + destinationOffset + (destinationWidth + 1) / 2 * destinationHeight) + 3 + val source = random.nextBytes(size) + val destination = if (shared) source else random.nextBytes(size) + val expectedSource = source.copyOf() + val expectedDestination = if (shared) expectedSource else destination.copyOf() + // Original per-pixel semantics, including cascading writes when + // views alias and ignoring shiftX when reading the source. + val referenceSource = DisplayListRenderer.Target(expectedSource, sourceWidth, sourceHeight, sourceOffset, 5) + val referenceDestination = DisplayListRenderer.Target(expectedDestination, destinationWidth, destinationHeight, destinationOffset, shift) + for (y in 0 until sourceHeight) for (x in 0 until sourceWidth) + referenceDestination.put(x, y, referenceSource.get(x, y)) + assertEquals(emptySet(), renderer.render(DrawProtocol.SCREEN, + DisplayListRenderer.Target(source, sourceWidth, sourceHeight, sourceOffset, 5), + DisplayListRenderer.Target(destination, destinationWidth, destinationHeight, destinationOffset, shift))) + assertContentEquals(expectedDestination, destination, "$sourceWidth/$destinationWidth shared=$shared shift=$shift") + assertContentEquals(expectedSource, source) + } + } + val source = random.nextBytes(640 * 480 / 2) + val destination = ByteArray(source.size) + renderer.render(DrawProtocol.SCREEN, DisplayListRenderer.Target(source,640,480), DisplayListRenderer.Target(destination,640,480)) + assertContentEquals(source, destination) + } + @Test fun invalidGraphDoesNotCopyScreenIntoComposition() { + val screen = hex("12345678"); val composition = hex("9abcdef0") + val resources = mapOf(1 to DrawProtocol.displayList(listOf(byteArrayOf(99,0)))) + assertFails { DisplayListRenderer(resources).render(1, DisplayListRenderer.Target(screen,8,1), DisplayListRenderer.Target(composition,8,1)) } + assertContentEquals(hex("9abcdef0"), composition) + } @Test fun oddPixelsRawRleClippingFontsNestedListsAndLut() { val raw = DrawProtocol.rawImage(3, 3, hex("123045607890")) val font = ByteArray(197); font[0]=1; font[67]=193.toByte(); hex("0801011f").copyInto(font,193) diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/PreviewPackingTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/PreviewPackingTest.kt new file mode 100644 index 00000000..7d197217 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/PreviewPackingTest.kt @@ -0,0 +1,67 @@ +package com.faceclaw.app + +import kotlin.test.* + +class PreviewPackingTest { + private fun level(value: Byte): Int = minOf(15, ((value.toInt() and 255) + 8) / 16) + + @Test + fun everyPairOfGrayValuesHasTheSameWireEncoding() { + val pixels = ByteArray(65536 * 2) { if (it % 2 == 0) (it / 512).toByte() else (it / 2).toByte() } + val packed = BmpUtil.pack4bppFromGray8(pixels, 512, 256) + for (i in packed.indices) { + assertEquals((level(pixels[i * 2]) * 16 + level(pixels[i * 2 + 1])).toByte(), packed[i], "pair $i") + } + for (v in 0..255) assertEquals(level(v.toByte()), BmpUtil.nibbleForGray(v)) + } + + @Test + fun oddWidthsPadEveryRowAndIgnoreTrailingInput() { + for (width in listOf(1, 3, 5, 639, 641)) { + val height = 7 + val pixels = ByteArray(width * height + 13) { (it * 73 + 19).toByte() } + val stride = (width + 1) / 2 + val packed = BmpUtil.pack4bppFromGray8(pixels, width, height) + assertEquals(stride * height, packed.size) + for (y in 0 until height) { + for (x in 0 until width) { + val byte = packed[y * stride + x / 2].toInt() and 255 + assertEquals(level(pixels[y * width + x]), if (x % 2 == 0) byte / 16 else byte % 16) + } + assertEquals(0, packed[(y + 1) * stride - 1].toInt() and 15) + } + } + assertContentEquals(ByteArray(4), BmpUtil.pack4bppFromGray8(null, 3, 2)) + assertContentEquals(ByteArray(4), BmpUtil.pack4bppFromGray8(ByteArray(5) { -1 }, 3, 2)) + assertTrue(BmpUtil.pack4bppFromGray8(null, 0, 2).isEmpty()) + } + + @Test + fun wireFramesSkipPreviewButExplicitReadbackIncludesLatestShellAndPixels() { + val compositor = SurfaceCompositor(false) + compositor.configureScreen(3, 2) + compositor.configureSurface("app", 0, 0, 3, 2, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE) + // One white 2x1 shell layer at (0,0), with no dimming or selection. + val header = listOf(1, 1, 0, 0, 2, 1, 256, 0, 0).flatMap { listOf(it.toByte(), (it shr 8).toByte()) }.toByteArray() + compositor.setShellScene(ArrayByteReader(header + byteArrayOf(-16, -16))) + val wire = compositor.applyAndComposite("app", ArrayByteReader(ByteArray(6) { 32 }), 0, 0, 3, 2, "first") + assertSame(wire.screenGray, wire.gray) // ShellScene.preview allocates its own pixels. + assertContentEquals(ByteArray(6) { 32 }, wire.screenGray) + assertEquals(1, wire.shellScene.layers.size) + val preview = assertNotNull(compositor.previewComposite()) + assertContentEquals(byteArrayOf(-16, -16, 32, 32, 32, 32), preview.gray) + assertEquals(wire.fingerprint, preview.fingerprint) + assertContentEquals(wire.screenGray, preview.screenGray) + + // Updates while hidden retain the newest frame. Preview reads don't + // consume the wire sequence or affect the next submitted frame. + val newer = compositor.applyAndComposite("app", ArrayByteReader(ByteArray(6) { 64 }), 0, 0, 3, 2, "second") + assertEquals(wire.seq + 1, newer.seq) + assertSame(newer.screenGray, newer.gray) + assertContentEquals(byteArrayOf(-16, -16, 64, 64, 64, 64), compositor.previewComposite()!!.gray) + compositor.setBlanked(true) + assertContentEquals(ByteArray(6), compositor.previewComposite()!!.gray) + compositor.setBlanked(false) + assertContentEquals(byteArrayOf(-16, -16, 64, 64, 64, 64), compositor.previewComposite()!!.gray) + } +} diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResourceCacheTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResourceCacheTest.kt index 2666890b..44c84e9c 100644 --- a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResourceCacheTest.kt +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/ResourceCacheTest.kt @@ -126,6 +126,51 @@ class ResourceCacheTest { assertNull(TexturePlanner.plan(null, ByteArray(packed.size), 8, 4, draws, cache, 1, true, 8, testPlatform())) } + @Test fun imagePixelLoopsPreserveClippingTransparencyAndNeighborNibbles() { + val width = 12 + val height = 8 + for (imageWidth in listOf(3, 4, 5, 9)) { + val imageHeight = 5 + val pixels = ByteArray(imageWidth * imageHeight) { i -> + if (i % 3 == 0) 0 else (((i % 15) + 1) * 16).toByte() + } + val id = ImageAtlas.ensure("pixel-loop-$imageWidth", imageWidth, imageHeight, ArrayByteReader(pixels)) + for (left in listOf(-1, 0, 1, 2, 10, 11, 14)) for (top in listOf(-1, 0, 1, 7, 9)) { + val gray = ByteArray(width * height) { ((it % 15 + 1) * 16).toByte() } + val visibleInk = ArrayList() + for (row in 0 until imageHeight) for (col in 0 until imageWidth) { + val x = left + col; val y = top + row; val value = pixels[row * imageWidth + col] + if (x in 0 until width && y in 0 until height && value.toInt() != 0) { + gray[y * width + x] = value + visibleInk.add(y * width + x) + } + } + val expected = BmpUtil.pack4bppFromGray8(gray, width, height) + val draws = arrayOf(SurfaceCompositor.ScreenDraw.image(id, left, top)) + val plan = TexturePlanner.plan(null, expected, width, height, draws, ResourceCacheState(), 1, true, 8, testPlatform()) + if (left < 0 || top < 0 || left >= width || top >= height) { + assertNull(plan) // The wire format cannot replay negative image origins. + continue + } + val result = assertNotNull(plan, "$imageWidth at $left,$top") + assertEquals(1, result.drawnImages) + val actual = ByteArray(expected.size) { 0x55 } + DisplayListRenderer(uploaded(result.resourceCommands)).execute( + DrawProtocol.sequence(DrawProtocol.fromOptimized(result.payload, width, height)), + DisplayListRenderer.Target(actual, width, height)) + assertContentEquals(expected, actual, "$imageWidth at $left,$top") + + // A conflicting visible ink pixel must reject the cached draw. + if (visibleInk.isNotEmpty()) { + val offset = visibleInk.first() + gray[offset] = 0 + assertNull(TexturePlanner.plan(null, BmpUtil.pack4bppFromGray8(gray, width, height), + width, height, draws, ResourceCacheState(), 1, true, 8, testPlatform())) + } + } + } + } + private fun registerFont(key: String, encodings: List, size: Int = 2): Int { val out = ByteSink(); val name = key.encodeToByteArray() out.write(name.size); out.write(name, 0, name.size); out.write(size) diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RleEncoderTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RleEncoderTest.kt new file mode 100644 index 00000000..6893b422 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/RleEncoderTest.kt @@ -0,0 +1,65 @@ +package com.faceclaw.app + +import kotlin.random.Random +import kotlin.test.* + +class RleEncoderTest { + /** Deliberately simple per-pixel reference, independent of the byte scan. */ + private fun reference(packed: ByteArray): ByteArray { + val output = ArrayList() + var color = -1 + var count = 0 + fun flush() { + while (count > 0) { + val n = minOf(count, 65535) + when { + n <= 15 -> output.add((n * 16 + color).toByte()) + n <= 255 -> { output.add(color.toByte()); output.add(n.toByte()) } + else -> { output.add(color.toByte()); output.add(0); output.add(n.toByte()); output.add((n / 256).toByte()) } + } + count -= n + } + } + for (byte in packed) for (pixel in listOf((byte.toInt() and 255) / 16, byte.toInt() and 15)) { + if (pixel != color) { flush(); color = pixel } + count++ + } + flush() + return output.toByteArray() + } + + @Test fun allTwoByteInputsPreserveExactTokensAcrossNibbleBoundaries() { + for (value in 0..65535) { + val packed = byteArrayOf((value / 256).toByte(), value.toByte()) + assertContentEquals(reference(packed), BleImageOptimizer.rleEncode(packed), "input $value") + } + assertContentEquals(ByteArray(0), BleImageOptimizer.rleEncode(ByteArray(0))) + } + + @Test fun longRunsAndEveryTokenBoundaryAtBothNibbleAlignments() { + for (length in listOf(1, 2, 14, 15, 16, 254, 255, 256, 65534, 65535, 65536, 131070, 131071)) { + for (prefix in 0..1) for (color in listOf(0, 7, 15)) { + val values = List(prefix) { (color + 1) % 16 } + List(length) { color } + listOf((color + 2) % 16) + val packed = ByteArray((values.size + 1) / 2) { i -> + (values[i * 2] * 16 + (values.getOrNull(i * 2 + 1) ?: 0)).toByte() + } + assertContentEquals(reference(packed), BleImageOptimizer.rleEncode(packed), "$prefix/$length/$color") + } + } + } + + @Test fun noiseAndMixedRunsPreserveExactWireOutput() { + val random = Random(123) + repeat(100) { + val data = random.nextBytes(random.nextInt(1, 4096)) + assertContentEquals(reference(data), BleImageOptimizer.rleEncode(data)) + repeat(20) { + val start = random.nextInt(data.size) + data.fill((random.nextInt(16) * 17).toByte(), start, minOf(data.size, start + random.nextInt(1, 200))) + } + assertContentEquals(reference(data), BleImageOptimizer.rleEncode(data)) + } + val screen = ByteArray(640 * 480 / 2) { 0x77 } + assertContentEquals(reference(screen), BleImageOptimizer.rleEncode(screen)) + } +} diff --git a/tests/phone-preview.test.cjs b/tests/phone-preview.test.cjs new file mode 100644 index 00000000..71a5709d --- /dev/null +++ b/tests/phone-preview.test.cjs @@ -0,0 +1,86 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const ts = require('typescript'); + +// Exercise the controller's real methods without starting BLE or the app shell. +function harness() { + const source = ts.createSourceFile('controller.ts', fs.readFileSync('app/g2/dashboard-controller.ts', 'utf8'), ts.ScriptTarget.Latest, true); + const controller = source.statements.find(node => ts.isClassDeclaration(node) && node.name.text === 'DashboardController'); + const names = ['attachPhonePreview', 'schedulePreviewUpdate', 'updateCompositePreview']; + const methods = names.map(name => controller.members.find(node => node.name?.getText(source) === name).getText(source)); + let now = 1000, focus = true, interactive = true, activityPresent = true, previews = 0, records = 0; + const timers = new Map(); + const activity = { hasWindowFocus: () => focus, getSystemService: () => ({ isInteractive: () => interactive }) }; + const context = { exports: {}, global: { isAndroid: true }, + Application: { android: { get foregroundActivity() { return activityPresent ? activity : null; } } }, + android: { content: { Context: { POWER_SERVICE: 'power' } } }, + Date: { now: () => now }, CONNECTED_PREVIEW_MIN_UPDATE_MS: 100, RECORDING_MIN_CAPTURE_MS: 200, + previewColorSetting: { get: () => 'green' }, + setTimeout: fn => { const id = timers.size + 1; timers.set(id, fn); return id; }, + clearTimeout: id => timers.delete(id), + }; + vm.runInNewContext(ts.transpileModule(`export class Harness { ${methods.join('\n')} }`, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); + const instance = new context.exports.Harness(); + Object.assign(instance, { lastConnectedPreviewUpdateAtMs: 0, lastRecordCaptureAtMs: 0, + phonePreviewVisible: null, previewTrailingTimer: null, screenRecordingActive: false, + display: { getCompositePreview: () => { previews++; return {}; }, recordScreenFrame: () => records++ }, + setDisplayPreview() {}, + }); + return { instance, timers, previews: () => previews, records: () => records, + tick: (ms = 1000) => { now += ms; instance.updateCompositePreview(); }, + focus: value => { focus = value; }, interactive: value => { interactive = value; }, + activity: value => { activityPresent = value; } }; +} + +test('phone preview skips hidden pages, background activity and screen off; resumes with latest frame', () => { + const h = harness(); + h.tick(); + assert.equal(h.previews(), 0); + let shown = true; + const detach = h.instance.attachPhonePreview(() => shown); + assert.equal(h.previews(), 1); + shown = false; h.tick(); + shown = true; h.focus(false); h.tick(); + h.focus(true); h.interactive(false); h.tick(); + h.interactive(true); h.activity(false); h.tick(); + assert.equal(h.previews(), 1); + h.activity(true); h.tick(); + assert.equal(h.previews(), 2); + detach(); h.tick(); + assert.equal(h.previews(), 2); +}); + +test('explicit GIF recording continues without rendering a hidden phone bitmap', () => { + const h = harness(); + h.instance.screenRecordingActive = true; + h.instance.attachPhonePreview(() => true); + h.interactive(false); + h.tick(); h.tick(100); h.tick(100); + assert.equal(h.records(), 3); + assert.equal(h.previews(), 1); + h.instance.phonePreviewVisible = null; + h.tick(); + assert.equal(h.records(), 4); + h.instance.screenRecordingActive = false; + h.tick(); + assert.equal(h.records(), 4); +}); + +test('page detach cancels trailing previews and cannot detach a newer page', () => { + const h = harness(); + const oldDetach = h.instance.attachPhonePreview(() => true); + const newDetach = h.instance.attachPhonePreview(() => true); + oldDetach(); + h.tick(); + assert.equal(h.previews(), 3); + h.instance.schedulePreviewUpdate(); + assert.equal(h.timers.size, 1); + newDetach(); + assert.equal(h.timers.size, 0); + h.instance.schedulePreviewUpdate(); + assert.equal(h.timers.size, 0); +}); diff --git a/tests/remote-input-native.cjs b/tests/remote-input-native.cjs index 392215aa..d39d1f43 100644 --- a/tests/remote-input-native.cjs +++ b/tests/remote-input-native.cjs @@ -21,17 +21,20 @@ int main(int argc, const char **argv) { @autoreleasepool { NSString *failure = [server start:atoi(argv[1])]; if (failure.length) return 3; puts("READY"); fflush(stdout); - for (;;) { @autoreleasepool { + __weak FaceclawRemoteInput *weakServer = server; + [server setRequestListener:^{ @autoreleasepool { + FaceclawRemoteInput *server = weakServer; + if (![NSThread isMainThread]) exit(4); NSString *raw = [server nextRequest]; if (raw) { NSDictionary *request = [NSJSONSerialization JSONObjectWithData:[raw dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; NSDictionary *body = [NSJSONSerialization JSONObjectWithData:[request[@"body"] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; - if ([body[@"text"] isEqual:@"stop"]) { [server stop]; usleep(300000); puts("STOPPED"); fflush(stdout); return 0; } + if ([body[@"text"] isEqual:@"stop"]) { [server stop]; usleep(300000); puts("STOPPED"); fflush(stdout); exit(0); } NSData *reply = [NSJSONSerialization dataWithJSONObject:@{@"ok":@YES, @"echo":body ?: @{}} options:0 error:nil]; [server complete:[request[@"id"] longLongValue] response:[[NSString alloc] initWithData:reply encoding:NSUTF8StringEncoding]]; } - usleep(1000); - }} + }}]; + [[NSRunLoop mainRunLoop] run]; }} `); const binary = path.join(directory, 'probe'); @@ -66,7 +69,7 @@ int main(int argc, const char **argv) { @autoreleasepool { await send(options, { action: 'input', gesture: 'click' }); await assert.rejects(send(options, { action: 'text', text: 'stop' })); assert.equal(await exited, 0); - console.log('Native socket probe passed: crypto, framing, UTF-8, multiple connections, shutdown.'); + console.log('Native socket probe passed: crypto, framing, UTF-8, main-thread request notifications, multiple connections, shutdown.'); } finally { child.kill(); } } main().catch(error => { console.error(error); process.exitCode = 1; }).finally(() => fs.rmSync(directory, { recursive: true, force: true })); diff --git a/tests/remote-service.test.cjs b/tests/remote-service.test.cjs new file mode 100644 index 00000000..3d7d75ba --- /dev/null +++ b/tests/remote-service.test.cjs @@ -0,0 +1,83 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const ts = require('typescript'); + +function fixture() { + let ready, changed, tokens = [{}], polls = 0, active = 0, maxActive = 0; + const queue = [], replies = [], timers = new Map(), handled = []; + const native = { + start() {}, addresses: () => ['127.0.0.1'], stop: () => { queue.length = 0; }, + nextRequest() { polls++; return queue.length ? JSON.stringify(queue.shift()) : null; }, + complete: (id, response) => replies.push({ id, ...JSON.parse(response) }), + }; + let handle = async body => ({ ok: true, body }); + const modules = { + '../native/settings-store': { getStringSetting: (_k, d) => d, setStringSetting() {}, + onSettingsStoreChanged: cb => { changed = cb; return () => { changed = null; }; } }, + '../native/remote-input': { remoteNative: cb => { ready = cb; return native; }, remoteInterfaces: () => [], + randomTokenSecret() {}, tokenHash() {} }, + './listeners': require('../.test-build/app/remote/listeners.js'), + './protocol': { TokenStore: class { list() { return tokens; } }, REMOTE_PORT: 8791, + handleRequest: async body => { + handled.push(body); active++; maxActive = Math.max(maxActive, active); + try { return await handle(body); } finally { active--; } + } }, + }; + const context = { exports: {}, require: id => modules[id], + setInterval: (fn, ms) => { timers.set(fn, ms); return fn; }, clearInterval: id => timers.delete(id) }; + vm.runInNewContext(ts.transpileModule(fs.readFileSync('app/remote/service.ts', 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); + const stop = context.exports.startRemoteInput({}); + return { stop, timers, replies, handled, get polls() { return polls; }, get maxActive() { return maxActive; }, + setHandle: fn => { handle = fn; }, + notify: () => ready(), enqueue: (id, expiresAt = Date.now() + 5000) => queue.push({ id, body: id, expiresAt }), + tokens: value => { tokens = value; changed('remoteInput.tokens.v1'); }, + }; +} +const settle = () => new Promise(resolve => setImmediate(resolve)); + +test('idle remote sockets do not poll native requests, including network reconciliation', () => { + const f = fixture(); + assert.deepEqual([...f.timers.values()], [3000]); + for (let i = 0; i < 100; i++) for (const tick of f.timers.keys()) tick(); + assert.equal(f.polls, 0); + f.stop(); assert.equal(f.timers.size, 0); +}); + +test('request notifications drain serially without losing wakeups while busy', async () => { + const f = fixture(); let release; + f.setHandle(body => body === 'a' ? new Promise(resolve => { release = resolve; }) : { ok: true }); + f.enqueue('a'); f.notify(); + f.enqueue('b'); f.notify(); f.enqueue('c'); f.notify(); + assert.deepEqual(f.handled, ['a']); + release({ ok: true }); await settle(); + assert.deepEqual(f.handled, ['a', 'b', 'c']); + assert.deepEqual(f.replies.map(r => r.id), ['a', 'b', 'c']); + assert.equal(f.maxActive, 1); + const idlePolls = f.polls; await settle(); assert.equal(f.polls, idlePolls); + f.stop(); +}); + +test('expired requests and failed handlers do not prevent queued requests from draining', async () => { + const f = fixture(); f.setHandle(body => { if (body === 'bad') throw Error('test'); return { ok: true }; }); + f.enqueue('expired', 0); f.enqueue('bad'); f.enqueue('good'); f.notify(); await settle(); + assert.deepEqual(f.handled, ['bad', 'good']); + assert.equal(f.replies[0].error, 'failed'); assert.equal(f.replies[1].ok, true); + f.stop(); +}); + +test('token removal and service shutdown stop dispatch, and re-enabling tokens resumes it', async () => { + const f = fixture(); + f.enqueue('removed'); f.tokens([]); f.notify(); await settle(); + assert.deepEqual(f.handled, []); assert.equal(f.timers.size, 0); + f.tokens([{}]); f.enqueue('resumed'); f.notify(); await settle(); + assert.deepEqual(f.handled, ['resumed']); + let release; f.setHandle(() => new Promise(resolve => { release = resolve; })); + f.enqueue('in-flight'); f.notify(); f.stop(); + f.enqueue('late'); f.notify(); release({ ok: true }); await settle(); + assert.deepEqual(f.replies.map(r => r.id), ['resumed']); + assert.deepEqual(f.handled, ['resumed', 'in-flight']); +});