From 2c325fa1d433b213ab421686ab5932ad12e12123 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Mon, 12 Oct 2026 17:08:41 -0700 Subject: [PATCH] Center the phone screen preview --- .../faceclaw/app/FaceclawBleCommunicator.kt | 9 ++ .../faceclaw/app/FaceclawPreviewCompositor.kt | 9 ++ .../com/faceclaw/app/PreviewBitmapUtil.kt | 28 +++++++ app/g2/dashboard-controller.ts | 19 +++-- app/g2/glance-host.ts | 10 ++- app/g2/ios-preview-controller.ts | 12 ++- app/g2/phone-preview-crop.ts | 33 ++++++++ app/native/faceclaw-communicator.ios.ts | 6 +- app/native/faceclaw-communicator.ts | 16 +++- app/native/ios-graphics.ts | 20 ++++- app/native/preview-display.ios.ts | 4 +- app/native/preview-display.ts | 13 ++- app/ui/dashboard-settings.ts | 13 +++ app/ui/dashboard/settings-menus.ts | 2 + app/ui/shell/geometry.ts | 52 ++++++++++++ tests/app-layout.test.cjs | 84 +++++++++++++++++++ 16 files changed, 307 insertions(+), 23 deletions(-) create mode 100644 app/g2/phone-preview-crop.ts diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 24ce8c53..cdff794d 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -242,6 +242,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? { + val composite = core.previewComposite() ?: return null + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green, + cropX, cropY, cropWidth, cropHeight) + } + /** * Posted to the main looper after each step of an animation the preview is replaying (menu * highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt index 6ae3b8ab..6c6d4a60 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt @@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) { java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? { + val composite = compositor.previewComposite() ?: return null + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green, + cropX, cropY, cropWidth, cropHeight) + } + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(): String { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt index 6c98b9b0..75135480 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() { return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) } + /** + * As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the + * mirror's Scaled preview). The area may extend past the frame's edges; it is black + * there. + */ + @JvmStatic + fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw IllegalArgumentException("invalid gray preview buffer") + } + if (cropWidth <= 0 || cropHeight <= 0) { + throw IllegalArgumentException("invalid preview crop") + } + val bytes = ByteArray(width * height) + gray.get(bytes) + val cropped = ByteArray(cropWidth * cropHeight) + val left = maxOf(cropX, 0) + val right = minOf(cropX + cropWidth, width) + if (right > left) { + for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) { + System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left) + } + } + val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green) + return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888) + } + /** * A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes * alpha; black, which the glasses show as unlit, is transparent), for diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 53ce6cb8..d172681a 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -46,6 +46,7 @@ const MIRROR_TOUCH_GESTURES: Record, WearRemoteI "swipe-right": "swipe-right", }; import { GlanceHost } from "./glance-host"; +import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop"; import { type GlanceEvent } from "./glance-state"; import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress"; @@ -69,7 +70,7 @@ import { ALL_APPS } from "../apps/all-apps"; import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; -import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry"; +import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; @@ -273,6 +274,8 @@ class DashboardController { private extractedFontsConfirmed = false; private screenRecordingActive = false; private displayPreview: ImageSource | null = createInitialDisplayPreview(); + /** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */ + private displayPreviewCrop: ScreenRect | null = null; private silentMode = false; // Brightness value last pushed to the glasses; see pushBrightness. private lastPushedBrightness: string | null = null; @@ -458,8 +461,9 @@ class DashboardController { this.applyAppLayoutsIfChanged(); this.syncAssistantBridgeIfChanged(); this.syncLockScreenSettingIfChanged(); - // A Preview color change should show on the mirror at once, not at the - // next composited frame (cheap: floor-limited, dedup'd on the Java side). + // A Preview color or scaling change should show on the mirror at once, + // not at the next composited frame (cheap: floor-limited, dedup'd on + // the Java side). this.schedulePreviewUpdate(); }); // Connect to the external agent bridge at boot if configured; the @@ -1931,8 +1935,9 @@ class DashboardController { await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch"); return; } - const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH); - const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT); + const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny); + const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH); + const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT); const foreground = shell.foregroundWindow(); const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId); this.appendLog(`mirror tap at ${x},${y}`); @@ -2883,8 +2888,10 @@ class DashboardController { const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager; if (!power?.isInteractive()) return; } - const preview = display.getCompositePreview(previewColorSetting.get() === "green"); + const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect()); + const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop); if (preview) { + this.displayPreviewCrop = crop; this.setDisplayPreview(preview); } } diff --git a/app/g2/glance-host.ts b/app/g2/glance-host.ts index e92613ad..cf5d05ff 100644 --- a/app/g2/glance-host.ts +++ b/app/g2/glance-host.ts @@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import * as frameTimings from "../native/frame-timings"; import { type DisplayTarget } from "../native/preview-display"; -import { minWindowTop } from "../ui/shell/geometry"; +import { minWindowTop, type ScreenRect } from "../ui/shell/geometry"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition"; import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state"; @@ -59,6 +59,8 @@ export class GlanceHost { private rendering = false; private renderPending = false; private nextRenderWantsFreshData = false; + /** Where the last render put the board on screen. */ + private boardRect: ScreenRect | null = null; constructor(private readonly options: GlanceHostOptions) {} @@ -66,6 +68,11 @@ export class GlanceHost { return this.state.visible; } + /** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */ + screenRect(): ScreenRect | null { + return this.state.visible ? this.boardRect : null; + } + /** Whether sleep-time gestures should reach the board at all. */ isEnabled(): boolean { return this.options.getProvider()?.isEnabled() ?? false; @@ -249,6 +256,7 @@ export class GlanceHost { const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0); const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop())); boardImage.composeInto(full, x, y); + this.boardRect = { x, y, width: boardImage.width, height: boardImage.height }; return full; }), ); diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index f59de83b..3405dd66 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -43,7 +43,8 @@ import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/g import { noopLayerActions, type LayerActions } from '../ui/layers' import { TextViewerLayer } from '../apps/files/text-viewer' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' -import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry' +import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry' +import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, @@ -116,6 +117,8 @@ export class IosPreviewController { appendLog: message => this.logBluetooth(message), }) private previewTimer: ReturnType | null = null + /** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */ + private previewCrop: ScreenRect | null = null private clockTimer: ReturnType | null = null private offSettings: (() => void) | null = null private lastBrightness: string | null = null @@ -506,8 +509,10 @@ export class IosPreviewController { this.previewTimer = null if (!this.active) return try { - const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green') + const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect()) + const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop) if (!image) return + this.previewCrop = crop this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' : `${shell.foregroundWindow()?.title ?? 'Apps'} ยท ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) } catch (error) { this.fail(error) } @@ -561,7 +566,8 @@ export class IosPreviewController { await shell.receiveInput(event) this.requestShellRender() } - private async mirrorTap(nx: number, ny: number): Promise { + private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise { + const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy) const x = Math.max(0, Math.min(639, Math.floor(nx * 640))) const y = Math.max(0, Math.min(479, Math.floor(ny * 480))) const window = shell.foregroundWindow() diff --git a/app/g2/phone-preview-crop.ts b/app/g2/phone-preview-crop.ts new file mode 100644 index 00000000..2eafaa51 --- /dev/null +++ b/app/g2/phone-preview-crop.ts @@ -0,0 +1,33 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image"; +import { previewScalingSetting } from "../ui/dashboard-settings"; +import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry"; +import { shell } from "../ui/shell/shell"; + +/** + * The screen area the phone's mirror shows (Settings > Phone display > + * Preview scaling), or null for the whole panel. Scaled zooms to the + * Glanceboard while it shows, else to the area the shell uses around the + * foreground window, centred at the panel's aspect. A locked display gets + * the whole panel: its message box is centred in the panel, not the band. + */ +export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null { + if (locked || previewScalingSetting.get() === "raw") return null; + const window = shell.foregroundWindow(); + const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId)); + const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT; + return wholePanel ? null : crop; +} + +/** + * A point on the mirror, as fractions of the mirror's width and height, + * as fractions of the panel's, given the crop the mirror is showing. + * Points in the black margins past the panel's edges come out below 0 or + * above 1. + */ +export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } { + if (!crop) return { nx, ny }; + return { + nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH, + ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT, + }; +} diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index 9bb7ed63..ed8937b4 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -489,13 +489,13 @@ export class FaceclawCommunicatorBridge { }); } - /** Phone-UI preview of the current composited screen, or null if none yet. */ - getCompositePreview(green = false): ImageSource | null { + /** Phone-UI preview of the current composited screen (or of `crop`'s area of it), or null if none yet. */ + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { const frame = this.communicator.previewComposite(); if (!frame) return null; const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight()); if (width <= 0 || height <= 0) return null; - return previewPixels(fromData(frame.pixels), width, height, green); + return previewPixels(fromData(frame.pixels), width, height, green, crop); } /** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */ diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index c469d2c2..481ae62b 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -507,11 +507,21 @@ export class FaceclawCommunicatorBridge { /** * Phone-UI preview of the current composited screen, or null if none yet. * `green` tints it green-on-black (the Preview color setting) instead of - * grayscale. + * grayscale; `crop` limits it to a screen area (the Preview scaling + * setting), black where that runs past the screen's edges. */ - getCompositePreview(green = false): ImageSource | null { + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { if (!global.isAndroid) return null; - const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); + const bitmap = crop + ? this.communicator.getCompositePreviewBitmap( + PREVIEW_BRIGHTEN_GAMMA, + green, + Math.round(crop.x), + Math.round(crop.y), + Math.round(crop.width), + Math.round(crop.height), + ) + : this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); return bitmap ? new ImageSource(bitmap) : null; } diff --git a/app/native/ios-graphics.ts b/app/native/ios-graphics.ts index 86e8e72d..1a57d828 100644 --- a/app/native/ios-graphics.ts +++ b/app/native/ios-graphics.ts @@ -3,11 +3,25 @@ import { GrayImage } from '../graphics/image' declare const FaceclawGraphics: any -export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource { +type CropRect = { x: number; y: number; width: number; height: number } + +/** The gray pixels of a screen area; black where it runs past the frame's edges. */ +function cropGray(pixels: Uint8Array, width: number, height: number, crop: CropRect): Uint8Array { + const cropped = new Uint8Array(crop.width * crop.height) + const left = Math.max(crop.x, 0), right = Math.min(crop.x + crop.width, width) + if (right <= left) return cropped + for (let y = Math.max(crop.y, 0); y < Math.min(crop.y + crop.height, height); y++) { + cropped.set(pixels.subarray(y * width + left, y * width + right), (y - crop.y) * crop.width + (left - crop.x)) + } + return cropped +} + +/** Gray pixels as the phone mirror's image, optionally cropped to a screen area (Preview scaling). */ +export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false, crop: CropRect | null = null): ImageSource { // NSData copies the buffer; the UIImage owns its pixels after this call. - const copy = new Uint8Array(pixels) + const copy = crop ? cropGray(pixels, width, height, crop) : new Uint8Array(pixels) const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) - const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green) + const native = FaceclawGraphics.previewWidthHeightGreen(data, crop?.width ?? width, crop?.height ?? height, green) if (!native) throw new Error('Could not create preview image') return new ImageSource(native) } diff --git a/app/native/preview-display.ios.ts b/app/native/preview-display.ios.ts index c25628a6..ce85084f 100644 --- a/app/native/preview-display.ios.ts +++ b/app/native/preview-display.ios.ts @@ -138,8 +138,8 @@ export class PreviewDisplayTarget implements DisplayTarget { return this.compositor.composite(); } - getCompositePreview(green = false): ImageSource | null { - return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green); + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { + return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green, crop); } /** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */ diff --git a/app/native/preview-display.ts b/app/native/preview-display.ts index 4f6d7531..8df3f5af 100644 --- a/app/native/preview-display.ts +++ b/app/native/preview-display.ts @@ -160,9 +160,18 @@ export class PreviewDisplayTarget implements DisplayTarget { return Promise.resolve("composited"); } - getCompositePreview(green = false): ImageSource | null { + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { if (!global.isAndroid) return null; - const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); + const bitmap = crop + ? this.compositor.getCompositePreviewBitmap( + PREVIEW_BRIGHTEN_GAMMA, + green, + Math.round(crop.x), + Math.round(crop.y), + Math.round(crop.width), + Math.round(crop.height), + ) + : this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); return bitmap ? new ImageSource(bitmap) : null; } diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index b0652865..51880c80 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -616,6 +616,19 @@ export const previewColorSetting = new ConfigSettingEnum({ "How the phone's mirror of the glasses display renders: white/grayscale (clearest), or green to match the physical glasses.", }); +export type PreviewScaling = "scaled" | "raw"; + +export const previewScalingSetting = new ConfigSettingEnum({ + id: "preview-scaling", + label: "Preview scaling", + storageKey: "phone.previewScaling", + defaultValue: "scaled", + values: ["scaled", "raw"], + formatValue: (value) => (value === "raw" ? "Raw" : "Scaled"), + description: + "How much of the glasses display the phone's mirror shows: Scaled centers and enlarges the part the current display mode uses; Raw shows the whole 640x480 panel.", +}); + export const mirrorTouchSetting = new ConfigSettingBoolean({ id: "mirror-touch", label: "Touch mirror", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index e8d8a7f7..92833593 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -57,6 +57,7 @@ import { mirrorTouchSetting, openAiApiKeySetting, previewColorSetting, + previewScalingSetting, phoneRotationSetting, ringConnectionModeSetting, sonioxApiKeySetting, @@ -162,6 +163,7 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(phoneRotationSetting), enumSettingMenuItem(previewColorSetting), + enumSettingMenuItem(previewScalingSetting), toggleSettingMenuItem(mirrorTouchSetting), ], }, diff --git a/app/ui/shell/geometry.ts b/app/ui/shell/geometry.ts index 5be3bfbb..9e6e738d 100644 --- a/app/ui/shell/geometry.ts +++ b/app/ui/shell/geometry.ts @@ -422,3 +422,55 @@ export function appViewportRect(mode: WindowHeightMode, appId?: string): { ...appViewportSize(mode, appId), }; } + +export type ScreenRect = { x: number; y: number; width: number; height: number }; + +function unionRect(a: ScreenRect, b: ScreenRect): ScreenRect { + const x = Math.min(a.x, b.x); + const y = Math.min(a.y, b.y); + return { + x, + y, + width: Math.max(a.x + a.width, b.x + b.width) - x, + height: Math.max(a.y + a.height, b.y + b.height) - y, + }; +} + +/** + * The part of the screen the shell draws in while the given (foreground) + * window is up: the window's band, plus the frame's side pixels; the + * min-height band the shell overlays align to; and a side strip or bottom + * row (a popup shows within the window's content). An immersive window has + * the whole panel. The phone mirror's Scaled preview zooms to this. + */ +export function usedScreenRect(heightMode: WindowHeightMode, appId?: string): ScreenRect { + if (isImmersive(appId)) return { x: 0, y: 0, width: G2_LENS_WIDTH, height: G2_LENS_HEIGHT }; + const frame = windowFramed(appId) ? 1 : 0; + const x = appViewportLeft(appId) - frame; + const width = appViewportWidth(appId) + 2 * frame; + let used = unionRect( + { x, y: windowTop(heightMode, appId), width, height: windowBandHeight(heightMode, appId) }, + { x, y: minWindowTop(appId), width, height: MIN_WINDOW_HEIGHT }, + ); + if (switcherPosition() !== "popup") used = unionRect(used, switcherRect(heightMode, appId)); + return used; +} + +/** + * The screen area the phone mirror shows to present `used` as large as it + * fits, centred, at the panel's own 4:3 aspect (so the mirror keeps its + * shape whatever the display mode): `used` grown along one axis, past the + * panel's edges if need be (the mirror shows black there). + */ +export function phonePreviewRect(used: ScreenRect): ScreenRect { + // Whole 4:3 units, so the aspect is exact. + const units = Math.ceil(Math.max(used.width / 4, used.height / 3)); + const width = units * 4; + const height = units * 3; + return { + x: Math.round(used.x + (used.width - width) / 2), + y: Math.round(used.y + (used.height - height) / 2), + width, + height, + }; +} diff --git a/tests/app-layout.test.cjs b/tests/app-layout.test.cjs index cc31f474..2128a2ad 100644 --- a/tests/app-layout.test.cjs +++ b/tests/app-layout.test.cjs @@ -263,6 +263,90 @@ test('an immersive app fills the panel without chrome rows, frame or depth, in a } }); +test('the phone mirror zooms to the area the shell uses, centred at the panel\'s 4:3', () => { + const g = geometry(); + const preview = (mode, appId) => { + const used = { ...g.usedScreenRect(mode, appId) }; + return { used, shown: { ...g.phonePreviewRect(used) } }; + }; + const panel = { x: 0, y: 0, width: 640, height: 480 }; + // A side strip and band: the panel's width, so only centring vertically. + assert.deepEqual(preview('min'), { used: { x: 0, y: 96, width: 640, height: 288 }, shown: panel }); + g.settings.displayLayoutSetting.value = 'band-top'; + assert.deepEqual(preview('min').shown, { x: 0, y: -96, width: 640, height: 480 }); + g.settings.appSwitcherPositionSetting.value = 'right'; + g.settings.displayLayoutSetting.value = 'band-high'; + assert.deepEqual(preview('min'), { used: { x: 0, y: 48, width: 640, height: 288 }, shown: { x: 0, y: -48, width: 640, height: 480 } }); + // A bottom row: the band with the frame's sides, and the row under it. + g.settings.appSwitcherPositionSetting.value = 'bottom'; + g.settings.displayLayoutSetting.value = 'band-middle'; + assert.deepEqual(preview('min'), { used: { x: 31, y: 78, width: 578, height: 324 }, shown: { x: 30, y: 23, width: 580, height: 435 } }); + // A tall window (the terminal's default) leaves just the frame's margins. + assert.deepEqual(preview('max', 'terminal'), { used: { x: 31, y: 0, width: 578, height: 480 }, shown: panel }); + // A popup shows within the window: just the band. + g.settings.appSwitcherPositionSetting.value = 'popup'; + assert.deepEqual(preview('min'), { used: { x: 31, y: 96, width: 578, height: 288 }, shown: { x: 30, y: 23, width: 580, height: 435 } }); + // Full Screen and immersive windows use the whole panel. + g.settings.displayLayoutSetting.value = 'fullscreen'; + assert.deepEqual(preview('min'), { used: panel, shown: panel }); + g.settings.displayLayoutSetting.value = 'band-middle'; + g.setAppImmersive('onboarding', true); + assert.deepEqual(preview('min', 'onboarding'), { used: panel, shown: panel }); + // Every layout's shown area is exactly 4:3 and contains the used area. + for (const position of ['left', 'right', 'bottom', 'popup']) { + g.settings.appSwitcherPositionSetting.value = position; + for (const layout of LAYOUTS) { + g.settings.displayLayoutSetting.value = layout; + for (const mode of ['min', 'medium', 'max']) { + const { used, shown } = preview(mode); + assert.equal(shown.width * 3, shown.height * 4); + assert.ok(shown.width <= 640 && shown.height <= 480); + assert.ok(shown.x <= used.x && shown.y <= used.y); + assert.ok(shown.x + shown.width >= used.x + used.width && shown.y + shown.height >= used.y + used.height); + } + } + } +}); + +test('the mirror\'s crop follows Preview scaling, the Glanceboard and the lock, and maps taps back to the panel', () => { + const g = geometry(); + const scaling = { value: 'scaled', get() { return this.value; } }; + let foreground = { heightMode: 'min', appId: 'launcher' }; + const modules = { + '../graphics/image': { G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 }, + '../ui/dashboard-settings': { previewScalingSetting: scaling }, + '../ui/shell/geometry': g, + '../ui/shell/shell': { shell: { foregroundWindow: () => foreground } }, + }; + const context = { exports: {}, require: (name) => modules[name] }; + vm.runInNewContext(js(read('app/g2/phone-preview-crop.ts')), context); + const { phonePreviewCrop, mirrorToPanelFraction } = context.exports; + const crop = (locked = false, glance = null) => { + const rect = phonePreviewCrop(locked, glance); + return rect && { ...rect }; + }; + g.settings.appSwitcherPositionSetting.value = 'bottom'; + const band = { x: 30, y: 23, width: 580, height: 435 }; + assert.deepEqual(crop(), band); + // The whole panel is no crop at all. + foreground = { heightMode: 'max', appId: 'terminal' }; + assert.equal(crop(), null); + foreground = null; + assert.deepEqual(crop(), band); + // The Glanceboard while it shows; a locked display's centred box needs the panel. + assert.deepEqual(crop(false, { x: 32, y: 48, width: 576, height: 432 }), { x: 32, y: 48, width: 576, height: 432 }); + assert.deepEqual(crop(false, { x: 32, y: 96, width: 576, height: 288 }), { x: 32, y: 24, width: 576, height: 432 }); + assert.equal(crop(true), null); + scaling.value = 'raw'; + assert.equal(crop(), null); + // Taps: the mirror's corners are the crop's, its margins past the panel's edges. + assert.deepEqual({ ...mirrorToPanelFraction(null, 0.25, 0.75) }, { nx: 0.25, ny: 0.75 }); + assert.deepEqual({ ...mirrorToPanelFraction(band, 0, 0) }, { nx: 30 / 640, ny: 23 / 480 }); + assert.deepEqual({ ...mirrorToPanelFraction(band, 1, 1) }, { nx: 610 / 640, ny: 458 / 480 }); + const above = mirrorToPanelFraction({ x: 0, y: -96, width: 640, height: 480 }, 0.5, 0.1); + assert.ok(above.ny < 0); +}); + // The real chrome layer over the given geometry, with platform pieces stubbed: // a 4px-per-character font that paints each character as a 2x6 block, up to // `notifications` 24px notification icons (keyed n0, n1, ...), and a phone