Terminal feature; raw screenshots.

This commit is contained in:
James Babcock committed 2026-07-04 17:38:53 -07:00
1 parent d2a942f5a3
commit fab6f6689e
19 files changed
+1075 -10

No files matched your search

+2
View File
@@ -3,6 +3,8 @@ import org.apache.tools.ant.taskdefs.condition.Os
// You can add your native dependencies here
dependencies {
// implementation 'androidx.multidex:multidex:2.0.1'
// WebSocket transport for the g2mirror terminal-mirroring client.
implementation 'com.squareup.okhttp3:okhttp:4.12.0'
}
def faceclawSherpaOnnxVersion = "1.13.0"
@@ -30,12 +30,16 @@
NameNotFoundException for any package we cannot see. -->
<uses-permission android:name="android.permission.QUERY_ALL_PACKAGES"/>
<!-- usesCleartextTraffic: the g2mirror terminal-mirroring protocol runs
plain ws:// over tailscale (transport security is the network's job),
which Android's default policy would otherwise block. -->
<application
android:name="com.tns.NativeScriptApplication"
android:allowBackup="true"
android:icon="@mipmap/ic_launcher"
android:label="@string/app_name"
android:theme="@style/AppTheme"
android:usesCleartextTraffic="true"
android:hardwareAccelerated="true">
<activity
@@ -0,0 +1,116 @@
package com.faceclaw.app;
import android.os.Handler;
import android.os.Looper;
import android.util.Log;
import java.util.concurrent.TimeUnit;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import okhttp3.WebSocket;
import okhttp3.WebSocketListener;
/**
* Thin okhttp WebSocket wrapper for the TypeScript side. Text frames only
* (the g2mirror protocol is JSON-in-text-frames). All listener callbacks are
* posted to the main thread, where the NativeScript runtime expects them.
*/
public class FaceclawWebSocket {
private static final String TAG = "FaceclawWebSocket";
private static final Handler mainHandler = new Handler(Looper.getMainLooper());
private static volatile OkHttpClient sharedClient;
private final WebSocket socket;
private volatile boolean closeRequested;
public FaceclawWebSocket(String url, final FaceclawWebSocketListener listener) {
if (url == null || url.trim().isEmpty()) {
throw new IllegalArgumentException("url is required");
}
if (listener == null) {
throw new IllegalArgumentException("listener is required");
}
Request request = new Request.Builder().url(url.trim()).build();
socket = getClient().newWebSocket(request, new WebSocketListener() {
@Override public void onOpen(WebSocket webSocket, Response response) {
mainHandler.post(() -> {
try {
listener.onOpen();
} catch (Throwable t) {
Log.w(TAG, "listener onOpen failed", t);
}
});
}
@Override public void onMessage(WebSocket webSocket, String text) {
mainHandler.post(() -> {
try {
listener.onTextMessage(text);
} catch (Throwable t) {
Log.w(TAG, "listener onTextMessage failed", t);
}
});
}
@Override public void onClosing(WebSocket webSocket, int code, String reason) {
webSocket.close(code, reason);
}
@Override public void onClosed(WebSocket webSocket, int code, String reason) {
mainHandler.post(() -> {
try {
listener.onClosed(code, reason == null ? "" : reason);
} catch (Throwable t) {
Log.w(TAG, "listener onClosed failed", t);
}
});
}
@Override public void onFailure(WebSocket webSocket, Throwable t, Response response) {
if (closeRequested) {
return;
}
final String message = t == null ? "unknown websocket failure" : String.valueOf(t);
mainHandler.post(() -> {
try {
listener.onFailure(message);
} catch (Throwable inner) {
Log.w(TAG, "listener onFailure failed", inner);
}
});
}
});
}
private static OkHttpClient getClient() {
OkHttpClient client = sharedClient;
if (client == null) {
synchronized (FaceclawWebSocket.class) {
if (sharedClient == null) {
sharedClient = new OkHttpClient.Builder()
.pingInterval(20, TimeUnit.SECONDS)
.build();
}
client = sharedClient;
}
}
return client;
}
public boolean sendText(String message) {
return socket.send(message == null ? "" : message);
}
public void close(int code, String reason) {
closeRequested = true;
try {
if (!socket.close(code, reason)) {
socket.cancel();
}
} catch (Throwable t) {
socket.cancel();
}
}
}
@@ -0,0 +1,9 @@
package com.faceclaw.app;
/** Callbacks for FaceclawWebSocket, delivered on the Android main thread. */
public interface FaceclawWebSocketListener {
void onOpen();
void onTextMessage(String message);
void onClosed(int code, String reason);
void onFailure(String message);
}
@@ -0,0 +1,44 @@
package com.faceclaw.app;
import android.content.Context;
import java.io.File;
import java.io.FileOutputStream;
import java.nio.ByteBuffer;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.Locale;
/**
* Saves raw screenshots for compression experiments. The output is the exact
* headerless top-down 4bpp frame buffer that gets zlib-compressed on the wire
* (build4bppBmp followed by pack4bppFromBmp), deliberately untransformed — no
* resize, no gamma, no header.
*/
public final class ScreenshotUtil {
private ScreenshotUtil() {}
/** Returns the absolute path of the written file. */
public static String saveRawScreenshot(Context context, ByteBuffer image8bpp, int width, int height) throws java.io.IOException {
if (image8bpp == null || width <= 0 || height <= 0 || image8bpp.remaining() < width * height) {
throw new IllegalArgumentException("invalid screenshot buffer");
}
byte[] gray = new byte[image8bpp.remaining()];
image8bpp.get(gray);
byte[] packed = BmpUtil.pack4bppFromBmp(BmpUtil.build4bppBmp(gray, width, height));
if (packed.length == 0) {
throw new IllegalStateException("failed to pack 4bpp frame");
}
File dir = new File(context.getExternalFilesDir(null), "screenshots");
if (!dir.exists() && !dir.mkdirs()) {
throw new java.io.IOException("failed to create " + dir);
}
String stamp = new SimpleDateFormat("yyyyMMdd-HHmmss-SSS", Locale.US).format(new Date());
File file = new File(dir, "screen-" + stamp + "-" + width + "x" + height + ".4bpp");
try (FileOutputStream out = new FileOutputStream(file)) {
out.write(packed);
}
return file.getAbsolutePath();
}
}
+23
View File
@@ -30,10 +30,13 @@ import {
import {
nightscoutApiTokenSetting,
nightscoutSiteUrlSetting,
onAnySettingChanged,
rawScreenshotsEnabledSetting,
systemCardNameSetting,
voiceControlEnabledSetting,
type ConfigSettingString,
} from "../ui/dashboard-settings";
import { saveRawScreenshot } from "../native/raw-screenshot";
type ConnectionPhase = "disconnected" | "connecting" | "connected" | "disconnecting";
@@ -49,6 +52,7 @@ export type DashboardSnapshot = {
evenAppConflictWarningVisible: boolean;
firmwareWarningMessage: string;
firmwareWarningVisible: boolean;
rawScreenshotsEnabled: boolean;
};
type DashboardListener = (snapshot: DashboardSnapshot) => void;
@@ -158,6 +162,11 @@ class DashboardController {
constructor() {
setDashboardActions({
requestRender: () => {
void this.requestRender("interval").catch((error) => {
this.appendLog(`requested render failed: ${this.formatError(error)}`);
});
},
disconnect: () => this.disconnect(),
startTextSettingEdit: (setting) => this.startTextSettingEdit(setting),
endTextSettingEdit: () => this.endTextSettingEdit(),
@@ -173,6 +182,19 @@ class DashboardController {
this.appendLog(`notification wake failed: ${this.formatError(error)}`);
});
});
// Settings toggled from the glasses can change what the phone UI shows
// (e.g. the raw-screenshot button), so re-emit the snapshot on any change.
onAnySettingChanged(() => this.emit());
}
/**
* Save the current dashboard as the raw headerless 4bpp frame buffer the
* wire compressor sees; test data for compression experiments.
*/
saveRawDashboardScreenshot(): string {
const path = saveRawScreenshot(drawDashboard());
this.appendLog(`raw screenshot saved: ${path}`);
return path;
}
subscribe(listener: DashboardListener): () => void {
@@ -196,6 +218,7 @@ class DashboardController {
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
firmwareWarningMessage: this.firmwareWarningMessage,
firmwareWarningVisible: this.firmwareWarningMessage.length > 0,
rawScreenshotsEnabled: rawScreenshotsEnabledSetting.get(),
};
}
+320
View File
@@ -0,0 +1,320 @@
import { toUint8Array } from "../util/array-util";
declare const com: any;
declare const android: any;
/**
* Client for the g2mirror terminal-mirroring protocol (see
* ../experiments/g2mirror/PROTOCOL.md). One instance per Terminal app
* session; talks JSON over a websocket to g2mirror-server, which relays to
* one wrapped CLI app at a time.
*/
const PROTOCOL_VERSION = 1;
const SESSION_LIST_REFRESH_MS = 3_000;
export type G2MirrorSession = {
socket: string;
pid: number;
cwdHint: string;
};
export type G2MirrorPhase =
| "idle"
| "connecting"
| "connected" // handshake accepted; can list/attach
| "attached" // relaying to one session
| "failed";
export type G2MirrorState = {
phase: G2MirrorPhase;
status: string;
sessions: G2MirrorSession[];
attachedCommand: string;
};
export type G2MirrorClientOptions = {
host: string;
port: number;
authToken: string;
deviceName: string;
cols: number;
rows: number;
};
type TerminalDataKind = "snapshot" | "output";
export class G2MirrorClient {
private ws: any = null;
private listenerProxy: any = null;
private phase: G2MirrorPhase = "idle";
private status = "Not connected.";
private sessions: G2MirrorSession[] = [];
private attachedCommand = "";
private listRefreshTimer: ReturnType<typeof setInterval> | null = null;
private stopped = false;
private readonly stateListeners = new Set<(state: G2MirrorState) => void>();
private readonly terminalDataListeners = new Set<(data: Uint8Array, kind: TerminalDataKind) => void>();
private readonly sessionAttachedListeners = new Set<(command: string) => void>();
private readonly sessionDetachedListeners = new Set<(reason: string) => void>();
constructor(private readonly options: G2MirrorClientOptions) {}
state(): G2MirrorState {
return {
phase: this.phase,
status: this.status,
sessions: this.sessions.slice(),
attachedCommand: this.attachedCommand,
};
}
onStateChange(listener: (state: G2MirrorState) => void): () => void {
this.stateListeners.add(listener);
return () => this.stateListeners.delete(listener);
}
/** Raw VT100/xterm bytes to feed the emulator. Snapshot implies reset-first. */
onTerminalData(listener: (data: Uint8Array, kind: TerminalDataKind) => void): () => void {
this.terminalDataListeners.add(listener);
return () => this.terminalDataListeners.delete(listener);
}
onSessionAttached(listener: (command: string) => void): () => void {
this.sessionAttachedListeners.add(listener);
return () => this.sessionAttachedListeners.delete(listener);
}
onSessionDetached(listener: (reason: string) => void): () => void {
this.sessionDetachedListeners.add(listener);
return () => this.sessionDetachedListeners.delete(listener);
}
start(): void {
if (this.ws) return;
this.stopped = false;
const url = `ws://${this.options.host}:${this.options.port}`;
this.setState("connecting", `Connecting to ${this.options.host}:${this.options.port}...`);
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
onOpen: () => {
if (this.stopped) return;
this.setState("connecting", "Authenticating...");
this.send({
type: "init",
version: PROTOCOL_VERSION,
auth_token: this.options.authToken,
device: this.options.deviceName,
width: this.options.cols,
height: this.options.rows,
});
},
onTextMessage: (message: string) => {
if (this.stopped) return;
this.handleMessage(String(message));
},
onClosed: (code: number, reason: string) => {
if (this.stopped) return;
this.handleConnectionLost(`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""}).`);
},
onFailure: (message: string) => {
if (this.stopped) return;
this.handleConnectionLost(`Connection failed: ${shortenError(String(message))}`);
},
});
try {
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy);
} catch (error) {
this.ws = null;
this.handleConnectionLost(`Connection failed: ${shortenError(String((error as Error)?.message ?? error))}`);
}
}
stop(): void {
this.stopped = true;
this.clearListRefreshTimer();
if (this.ws) {
// Best effort: let the wrapped app resize back before we go away.
if (this.phase === "attached") {
this.send({ type: "unview" });
this.send({ type: "disconnect" });
}
try {
this.ws.close(1000, "bye");
} catch {
// ignore
}
this.ws = null;
}
this.listenerProxy = null;
this.setState("idle", "Not connected.");
}
listSessions(): void {
if (this.phase === "connected" || this.phase === "attached") {
this.send({ type: "list" });
}
}
connectSession(socket: string): void {
if (this.phase !== "connected") return;
this.setState(this.phase, "Attaching to session...");
this.send({ type: "connect", socket });
}
disconnectSession(): void {
if (this.phase !== "attached") return;
this.send({ type: "unview" });
this.send({ type: "disconnect" });
}
view(): void {
if (this.phase !== "attached") return;
this.send({ type: "view" });
}
unview(): void {
if (this.phase !== "attached") return;
this.send({ type: "unview" });
}
private handleMessage(text: string): void {
let message: any;
try {
message = JSON.parse(text);
} catch {
return;
}
if (!message || typeof message.type !== "string") return;
switch (message.type) {
case "init":
this.setState("connected", "Connected.");
this.listSessions();
this.ensureListRefreshTimer();
return;
case "error": {
const errorText = String(message.message ?? "unknown error");
if (this.phase === "connecting") {
// Handshake rejection; the server closes the socket after this.
this.setState("failed", `Rejected: ${errorText}`);
} else {
this.setState(this.phase, `Server error: ${errorText}`);
}
return;
}
case "sessions": {
const raw = Array.isArray(message.sessions) ? message.sessions : [];
this.sessions = raw
.map((item: any): G2MirrorSession => ({
socket: String(item?.socket ?? ""),
pid: Number(item?.pid) || 0,
cwdHint: String(item?.cwd_hint ?? ""),
}))
.filter((session: G2MirrorSession) => session.socket.length > 0);
this.emitState();
return;
}
case "connect": {
// Session accepted us (relayed from the wrapper).
this.clearListRefreshTimer();
this.attachedCommand = String(message.command ?? "");
this.setState("attached", `Attached: ${this.attachedCommand || "session"}`);
for (const listener of Array.from(this.sessionAttachedListeners)) {
listener(this.attachedCommand);
}
return;
}
case "snapshot":
case "output": {
const data = decodeBase64(String(message.data ?? ""));
for (const listener of Array.from(this.terminalDataListeners)) {
listener(data, message.type as TerminalDataKind);
}
return;
}
case "exit": {
const status = message.status === null || message.status === undefined ? "signal" : String(message.status);
this.setState(this.phase, `Session exited (status ${status}).`);
return;
}
case "disconnected": {
const reason = String(message.reason ?? "unknown");
this.attachedCommand = "";
if (this.phase === "attached") {
this.setState("connected", `Detached (${reason}).`);
this.ensureListRefreshTimer();
this.listSessions();
for (const listener of Array.from(this.sessionDetachedListeners)) {
listener(reason);
}
}
return;
}
default:
// Unknown message types are expected; ignore for forward compatibility.
return;
}
}
private handleConnectionLost(statusText: string): void {
this.clearListRefreshTimer();
this.ws = null;
this.listenerProxy = null;
const wasAttached = this.phase === "attached";
this.attachedCommand = "";
this.setState("failed", statusText);
if (wasAttached) {
for (const listener of Array.from(this.sessionDetachedListeners)) {
listener("connection lost");
}
}
}
private ensureListRefreshTimer(): void {
if (this.listRefreshTimer) return;
this.listRefreshTimer = setInterval(() => {
if (this.phase === "connected") {
this.listSessions();
}
}, SESSION_LIST_REFRESH_MS);
}
private clearListRefreshTimer(): void {
if (this.listRefreshTimer) {
clearInterval(this.listRefreshTimer);
this.listRefreshTimer = null;
}
}
private send(message: object): void {
if (!this.ws) return;
try {
this.ws.sendText(JSON.stringify(message));
} catch (error) {
console.warn("g2mirror send failed", error);
}
}
private setState(phase: G2MirrorPhase, status: string): void {
this.phase = phase;
this.status = status;
this.emitState();
}
private emitState(): void {
const state = this.state();
for (const listener of Array.from(this.stateListeners)) {
listener(state);
}
}
}
function decodeBase64(data: string): Uint8Array {
if (!data) return new Uint8Array(0);
return toUint8Array(android.util.Base64.decode(data, android.util.Base64.DEFAULT));
}
function shortenError(message: string): string {
const collapsed = message.replace(/\s+/g, " ").trim();
return collapsed.length <= 120 ? collapsed : `${collapsed.slice(0, 117)}...`;
}
+24
View File
@@ -0,0 +1,24 @@
import { Utils } from "@nativescript/core";
import { type GrayImage } from "../graphics/image";
declare const com: any;
/**
* Save an image as the raw headerless 576x288 4bpp frame buffer (the exact
* bytes the wire compressor sees) for compression experiments. Returns the
* absolute path of the written file; retrieve with
* adb pull /sdcard/Android/data/com.faceclaw.app/files/screenshots/
*/
export function saveRawScreenshot(image: GrayImage): string {
const context = Utils.android.getApplicationContext();
if (!context) throw new Error("Android application context unavailable");
return String(
com.faceclaw.app.ScreenshotUtil.saveRawScreenshot(
context,
image.pixels.buffer,
image.width,
image.height,
),
);
}
+10 -6
View File
@@ -25,6 +25,7 @@
<Button row="1" col="2" text="Click" tap="{{ onSyntheticRightTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<Button text="Save raw screenshot" tap="{{ onScreenshotTap }}" visibility="{{ screenshotButtonVisibility }}" />
</StackLayout>
<Label row="2" text="{{ status }}" class="m-t-16" textWrap="true" />
<ScrollView id="logScrollView" row="3" class="m-t-16" visibility="{{ showLogVisibility }}">
@@ -49,12 +50,15 @@
width="{{ landscapeDisplayPreviewWidth }}"
height="{{ landscapeDisplayPreviewHeight }}" />
</StackLayout>
<GridLayout row="0" col="1" rows="auto,auto,auto" columns="*,*,*" class="m-l-16">
<Button row="0" col="1" text="Swipe up" tap="{{ onSyntheticUpTap }}" />
<Button row="1" col="0" text="Double-click" tap="{{ onSyntheticLeftTap }}" />
<Button row="1" col="2" text="Click" tap="{{ onSyntheticRightTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<StackLayout row="0" col="1" class="m-l-16">
<GridLayout rows="auto,auto,auto" columns="*,*,*">
<Button row="0" col="1" text="Swipe up" tap="{{ onSyntheticUpTap }}" />
<Button row="1" col="0" text="Double-click" tap="{{ onSyntheticLeftTap }}" />
<Button row="1" col="2" text="Click" tap="{{ onSyntheticRightTap }}" />
<Button row="2" col="1" text="Swipe down" tap="{{ onSyntheticDownTap }}" />
</GridLayout>
<Button text="Save raw screenshot" tap="{{ onScreenshotTap }}" visibility="{{ screenshotButtonVisibility }}" />
</StackLayout>
<Label row="1" col="0" colSpan="2" text="{{ status }}" class="m-t-16" textWrap="true" />
<ScrollView id="logScrollViewLandscape" row="2" col="0" colSpan="2" class="m-t-16" visibility="{{ showLogVisibility }}">
<Label text="{{ log }}" textWrap="true" verticalAlignment="top" />
+26
View File
@@ -15,6 +15,7 @@ export class MainViewModel extends Observable {
private _evenAppConflictWarningVisible = false;
private _firmwareWarningMessage = "";
private _firmwareWarningVisible = false;
private _rawScreenshotsEnabled = false;
private _showLog = false;
private _phase: "disconnected" | "connecting" | "connected" | "disconnecting" = "disconnected";
@@ -32,6 +33,7 @@ export class MainViewModel extends Observable {
this.evenAppConflictWarningVisible = snapshot.evenAppConflictWarningVisible;
this.firmwareWarningMessage = snapshot.firmwareWarningMessage;
this.firmwareWarningVisible = snapshot.firmwareWarningVisible;
this.rawScreenshotsEnabled = snapshot.rawScreenshotsEnabled;
});
}
@@ -230,6 +232,30 @@ export class MainViewModel extends Observable {
return this._firmwareWarningVisible ? "visible" : "collapse";
}
get rawScreenshotsEnabled(): boolean {
return this._rawScreenshotsEnabled;
}
set rawScreenshotsEnabled(value: boolean) {
if (this._rawScreenshotsEnabled !== value) {
this._rawScreenshotsEnabled = value;
this.notifyPropertyChange("rawScreenshotsEnabled", value);
this.notifyPropertyChange("screenshotButtonVisibility", this.screenshotButtonVisibility);
}
}
get screenshotButtonVisibility(): "visible" | "collapse" {
return this._rawScreenshotsEnabled ? "visible" : "collapse";
}
onScreenshotTap(): void {
try {
dashboardController.saveRawDashboardScreenshot();
} catch (error) {
console.error("raw screenshot failed", error);
}
}
get phase(): "disconnected" | "connecting" | "connected" | "disconnecting" {
return this._phase;
}
+73
View File
@@ -0,0 +1,73 @@
import type { Terminal as XtermTerminal } from "@xterm/headless";
// xterm-headless expects a few browser globals (navigator for platform
// sniffing, window for timer/scheduling lookups) that the NativeScript
// runtime doesn't define. Provide minimal stand-ins before the library is
// evaluated (hence require, not a hoisted import).
const runtimeGlobal = globalThis as any;
if (typeof runtimeGlobal.navigator === "undefined") {
runtimeGlobal.navigator = {
userAgent: "NativeScript",
platform: "Android",
language: "en",
languages: ["en"],
};
}
if (typeof runtimeGlobal.window === "undefined") {
runtimeGlobal.window = runtimeGlobal;
}
if (typeof runtimeGlobal.self === "undefined") {
runtimeGlobal.self = runtimeGlobal;
}
// eslint-disable-next-line @typescript-eslint/no-var-requires
const { Terminal } = require("@xterm/headless") as typeof import("@xterm/headless");
/**
* Headless terminal emulator for g2mirror streams: feed it the raw
* VT100/xterm bytes from snapshot/output messages and read back the visible
* text grid. Formatting (color/bold/inverse) is deliberately dropped for
* now — we only mirror the text.
*/
export class TerminalEmulator {
private readonly term: XtermTerminal;
constructor(
readonly cols: number,
readonly rows: number,
) {
this.term = new Terminal({
cols,
rows,
scrollback: 0,
allowProposedApi: true,
});
}
/**
* Apply terminal bytes. onProcessed fires once the emulator has actually
* parsed them (xterm buffers writes internally), so schedule repaints there.
*/
write(data: Uint8Array, onProcessed: () => void): void {
this.term.write(data, onProcessed);
}
/** Full reset; snapshots are defined as repainting from a cleared screen. */
reset(): void {
this.term.reset();
}
visibleLines(): string[] {
const buffer = this.term.buffer.active;
const lines: string[] = [];
for (let row = 0; row < this.rows; row++) {
const line = buffer.getLine(buffer.viewportY + row);
lines.push(line ? line.translateToString(true) : "");
}
return lines;
}
cursor(): { x: number; y: number } {
const buffer = this.term.buffer.active;
return { x: buffer.cursorX, y: buffer.cursorY };
}
}
+316
View File
@@ -0,0 +1,316 @@
import { getFont } from "~/graphics/bdffont";
import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "~/graphics/image";
import {
G2MirrorClient,
type G2MirrorSession,
type G2MirrorState,
} from "~/native/g2mirror-client";
import {
terminalAuthTokenSetting,
terminalHostSetting,
terminalPortSetting,
textSettingMenuItem,
} from "../dashboard-settings";
import { TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { MenuLayer } from "../menu";
import { type DashboardInputEvent, type Layer, type LayerActions, type LayerContext } from "../layers";
// Terminus-12 has a 6x12 cell, so the full 576x288 lens is a 96x24 grid —
// this is the size we declare in the g2mirror init handshake, fixed per
// websocket connection.
const terminalFont = getFont("terminus12");
const CELL_WIDTH = 6;
const CELL_HEIGHT = 12;
const TERMINAL_COLS = G2_LENS_WIDTH / CELL_WIDTH;
const TERMINAL_ROWS = G2_LENS_HEIGHT / CELL_HEIGHT;
const DEVICE_NAME = "Faceclaw G2";
const HUB_ROW_HEIGHT = 20;
const HUB_LIST_TOP = 64;
type HubItem = {
label: string;
onSelect: (ctx: LayerContext) => void;
};
/**
* The Terminal app hub: connection status, list of live g2mirror sessions,
* and the app-local settings menu. Selecting a session attaches and opens
* the full-screen readonly terminal view.
*/
export class TerminalAppLayer implements Layer {
private client: G2MirrorClient | null = null;
private clientState: G2MirrorState | null = null;
private selectedIndex = 0;
private lastCtx: LayerContext | null = null;
private activeViewLayer: TerminalViewLayer | null = null;
private unsubscribers: Array<() => void> = [];
constructor(private readonly actions: LayerActions) {
selfTestEmulator();
this.startClient();
}
private startClient(): void {
this.stopClient();
const host = terminalHostSetting.get().trim();
if (!host) return;
const port = parseInt(terminalPortSetting.get(), 10) || 8737;
const client = new G2MirrorClient({
host,
port,
authToken: terminalAuthTokenSetting.get(),
deviceName: DEVICE_NAME,
cols: TERMINAL_COLS,
rows: TERMINAL_ROWS,
});
this.client = client;
this.clientState = client.state();
this.unsubscribers.push(
client.onStateChange((state) => {
this.clientState = state;
this.actions.requestRender();
}),
client.onSessionAttached(() => {
this.openViewLayer();
}),
client.onSessionDetached(() => {
this.closeViewLayer();
}),
);
client.start();
}
private stopClient(): void {
for (const unsubscribe of this.unsubscribers.splice(0)) {
unsubscribe();
}
this.client?.stop();
this.client = null;
this.clientState = null;
}
private openViewLayer(): void {
const ctx = this.lastCtx;
if (!ctx || !this.client || this.activeViewLayer) return;
const viewLayer = new TerminalViewLayer(this.client, this.actions, () => {
if (this.activeViewLayer === viewLayer) {
this.activeViewLayer = null;
}
});
this.activeViewLayer = viewLayer;
ctx.stack.push(viewLayer);
this.actions.requestRender();
}
private closeViewLayer(): void {
const ctx = this.lastCtx;
const viewLayer = this.activeViewLayer;
if (!ctx || !viewLayer) return;
ctx.stack.popIfTop((layer) => layer === viewLayer);
this.actions.requestRender();
}
paint(ctx: LayerContext): GrayImage {
this.lastCtx = ctx;
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
image.drawText(terminalFont, 18, 14, "Terminal", 220);
image.drawRect(12, 12, G2_LENS_WIDTH - 24, G2_LENS_HEIGHT - 24, 52);
image.drawText(terminalFont, 24, 38, this.statusLine(), 170);
let listTop = HUB_LIST_TOP;
if (!terminalHostSetting.get().trim()) {
image.drawText(terminalFont, 24, 56, "To set up a server, see:", 150);
image.drawText(terminalFont, 24, 70, "https://github.com/jimrandomh/g2mirror", 190);
listTop += 32;
}
const items = this.hubItems();
this.selectedIndex = Math.max(0, Math.min(this.selectedIndex, items.length - 1));
for (let index = 0; index < items.length; index++) {
const y = listTop + index * HUB_ROW_HEIGHT;
if (y + HUB_ROW_HEIGHT > G2_LENS_HEIGHT - 40) break;
const selected = index === this.selectedIndex;
if (selected) {
image.fillRoundedRect(20, y - 2, G2_LENS_WIDTH - 40, HUB_ROW_HEIGHT - 1, 15);
image.drawRoundedRect(20, y - 2, G2_LENS_WIDTH - 40, HUB_ROW_HEIGHT - 1, 45);
}
image.drawText(terminalFont, 32, y + 2, items[index]!.label, selected ? 255 : 200);
}
image.drawText(terminalFont, 24, G2_LENS_HEIGHT - 36, "Double-click to go back", 110);
return image;
}
async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> {
this.lastCtx = ctx;
const items = this.hubItems();
switch (event.type) {
case "double-click":
ctx.stack.pop();
return;
case "scroll-up":
this.selectedIndex = Math.max(0, this.selectedIndex - 1);
return;
case "scroll-down":
this.selectedIndex = Math.min(items.length - 1, this.selectedIndex + 1);
return;
case "click": {
const item = items[Math.max(0, Math.min(this.selectedIndex, items.length - 1))];
item?.onSelect(ctx);
return;
}
default:
return;
}
}
onRemoved(): void {
this.stopClient();
}
private statusLine(): string {
const host = terminalHostSetting.get().trim();
if (!host) {
return "No host configured. Open Settings below.";
}
return this.clientState?.status ?? "Not connected.";
}
private hubItems(): HubItem[] {
const items: HubItem[] = [];
const state = this.clientState;
const phase = state?.phase ?? "idle";
if (phase === "connected" || phase === "attached") {
for (const session of state?.sessions ?? []) {
items.push({
label: sessionLabel(session),
onSelect: () => {
this.client?.connectSession(session.socket);
},
});
}
if (!state?.sessions.length) {
items.push({
label: "(no live sessions; run g2mirror <command>)",
onSelect: () => {
this.client?.listSessions();
},
});
}
}
if (phase === "idle" || phase === "failed") {
items.push({
label: terminalHostSetting.get().trim() ? "Connect" : "Connect (host not set)",
onSelect: () => {
this.startClient();
},
});
}
items.push({
label: "Settings",
onSelect: (ctx) => {
ctx.stack.push(createTerminalSettingsMenuLayer());
},
});
return items;
}
}
/** Full-screen readonly mirror of the attached session. */
class TerminalViewLayer implements Layer {
private readonly emulator: import("./terminal-emulator").TerminalEmulator;
private receivedData = false;
private unsubscribeData: (() => void) | null = null;
constructor(
private readonly client: G2MirrorClient,
private readonly actions: LayerActions,
private readonly onClosed: () => void,
) {
// Lazy import keeps xterm out of the startup path; it is only needed
// once a terminal is actually viewed.
const { TerminalEmulator } = require("./terminal-emulator") as typeof import("./terminal-emulator");
this.emulator = new TerminalEmulator(TERMINAL_COLS, TERMINAL_ROWS);
this.unsubscribeData = client.onTerminalData((data, kind) => {
if (kind === "snapshot") {
this.emulator.reset();
}
this.receivedData = true;
this.emulator.write(data, () => this.actions.requestRender());
});
client.view();
}
paint(): GrayImage {
const image = new GrayImage(G2_LENS_WIDTH, G2_LENS_HEIGHT, 0);
if (!this.receivedData) {
image.drawText(terminalFont, 24, 130, "Waiting for terminal output...", 170);
image.drawText(terminalFont, 24, 150, "Double-click to go back", 110);
return image;
}
const cursor = this.emulator.cursor();
image.fillRect(cursor.x * CELL_WIDTH, cursor.y * CELL_HEIGHT, CELL_WIDTH, CELL_HEIGHT, 70);
const lines = this.emulator.visibleLines();
for (let row = 0; row < lines.length; row++) {
const line = lines[row]!;
if (line.length === 0) continue;
image.drawText(terminalFont, 0, row * CELL_HEIGHT, line, 200);
}
return image;
}
handleInput(event: DashboardInputEvent, ctx: LayerContext): void {
// Readonly for now: protocol v1 has no keyboard input, so scrolls and
// clicks are ignored rather than forwarded.
if (event.type === "double-click") {
ctx.stack.pop();
}
}
onRemoved(): void {
this.unsubscribeData?.();
this.unsubscribeData = null;
// Let the wrapped app resize back for the person at the real keyboard,
// and free the session for other clients.
this.client.unview();
this.client.disconnectSession();
this.onClosed();
}
}
// One-time check that the xterm-headless emulator evaluates and parses in
// this JS runtime, run at app open so a broken dependency surfaces in the log
// immediately instead of mid-session-attach.
let emulatorSelfTested = false;
function selfTestEmulator(): void {
if (emulatorSelfTested) return;
emulatorSelfTested = true;
try {
const { TerminalEmulator } = require("./terminal-emulator") as typeof import("./terminal-emulator");
const emulator = new TerminalEmulator(8, 2);
emulator.write(Uint8Array.from([0x68, 0x69, 0x1b, 0x5b, 0x31, 0x6d, 0x21]), () => {
console.log(`terminal emulator self-test: ${JSON.stringify(emulator.visibleLines())}`);
});
} catch (error) {
console.error("terminal emulator failed to load:", error);
}
}
function createTerminalSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Terminal > Settings",
[
textSettingMenuItem(terminalHostSetting),
textSettingMenuItem(terminalPortSetting),
textSettingMenuItem(terminalAuthTokenSetting),
],
TOP_LEFT_MENU_LAYOUT,
);
}
function sessionLabel(session: G2MirrorSession): string {
const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/");
return `${hint || "session"} (pid ${session.pid})`;
}
+55
View File
@@ -34,6 +34,24 @@ type ConfigSettingOptions<TValue, TId extends string> = {
formatValue?: (value: TValue) => string;
};
// Fired after any setting's set(); lets phone-side UI that depends on
// settings toggled from the glasses (e.g. the raw-screenshot button) update
// without waiting for an unrelated snapshot emit.
const settingChangeListeners = new Set<() => void>();
export function onAnySettingChanged(listener: () => void): () => void {
settingChangeListeners.add(listener);
return () => {
settingChangeListeners.delete(listener);
};
}
function notifySettingChanged(): void {
for (const listener of Array.from(settingChangeListeners)) {
listener();
}
}
export abstract class ConfigSetting<TValue, TId extends string = string> {
readonly id: TId;
readonly label: string;
@@ -69,6 +87,7 @@ export class ConfigSettingBoolean<TId extends string = string> extends ConfigSet
set(value: boolean): boolean {
ApplicationSettings.setBoolean(this.storageKey, value);
notifySettingChanged();
return value;
}
@@ -107,6 +126,7 @@ export class ConfigSettingEnum<TValue extends string, TId extends string = strin
set(value: TValue): TValue {
const normalized = this.normalizer(value);
ApplicationSettings.setString(this.storageKey, normalized);
notifySettingChanged();
return normalized;
}
@@ -141,6 +161,7 @@ export class ConfigSettingString<TId extends string = string> extends ConfigSett
set(value: string): string {
const normalized = this.normalizer(value);
ApplicationSettings.setString(this.storageKey, normalized);
notifySettingChanged();
return normalized;
}
}
@@ -239,6 +260,40 @@ export const voiceControlEnabledSetting = new ConfigSettingBoolean({
defaultValue: false,
});
export const rawScreenshotsEnabledSetting = new ConfigSettingBoolean({
id: "raw-screenshots-enabled",
label: "Take raw screenshots",
storageKey: "developer.rawScreenshots",
defaultValue: false,
});
export const terminalHostSetting = new ConfigSettingString({
id: "terminal-host",
label: "Host",
storageKey: "terminal.host",
defaultValue: "",
editorTitle: "g2mirror host (tailscale IP)",
glassesEditTitle: "Edit terminal host",
});
export const terminalPortSetting = new ConfigSettingString({
id: "terminal-port",
label: "Port",
storageKey: "terminal.port",
defaultValue: "8737",
editorTitle: "g2mirror port",
glassesEditTitle: "Edit terminal port",
});
export const terminalAuthTokenSetting = new ConfigSettingString({
id: "terminal-auth-token",
label: "Auth token",
storageKey: "terminal.authToken",
defaultValue: "",
editorTitle: "g2mirror auth token",
glassesEditTitle: "Edit terminal auth token",
});
export const nightscoutSiteUrlSetting = new ConfigSettingString({
id: "nightscout-site-url",
label: "Nightscout site URL",
+1
View File
@@ -40,6 +40,7 @@ export let dashboardState: DashboardState = {
};
const dashboardActions: LayerActions = {
requestRender: () => {},
disconnect: () => {},
startTextSettingEdit: () => {},
endTextSettingEdit: () => {},
+7
View File
@@ -7,6 +7,7 @@ import { MenuLayer } from "../menu";
import { NotificationsListLayer } from "../notifications";
import { createNightscoutSettingsMenuLayer, createSettingsMenuLayer } from "./settings-menus";
import { TelepromptLayer } from "../apps/teleprompt";
import { TerminalAppLayer } from "../apps/terminal";
import { StopwatchLayer } from "../apps/stopwatch";
import { TranscribeLayer } from "../apps/transcribe";
import { ScreenTestLayer } from "../apps/screen-test";
@@ -100,6 +101,12 @@ function createAppsMenuLayer(): MenuLayer {
layer.start(ctx);
},
},
{
label: "Terminal",
onSelect: (ctx) => {
ctx.stack.push(new TerminalAppLayer(ctx.actions));
},
},
{
label: "Screen test",
onSelect: (ctx) => {
+17 -1
View File
@@ -1,5 +1,5 @@
import { dashboardState, TOP_LEFT_MENU_LAYOUT } from "../dashboard";
import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { batteryDisplayModeSetting, dashboardSlotIds, enumSettingMenuItem, rawScreenshotsEnabledSetting, showSignalStrengthSetting, showAndroidNotificationsSetting, showBatteryIndicatorsSetting, showFaceclawLogoSetting, systemCardNameSetting, textSettingMenuItem, toggleSettingMenuItem, voiceControlEnabledSetting, dashboardSlotSettings, nightscoutSiteUrlSetting, nightscoutApiTokenSetting, screenTimeoutSetting } from "../dashboard-settings";
import { MenuLayer } from "../menu";
export function createSettingsMenuLayer(): MenuLayer {
@@ -30,6 +30,22 @@ export function createSettingsMenuLayer(): MenuLayer {
ctx.stack.push(createIntegrationsMenuLayer());
},
},
{
label: "Developer",
onSelect: (ctx) => {
ctx.stack.push(createDeveloperSettingsMenuLayer());
},
},
],
TOP_LEFT_MENU_LAYOUT,
);
}
function createDeveloperSettingsMenuLayer(): MenuLayer {
return new MenuLayer(
"Settings > Developer",
[
toggleSettingMenuItem(rawScreenshotsEnabledSetting),
],
TOP_LEFT_MENU_LAYOUT,
);
+17 -3
View File
@@ -11,6 +11,8 @@ export type DashboardInputEvent =
| { type: "unknown"; kind: string; eventSource: number; eventType: number };
export type LayerActions = {
/** Ask for a dashboard repaint+transmit, e.g. when async data arrives. */
requestRender: () => void;
disconnect: () => Promise<void> | void;
startTextSettingEdit: (setting: ConfigSettingString) => Promise<void> | void;
endTextSettingEdit: () => Promise<void> | void;
@@ -24,6 +26,14 @@ export type LayerActions = {
export type PaintBelow = () => GrayImage;
function notifyRemoved(layer: Layer | undefined): void {
try {
layer?.onRemoved?.();
} catch (error) {
console.warn("layer onRemoved failed", error);
}
}
export interface LayerContext {
readonly stack: LayerStack;
readonly actions: LayerActions;
@@ -33,6 +43,8 @@ export interface Layer {
readonly paintOverBase?: boolean;
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage;
handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise<void> | void;
/** Called when the layer leaves the stack by any path (pop or clearToBase). */
onRemoved?(): void;
}
export class LayerStack {
@@ -53,21 +65,23 @@ export class LayerStack {
pop(): void {
if (this.layers.length > 1) {
this.layers.pop();
notifyRemoved(this.layers.pop());
}
}
/** Pop the top layer only if it matches; returns whether a layer was popped. */
popIfTop(predicate: (layer: Layer) => boolean): boolean {
if (this.layers.length > 1 && predicate(this.layers[this.layers.length - 1]!)) {
this.layers.pop();
notifyRemoved(this.layers.pop());
return true;
}
return false;
}
clearToBase(): void {
this.layers.splice(1);
for (const layer of this.layers.splice(1)) {
notifyRemoved(layer);
}
}
isAtBase(): boolean {
+10
View File
@@ -10,6 +10,7 @@
"dependencies": {
"@nativescript/core": "~9.0.0",
"@nativescript/theme": "^3.1.0",
"@xterm/headless": "^6.0.0",
"lodash": "^4.18.1",
"lz4js": "^0.2.0",
"pako": "^2.1.0",
@@ -1288,6 +1289,15 @@
}
}
},
"node_modules/@xterm/headless": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/@xterm/headless/-/headless-6.0.0.tgz",
"integrity": "sha512-5Yj1QINYCyzrZtf8OFIHi47iQtI+0qYFPHmouEfG8dHNxbZ9Tb9YGSuLcsEwj9Z+OL75GJqPyJbyoFer80a2Hw==",
"license": "MIT",
"workspaces": [
"addons/*"
]
},
"node_modules/@xtuc/ieee754": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/@xtuc/ieee754/-/ieee754-1.2.0.tgz",
+1
View File
@@ -6,6 +6,7 @@
"dependencies": {
"@nativescript/core": "~9.0.0",
"@nativescript/theme": "^3.1.0",
"@xterm/headless": "^6.0.0",
"lodash": "^4.18.1",
"lz4js": "^0.2.0",
"pako": "^2.1.0",