From ccb320c66fd5d0e8ad486b8084c1bb5b95bfe7a7 Mon Sep 17 00:00:00 2001 From: James Babcock Date: Sun, 20 Sep 2026 00:34:58 -0700 Subject: [PATCH] Add tokens and faceclaw-input script, which allows using a CLI utility as an input device for the glasses --- .../com/faceclaw/app/FaceclawRemoteInput.java | 124 ++++++++++ App_Resources/iOS/src/FaceclawRemoteInput.h | 12 + App_Resources/iOS/src/FaceclawRemoteInput.m | 150 +++++++++++ App_Resources/iOS/src/module.modulemap | 1 + CHANGELOG | 5 + app/apps/terminal/terminal-app.worker.ts | 3 +- app/g2/dashboard-controller.ts | 10 + app/g2/ios-preview-controller.ts | 18 ++ app/native/remote-input.ios.ts | 13 + app/native/remote-input.ts | 17 ++ app/remote/listeners.ts | 70 ++++++ app/remote/protocol.ts | 99 ++++++++ app/remote/service.ts | 54 ++++ app/ui/dashboard/remote-input-menu.ts | 90 +++++++ app/ui/dashboard/settings-menus.ts | 2 + app/ui/shell/shell.ts | 6 +- app/ui/shell/worker-window.ts | 6 +- native/kotlin/android-java-boundaries.json | 6 + package-lock.json | 17 +- package.json | 2 + scripts/faceclaw-input-editor.cjs | 67 +++++ scripts/faceclaw-input.cjs | 204 +++++++++++++++ scripts/faceclaw-input.md | 162 ++++++++++++ tests/ios-app-host.test.cjs | 2 + tests/ios-glanceboard.test.cjs | 21 +- tests/remote-input-compose.test.cjs | 105 ++++++++ tests/remote-input-native.cjs | 72 ++++++ tests/remote-input-pty.py | 140 +++++++++++ tests/remote-input.test.cjs | 234 ++++++++++++++++++ tests/remote-listeners.test.cjs | 40 +++ tests/remote-text.test.cjs | 43 ++++ tests/tsconfig.json | 2 + 32 files changed, 1785 insertions(+), 12 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java create mode 100644 App_Resources/iOS/src/FaceclawRemoteInput.h create mode 100644 App_Resources/iOS/src/FaceclawRemoteInput.m create mode 100644 app/native/remote-input.ios.ts create mode 100644 app/native/remote-input.ts create mode 100644 app/remote/listeners.ts create mode 100644 app/remote/protocol.ts create mode 100644 app/remote/service.ts create mode 100644 app/ui/dashboard/remote-input-menu.ts create mode 100644 scripts/faceclaw-input-editor.cjs create mode 100755 scripts/faceclaw-input.cjs create mode 100644 scripts/faceclaw-input.md create mode 100644 tests/remote-input-compose.test.cjs create mode 100644 tests/remote-input-native.cjs create mode 100644 tests/remote-input-pty.py create mode 100644 tests/remote-input.test.cjs create mode 100644 tests/remote-listeners.test.cjs create mode 100644 tests/remote-text.test.cjs diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java new file mode 100644 index 00000000..1b602978 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawRemoteInput.java @@ -0,0 +1,124 @@ +package com.faceclaw.app; + +import java.io.ByteArrayOutputStream; +import java.net.InetAddress; +import java.net.NetworkInterface; +import java.util.Collections; +import org.json.JSONArray; +import java.net.ServerSocket; +import java.net.Socket; +import java.nio.charset.StandardCharsets; +import java.nio.ByteBuffer; +import java.security.MessageDigest; +import java.security.SecureRandom; +import org.json.JSONObject; + +/** Explicit-address, bounded, one-request-per-connection input port. JS polls on its own thread. */ +public final class FaceclawRemoteInput { + private static final FaceclawRemoteInput INSTANCE = new FaceclawRemoteInput(); + public static FaceclawRemoteInput getInstance() { return INSTANCE; } + private ServerSocket server; + private Socket client; + private String pending, reply; + private long nextId, requestId; + private long deadline; + + public synchronized String start(int port) { return startAddress(port, "127.0.0.1"); } + public synchronized String startAddress(int port, String address) { + if (server != null) return ""; + try { + // Numeric local addresses only; never resolve hostnames or bind a wildcard. + if (!address.matches("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?")) return "Invalid local address."; + InetAddress local = InetAddress.getByName(address); + if (local.isAnyLocalAddress() || local.isMulticastAddress() || NetworkInterface.getByInetAddress(local) == null) + return "Address is not assigned to a local interface."; + ServerSocket listener = new ServerSocket(port, 8, local); + server = listener; + Thread thread = new Thread(() -> serve(listener), "FaceclawInput"); + thread.setDaemon(true); + thread.start(); + return ""; + } catch (Exception e) { return "Could not open local input port " + port + "."; } + } + public synchronized void stop() { + try { if (server != null) server.close(); } catch (Exception ignored) {} + try { if (client != null) client.close(); } catch (Exception ignored) {} + server = null; client = null; pending = null; requestId = 0; reply = null; + notifyAll(); + } + public synchronized String nextRequest() { + if (System.currentTimeMillis() >= deadline) { pending = null; return null; } + String result = pending; pending = null; return result; + } + public synchronized void complete(long id, String value) { + if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; notifyAll(); } + } + private void serve(ServerSocket listener) { + while (!listener.isClosed()) { + try (Socket socket = listener.accept()) { + synchronized (this) { if (server != listener) return; client = socket; } + socket.setSoTimeout(5000); + socket.setTcpNoDelay(true); + long readDeadline = System.currentTimeMillis() + 5000; + ByteArrayOutputStream body = new ByteArrayOutputStream(); + int b; + for (;;) { + long remaining = readDeadline - System.currentTimeMillis(); + if (remaining <= 0) throw new Exception("Read timeout"); + socket.setSoTimeout((int) remaining); + b = socket.getInputStream().read(); + if (b == -1 || b == '\n') break; + if (body.size() >= 65536) throw new Exception("Invalid frame"); + body.write(b); + } + if (b != '\n') continue; + String text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString(); + String response; + synchronized (this) { + if (server != listener) return; + requestId = ++nextId; + deadline = System.currentTimeMillis() + 5000; + reply = null; + pending = new JSONObject().put("id", requestId).put("expiresAt", deadline) + .put("body", text).toString(); + while (reply == null && server == listener && System.currentTimeMillis() < deadline) { + wait(Math.max(1, deadline - System.currentTimeMillis())); + } + response = reply != null ? reply : "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"; + if (server == listener) { pending = null; reply = null; requestId = 0; } + } + socket.getOutputStream().write((response + "\n").getBytes(StandardCharsets.UTF_8)); + } catch (Exception ignored) { + // Never log request bodies: they contain credentials and possibly private text. + } finally { + synchronized (this) { if (server == listener) client = null; } + } + } + } + public static String interfaces() { + JSONArray result = new JSONArray(); + try { + for (NetworkInterface device : Collections.list(NetworkInterface.getNetworkInterfaces())) { + if (!device.isUp()) continue; + for (InetAddress address : Collections.list(device.getInetAddresses())) { + if (address.isLoopbackAddress() || address.isLinkLocalAddress() || address.isAnyLocalAddress()) continue; + result.put(new JSONObject().put("name", device.getName()).put("address", address.getHostAddress()) + .put("pointToPoint", device.isPointToPoint())); + } + } + } catch (Exception ignored) {} + return result.toString(); + } + public String randomSecret() { + byte[] bytes = new byte[32]; new SecureRandom().nextBytes(bytes); return hex(bytes); + } + public String hash(String value) { + try { return hex(MessageDigest.getInstance("SHA-256").digest(value.getBytes(StandardCharsets.UTF_8))); } + catch (Exception e) { throw new IllegalStateException("SHA-256 unavailable", e); } + } + private static String hex(byte[] bytes) { + StringBuilder result = new StringBuilder(bytes.length * 2); + for (byte b : bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b & 255)); + return result.toString(); + } +} diff --git a/App_Resources/iOS/src/FaceclawRemoteInput.h b/App_Resources/iOS/src/FaceclawRemoteInput.h new file mode 100644 index 00000000..dbbce5e0 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawRemoteInput.h @@ -0,0 +1,12 @@ +#import +@interface FaceclawRemoteInput : NSObject ++ (instancetype)shared; ++ (NSString *)interfaces; +- (NSString *)start:(int)port; +- (NSString *)start:(int)port address:(NSString *)address; +- (void)stop; +- (NSString *)nextRequest; +- (void)complete:(long long)identifier response:(NSString *)response; +- (NSString *)randomSecret; +- (NSString *)tokenDigest:(NSString *)value; +@end diff --git a/App_Resources/iOS/src/FaceclawRemoteInput.m b/App_Resources/iOS/src/FaceclawRemoteInput.m new file mode 100644 index 00000000..0c1c46af --- /dev/null +++ b/App_Resources/iOS/src/FaceclawRemoteInput.m @@ -0,0 +1,150 @@ +#import "FaceclawRemoteInput.h" +#import "FaceclawCrypto.h" +#import +#import +#import +#import +#import +#import +#import +#import +#import + +@implementation FaceclawRemoteInput { + NSCondition *_condition; + int _server, _client; + NSUInteger _generation; + long long _nextId, _requestId; + NSTimeInterval _deadline; + NSString *_pending, *_reply; +} ++ (instancetype)shared { + static FaceclawRemoteInput *instance; static dispatch_once_t once; + dispatch_once(&once, ^{ instance = [self new]; }); return instance; +} +- (instancetype)init { + if ((self = [super init])) { _condition = [NSCondition new]; _server = _client = -1; } + return self; +} +- (NSString *)start:(int)port { return [self start:port address:@"127.0.0.1"]; } +- (NSString *)start:(int)port address:(NSString *)address { + [_condition lock]; + if (_server >= 0) { [_condition unlock]; return @""; } + struct addrinfo hints = {0}, *resolved = NULL; + hints.ai_flags = AI_NUMERICHOST; hints.ai_socktype = SOCK_STREAM; + NSString *service = [NSString stringWithFormat:@"%d", port]; + if (getaddrinfo(address.UTF8String, service.UTF8String, &hints, &resolved) != 0) { + [_condition unlock]; return @"Invalid local address."; + } + BOOL wildcard = resolved->ai_family == AF_INET ? + ((struct sockaddr_in *)resolved->ai_addr)->sin_addr.s_addr == INADDR_ANY : + IN6_IS_ADDR_UNSPECIFIED(&((struct sockaddr_in6 *)resolved->ai_addr)->sin6_addr); + if (wildcard) { freeaddrinfo(resolved); [_condition unlock]; return @"Wildcard addresses are not allowed."; } + int fd = socket(resolved->ai_family, SOCK_STREAM, 0), yes = 1; + setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); + int result = fd < 0 ? -1 : bind(fd, resolved->ai_addr, (socklen_t)resolved->ai_addrlen); + freeaddrinfo(resolved); + if (result || listen(fd, 8)) { + if (fd >= 0) close(fd); [_condition unlock]; return @"Could not open input port on this address."; + } + _server = fd; NSUInteger generation = ++_generation; + [_condition unlock]; + dispatch_async(dispatch_get_global_queue(QOS_CLASS_UTILITY, 0), ^{ [self serve:fd generation:generation]; }); + return @""; +} +- (void)stop { + [_condition lock]; ++_generation; + // The serving thread owns close(); shutdown interrupts its blocking I/O. + if (_server >= 0) shutdown(_server, SHUT_RDWR); + if (_client >= 0) shutdown(_client, SHUT_RDWR); + _server = _client = -1; _requestId = 0; _pending = _reply = nil; + [_condition broadcast]; [_condition unlock]; +} +- (NSString *)nextRequest { + [_condition lock]; + NSString *result = NSDate.date.timeIntervalSince1970 < _deadline ? _pending : nil; + _pending = nil; [_condition unlock]; return result; +} +- (void)complete:(long long)identifier response:(NSString *)response { + [_condition lock]; + if (_requestId == identifier && NSDate.date.timeIntervalSince1970 < _deadline) { _reply = response; [_condition signal]; } + [_condition unlock]; +} +- (void)serve:(int)listener generation:(NSUInteger)generation { + for (;;) { + [_condition lock]; BOOL current = generation == _generation; [_condition unlock]; + if (!current) break; + // Poll also guarantees stop is observed on platforms where shutdown does not wake accept. + struct pollfd p = { .fd = listener, .events = POLLIN }; + if (poll(&p, 1, 250) <= 0) continue; + int client = accept(listener, NULL, NULL); + if (client < 0) break; + @autoreleasepool { + [_condition lock]; current = generation == _generation; + if (current) _client = client; + [_condition unlock]; + if (!current) { close(client); break; } + struct timeval timeout = {5, 0}; int yes = 1; + setsockopt(client, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)); + setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout)); + setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes)); + NSTimeInterval readDeadline = NSDate.date.timeIntervalSince1970 + 5; + NSMutableData *body = [NSMutableData new]; unsigned char byte; BOOL complete = NO; + while (body.length <= 65536) { + NSTimeInterval remaining = readDeadline - NSDate.date.timeIntervalSince1970; + struct pollfd input = { .fd = client, .events = POLLIN }; + if (remaining <= 0 || poll(&input, 1, (int)(remaining * 1000)) <= 0 || recv(client, &byte, 1, 0) != 1) break; + if (byte == '\n') { complete = YES; break; } + [body appendBytes:&byte length:1]; + } + NSString *text = complete ? [[NSString alloc] initWithData:body encoding:NSUTF8StringEncoding] : nil; + if (text) { + [_condition lock]; + if (generation == _generation) { + _requestId = ++_nextId; _deadline = NSDate.date.timeIntervalSince1970 + 5; _reply = nil; + NSData *json = [NSJSONSerialization dataWithJSONObject:@{@"id":@(_requestId), @"expiresAt":@(_deadline * 1000), @"body":text} options:0 error:nil]; + _pending = [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]; + while (!_reply && generation == _generation && NSDate.date.timeIntervalSince1970 < _deadline) + [_condition waitUntilDate:[NSDate dateWithTimeIntervalSince1970:_deadline]]; + NSString *reply = _reply ?: @"{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"; + if (generation == _generation) { _pending = _reply = nil; _requestId = 0; } + [_condition unlock]; + NSData *data = [[reply stringByAppendingString:@"\n"] dataUsingEncoding:NSUTF8StringEncoding]; + NSUInteger offset = 0; + while (offset < data.length) { + ssize_t n = send(client, (const char *)data.bytes + offset, data.length - offset, 0); + if (n <= 0) break; offset += n; + } + } else [_condition unlock]; + } + [_condition lock]; if (generation == _generation) _client = -1; [_condition unlock]; + close(client); + } + } + close(listener); + [_condition lock]; if (generation == _generation) _server = -1; [_condition unlock]; +} ++ (NSString *)interfaces { + NSMutableArray *result = [NSMutableArray new]; struct ifaddrs *interfaces = NULL; + if (getifaddrs(&interfaces) == 0) { + for (struct ifaddrs *item = interfaces; item; item = item->ifa_next) { + if (!item->ifa_addr || !(item->ifa_flags & IFF_UP) || (item->ifa_flags & IFF_LOOPBACK)) continue; + int family = item->ifa_addr->sa_family; + if (family != AF_INET && family != AF_INET6) continue; + char address[NI_MAXHOST]; + if (getnameinfo(item->ifa_addr, item->ifa_addr->sa_len, address, sizeof(address), NULL, 0, NI_NUMERICHOST)) continue; + [result addObject:@{@"name":@(item->ifa_name), @"address":@(address), @"pointToPoint":@((item->ifa_flags & IFF_POINTOPOINT) != 0)}]; + } + freeifaddrs(interfaces); + } + return [[NSString alloc] initWithData:[NSJSONSerialization dataWithJSONObject:result options:0 error:nil] encoding:NSUTF8StringEncoding]; +} +- (NSString *)randomSecret { + unsigned char bytes[32]; + if (SecRandomCopyBytes(kSecRandomDefault, sizeof(bytes), bytes) != errSecSuccess) return @""; + NSMutableString *hex = [NSMutableString new]; + for (NSUInteger i = 0; i < sizeof(bytes); i++) [hex appendFormat:@"%02x", bytes[i]]; + return hex; +} +- (NSString *)tokenDigest:(NSString *)value { return [FaceclawCrypto sha256:[value dataUsingEncoding:NSUTF8StringEncoding]]; } +@end diff --git a/App_Resources/iOS/src/module.modulemap b/App_Resources/iOS/src/module.modulemap index 7287d6bf..73b6e464 100644 --- a/App_Resources/iOS/src/module.modulemap +++ b/App_Resources/iOS/src/module.modulemap @@ -11,6 +11,7 @@ module FaceclawGraphics { header "FaceclawConfigPort.h" header "FaceclawEvenHubWebView.h" header "FaceclawCrypto.h" + header "FaceclawRemoteInput.h" header "FaceclawQrScanner.h" export * } diff --git a/CHANGELOG b/CHANGELOG index 2681bcf1..4d81fb5a 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -2,7 +2,12 @@ ## 0.7.3 - Future +### Other Features + +* Add tokens and faceclaw-input script, which allows using a CLI utility as an input device for the glasses + ### iOS port: + * Add iOS calendar integration * iOS main screen matches the Android version's styling * Hub apps on iOS no longer have their timers stopped while the phone app is in the background diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index ac1c57ba..cf914747 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -362,7 +362,8 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { // Enter ("\r") after a wrapper-side pause so paste-detecting apps // (e.g. Claude Code) submit instead of inserting a newline. if (window && window.kind === "view") { - window.client.submitInput(message.text); + if (message.submit === false) window.client.sendInput(message.text); + else window.client.submitInput(message.text); } else if (window && window.kind === "hub" && window.mode === "add") { terminalNewConnectionSetting.set(message.text); scheduleRender(window); diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 3f0d2b63..519d9a71 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1,3 +1,4 @@ +import { startRemoteInput } from "../remote/service"; import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { Application, ImageSource } from "@nativescript/core"; import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events"; @@ -404,6 +405,15 @@ class DashboardController { this.syncAssistantBridge(); // The watch drives the same synthetic-input path as the phone UI's test // buttons, plus app/window/lock commands; it mirrors the state below. + startRemoteInput({ + ready: () => this.phase === "connected" || this.phase === "charging", + locked: () => this.glassesLocked, + input: (gesture, source) => this.injectSyntheticRingInput(gesture, source), + acceptsText: () => !!shell.foregroundWindow()?.receiveTextInput, + text: (text, submit) => { if (!shell.isScreenOn()) shell.wake("window"); shell.sendTextToForegroundWindow(text, { submit }); this.requestShellRender(); }, + assistantAvailable: () => shell.isAssistantAvailable(), + assistant: text => shell.sendToAssistant(text), + }); this.wearRemote = new WearRemote({ apps: LAUNCHABLE_APPS, injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"), diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 7e4fac3e..02045a01 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -1,3 +1,4 @@ +import { startRemoteInput } from "../remote/service" import type { KeyboardInputSession } from '../ui/shell/keyboard-input' import { acceptInput, resetRingInputFilter } from "../ui/input-monitor"; import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup } from '../native/notification-icons.ios' @@ -170,6 +171,23 @@ export class IosPreviewController { shell.openNotificationModal(key, woke) this.requestShellRender() }) + startRemoteInput({ + ready: () => this.runtimeRunning, + locked: () => this.glassesLocked, + input: (gesture, source) => { + // Share the controller queue with physical input. A discrete hold includes its release. + const task = this.inputQueue.then(async () => { + await this.receiveInput(makeInputEvent({ type: gesture, source } as InputEventPayload)) + if (gesture === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source })) + }) + this.inputQueue = task.catch(error => this.fail(error)) + return task + }, + acceptsText: () => !!shell.foregroundWindow()?.receiveTextInput, + text: (text, submit) => { if (!shell.isScreenOn()) shell.wake('window'); shell.sendTextToForegroundWindow(text, { submit }); this.requestShellRender() }, + assistantAvailable: () => shell.isAssistantAvailable(), + assistant: text => shell.sendToAssistant(text), + }) registerSystemTools() registerWindowTools({ apps: ALL_APPS.filter(app => !iosAppUnavailableReason(app.appId)), launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() }) diff --git a/app/native/remote-input.ios.ts b/app/native/remote-input.ios.ts new file mode 100644 index 00000000..d20e9e5c --- /dev/null +++ b/app/native/remote-input.ios.ts @@ -0,0 +1,13 @@ +import { createListenerPool, type InterfaceAddress } from '../remote/listeners'; +declare const FaceclawRemoteInput: any; +export function remoteNative() { + return createListenerPool(() => { + const native = FaceclawRemoteInput.alloc().init(); + return { start: (port, address) => String(native.startAddress(port, address)), stop: () => native.stop(), + nextRequest: () => native.nextRequest(), complete: (id, response) => native.completeResponse(id, response) }; + }); +} +export function remoteInterfaces(): InterfaceAddress[] { return JSON.parse(String(FaceclawRemoteInput.interfaces())); } +export function randomTokenSecret(): string { return String(FaceclawRemoteInput.shared().randomSecret()); } +export function tokenHash(value: string): string { return String(FaceclawRemoteInput.shared().tokenDigest(value)); } +export function copyRemoteToken(value: string): void { UIPasteboard.generalPasteboard.string = value; } diff --git a/app/native/remote-input.ts b/app/native/remote-input.ts new file mode 100644 index 00000000..8650ef07 --- /dev/null +++ b/app/native/remote-input.ts @@ -0,0 +1,17 @@ +import { createListenerPool, type InterfaceAddress } from '../remote/listeners'; +import { Utils } from '@nativescript/core'; +declare const com: any; +export function remoteNative() { + return createListenerPool(() => { + const native = new com.faceclaw.app.FaceclawRemoteInput(); + return { start: (port, address) => String(native.startAddress(port, address)), stop: () => native.stop(), + nextRequest: () => native.nextRequest(), complete: (id, response) => native.complete(id, response) }; + }); +} +export function remoteInterfaces(): InterfaceAddress[] { return JSON.parse(String(com.faceclaw.app.FaceclawRemoteInput.interfaces())); } +export function randomTokenSecret(): string { return String(com.faceclaw.app.FaceclawRemoteInput.getInstance().randomSecret()); } +export function tokenHash(value: string): string { return String(com.faceclaw.app.FaceclawRemoteInput.getInstance().hash(value)); } +export function copyRemoteToken(value: string): void { + const context = Utils.android.getApplicationContext(); + context.getSystemService('clipboard').setPrimaryClip(android.content.ClipData.newPlainText('Faceclaw input token', value)); +} diff --git a/app/remote/listeners.ts b/app/remote/listeners.ts new file mode 100644 index 00000000..66e2ea49 --- /dev/null +++ b/app/remote/listeners.ts @@ -0,0 +1,70 @@ +/** Interface selection never returns wildcard, Wi-Fi, cellular or LAN addresses. */ +export type InterfaceAddress = { name: string; address: string; pointToPoint: boolean }; +export const INTERFACE_STORAGE_KEY = 'remoteInput.interface'; +export function isTunnelInterface(item: InterfaceAddress): boolean { + if (/^(wlan|wifi|wl|en\d|eth\d|rmnet|ccmni|pdp_ip|ap\d|p2p)/i.test(item.name)) return false; + return item.pointToPoint || /^(tailscale|utun\d|tun\d|wg\d)/i.test(item.name); +} +function isTailnetAddress(address: string): boolean { + const parts = address.split('.').map(Number); + return /^fd7a:115c:a1e0:/i.test(address) || (parts.length === 4 && parts[0] === 100 && parts[1] >= 64 && parts[1] <= 127); +} +export function listenerAddresses(interfaces: InterfaceAddress[], selection: string): string[] { + const addresses = ['127.0.0.1']; + if (selection === 'localhost') return addresses; + const tunnels = interfaces.filter(isTunnelInterface); + // A generic tun/utun name alone does not identify Tailscale. Its dedicated + // IPv6 prefix does; CGNAT IPv4 alone could belong to an unrelated VPN. + const tailscaleNames = new Set(tunnels.filter(item => /^tailscale/i.test(item.name) || + /^fd7a:115c:a1e0:/i.test(item.address)).map(item => item.name)); + for (const item of tunnels) { + const selected = selection === 'tailscale' + ? tailscaleNames.has(item.name) && isTailnetAddress(item.address) + : selection === `interface:${item.name}`; + if (selected && item.address !== '0.0.0.0' && item.address !== '::' && !item.address.startsWith('fe80:')) addresses.push(item.address); + } + return [...new Set(addresses)].sort(); +} + +export type NativeListener = { + start(port: number, address: string): string; + stop(): void; + nextRequest(): string | null; + complete(id: number, response: string): void; +}; +/** Independent sockets preserve localhost access while adding/removing a VPN. */ +export function createListenerPool(create: () => NativeListener) { + let nextId = 0, cursor = 0; + const entries = new Map(); + return { + start(port: number, addresses: string[]): string { + for (const [address, entry] of entries) if (!addresses.includes(address)) { entry.listener.stop(); entries.delete(address); } + const failures: string[] = []; + for (const address of addresses) if (!entries.has(address)) { + const listener = create(); + const error = listener.start(port, address); + if (error) { listener.stop(); failures.push(`${address}: ${error}`); } + else entries.set(address, { id: ++nextId, listener }); + } + return failures.join('\n'); + }, + addresses(): string[] { return [...entries.keys()]; }, + stop(): void { for (const entry of entries.values()) entry.listener.stop(); entries.clear(); }, + nextRequest(): string | null { + const active = [...entries.values()]; + for (let i = 0; i < active.length; i++) { + const entry = active[cursor++ % active.length]; + const raw = entry.listener.nextRequest(); + if (raw) { + const request = JSON.parse(String(raw)); + return JSON.stringify({ ...request, id: `${entry.id}/${request.id}` }); + } + } + return null; + }, + complete(id: string, response: string): void { + const [entryId, requestId] = id.split('/').map(Number); + for (const entry of entries.values()) if (entry.id === entryId) entry.listener.complete(requestId, response); + }, + }; +} diff --git a/app/remote/protocol.ts b/app/remote/protocol.ts new file mode 100644 index 00000000..0c1fc492 --- /dev/null +++ b/app/remote/protocol.ts @@ -0,0 +1,99 @@ +/** Local input API v1. Transport: one UTF-8 JSON line per TCP connection. */ +export const REMOTE_PORT = 8791; +export const PERMISSIONS = ['input', 'text', 'assistant'] as const; +export type Permission = typeof PERMISSIONS[number]; +export const GESTURES = ['click', 'double-click', 'long-press', 'short-then-long-press', + 'scroll-up', 'scroll-down', 'swipe-up', 'swipe-down', 'swipe-left', 'swipe-right'] as const; +export type RemoteGesture = typeof GESTURES[number]; +export type TokenRecord = { id: string; name: string; hash: string; permissions: Permission[]; createdAt: number }; +export type RemoteHost = { + ready(): boolean; + locked(): boolean; + input(gesture: RemoteGesture, source: 'watch' | 'ring'): Promise; + acceptsText(): boolean; + text(text: string, submit: boolean): void; + assistantAvailable(): boolean; + assistant(text: string): void; +}; +export type Reply = { ok: true } | { ok: false; error: string; message: string }; +const error = (code: string, message: string): Reply => ({ ok: false, error: code, message }); + +export class TokenStore { + constructor(private readonly read: () => string, private readonly write: (json: string) => void, + private readonly random: () => string, private readonly hash: (value: string) => string) {} + list(): TokenRecord[] { + try { + const parsed = JSON.parse(this.read()); + if (!Array.isArray(parsed)) return []; + return parsed.filter((r): r is TokenRecord => r && typeof r.id === 'string' && + typeof r.name === 'string' && /^[a-f0-9]{64}$/.test(r.hash) && typeof r.createdAt === 'number' && + Array.isArray(r.permissions) && r.permissions.every((p: Permission) => PERMISSIONS.includes(p))); + } catch { return []; } + } + create(name: string, permissions: Permission[]): { token: string; record: TokenRecord } { + if (!name.trim() || name.length > 80) throw new Error('Enter a token name (1–80 characters).'); + if (!permissions.length || !permissions.every(p => PERMISSIONS.includes(p))) throw new Error('Select at least one permission.'); + const records = this.list(); + if (records.length >= 32) throw new Error('Revoke a token before creating more (maximum 32).'); + const secret = this.random(); + if (!/^[a-f0-9]{64}$/.test(secret)) throw new Error('Secure random token generation failed.'); + const token = `fc1_${secret}`; + const hash = this.hash(token); + const record = { id: hash.slice(0, 16), name: name.trim(), hash, permissions: [...new Set(permissions)], createdAt: Date.now() }; + this.write(JSON.stringify([...records, record])); + return { token, record }; + } + permissions(id: string, permissions: Permission[]): void { + if (!permissions.every(p => PERMISSIONS.includes(p))) throw new Error('Invalid permission.'); + this.write(JSON.stringify(this.list().map(r => r.id === id ? { ...r, permissions: [...new Set(permissions)] } : r))); + } + revoke(id: string): void { this.write(JSON.stringify(this.list().filter(r => r.id !== id))); } + authenticate(token: unknown): TokenRecord | undefined { + if (typeof token !== 'string' || !/^fc1_[a-f0-9]{64}$/.test(token)) return undefined; + const digest = this.hash(token); + // Compare fixed-size digests without an early exit on mismatching bytes. + return this.list().find(r => { + let difference = r.hash.length ^ digest.length; + for (let i = 0; i < 64; i++) difference |= r.hash.charCodeAt(i) ^ digest.charCodeAt(i); + return difference === 0; + }); + } +} + +export async function handleRequest(body: string, tokens: TokenStore, host: RemoteHost): Promise { + let request: any; + try { request = JSON.parse(body); } catch { return error('bad_request', 'Expected a JSON object.'); } + if (!request || typeof request !== 'object' || Array.isArray(request)) return error('bad_request', 'Expected a JSON object.'); + const token = tokens.authenticate(request.token); + if (!token) return error('unauthorized', 'Invalid or revoked token.'); + if (request.version !== 1 || (request.action !== 'ping' && !PERMISSIONS.includes(request.action))) return error('bad_request', 'Expected version 1 and action ping, input, text or assistant.'); + if (request.action === 'ping') { + if (request.permission !== undefined && !PERMISSIONS.includes(request.permission)) return error('bad_request', 'Invalid permission.'); + if (request.permission && !token.permissions.includes(request.permission)) return error('forbidden', `Token lacks ${request.permission} permission.`); + return { ok: true }; + } + if (request.action === 'text' && request.submit !== undefined && typeof request.submit !== 'boolean') return error('bad_request', 'submit must be a boolean.'); + if (!token.permissions.includes(request.action)) return error('forbidden', `Token lacks ${request.action} permission.`); + if (request.action === 'input') { + if (!GESTURES.includes(request.gesture) || !['watch', 'ring'].includes(request.source ?? 'watch') || + (request.source === 'ring' && request.gesture.startsWith('swipe-'))) { + return error('bad_request', 'Invalid gesture or source; directions require watch input.'); + } + } else if (typeof request.text !== 'string' || !request.text.trim() || request.text.length > 8000 || request.text.includes('\0')) { + return error('bad_request', 'Text must contain 1–8000 characters without NUL.'); + } + if (!host.ready()) return error('unavailable', 'Faceclaw is not ready for input.'); + // Gestures follow the normal lock-screen path. Text must never bypass it. + if (request.action !== 'input' && host.locked()) return error('locked', 'The glasses are locked.'); + try { + if (request.action === 'input') await host.input(request.gesture, request.source ?? 'watch'); + else if (request.action === 'text') { + if (!host.acceptsText()) return error('unavailable', 'The foreground window does not accept text.'); + host.text(request.text, request.submit !== false); + } else { + if (!host.assistantAvailable()) return error('unavailable', 'Set up the voice assistant in Settings first.'); + host.assistant(request.text); + } + return { ok: true }; + } catch { return error('failed', 'Faceclaw could not deliver the request.'); } +} diff --git a/app/remote/service.ts b/app/remote/service.ts new file mode 100644 index 00000000..02e64c5b --- /dev/null +++ b/app/remote/service.ts @@ -0,0 +1,54 @@ +import { getStringSetting, setStringSetting, onSettingsStoreChanged } from '../native/settings-store'; +import { remoteNative, remoteInterfaces, randomTokenSecret, tokenHash } from '../native/remote-input'; +import { INTERFACE_STORAGE_KEY, listenerAddresses } from './listeners'; +import { TokenStore, handleRequest, REMOTE_PORT, type RemoteHost } from './protocol'; +export const TOKEN_STORAGE_KEY = 'remoteInput.tokens.v1'; +export const remoteTokens = new TokenStore(() => getStringSetting(TOKEN_STORAGE_KEY, '[]'), + value => setStringSetting(TOKEN_STORAGE_KEY, value), randomTokenSecret, tokenHash); +let status = 'No tokens; listener stopped.'; +export function remoteInputStatus(): string { return status; } +export function remoteInterfaceSelection(): string { return getStringSetting(INTERFACE_STORAGE_KEY, 'tailscale'); } +export function selectRemoteInterface(value: string): void { setStringSetting(INTERFACE_STORAGE_KEY, value); } + +/** Owned by the main controller. Native socket work never enters a JS isolate. */ +export function startRemoteInput(host: RemoteHost): () => void { + const native = remoteNative(); + let timer: ReturnType | null = null; + let busy = false; + let networkTimer: ReturnType | null = null; + const sync = () => { + if (!remoteTokens.list().length) { + native.stop(); + if (timer !== null) clearInterval(timer); + timer = null; + if (networkTimer !== null) clearInterval(networkTimer); + networkTimer = null; + status = 'No tokens; listener stopped.'; + return; + } + const selection = remoteInterfaceSelection(); + const addresses = listenerAddresses(remoteInterfaces(), selection); + const failure = native.start(REMOTE_PORT, addresses); + status = `Listening on ${native.addresses().map(address => `${address.includes(':') ? `[${address}]` : address}:${REMOTE_PORT}`).join(', ') || 'no addresses'}.`; + if (selection !== 'localhost' && addresses.length === 1) status += '\nSelected tunnel not found; localhost only. Enable Tailscale or select its tunnel interface.'; + if (failure) status += `\n${failure}`; + // Reconcile after VPN reconnects/address changes; unchanged sockets stay open. + if (networkTimer === null) networkTimer = setInterval(sync, 3000); + if (timer !== null) return; + timer = setInterval(() => { + if (busy) return; + const raw = native.nextRequest(); + if (!raw) return; + const request = JSON.parse(String(raw)); + if (Date.now() >= request.expiresAt) return; + busy = true; + void handleRequest(request.body, remoteTokens, host) + .then(reply => native.complete(request.id, JSON.stringify(reply))) + .catch(() => native.complete(request.id, JSON.stringify({ ok: false, error: 'failed', message: 'Request failed.' }))) + .finally(() => { busy = false; }); + }, 15); + }; + sync(); + const off = onSettingsStoreChanged(key => { if (key === TOKEN_STORAGE_KEY || key === INTERFACE_STORAGE_KEY) sync(); }); + return () => { off(); if (timer !== null) clearInterval(timer); if (networkTimer !== null) clearInterval(networkTimer); native.stop(); }; +} diff --git a/app/ui/dashboard/remote-input-menu.ts b/app/ui/dashboard/remote-input-menu.ts new file mode 100644 index 00000000..eb046096 --- /dev/null +++ b/app/ui/dashboard/remote-input-menu.ts @@ -0,0 +1,90 @@ +import { remoteTokens, remoteInputStatus, remoteInterfaceSelection, selectRemoteInterface } from '../../remote/service'; +import { PERMISSIONS, type Permission } from '../../remote/protocol'; +import { isTunnelInterface } from '../../remote/listeners'; +import { copyRemoteToken, remoteInterfaces } from '../../native/remote-input'; +import { TextViewerLayer } from '../../apps/files/text-viewer'; +import { ConfigSettingString, textSettingMenuItem } from '../dashboard-settings'; +import { getDefaultSmallFont } from '../../graphics/ui-fonts'; +import { drawRightValueMenuItem, type MenuItem } from '../menu'; +import { type LayerContext } from '../layers'; +import { openSettingsSubMenu } from './settings-panel'; + +const labels: Record = { input: 'Provide input', text: 'Text to foreground window', assistant: 'Text to voice assistant' }; +const nameSetting = new ConfigSettingString({ id: 'remoteInputNewName', storageKey: 'remoteInput.newName', + label: 'Token name', defaultValue: 'My app', normalize: value => (value ?? '').slice(0, 80) }); + +function permissionItem(permission: Permission, get: () => Permission[], set: (values: Permission[]) => void): MenuItem { + return { label: labels[permission], onSelect: ctx => { + const current = get(); + set(current.includes(permission) ? current.filter(p => p !== permission) : [...current, permission]); + ctx.actions.requestRender(); + }, render: ({ image, x, y, width }) => drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, + labels[permission], get().includes(permission) ? 'On' : 'Off') }; +} +function showError(ctx: LayerContext, error: unknown): void { + ctx.stack.push(new TextViewerLayer(error instanceof Error ? error.message : 'Could not update token.', 'Input tokens')); +} +function tokenDetails(ctx: LayerContext, id: string): void { + const record = remoteTokens.list().find(r => r.id === id); + if (!record) return; + openSettingsSubMenu(ctx, record.name, [ + ...PERMISSIONS.map(p => permissionItem(p, () => remoteTokens.list().find(r => r.id === id)?.permissions ?? [], + values => remoteTokens.permissions(id, values))), + { label: 'Revoke token', onSelect: inner => openSettingsSubMenu(inner, `Revoke ${record.name}?`, [ + { label: 'Cancel', onSelect: c => { c.stack.pop(); } }, + { label: 'Revoke', onSelect: c => { + remoteTokens.revoke(id); + c.stack.pop(); c.stack.pop(); c.stack.pop(); + openTokens(c); + } }, + ]) }, + ]); +} +function createToken(ctx: LayerContext): void { + let permissions: Permission[] = ['input']; + openSettingsSubMenu(ctx, 'Create input token', [ + textSettingMenuItem(nameSetting), + ...PERMISSIONS.map(p => permissionItem(p, () => permissions, values => { permissions = values; })), + { label: 'Generate token', onSelect: inner => { + try { + const { token } = remoteTokens.create(nameSetting.get(), permissions); + inner.stack.pop(); inner.stack.pop(); + openTokens(inner); + // The secret lives only in this temporary menu; it cannot be recovered after closing it. + openSettingsSubMenu(inner, 'Save your token now', [ + { label: 'Copy token to phone clipboard', onSelect: c => { + try { copyRemoteToken(token); c.stack.push(new TextViewerLayer('Token copied. Save it in the calling app or FACECLAW_TOKEN. The secret cannot be shown again after leaving this menu.', 'Copied')); } + catch (error) { showError(c, error); } + } }, + { label: 'Show token (only available now)', onSelect: c => c.stack.push(new TextViewerLayer(token, 'Input token')) }, + { label: 'Done', onSelect: c => { c.stack.pop(); } }, + ]); + } catch (error) { showError(inner, error); } + } }, + ]); +} +function interfaceMenu(ctx: LayerContext): void { + const interfaces = remoteInterfaces().filter(isTunnelInterface); + const names = [...new Set(interfaces.map(item => item.name))]; + const choices = [ + { value: 'tailscale', label: 'Localhost + Tailscale (auto)' }, + { value: 'localhost', label: 'Localhost / USB only' }, + ...names.map(name => ({ value: `interface:${name}`, label: `${name}: ${interfaces.filter(item => item.name === name).map(item => item.address).join(', ')}` })), + ]; + openSettingsSubMenu(ctx, 'Listening interfaces', choices.map(choice => ({ + label: `${choice.label}${remoteInterfaceSelection() === choice.value ? ' *' : ''}`, + onSelect: inner => { selectRemoteInterface(choice.value); inner.stack.pop(); }, + }))); +} +function openTokens(ctx: LayerContext): void { + openSettingsSubMenu(ctx, 'Input tokens', [ + { label: 'Create token', onSelect: createToken }, + { label: 'Listening interfaces', onSelect: interfaceMenu }, + { label: 'Connection information', onSelect: c => c.stack.push(new TextViewerLayer( + `${remoteInputStatus()}\n\nAndroid USB: adb forward tcp:8791 tcp:8791\n\niOS USB: iproxy 8791 8791\n\nTailscale: use --host with the displayed tunnel IP. Listening interfaces lets you select a tunnel if auto detection fails. Wi-Fi/LAN addresses are never selected.\n\nUse scripts/faceclaw-input.cjs with FACECLAW_TOKEN set. Tokens grant only the selected permissions. Revoking a token takes effect on the next request. Text is blocked while the glasses are locked.`, 'Input API')) }, + ...remoteTokens.list().map(record => ({ label: record.name, onSelect: (c: LayerContext) => tokenDetails(c, record.id) })), + ]); +} +export function remoteInputMenuItem(): MenuItem { + return { label: 'Input tokens', description: 'Allow another app to provide ring/watch input, type into the foreground window, or send messages to the voice assistant. Create tokens with individual permissions and revoke them here.', onSelect: openTokens }; +} diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index 938ba9df..890d3607 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -1,3 +1,4 @@ +import { remoteInputMenuItem } from "./remote-input-menu"; import { knownFolders } from "@nativescript/core"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import type { GrayImage } from "../../graphics/image"; @@ -144,6 +145,7 @@ function settingsSections(): SettingsSection[] { { label: "API Keys", items: [ + remoteInputMenuItem(), textSettingMenuItem(elevenLabsApiKeySetting), textSettingMenuItem(openAiApiKeySetting), textSettingMenuItem(sonioxApiKeySetting), diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 196b4766..e385ab54 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -137,7 +137,7 @@ export type ShellWindow = { * Deliver a text string to the window (e.g. finalized voice input). Optional: * only windows that consume typed text (the terminal) implement it. */ - receiveTextInput?: (text: string) => void; + receiveTextInput?: (text: string, options?: { submit?: boolean }) => void; /** Foreground state changed: this window's surface is (not) the visible one. */ setForeground?: (foreground: boolean) => void; /** @@ -1105,8 +1105,8 @@ class Shell { } /** Deliver a text string to the foreground window (e.g. finalized voice input). */ - sendTextToForegroundWindow(text: string): void { - this.foregroundWindow()?.receiveTextInput?.(text); + sendTextToForegroundWindow(text: string, options?: { submit?: boolean }): void { + this.foregroundWindow()?.receiveTextInput?.(text, options); } /** diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index 55687b26..a2c0cf88 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -20,7 +20,7 @@ export type WorkerAppMessage = | { type: "resize-window"; windowId: string; viewport: { width: number; height: number } } | { type: "close-window"; windowId: string } | { type: "input"; windowId: string; event: unknown; frameId: number; focused: boolean } - | { type: "text-input"; windowId: string; text: string } + | { type: "text-input"; windowId: string; text: string; submit?: boolean } | { type: "render"; windowId: string; focused: boolean } | { type: "foreground"; windowId: string; foreground: boolean; focused: boolean } /** @@ -441,8 +441,8 @@ export class WorkerAppHost { requestRender: () => { this.post({ type: "render", windowId: spec.windowId, focused: shell.isWindowFocused(spec.windowId) }); }, - receiveTextInput: (text) => { - this.post({ type: "text-input", windowId: spec.windowId, text }); + receiveTextInput: (text, options) => { + this.post({ type: "text-input", windowId: spec.windowId, text, ...(options?.submit === undefined ? {} : { submit: options.submit }) }); }, setForeground: (foreground) => { this.options.setSurfaceVisible(surfaceId, foreground); diff --git a/native/kotlin/android-java-boundaries.json b/native/kotlin/android-java-boundaries.json index 9bef130a..cfceae10 100644 --- a/native/kotlin/android-java-boundaries.json +++ b/native/kotlin/android-java-boundaries.json @@ -82,6 +82,12 @@ "com/faceclaw/app/FaceclawWebSocket.java" ] }, + { + "reason": "Android JVM loopback sockets, SecureRandom and SHA-256; iOS uses its native socket and Security implementations. Protocol and permissions stay in shared TypeScript.", + "files": [ + "com/faceclaw/app/FaceclawRemoteInput.java" + ] + }, { "reason": "Android Bitmap/Canvas/Paint/typeface, SVG rasterization and camera scanning", "files": [ diff --git a/package-lock.json b/package-lock.json index 8446d8ae..aeec6fa8 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,6 +13,7 @@ "@nativescript/core": "~9.0.0", "@nativescript/theme": "^3.1.0", "@xterm/headless": "^6.0.0", + "enquirer": "2.4.1", "fzstd": "^0.1.1", "lodash": "^4.18.1", "lz4js": "^0.2.0", @@ -1513,7 +1514,6 @@ "version": "4.1.3", "resolved": "https://registry.npmjs.org/ansi-colors/-/ansi-colors-4.1.3.tgz", "integrity": "sha512-/6w/C21Pm1A7aZitlI5Ni/2J6FFQN8i1Cvz3kHABAAbw93v/NlvKdVOqz7CCWz/3iv/JplRSEEZ83XION15ovw==", - "dev": true, "license": "MIT", "engines": { "node": ">=6" @@ -1523,7 +1523,6 @@ "version": "5.0.1", "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.1.tgz", "integrity": "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ==", - "dev": true, "license": "MIT", "engines": { "node": ">=8" @@ -2177,6 +2176,19 @@ "node": ">=10.13.0" } }, + "node_modules/enquirer": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", + "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", + "license": "MIT", + "dependencies": { + "ansi-colors": "^4.1.1", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8.6" + } + }, "node_modules/entities": { "version": "7.0.1", "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz", @@ -4154,7 +4166,6 @@ "version": "6.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dev": true, "license": "MIT", "dependencies": { "ansi-regex": "^5.0.1" diff --git a/package.json b/package.json index d0d81931..81a0f3ac 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,7 @@ "version": "1.0.0", "private": false, "scripts": { + "input": "node scripts/faceclaw-input.cjs", "native:android": "node scripts/kotlin-build.cjs android", "native:ios": "node scripts/kotlin-build.cjs ios", "test:kotlin": "bash scripts/test-kotlin.sh", @@ -24,6 +25,7 @@ "@nativescript/core": "~9.0.0", "@nativescript/theme": "^3.1.0", "@xterm/headless": "^6.0.0", + "enquirer": "2.4.1", "fzstd": "^0.1.1", "lodash": "^4.18.1", "lz4js": "^0.2.0", diff --git a/scripts/faceclaw-input-editor.cjs b/scripts/faceclaw-input-editor.cjs new file mode 100644 index 00000000..8335a26a --- /dev/null +++ b/scripts/faceclaw-input-editor.cjs @@ -0,0 +1,67 @@ +'use strict'; +const { Input } = require('enquirer'); + +// Enquirer owns the editable line and its rendering. The remote's single +// keypress listener routes keys here so starting/closing a prompt cannot leak +// an Enter, arrow or Esc into the watch controls, even within one input chunk. +class MessageInput extends Input { + start() {} // Keep terminal ownership (raw mode and signals) with the remote. + cursorHide() { + if (this.cursorHidden) return; + this.cursorHidden = true; + this.stdout.write('\x1b[?25l'); + const show = () => this.cursorShow(); + // Enquirer's default cursor cleanup installs signal handlers that call + // process.exit immediately. Let the remote restore raw mode first instead. + process.once('exit', show); + this.once('close', () => { show(); process.removeListener('exit', show); }); + } + render() { + this.paint = (this.paint || Promise.resolve()).then(() => super.render()); + return this.paint; + } + home() { return this.first(); } + end() { return this.last(); } + last() { this.cursor = this.input.length; return this.render(); } + async cancel() { + this.input = this.value = ''; + this.cursor = 0; + this.state.clipboard = []; + return super.cancel(); + } +} + +function createEditor(action, input, output) { + const prompt = new MessageInput({ + name: action, message: action === 'text' ? 'Foreground text' : 'Assistant message', + footer: 'Enter: send | Esc: discard | Ctrl-C: quit', + stdin: input, stdout: output, initial: '', + validate: value => value.length > 8000 ? 'Maximum 8000 characters.' : value.includes('\0') ? 'NUL is not allowed.' : true, + }); + let resolve, reject, closing = false; + const result = new Promise((yes, no) => { resolve = yes; reject = no; }); + prompt.once('submit', text => { + resolve(text); + prompt.input = prompt.value = ''; + prompt.state.clipboard = []; + }); + prompt.once('cancel', () => resolve(null)); + // Queue keys during async initialization, including typing/paste that arrives + // in the same terminal chunk as the mode-switching 'i' or 'a'. + let keys = prompt.initialize().catch(reject); + return { + result, + keypress(text, key) { + keys = keys.then(async () => { + if (closing || prompt.state.closed) return; + await prompt.keypress(text, key); + await prompt.paint; + }).catch(reject); + }, + cancel() { + closing = true; + return keys.then(() => prompt.state.closed ? undefined : prompt.cancel()).catch(reject); + }, + }; +} +module.exports = { createEditor }; diff --git a/scripts/faceclaw-input.cjs b/scripts/faceclaw-input.cjs new file mode 100755 index 00000000..ab343b42 --- /dev/null +++ b/scripts/faceclaw-input.cjs @@ -0,0 +1,204 @@ +#!/usr/bin/env node +'use strict'; +const net = require('node:net'); +const readline = require('node:readline'); +const fs = require('node:fs'); + +const GESTURES = new Set(['click', 'double-click', 'long-press', 'short-then-long-press', 'scroll-up', 'scroll-down', + 'swipe-up', 'swipe-down', 'swipe-left', 'swipe-right']); +const ALIASES = { tap: 'click', 'double-tap': 'double-click', up: 'swipe-up', down: 'swipe-down', left: 'swipe-left', right: 'swipe-right' }; +const HELP = `Usage: faceclaw-input [--host HOST] [--port PORT] [--token-file FILE] COMMAND + + input GESTURE [--source watch|ring] Send one gesture (default source: watch) + text [-n] MESSAGE Type into the foreground window (-n: no Enter) + assistant MESSAGE Send a message to the voice assistant + interactive Use this terminal as a watch input device + +Gestures: tap, double-tap, long-press, short-then-long-press, + up, down, left, right, scroll-up, scroll-down +Interactive: arrows = watch directions; Enter = tap; Esc = double-tap; + Tab = long-press; i = compose text; a = compose assistant message; + Enter sends a draft; Esc discards it; Ctrl-C quits in either mode. +Token: FACECLAW_TOKEN or --token-file FILE. Host/port default to 127.0.0.1:8791 +(or FACECLAW_HOST/FACECLAW_PORT). Android USB: adb forward tcp:8791 tcp:8791 +Keep Faceclaw running. Each token needs the permission for its command. +`; +function parseArgs(args, env = process.env) { + const options = { host: env.FACECLAW_HOST || '127.0.0.1', port: Number(env.FACECLAW_PORT || 8791), token: env.FACECLAW_TOKEN || '', source: 'watch' }; + const positional = []; + let noSubmit = false; + for (let i = 0; i < args.length; i++) { + const arg = args[i]; + if (arg === '--') { positional.push(...args.slice(i + 1)); break; } + if (arg === '-n') { noSubmit = true; continue; } + if (arg === '--help' || arg === '-h') return { help: true }; + if (['--host', '--port', '--token-file', '--source'].includes(arg)) { + const value = args[++i]; + if (!value || value.startsWith('--')) throw new Error(`Missing value for ${arg}.`); + if (arg === '--port') options.port = Number(value); + else if (arg === '--token-file') options.token = fs.readFileSync(value, 'utf8').trim(); + else options[arg.slice(2)] = value; + } else if (arg.startsWith('--')) throw new Error(`Unknown option: ${arg}`); + else positional.push(arg); + } + if (!Number.isInteger(options.port) || options.port < 1 || options.port > 65535) throw new Error('Invalid port.'); + if (!/^fc1_[a-f0-9]{64}$/.test(options.token)) throw new Error('Set FACECLAW_TOKEN or --token-file to a generated Faceclaw token.'); + if (!['watch', 'ring'].includes(options.source)) throw new Error('Source must be watch or ring.'); + const [command, ...values] = positional; + if (noSubmit && command !== 'text') throw new Error('-n is only valid with text.'); + if (command === 'interactive') { + if (values.length || options.source !== 'watch') throw new Error('Interactive mode uses watch input and takes no message.'); + return { options, interactive: true }; + } + let payload; + if (command === 'input') { + const gesture = ALIASES[values[0]] || values[0]; + if (values.length !== 1 || !GESTURES.has(gesture)) throw new Error('Expected one valid gesture. See --help.'); + if (options.source === 'ring' && gesture.startsWith('swipe-')) throw new Error('Ring input uses scroll-up/down; directions require watch input.'); + payload = { action: 'input', gesture, source: options.source }; + } else if (command === 'text' || command === 'assistant') { + const text = values.join(' '); + if (!text.trim() || text.length > 8000 || text.includes('\0')) throw new Error('Message must contain 1–8000 characters without NUL.'); + payload = { action: command, text, ...(noSubmit ? { submit: false } : {}) }; + } else throw new Error('Expected input, text, assistant or interactive. See --help.'); + return { options, payload }; +} +function send(options, payload, signal) { + return new Promise((resolve, reject) => { + const socket = net.createConnection({ host: options.host, port: options.port }); + let buffer = Buffer.alloc(0), settled = false; + const abort = () => finish(new Error('Request cancelled.')); + const deadline = setTimeout(() => finish(new Error('Faceclaw request timed out.')), 6500); + function finish(error, value) { + if (settled) return; + settled = true; clearTimeout(deadline); signal?.removeEventListener('abort', abort); socket.destroy(); + if (error) reject(error); else resolve(value); + } + socket.on('connect', () => socket.write(JSON.stringify({ ...payload, version: 1, token: options.token }) + '\n')); + socket.on('error', error => finish(new Error(`Could not reach Faceclaw: ${error.code || 'connection error'}. Check the app, host address and USB forwarding or Tailscale.`))); + socket.on('data', chunk => { + buffer = Buffer.concat([buffer, chunk]); + if (buffer.length > 65536) return finish(new Error('Oversized Faceclaw response.')); + const end = buffer.indexOf(10); + if (end < 0) return; + let response; + try { response = JSON.parse(buffer.subarray(0, end).toString('utf8')); } + catch { return finish(new Error('Invalid Faceclaw response.')); } + if (response?.ok !== true) { + const error = new Error(response?.message || 'Faceclaw rejected the request.'); + error.code = response?.error; + return finish(error); + } + finish(null, response); + }); + socket.on('end', () => { if (!settled) finish(new Error('Faceclaw closed the connection without a response.')); }); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + }); +} +function keyGesture(key) { + return { up: 'swipe-up', down: 'swipe-down', left: 'swipe-left', right: 'swipe-right', + return: 'click', enter: 'click', escape: 'double-click', tab: 'long-press' }[key?.name]; +} +async function interactive(options, input = process.stdin, output = process.stdout, transmit = send) { + if (!input.isTTY || typeof input.setRawMode !== 'function') throw new Error('Interactive mode requires a terminal.'); + const wasRaw = !!input.isRaw; + readline.emitKeypressEvents(input, { escapeCodeTimeout: 75 }); + const controls = 'Arrows: directions | Enter: tap | Esc: double-tap | Tab: long-press | i: text | a: assistant | Ctrl-C: quit\n'; + output.write(controls); + input.setRawMode(true); + await new Promise(resolve => { + let active = true, sending = false, connected = false; + let heartbeat = null, editor = null; + const deferredStatus = []; + const status = message => editor ? deferredStatus.push(message) : output.write(message); + const queue = []; + const abort = new AbortController(); + const stop = () => { + if (!active) return; + active = false; queue.length = 0; abort.abort(); + if (heartbeat !== null) clearInterval(heartbeat); + input.removeListener('keypress', onKey); input.removeListener('end', stop); + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.removeListener(signal, stop); + Promise.resolve(editor?.cancel()).catch(() => {}).finally(() => { + input.setRawMode(wasRaw); input.pause(); output.write('\n'); resolve(); + }); + }; + const reportFailure = error => { + if (!active) return; + const disconnected = !['forbidden', 'locked', 'unavailable', 'bad_request', 'failed'].includes(error.code); + if (disconnected) connected = false; + queue.length = 0; + status(`\n${error.message}${disconnected ? ' Waiting for connection; Ctrl-C to quit.' : ''}\n`); + }; + const checkConnection = async () => { + if (!active || sending || editor) return; + sending = true; + try { + await transmit(options, { action: 'ping' }, abort.signal); + if (!active) return; + if (!connected) status('Connected.\n'); + connected = true; + } catch (error) { reportFailure(error); } + finally { sending = false; if (active && connected && queue.length) void drain(); } + }; + const drain = async () => { + if (sending || !connected) return; + sending = true; + try { + while (active && queue.length) { + const payload = queue.shift(); + await transmit(options, payload, abort.signal); + if (active && payload.action !== 'input') status(payload.action === 'text' ? 'Sent to foreground window.\n' : 'Sent to assistant.\n'); + } + } catch (error) { reportFailure(error); } + finally { sending = false; } + }; + const compose = action => { + try { + editor = require('./faceclaw-input-editor.cjs').createEditor(action, input, output); + } catch (error) { output.write(`Could not open editor: ${error.message}\nRun npm install in the Faceclaw checkout.\n`); return; } + editor.result.then(text => { + editor = null; + if (!active) return; + for (const message of deferredStatus.splice(0)) output.write(message); + if (text === null) output.write('Draft discarded.\n'); + else if (!text.trim()) output.write('Empty draft; nothing sent.\n'); + else if (!connected) output.write('Not sent: connection was lost. Reconnect before sending a new message.\n'); + else { queue.push({ action, text }); void drain(); } + output.write(controls); + }).catch(error => { + editor = null; + if (active) { for (const message of deferredStatus.splice(0)) output.write(message); output.write(`Editor failed: ${error.message}\n${controls}`); } + }); + }; + const onKey = (text, key) => { + if (key?.ctrl && key.name === 'c') return stop(); + if (editor) { editor.keypress(text, key); return; } + // Node marks a standalone Esc as meta=true as well as name='escape'. + if (key?.ctrl || (key?.meta && key.name !== 'escape')) return; + if (connected && !key?.shift && (key?.name === 'i' || key?.name === 'a')) { + compose(key.name === 'i' ? 'text' : 'assistant'); return; + } + const gesture = keyGesture(key); + // Bound key repeat lag; discard excess while the remote is slow. + if (gesture && connected && queue.length < 16) { queue.push({ action: 'input', gesture, source: 'watch' }); void drain(); } + }; + input.on('keypress', onKey); input.on('end', stop); + for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) process.on(signal, stop); + input.resume(); + heartbeat = setInterval(() => { void checkConnection(); }, 3000); + output.write('Connecting...\n'); + void checkConnection(); + }); +} +async function main() { + try { + const args = parseArgs(process.argv.slice(2)); + if (args.help) return process.stdout.write(HELP); + if (args.interactive) await interactive(args.options); + else { await send(args.options, args.payload); process.stdout.write('Sent.\n'); } + } catch (error) { process.stderr.write(`${error.message}\n`); process.exitCode = 1; } +} +module.exports = { parseArgs, send, keyGesture, interactive }; +if (require.main === module) void main(); diff --git a/scripts/faceclaw-input.md b/scripts/faceclaw-input.md new file mode 100644 index 00000000..30fa17fb --- /dev/null +++ b/scripts/faceclaw-input.md @@ -0,0 +1,162 @@ +# Faceclaw input tokens and CLI + +In **Settings → API Keys → Input tokens**, choose **Create token**, give it a +name, select its permissions, and choose **Generate token**. Copy or save the +secret before leaving the result menu: Faceclaw stores only its SHA-256 hash. +Select an existing token to change permissions or revoke it. A token with no +permissions is disabled. Up to 32 tokens can exist at once. + +Permissions are independent: + +- **Provide input** simulates watch or ring gestures, including navigation, + taps and the system menu. It has the same reach as those input devices. +- **Text to foreground window** delivers text to the selected app's text input. +- **Text to voice assistant** submits a query to the configured assistant. + +Text delivery is rejected while the glasses are locked. Gestures pass through +the existing lock-screen and sleep behavior. The assistant must be configured; +the selected window must accept text. Android requires connected glasses; iOS +also allows the active phone preview. An assistant acknowledgment means the +query was submitted, not that the assistant has finished answering. + +## Connect + +While at least one token exists, Faceclaw listens on **127.0.0.1:8791** and, +by default, detected **Tailscale addresses**. It never binds a wildcard or a +Wi-Fi/LAN address. Under **Input tokens → Listening interfaces**, choose: + +- **Localhost + Tailscale (auto)**: detect a named Tailscale tunnel or a tunnel + with Tailscale's dedicated IPv6 prefix; bind its tailnet IPv4/IPv6 addresses. +- **Localhost / USB only**: disable remote network access. +- A specific **tunnel interface** (such as `tun0` or `utun3`): use this when + automatic identification is unavailable, including IPv4-only VPN setups. + +Automatic detection uses the prefixes in [Tailscale's address documentation](https://tailscale.com/docs/concepts/ip-and-dns-addresses). +A CGNAT IPv4 address by itself is not sufficient to identify Tailscale. The +interface picker shows names and addresses of tunnel interfaces; choose the +one belonging to Tailscale. Physical Wi-Fi, Ethernet, and cellular interfaces +are excluded. Interface selection is saved; addresses are refreshed every +three seconds after a tunnel connects, disconnects, or changes address. +Localhost remains available even when the selected tunnel is absent. + +**Connection information** shows the actual listening addresses. From a +computer on your tailnet, pass the phone's displayed Tailscale address: + +```sh +node scripts/faceclaw-input.cjs --host 100.x.y.z interactive +``` + +Apps on the same phone can connect directly to localhost. From a computer, +you can also forward the port over USB: + +```sh +# Android (ADB must already be paired/authorized) +adb forward tcp:8791 tcp:8791 + +# iOS, using libimobiledevice's iproxy with a trusted, connected phone +iproxy 8791 8791 +``` + +Keep Faceclaw running; iOS +can suspend it when backgrounded without an active glasses session. Revoking +the last token stops the listener. + +## Command-line utility + +The utility requires Node.js 18+. Run `npm install` in this checkout to install +its [Enquirer](https://github.com/enquirer/enquirer) text editor, then run the +CLI below. For use on another computer, copy both `faceclaw-input.cjs` and +`faceclaw-input-editor.cjs` into one directory and run +`npm install enquirer@2.4.1` there. One-off commands do not load the editor. + +```sh +export FACECLAW_TOKEN='fc1_…' # Paste the generated token +node scripts/faceclaw-input.cjs input tap +node scripts/faceclaw-input.cjs input up +node scripts/faceclaw-input.cjs input scroll-down --source ring +node scripts/faceclaw-input.cjs text 'Text for the foreground app' +node scripts/faceclaw-input.cjs text -n 'Type without pressing Enter' +node scripts/faceclaw-input.cjs assistant 'What is on my calendar?' +node scripts/faceclaw-input.cjs interactive +``` + +`npm run input -- …` is also available. Use `--token-file FILE` instead of the +environment variable to read a token from a file. `--host` and `--port` (or +`FACECLAW_HOST` and `FACECLAW_PORT`) select a forwarded endpoint. Messages +starting with `--` can follow an option terminator, e.g. `text -- '--help'`. +For terminal windows, text normally appends Enter; `text -n` suppresses that +submission, like `echo -n`. Embedded newlines in the message are preserved. +Other apps receive the text unchanged. Assistant messages do not accept `-n`. +One-off commands return a nonzero exit code on rejection or connection failure. +The CLI never automatically replays an input or message after a failure, +since delivery may already have occurred. + +Interactive mode connects and authenticates immediately, without waiting for a +keypress. It validates the token using a side-effect-free `ping`, then checks +connectivity periodically while idle. Each action checks its own permission, +so a token with only text or assistant permission can also use composition. If a request fails, it reports +the error, drops queued keys, and keeps trying to reconnect until Ctrl-C. +It never replays a possibly-delivered gesture. Each request uses a fresh socket, +kept open until the reply arrives (including when using USB forwarding). +Update the phone app alongside the CLI so it supports `ping`. + +Interactive mode maps arrow keys to watch directions, Enter to tap, Esc to +double-tap, and Tab to a complete long-press (press and release). Ctrl-C exits +and restores terminal settings. Esc uses a short disambiguation delay to tell +it apart from arrow-key escape sequences. Events are serialized, with a bounded +queue to prevent excessive key-repeat lag. + +### Compose text or an assistant message + +From interactive controls, press **i** to compose text for the foreground +window, or **a** to compose a message for the assistant. A labeled Enquirer +prompt replaces the gesture controls until you finish: + +- **Enter** sends the draft once and returns to controls. Foreground text uses + the normal submission behavior, including Enter in terminal windows. +- **Esc** discards and clears the draft, then returns to controls. +- **Ctrl-C** discards the draft and quits the utility. + +While composing, arrows edit the line; Tab, Enter and Esc never send watch +gestures. The letters `i` and `a` are ordinary text. The editor supports +Left/Right, Home/End, Backspace/Delete, Ctrl-A/E (start/end), Ctrl-W (delete the +previous word), and Ctrl-K (delete to the end). Input is a single line, with +an 8000-character limit; an empty draft sends nothing. Every new draft starts +empty, and no message history is stored. Connection-status output is deferred +while the editor is visible so it cannot corrupt the prompt. + +The token needs **Text to foreground window** for `i` and **Text to voice +assistant** for `a`. A permission rejection is reported and returns to controls; +it does not end the interactive session or resend the message. + +## Protocol for other apps + +Open a TCP connection, send one UTF-8 JSON object terminated by a newline, and +read a newline-terminated JSON reply. Each connection carries one request. +Authentication and current permissions are checked for every request. + +```json +{"version":1,"token":"fc1_…","action":"input","gesture":"click","source":"watch"} +{"version":1,"token":"fc1_…","action":"text","text":"hello","submit":false} +{"version":1,"token":"fc1_…","action":"ping","permission":"input"} +{"version":1,"token":"fc1_…","action":"assistant","text":"Hello"} +``` + +Input gestures: `click`, `double-click`, `long-press`, `short-then-long-press`, +`scroll-up`, `scroll-down`, `swipe-up`, `swipe-down`, `swipe-left`, `swipe-right`. +Source defaults to `watch`; `ring` supports all except the four swipes. +The CLI also accepts `tap`, `double-tap`, `up`, `down`, `left`, `right` aliases. + +`submit` is an optional boolean for `text`, defaulting to `true`; `false` +suppresses the terminal app's appended Enter. `ping` validates the token and, +if supplied, the requested `permission`, without dispatching any input or +requiring connected glasses. + +Successful reply: `{"ok":true}`. Rejections contain `ok:false`, `error`, and +`message`. Error codes: `bad_request`, `unauthorized`, `forbidden`, `locked`, +`unavailable`, `failed`, `timeout`. Malformed framing closes the connection. +Frames are limited to 64 KiB; text to 8000 JavaScript UTF-16 code units and must +contain non-whitespace text without NUL. Whitespace in valid text is preserved. +Connections have a five-second read deadline and five-second dispatch deadline. +Each listening address serves one connection at a time with a bounded socket backlog. +Token values and request text are never written to the server log. diff --git a/tests/ios-app-host.test.cjs b/tests/ios-app-host.test.cjs index b0636ded..7217df57 100644 --- a/tests/ios-app-host.test.cjs +++ b/tests/ios-app-host.test.cjs @@ -76,6 +76,7 @@ for (const isIOS of [true, false]) { } }); const menus = load('app/ui/dashboard/settings-menus.ts', { global: { isIOS }, require: id => { if (id === '../dashboard-settings') return settings; + if (id === './remote-input-menu') return { remoteInputMenuItem: () => ({ label: 'Input tokens' }) }; if (id === './settings-panel') return { SettingsPanelLayer: class { constructor(sections) { this.sections = sections; } } }; return { LOCAL_MODEL: { sizeBytes: 1000 }, ASR_MODELS: { moonshine: {}, 'whisper-base-en': {} }, uiFontPickerMenuItem: () => ({ label: 'Font' }), terminalFontPickerMenuItem: () => ({ label: 'Terminal font' }) }; @@ -193,6 +194,7 @@ test('background glasses input still composites frames; phone resume preserves t const settings = { brightnessSetting: { get: () => 'auto' }, brightnessSettingToLevel: () => null, lockScreenEnabledSetting: { get: () => true }, onAnySettingChanged: () => () => {}, previewColorSetting: { get: () => 'white' } }; const modules = { "../ui/input-monitor": load("app/ui/input-monitor.ts", {}), + '../remote/service': { startRemoteInput() {} }, '../assistant/system-tools': { registerSystemTools() {} }, '../assistant/window-tools': { registerWindowTools() {} }, '../assistant/navigate-tools': { registerNavigateTools() {} }, diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index c3116b9d..76d065a0 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -24,7 +24,7 @@ const timings = load('app/native/frame-timings.ts', {}); function fixture() { const observers = new Map(), phoneState = { protectedDataAvailable: true }; - let settingsChanged; + let settingsChanged, remoteHost; let now = 1000, nextTask = 0, screenOn = true, shellOptions, session; const tasks = new Map(), sent = [], previews = [], received = [], errors = []; const boardStats = { starts: 0, stops: 0, paints: 0 }; @@ -74,6 +74,7 @@ function fixture() { const inputMonitor = load("app/ui/input-monitor.ts", {}); const modules = { "../ui/input-monitor": inputMonitor, + '../remote/service': { startRemoteInput(host) { remoteHost = host; } }, '../assistant/system-tools': { registerSystemTools() {} }, '../assistant/window-tools': { registerWindowTools() {} }, '../assistant/navigate-tools': { registerNavigateTools() {} }, @@ -139,7 +140,7 @@ function fixture() { await render(); } async function phone(type, origin = 'ring') { controller.gesture(type, origin); await render(); } - return { controller, shell, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render }; + return { controller, shell, remoteHost, keyboardChanged: session => shellOptions.onKeyboardInputChanged(session), settings, phoneState, observers, wear: wearing => session.onWear(wearing), settingsChanged: () => settingsChanged(), sent, previews, received, boardStats, hardware, phone, advance, render }; } const assertBlank = pixels => assert.ok(pixels.every(p => p === 0)); const assertBoard = pixels => { @@ -319,3 +320,19 @@ test('keyboard sessions cannot deliver text after the glasses lock or phone leav assert.equal(calls.length, 2); editor.discard(); assert.deepEqual(calls.at(-1), ['discard']); }); + + +test('external input shares iOS dispatch, including hold release and phone-lock gating', async () => { + const f = fixture(); await f.controller.connect(); f.wear(false); + assert.equal(f.remoteHost.ready(), true); + assert.equal(f.remoteHost.locked(), false); + await f.remoteHost.input('swipe-left', 'watch'); + await f.remoteHost.input('long-press', 'ring'); + assert.deepEqual(f.received.map(e => [e.type, e.source]), [ + ['swipe-left', 'watch'], ['long-press', 'ring'], ['long-press-release', 'ring'], + ]); + f.observers.get('lock')(); + assert.equal(f.remoteHost.locked(), true); + await f.remoteHost.input('click', 'watch'); + assert.equal(f.received.length, 3); +}); diff --git a/tests/remote-input-compose.test.cjs b/tests/remote-input-compose.test.cjs new file mode 100644 index 00000000..02311226 --- /dev/null +++ b/tests/remote-input-compose.test.cjs @@ -0,0 +1,105 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { PassThrough } = require('node:stream'); +const { interactive } = require('../scripts/faceclaw-input.cjs'); +const { createEditor } = require('../scripts/faceclaw-input-editor.cjs'); + +async function until(predicate) { + for (let i = 0; i < 100; i++) { + if (predicate()) return; + await new Promise(resolve => setTimeout(resolve, 5)); + } + assert.fail('Timed out waiting for interactive state'); +} +async function fixture(t, transmit) { + const input = new PassThrough(), output = new PassThrough(), modes = [], requests = []; + let printed = '', stopped = false; + input.isTTY = output.isTTY = true; input.isRaw = false; output.columns = 100; + input.setRawMode = value => { input.isRaw = value; modes.push(value); }; + output.on('data', data => { printed += data.toString(); }); + const task = interactive({}, input, output, async (_options, payload) => { + requests.push(payload); if (transmit) return transmit(payload); + }).then(() => { stopped = true; }); + t.after(async () => { if (!stopped) input.write('\x03'); await task; input.destroy(); output.destroy(); }); + await until(() => printed.includes('Connected.')); + return { input, requests, modes, task, printed: () => printed, stopped: () => stopped, + messages: () => requests.filter(r => r.action !== 'ping') }; +} + +test('i composes with Enquirer editing and sends text once; editing keys never become gestures', async t => { + const f = await fixture(t); + // Start and type in one input chunk. Left/right, delete/backspace, Ctrl-A/E, + // and Ctrl-W exercise the actual Enquirer input editor, not a stub. + f.input.write('ihello worxd\x1b[D\x1b[D\x1b[3~l\x1b[CX\x7f\x05!\x01Say \x05 extra\x17\r'); + await until(() => f.messages().length === 1); + assert.deepEqual(f.messages(), [{ action: 'text', text: 'Say hello world!' }]); + assert.match(f.printed(), /Sent to foreground window/); + f.input.write('\x1b[A'); + await until(() => f.messages().length === 2); + assert.deepEqual(f.messages()[1], { action: 'input', gesture: 'swipe-up', source: 'watch' }); +}); + +test('a sends to assistant and i/a remain literal while composing; Unicode is preserved', async t => { + const f = await fixture(t); + f.input.write('aAssistant: café 世界 👋 i a\r'); + await until(() => f.messages().length === 1); + assert.deepEqual(f.messages(), [{ action: 'assistant', text: 'Assistant: café 世界 👋 i a' }]); +}); + +test('Esc discards and clears a draft, then returns to controls with a fresh next draft', async t => { + const f = await fixture(t); + f.input.write('isecret draft\t\x1b[A\x1b[B'); + await until(() => f.printed().includes('secret draft')); + f.input.write('\x1b'); + await until(() => f.printed().includes('Draft discarded.')); + assert.deepEqual(f.messages(), []); + f.input.write('a\r'); + await until(() => f.printed().includes('Empty draft; nothing sent.')); + assert.deepEqual(f.messages(), []); + f.input.write('\r'); + await until(() => f.messages().length === 1); + assert.deepEqual(f.messages()[0], { action: 'input', gesture: 'click', source: 'watch' }); + f.input.write('ifresh\r'); + await until(() => f.messages().length === 2); + assert.equal(f.messages()[1].text, 'fresh'); +}); + +test('Ctrl-C while composing clears the prompt and restores raw mode without sending a draft', async t => { + const f = await fixture(t); + f.input.write('aunfinished'); + await until(() => f.printed().includes('unfinished')); + f.input.write('\x03'); await f.task; + assert.deepEqual(f.messages(), []); + assert.deepEqual(f.modes, [true, false]); + assert.equal(f.input.listenerCount('keypress'), 0); +}); + +test('composition-only token works without input permission and a rejected text does not kill the session', async t => { + let rejectText = true; + const f = await fixture(t, payload => { + assert.notEqual(payload.permission, 'input'); + if (payload.action === 'text' && rejectText) { + rejectText = false; + throw Object.assign(new Error('Token lacks text permission.'), { code: 'forbidden' }); + } + }); + f.input.write('ino access\r'); + await until(() => f.printed().includes('Token lacks text permission.')); + assert.equal(f.stopped(), false); + f.input.write('aallowed\r'); + await until(() => f.printed().includes('Sent to assistant.')); + assert.deepEqual(f.messages(), [{ action: 'text', text: 'no access' }, { action: 'assistant', text: 'allowed' }]); +}); + +test('editor validates the API length limit and lets the user fix the draft', async () => { + const input = new PassThrough(), output = new PassThrough(); output.columns = 80; + let printed = ''; output.on('data', data => { printed += data; }); + const editor = createEditor('text', input, output); + editor.keypress('x'.repeat(8001), {}); editor.keypress('\r', { name: 'return' }); + await until(() => printed.includes('Maximum 8000 characters.')); + editor.keypress('\x01', { name: 'a', ctrl: true }); + editor.keypress('\x0b', { name: 'k', ctrl: true }); + editor.keypress('valid', {}); editor.keypress('\r', { name: 'return' }); + assert.equal(await editor.result, 'valid'); + input.destroy(); output.destroy(); +}); diff --git a/tests/remote-input-native.cjs b/tests/remote-input-native.cjs new file mode 100644 index 00000000..392215aa --- /dev/null +++ b/tests/remote-input-native.cjs @@ -0,0 +1,72 @@ +// macOS host probe for the production iOS socket implementation. Run separately: +// node tests/remote-input-native.cjs (requires Xcode command-line tools). +const assert = require('node:assert/strict'); +const { execFileSync, spawn } = require('node:child_process'); +const fs = require('node:fs'); +const os = require('node:os'); +const path = require('node:path'); +const net = require('node:net'); +const crypto = require('node:crypto'); +const { send } = require('../scripts/faceclaw-input.cjs'); +const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'faceclaw-input-')); +async function main() { + const harness = path.join(directory, 'main.m'); + fs.writeFileSync(harness, ` +#import "FaceclawRemoteInput.h" +#import +int main(int argc, const char **argv) { @autoreleasepool { + FaceclawRemoteInput *server = [FaceclawRemoteInput shared]; + NSString *secret = [server randomSecret]; + if (secret.length != 64 || ![[server tokenDigest:@"test"] isEqual:@"9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"]) return 2; + NSString *failure = [server start:atoi(argv[1])]; + if (failure.length) return 3; + puts("READY"); fflush(stdout); + for (;;) { @autoreleasepool { + NSString *raw = [server nextRequest]; + if (raw) { + NSDictionary *request = [NSJSONSerialization JSONObjectWithData:[raw dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; + NSDictionary *body = [NSJSONSerialization JSONObjectWithData:[request[@"body"] dataUsingEncoding:NSUTF8StringEncoding] options:0 error:nil]; + if ([body[@"text"] isEqual:@"stop"]) { [server stop]; usleep(300000); puts("STOPPED"); fflush(stdout); return 0; } + NSData *reply = [NSJSONSerialization dataWithJSONObject:@{@"ok":@YES, @"echo":body ?: @{}} options:0 error:nil]; + [server complete:[request[@"id"] longLongValue] response:[[NSString alloc] initWithData:reply encoding:NSUTF8StringEncoding]]; + } + usleep(1000); + }} +}} +`); + const binary = path.join(directory, 'probe'); + execFileSync('xcrun', ['clang', '-fobjc-arc', '-framework', 'Foundation', '-framework', 'Security', + '-IApp_Resources/iOS/src', harness, 'App_Resources/iOS/src/FaceclawRemoteInput.m', + 'App_Resources/iOS/src/FaceclawCrypto.m', '-o', binary], { stdio: 'inherit' }); + const reservation = net.createServer(); + await new Promise((resolve, reject) => { reservation.once('error', reject); reservation.listen(0, '127.0.0.1', resolve); }); + const port = reservation.address().port; + await new Promise(resolve => reservation.close(resolve)); + const child = spawn(binary, [String(port)], { stdio: ['ignore', 'pipe', 'inherit'] }); + const exited = new Promise(resolve => child.once('exit', resolve)); + try { + await new Promise((resolve, reject) => { + child.stdout.once('data', data => data.toString().includes('READY') ? resolve() : reject(new Error('Probe not ready'))); + child.once('exit', code => reject(new Error(`Probe exited: ${code}`))); + child.once('error', reject); + }); + const options = { host: '127.0.0.1', port, token: 'fc1_' + crypto.randomBytes(32).toString('hex') }; + const reply = await send(options, { action: 'assistant', text: 'Native round trip 👋 世界' }); + assert.equal(reply.echo.text, 'Native round trip 👋 世界'); + assert.equal(reply.echo.token, options.token); + // Invalid UTF-8 and oversized frames close without becoming JS requests. + for (const frame of [Buffer.from([0xff, 0x0a]), Buffer.from('x'.repeat(65537) + '\n')]) { + await new Promise((resolve, reject) => { + const socket = net.createConnection({ host: options.host, port }, () => socket.end(frame)); + socket.on('data', () => reject(new Error('Invalid frame was dispatched'))); + socket.on('error', error => error.code === 'ECONNRESET' ? resolve() : reject(error)); + socket.on('close', resolve); + }); + } + await send(options, { action: 'input', gesture: 'click' }); + await assert.rejects(send(options, { action: 'text', text: 'stop' })); + assert.equal(await exited, 0); + console.log('Native socket probe passed: crypto, framing, UTF-8, multiple connections, shutdown.'); + } finally { child.kill(); } +} +main().catch(error => { console.error(error); process.exitCode = 1; }).finally(() => fs.rmSync(directory, { recursive: true, force: true })); diff --git a/tests/remote-input-pty.py b/tests/remote-input-pty.py new file mode 100644 index 00000000..76fcde27 --- /dev/null +++ b/tests/remote-input-pty.py @@ -0,0 +1,140 @@ +#!/usr/bin/env python3 +"""Exercise the real CLI through a pseudo-terminal and a FIN-sensitive TCP peer.""" +import json +import os +import pty +import select +import signal +import socket +import subprocess +import termios +import threading +import time +from pathlib import Path + +root = Path(__file__).resolve().parents[1] +server = socket.socket() +server.bind(('127.0.0.1', 0)) +server.listen(8) +server.settimeout(.1) +requests, failures = [], [] +finished = threading.Event() +drop_next = threading.Event() + + +def serve(): + while not finished.is_set(): + try: + conn, _ = server.accept() + except socket.timeout: + continue + with conn: + conn.settimeout(2) + data = b'' + while b'\n' not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + request = json.loads(data) + requests.append(request) + # A forwarder may tear down both directions on FIN. Do likewise. + conn.settimeout(.04) + try: + if conn.recv(1, socket.MSG_PEEK) == b'': + failures.append('CLI sent FIN before reply') + continue + except socket.timeout: + pass + if request['action'] == 'input' and drop_next.is_set(): + drop_next.clear() + continue + conn.sendall(b'{"ok":true}\n') + + +thread = threading.Thread(target=serve, daemon=True) +thread.start() +master, slave = pty.openpty() +termios.tcsetwinsize(slave, (24, 100)) +original = termios.tcgetattr(slave) +env = {**os.environ, 'FACECLAW_TOKEN': 'fc1_' + 'a' * 64} +child = subprocess.Popen(['node', str(root / 'scripts/faceclaw-input.cjs'), '--port', str(server.getsockname()[1]), + 'interactive'], stdin=slave, stdout=slave, stderr=slave, env=env, cwd=root) +output = bytearray() + + +def wait_for(predicate, timeout=7): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if select.select([master], [], [], .03)[0]: + output.extend(os.read(master, 8192)) + if predicate(): + return + assert child.poll() is None, output.decode(errors='replace') + raise AssertionError(output.decode(errors='replace')) + + +try: + wait_for(lambda: b'Connected.' in output) + assert requests[0]['action'] == 'ping', requests + for sequence, gesture in [(b'\x1b[A', 'swipe-up'), (b'\r', 'click'), (b'\x1b', 'double-click'), (b'\t', 'long-press')]: + os.write(master, sequence) + wait_for(lambda: any(r.get('gesture') == gesture for r in requests)) + time.sleep(.1) # Each key follows the previous request's complete lifecycle. + assert child.poll() is None + before = len(requests) + os.write(master, b'ihello worxd\x1b[D\x1b[D\x1b[3~l\x1b[C\x01Say \x05!\r') + wait_for(lambda: b'Sent to foreground window.' in output) + assert [r for r in requests[before:] if r['action'] != 'ping'] == [ + {'action': 'text', 'text': 'Say hello world!', 'version': 1, 'token': env['FACECLAW_TOKEN']}] + os.write(master, 'aHello assistant 世界\r'.encode()) + wait_for(lambda: b'Sent to assistant.' in output) + assert requests[-1]['action'] == 'assistant' and requests[-1]['text'] == 'Hello assistant 世界' + before = len(requests) + os.write(master, b'idiscard this draft') + wait_for(lambda: b'discard this draft' in output) + os.write(master, b'\x1b') + wait_for(lambda: b'Draft discarded.' in output) + assert not [r for r in requests[before:] if r['action'] != 'ping'], 'Esc leaked an input or sent a draft' + drop_next.set() + os.write(master, b'\x1b[B') + wait_for(lambda: b'without a response' in output) + assert child.poll() is None + wait_for(lambda: output.count(b'Connected.') >= 2) + before = len(requests) + os.write(master, b'\x1b[C') + wait_for(lambda: any(r.get('gesture') == 'swipe-right' for r in requests[before:])) + time.sleep(.1) # Let the final reply arrive before testing orderly exit. + before = len(requests) + os.write(master, b'aunfinished final draft') + wait_for(lambda: b'unfinished final draft' in output) + os.write(master, b'\x03') + # Keep draining prompt cleanup output while waiting for process exit. + wait_for(lambda: child.poll() is not None, timeout=2) + assert child.returncode == 0 + assert termios.tcgetattr(slave) == original, 'Terminal mode was not restored' + assert not failures, failures + assert not [r for r in requests[before:] if r['action'] != 'ping'], 'Ctrl-C sent a draft' + assert len([r for r in requests if r.get('gesture') == 'swipe-down']) == 1, 'Failed input was replayed' + # Signals must also restore the terminal while the editor owns the prompt. + # Enquirer's default signal handling exits before our async cleanup runs. + output.clear() + child = subprocess.Popen(['node', str(root / 'scripts/faceclaw-input.cjs'), '--port', str(server.getsockname()[1]), + 'interactive'], stdin=slave, stdout=slave, stderr=slave, env=env, cwd=root) + wait_for(lambda: b'Connected.' in output) + os.write(master, b'isignal cleanup draft') + wait_for(lambda: b'signal cleanup draft' in output) + child.send_signal(signal.SIGTERM) + wait_for(lambda: child.poll() is not None, timeout=2) + assert child.returncode == 0 + assert termios.tcgetattr(slave) == original, 'Signal exit left the terminal in raw mode' + print('PTY probe passed: immediate connection, repeated keys, text/assistant editing, discard, recovery, Ctrl-C/SIGTERM and terminal restoration.') +finally: + if child.poll() is None: + child.kill() + child.wait() + finished.set() + thread.join(timeout=3) + server.close() + os.close(master) + os.close(slave) diff --git a/tests/remote-input.test.cjs b/tests/remote-input.test.cjs new file mode 100644 index 00000000..ef351ef6 --- /dev/null +++ b/tests/remote-input.test.cjs @@ -0,0 +1,234 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const net = require('node:net'); +const { EventEmitter } = require('node:events'); +const { TokenStore, handleRequest, GESTURES } = require('../.test-build/app/remote/protocol.js'); +const cli = require('../scripts/faceclaw-input.cjs'); +function fixture(permissions = ['input', 'text', 'assistant']) { + let stored = '[]'; const calls = []; + const tokens = new TokenStore(() => stored, value => { stored = value; }, + () => crypto.randomBytes(32).toString('hex'), value => crypto.createHash('sha256').update(value).digest('hex')); + const { token, record } = tokens.create('Test app', permissions); + const host = { ready: () => true, locked: () => false, input: async (...args) => calls.push(args), + acceptsText: () => true, text: text => calls.push(['text', text]), assistantAvailable: () => true, + assistant: text => calls.push(['assistant', text]) }; + const request = payload => handleRequest(JSON.stringify({ version: 1, token, ...payload }), tokens, host); + return { tokens, token, record, host, calls, request, stored: () => stored }; +} +test('tokens are random, persisted only as hashes, and survive reload', () => { + const f = fixture(); + assert.match(f.token, /^fc1_[a-f0-9]{64}$/); + assert.ok(!f.stored().includes(f.token)); + const another = f.tokens.create('Second', ['input']); + assert.notEqual(another.token, f.token); + assert.equal(f.tokens.authenticate(f.token).name, 'Test app'); + assert.equal(f.tokens.authenticate(f.token + 'x'), undefined); + assert.equal(f.tokens.authenticate('fc1_' + '0'.repeat(64)), undefined); + assert.throws(() => f.tokens.create('', ['input'])); + assert.throws(() => f.tokens.create('Empty', [])); +}); +test('permission matrix independently authorizes each operation; edits and revocation take effect immediately', async () => { + for (const permission of ['input', 'text', 'assistant']) { + const f = fixture([permission]); + for (const action of ['input', 'text', 'assistant']) { + const result = await f.request({ action, gesture: 'click', text: 'hello' }); + assert.equal(result.ok, action === permission); + if (!result.ok) assert.equal(result.error, 'forbidden'); + } + f.tokens.permissions(f.record.id, []); + assert.equal((await f.request({ action: permission, gesture: 'click', text: 'hello' })).error, 'forbidden'); + f.tokens.revoke(f.record.id); + assert.equal((await f.request({ action: permission })).error, 'unauthorized'); + } +}); +test('every supported gesture routes with the selected source, directions require watch', async () => { + const f = fixture(); + for (const gesture of GESTURES) { + assert.equal((await f.request({ action: 'input', gesture })).ok, true); + assert.deepEqual(f.calls.at(-1), [gesture, 'watch']); + assert.equal((await f.request({ action: 'input', gesture, source: 'ring' })).ok, !gesture.startsWith('swipe-')); + } +}); +test('malformed and unauthorized requests have no side effects', async () => { + const f = fixture(); + for (const body of ['null', '[]', '{', 'true']) assert.equal((await handleRequest(body, f.tokens, f.host)).ok, false); + for (const payload of [ { token: 'bad' }, { version: 2 }, { action: 'unlock' }, + { action: 'input', gesture: 'wakeword' }, { action: 'input', gesture: 'click', source: 'phone' }, + { action: 'text', text: '' }, { action: 'text', text: 123 }, { action: 'text', text: 'x'.repeat(8001) }, + { action: 'assistant', text: '\0' } ]) assert.equal((await f.request(payload)).ok, false); + assert.deepEqual(f.calls, []); +}); +test('locks block text and assistant; gestures still use lock-screen dispatch', async () => { + const f = fixture(); f.host.locked = () => true; + for (const action of ['text', 'assistant']) assert.equal((await f.request({ action, text: 'hello' })).error, 'locked'); + assert.equal((await f.request({ action: 'input', gesture: 'double-click' })).ok, true); + assert.deepEqual(f.calls, [['double-click', 'watch']]); +}); +test('readiness and unavailable destinations fail; text whitespace is preserved', async () => { + const f = fixture(); + f.host.ready = () => false; + assert.equal((await f.request({ action: 'input', gesture: 'click' })).error, 'unavailable'); + f.host.ready = () => true; f.host.acceptsText = () => false; + assert.equal((await f.request({ action: 'text', text: 'hello' })).error, 'unavailable'); + f.host.assistantAvailable = () => false; + assert.equal((await f.request({ action: 'assistant', text: 'hello' })).error, 'unavailable'); + f.host.acceptsText = () => true; + assert.equal((await f.request({ action: 'text', text: ' a\nb ' })).ok, true); + assert.deepEqual(f.calls, [['text', ' a\nb ']]); + f.host.input = async () => { throw new Error('private detail'); }; + assert.equal((await f.request({ action: 'input', gesture: 'click' })).error, 'failed'); +}); +test('CLI parses one-off commands and maps keyboard input', () => { + const env = { FACECLAW_TOKEN: fixture().token }; + assert.deepEqual(cli.parseArgs(['input', 'up'], env).payload, { action: 'input', gesture: 'swipe-up', source: 'watch' }); + assert.equal(cli.parseArgs(['assistant', 'a message'], env).payload.text, 'a message'); + assert.equal(cli.parseArgs(['interactive'], env).interactive, true); + assert.throws(() => cli.parseArgs(['input', 'up', '--source', 'ring'], env)); + assert.throws(() => cli.parseArgs(['text', ''], env)); + assert.throws(() => cli.parseArgs(['input', 'tap', '--port', '0'], env)); + assert.equal(cli.parseArgs(['--help'], {}).help, true); + assert.deepEqual(['up','down','left','right','return','escape','tab'].map(name => cli.keyGesture({name})), + ['swipe-up','swipe-down','swipe-left','swipe-right','click','double-click','long-press']); +}); +test('CLI exchanges framed UTF-8 requests with the permission-checking handler', async t => { + const f = fixture(['assistant']); + const server = net.createServer({ allowHalfOpen: true }, socket => { + let text = ''; + socket.setEncoding('utf8'); + socket.on('data', async chunk => { + text += chunk; + if (!text.endsWith('\n')) return; + const response = JSON.stringify(await handleRequest(text.trimEnd(), f.tokens, f.host)) + '\n'; + socket.write(response.slice(0, 4)); socket.end(response.slice(4)); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const options = { host: '127.0.0.1', port: server.address().port, token: f.token }; + await cli.send(options, { action: 'assistant', text: 'hello 世界 👋' }); + assert.deepEqual(f.calls, [['assistant', 'hello 世界 👋']]); + await assert.rejects(cli.send(options, { action: 'input', gesture: 'click' }), /lacks input permission/); +}); +test('interactive mode orders inputs and restores raw mode on Ctrl-C', async () => { + const input = new EventEmitter(); input.isTTY = true; input.isRaw = false; + const modes = []; input.setRawMode = value => modes.push(value); input.resume = () => {}; input.pause = () => {}; + const sent = []; + const task = cli.interactive({}, input, { write: () => {} }, async (_opts, payload) => { if (payload.action === 'input') sent.push(payload.gesture); }); + await new Promise(resolve => setImmediate(resolve)); + input.emit('keypress', '', { name: 'up' }); + input.emit('keypress', '', { name: 'return' }); + await new Promise(resolve => setImmediate(resolve)); + input.emit('keypress', '', { name: 'c', ctrl: true }); + await task; + assert.deepEqual(sent, ['swipe-up', 'click']); + assert.deepEqual(modes, [true, false]); + assert.equal(input.listenerCount('keypress'), 0); +}); + +test('interactive terminal escape sequences distinguish arrows from standalone Esc', async () => { + const { PassThrough } = require('node:stream'); + const input = new PassThrough(); input.isTTY = true; + input.setRawMode = () => {}; + const sent = []; + const task = cli.interactive({}, input, { write() {} }, async (_opts, payload) => { if (payload.action === 'input') sent.push(payload.gesture); }); + await new Promise(resolve => setImmediate(resolve)); + input.write('\x1b[A\r\t'); + await new Promise(resolve => setTimeout(resolve, 10)); + input.write('\x1b'); + await new Promise(resolve => setTimeout(resolve, 100)); + input.write('\x03'); + await task; + assert.deepEqual(sent, ['swipe-up', 'click', 'long-press', 'double-click']); + input.destroy(); +}); + +test('quitting interactive mode cancels the in-flight request and drops queued keys', async () => { + const input = new EventEmitter(); input.isTTY = true; + const modes = []; input.setRawMode = value => modes.push(value); input.resume = input.pause = () => {}; + let signal; + const task = cli.interactive({}, input, { write() {} }, (_opts, payload, abortSignal) => { + if (payload.action === 'ping') return Promise.resolve(); + signal = abortSignal; + return new Promise((_resolve, reject) => signal.addEventListener('abort', () => reject(new Error('cancelled')))); + }); + await new Promise(resolve => setImmediate(resolve)); + input.emit('keypress', '', { name: 'up' }); + input.emit('keypress', '', { name: 'down' }); + input.emit('keypress', '', { name: 'c', ctrl: true }); + await task; + assert.equal(signal.aborted, true); + assert.deepEqual(modes, [true, false]); +}); + + +test('text -n preserves the message and suppresses terminal submission only', async () => { + const f = fixture(); const env = { FACECLAW_TOKEN: f.token }; + const received = []; f.host.text = (text, submit) => received.push({ text, submit }); + for (const args of [['text', 'hello'], ['text', '-n', 'hello'], ['text', '-n', 'line one\nline two'], ['text', '--', '-n']]) { + const { payload } = cli.parseArgs(args, env); + assert.equal((await f.request(payload)).ok, true); + } + assert.deepEqual(received, [{ text: 'hello', submit: true }, { text: 'hello', submit: false }, + { text: 'line one\nline two', submit: false }, { text: '-n', submit: true }]); + assert.throws(() => cli.parseArgs(['assistant', '-n', 'hello'], env), /only valid with text/); + assert.equal((await f.request({ action: 'text', text: 'hello', submit: 'false' })).error, 'bad_request'); +}); + +test('ping authenticates and checks input permission without generating an input or requiring connected glasses', async () => { + const f = fixture(['input']); f.host.ready = () => false; + assert.deepEqual(await f.request({ action: 'ping', permission: 'input' }), { ok: true }); + assert.equal((await f.request({ action: 'ping', token: 'bad' })).error, 'unauthorized'); + f.tokens.permissions(f.record.id, ['text']); + assert.equal((await f.request({ action: 'ping', permission: 'input' })).error, 'forbidden'); + assert.deepEqual(f.calls, []); +}); + +test('interactive checks connectivity immediately, reports failures and remains alive until Ctrl-C', async () => { + const input = new EventEmitter(); input.isTTY = true; input.setRawMode = input.resume = input.pause = () => {}; + const requests = [], output = []; let stopped = false; + const task = cli.interactive({}, input, { write: text => output.push(text) }, async (_opts, payload) => { + requests.push(payload); throw new Error('Connection refused'); + }).then(() => { stopped = true; }); + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(requests, [{ action: 'ping' }]); + assert.match(output.join(''), /Connection refused/); assert.equal(stopped, false); + input.emit('keypress', '', { name: 'c', ctrl: true }); await task; +}); + +test('a lost response does not end interactive mode or replay an input on reconnect', async () => { + const input = new EventEmitter(); input.isTTY = true; input.setRawMode = input.resume = input.pause = () => {}; + const requests = [], output = []; let fail = true, stopped = false; + const task = cli.interactive({}, input, { write: text => output.push(text) }, async (_opts, payload) => { + requests.push(payload); + if (payload.action === 'input' && fail) { fail = false; throw new Error('Faceclaw closed the connection without a response.'); } + }).then(() => { stopped = true; }); + try { + await new Promise(resolve => setImmediate(resolve)); + input.emit('keypress', '', { name: 'up' }); + await new Promise(resolve => setImmediate(resolve)); + assert.equal(stopped, false); assert.match(output.join(''), /without a response/); + input.emit('keypress', '', { name: 'down' }); // Discarded while offline. + await new Promise(resolve => setTimeout(resolve, 3100)); + input.emit('keypress', '', { name: 'return' }); + await new Promise(resolve => setImmediate(resolve)); + assert.deepEqual(requests.filter(r => r.action === 'input').map(r => r.gesture), ['swipe-up', 'click']); + } finally { input.emit('keypress', '', { name: 'c', ctrl: true }); await task; } +}); + +test('client keeps the sending side open until the reply, including across a forwarding proxy', async t => { + let prematureEnd = false; + const server = net.createServer(socket => { + let text = '', replied = false; + socket.on('end', () => { if (!replied) prematureEnd = true; }); + socket.on('data', chunk => { + text += chunk.toString(); + if (text.includes('\n')) setTimeout(() => { replied = true; if (!socket.destroyed) socket.end('{"ok":true}\n'); }, 25); + }); + }); + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); + t.after(() => new Promise(resolve => server.close(resolve))); + const options = { host: '127.0.0.1', port: server.address().port, token: fixture().token }; + for (let i = 0; i < 4; i++) await cli.send(options, { action: 'input', gesture: 'click' }); + assert.equal(prematureEnd, false); +}); diff --git a/tests/remote-listeners.test.cjs b/tests/remote-listeners.test.cjs new file mode 100644 index 00000000..a70b78c4 --- /dev/null +++ b/tests/remote-listeners.test.cjs @@ -0,0 +1,40 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const { listenerAddresses, createListenerPool } = require('../.test-build/app/remote/listeners.js'); +const address = (name, address, pointToPoint = true) => ({ name, address, pointToPoint }); +const interfaces = [address('wlan0', '192.168.1.2', false), address('wlan0', 'fd7a:115c:a1e0::2', false), + address('rmnet0', '100.70.1.2'), address('tun0', '100.70.2.3'), address('tun0', 'fd7a:115c:a1e0::1234'), + address('utun2', '10.0.1.2'), address('utun2', 'fe80::1')]; +test('automatic detection selects only tailnet addresses of the Tailscale tunnel, plus localhost', () => { + assert.deepEqual(listenerAddresses(interfaces, 'tailscale'), ['100.70.2.3', '127.0.0.1', 'fd7a:115c:a1e0::1234']); + assert.deepEqual(listenerAddresses([address('tun1', '100.70.1.2')], 'tailscale'), ['127.0.0.1']); + assert.deepEqual(listenerAddresses([address('tailscale0', '100.70.1.2', false)], 'tailscale'), ['100.70.1.2', '127.0.0.1']); +}); +test('explicit tunnel selection persists by name; absent tunnel never falls back to LAN or wildcard', () => { + assert.deepEqual(listenerAddresses(interfaces, 'interface:tun0'), ['100.70.2.3', '127.0.0.1', 'fd7a:115c:a1e0::1234']); + assert.deepEqual(listenerAddresses(interfaces, 'interface:utun2'), ['10.0.1.2', '127.0.0.1']); + for (const choice of ['localhost', 'interface:wlan0', 'interface:rmnet0', 'interface:missing', '0.0.0.0']) + assert.deepEqual(listenerAddresses(interfaces, choice), ['127.0.0.1']); + assert.deepEqual(listenerAddresses([address('tun0', '0.0.0.0'), address('tun0', '::')], 'interface:tun0'), ['127.0.0.1']); +}); +test('listener pool preserves USB, adds/removes tunnel sockets and routes replies with unique IDs', () => { + const sockets = []; + const pool = createListenerPool(() => { + const socket = { start: (port, address) => { socket.address = address; return ''; }, stop: () => { socket.stopped = true; }, + nextRequest: () => socket.pending ? (socket.pending = false, JSON.stringify({ id: 1, body: '{}', expiresAt: Date.now() + 5000 })) : null, + complete: (id, response) => socket.reply = { id, response }, pending: true }; + sockets.push(socket); return socket; + }); + pool.start(8791, ['127.0.0.1']); + pool.start(8791, ['127.0.0.1', '100.70.2.3']); + assert.equal(sockets.length, 2); + const first = JSON.parse(pool.nextRequest()), second = JSON.parse(pool.nextRequest()); + assert.notEqual(first.id, second.id); + pool.complete(first.id, 'first'); pool.complete(second.id, 'second'); + assert.deepEqual(sockets.map(s => s.reply.response), ['first', 'second']); + pool.start(8791, ['127.0.0.1']); + assert.equal(sockets[0].stopped, undefined); assert.equal(sockets[1].stopped, true); + pool.start(8791, ['127.0.0.1', '100.70.2.3']); + pool.complete(second.id, 'late'); assert.equal(sockets[2].reply, undefined); + pool.stop(); assert.ok(sockets.every(s => s.stopped)); +}); diff --git a/tests/remote-text.test.cjs b/tests/remote-text.test.cjs new file mode 100644 index 00000000..df89912b --- /dev/null +++ b/tests/remote-text.test.cjs @@ -0,0 +1,43 @@ +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const ts = require('typescript'); +const crypto = require('node:crypto'); +const { parseArgs } = require('../scripts/faceclaw-input.cjs'); +const { TokenStore, handleRequest } = require('../.test-build/app/remote/protocol.js'); + +function evaluate(source, context) { + vm.runInNewContext(ts.transpileModule(source, { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, context); +} +test('CLI text and text -n reach the real terminal client with exactly one Enter or none', async () => { + const sent = [], native = { exports: {}, require: () => ({}), global: { isIOS: true }, NSUTF8StringEncoding: 4, + NSString: { stringWithString: text => ({ lengthOfBytesUsingEncoding: () => Buffer.byteLength(text), + dataUsingEncoding: () => ({ base64EncodedStringWithOptions: () => Buffer.from(text).toString('base64') }) }) } }; + evaluate(fs.readFileSync('app/native/g2mirror-client.ts', 'utf8'), native); + const client = new native.exports.G2MirrorClient({}); + client.ws = { sendText: json => sent.push(JSON.parse(json)) }; + client.handleMessage(JSON.stringify({ type: 'connect', command: 'test' })); + // Run the production worker message handler, without starting its network/render loops. + const source = ts.createSourceFile('worker.ts', fs.readFileSync('app/apps/terminal/terminal-app.worker.ts', 'utf8'), ts.ScriptTarget.Latest, true); + const handler = source.statements.find(node => ts.isExpressionStatement(node) && ts.isBinaryExpression(node.expression) && node.expression.left.getText(source) === 'global.onmessage'); + assert.ok(handler); + const worker = { global: {}, windows: new Map([['terminal', { kind: 'view', client }]]) }; + evaluate(handler.getText(source), worker); + let stored = '[]'; + const tokens = new TokenStore(() => stored, value => { stored = value; }, () => 'a'.repeat(64), + value => crypto.createHash('sha256').update(value).digest('hex')); + const { token } = tokens.create('CLI', ['text']); + const host = { ready: () => true, locked: () => false, acceptsText: () => true, + text: (text, submit) => worker.global.onmessage({ data: { type: 'text-input', windowId: 'terminal', text, submit } }) }; + for (const noSubmit of [false, true]) { + const text = 'café 世界'; + const args = parseArgs(['text', ...(noSubmit ? ['-n'] : []), text], { FACECLAW_TOKEN: token }); + assert.equal((await handleRequest(JSON.stringify({ version: 1, token, ...args.payload }), tokens, host)).ok, true); + const message = sent.at(-1); + assert.equal(Buffer.from(message.data, 'base64').toString('utf8'), text + (noSubmit ? '' : '\r')); + assert.equal(message.delays === undefined, noSubmit); + } +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index 7edc1895..57e77a20 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -11,6 +11,8 @@ "outDir": "../.test-build" }, "include": [ + "../app/remote/protocol.ts", + "../app/remote/listeners.ts", "../app/g2/ancs-client.ts", "../app/g2/ble-protocol.ts", "../app/graphics/lvgl-font.ts",