Add an Apple Watch app that can control the glasses

This commit is contained in:
James Babcock committed 2026-11-04 02:20:26 -08:00
1 parent c896ce9ac8
commit 2696f3b76c
46 files changed
+3877 -88

No files matched your search

+6
View File
@@ -70,3 +70,9 @@ tests/kotlin/build/
# tools/health-preview.cjs output (rendered PNGs, regenerate with the tool)
preview/
# Apple Watch app: build output, and the settings build_apple_watch.sh writes from build_paths.sh
apple-watch/build/
apple-watch/Local.xcconfig
App_Resources/iOS/watchapp/
xcuserdata/
+20
View File
@@ -57,6 +57,26 @@
<string>Faceclaw connects to the terminal servers and developer app URLs you configure on your local network.</string>
<key>UILaunchScreen</key>
<dict/>
<!-- Required: an app built with the iOS 27 SDK crashes at launch without the scene
life cycle. @nativescript/core switches to it when it sees this key. Its side effect:
the root page only loads once a scene connects, not on a background launch. -->
<key>UIApplicationSceneManifest</key>
<dict>
<key>UIApplicationSupportsMultipleScenes</key>
<false/>
<key>UISceneConfigurations</key>
<dict>
<key>UIWindowSceneSessionRoleApplication</key>
<array>
<dict>
<key>UISceneConfigurationName</key>
<string>Default Configuration</string>
<key>UISceneDelegateClassName</key>
<string>SceneDelegate</string>
</dict>
</array>
</dict>
</dict>
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
+1 -1
View File
@@ -5,7 +5,7 @@ CODE_SIGN_STYLE = Automatic;
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -framework WatchConnectivity -weak_framework FoundationModels
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
@@ -0,0 +1,43 @@
#import <Foundation/Foundation.h>
NS_ASSUME_NONNULL_BEGIN
/**
* Phone side of the Apple Watch remote: the single place that talks to
* WatchConnectivity, so JS only ever sees path + JSON strings. The twin of the
* Android FaceclawWearBridge; the messages are in wear/PROTOCOL.md.
*
* The session activates during launch, ahead of JS, because the watch can wake
* this app in the background and messages arriving before a delegate exists
* are lost. Until JS registers its handlers, messages are answered with
* jsReady:NO so the watch can say "open Faceclaw on the phone".
*
* The public methods and both handlers belong to the main thread.
*/
@interface FaceclawWatchBridge : NSObject
+ (instancetype)shared;
/** WCSession works on this device (every iPhone; no iPad). */
- (BOOL)isAvailable;
/** A paired Apple Watch has the Faceclaw app. It needn't be running. */
- (BOOL)isWatchAppAvailable;
/**
* Register JS. messageHandler gets every watch message; connectionHandler fires
* when isWatchAppAvailable or the watch app's live reachability changes.
*/
- (void)setMessageHandler:(void (^)(NSString *path, NSString *json))messageHandler
connectionHandler:(void (^)(void))connectionHandler;
/**
* Mirror the dashboard state: kept in the application context for the app's
* next launch, and sent live too while it's in front. An unchanged state is
* dropped unless `force`.
*/
- (void)publishState:(NSString *)json force:(BOOL)force;
/** Send an event live. A sticky one also stays in the application context for a suspended app. */
- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky;
/** A live message on any phone -> watch path; dropped if the watch app isn't in front. */
- (void)sendToWatch:(NSString *)path json:(NSString *)json;
/** Reply to a watch message; `seq` echoes the watch's sequence number. */
- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message;
@end
NS_ASSUME_NONNULL_END
+272
View File
@@ -0,0 +1,272 @@
#import "FaceclawWatchBridge.h"
#import <UIKit/UIKit.h>
#import <WatchConnectivity/WatchConnectivity.h>
// Live messages are {path, json, ...}; the watch app's Protocol.swift mirrors these keys.
static NSString *const kKeyPath = @"path";
static NSString *const kKeyJson = @"json";
static NSString *const kKeyUpdatedAt = @"updatedAt";
static NSString *const kKeyEvent = @"event";
static NSString *const kKeyEventAt = @"eventAt";
static NSString *const kKeyAt = @"at";
static NSString *const kKeyBattery = @"battery";
static NSString *const kPathState = @"/faceclaw/state";
static NSString *const kPathStateRequest = @"/faceclaw/state/request";
static NSString *const kPathBattery = @"/faceclaw/battery";
static NSString *const kPathPress = @"/faceclaw/press";
static NSString *const kPathAck = @"/faceclaw/ack";
static NSString *const kPathEvent = @"/faceclaw/event";
/** State requests kept for a JS handler that isn't registered yet. */
static const NSUInteger kPendingLimit = 16;
static const NSTimeInterval kPendingMaxAge = 30;
static long long NowMs(void) {
return (long long)(NSDate.date.timeIntervalSince1970 * 1000);
}
static double ReadSeq(NSString *json) {
NSData *data = [json dataUsingEncoding:NSUTF8StringEncoding];
id parsed = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;
id seq = [parsed isKindOfClass:NSDictionary.class] ? parsed[@"seq"] : nil;
return [seq isKindOfClass:NSNumber.class] ? [seq doubleValue] : 0;
}
@interface FaceclawWatchPendingMessage : NSObject
@property(nonatomic, copy) NSString *path;
@property(nonatomic, copy) NSString *json;
@property(nonatomic, strong) NSDate *receivedAt;
@end
@implementation FaceclawWatchPendingMessage
@end
@interface FaceclawWatchBridge () <WCSessionDelegate>
@property(nonatomic, copy, nullable) void (^messageHandler)(NSString *, NSString *);
@property(nonatomic, copy, nullable) void (^connectionHandler)(void);
@property(nonatomic, strong) NSMutableArray<FaceclawWatchPendingMessage *> *pending;
@property(nonatomic, copy, nullable) NSString *lastState;
@property(nonatomic) long long lastStateAt;
@property(nonatomic, copy, nullable) NSString *heldEvent;
@property(nonatomic) long long heldEventAt;
/** Sends and context updates run here: they cost IPC, and the main thread also runs BLE. */
@property(nonatomic, strong) dispatch_queue_t sendQueue;
/** Latest context waiting for sendQueue; nil when none is queued. Guarded by @synchronized(self). */
@property(nonatomic, copy, nullable) NSDictionary *queuedContext;
@end
@implementation FaceclawWatchBridge
+ (void)load {
// Registering here is what puts the delegate in place during launch, ahead of JS.
[NSNotificationCenter.defaultCenter addObserverForName:UIApplicationDidFinishLaunchingNotification
object:nil
queue:NSOperationQueue.mainQueue
usingBlock:^(NSNotification *note) {
[[FaceclawWatchBridge shared] activate];
}];
}
+ (instancetype)shared {
static FaceclawWatchBridge *instance;
static dispatch_once_t once;
dispatch_once(&once, ^{ instance = [FaceclawWatchBridge new]; });
return instance;
}
- (instancetype)init {
if ((self = [super init])) {
_pending = [NSMutableArray new];
_sendQueue = dispatch_queue_create("com.faceclaw.watch.send", DISPATCH_QUEUE_SERIAL);
}
return self;
}
- (void)activate {
if (!WCSession.isSupported) return;
WCSession *session = WCSession.defaultSession;
if (session.delegate == self) return;
session.delegate = self;
[session activateSession];
}
#pragma mark - Public API (main thread)
- (BOOL)isAvailable {
return WCSession.isSupported;
}
- (BOOL)isWatchAppAvailable {
if (!WCSession.isSupported) return NO;
WCSession *session = WCSession.defaultSession;
if (session.activationState != WCSessionActivationStateActivated) return NO;
return session.isReachable || (session.isPaired && session.isWatchAppInstalled);
}
- (void)setMessageHandler:(void (^)(NSString *, NSString *))messageHandler
connectionHandler:(void (^)(void))connectionHandler {
self.messageHandler = messageHandler;
self.connectionHandler = connectionHandler;
[self activate];
NSArray<FaceclawWatchPendingMessage *> *replay = [self.pending copy];
[self.pending removeAllObjects];
for (FaceclawWatchPendingMessage *message in replay) {
if (-message.receivedAt.timeIntervalSinceNow <= kPendingMaxAge) messageHandler(message.path, message.json);
}
// The watch's last battery report may have arrived while no one was listening.
if (WCSession.isSupported && WCSession.defaultSession.activationState == WCSessionActivationStateActivated) {
[self deliverBatteryFrom:WCSession.defaultSession.receivedApplicationContext];
}
}
- (void)publishState:(NSString *)json force:(BOOL)force {
if (!force && [json isEqualToString:self.lastState]) return;
self.lastState = json;
self.lastStateAt = NowMs();
[self pushContext];
[self sendLive:@{kKeyPath: kPathState, kKeyJson: json, kKeyUpdatedAt: @(self.lastStateAt)}];
}
- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky {
long long at = NowMs();
if (sticky) {
self.heldEvent = json;
self.heldEventAt = at;
[self pushContext];
}
[self sendLive:@{kKeyPath: kPathEvent, kKeyJson: json, kKeyAt: @(at)}];
}
- (void)sendToWatch:(NSString *)path json:(NSString *)json {
[self sendLive:@{kKeyPath: path, kKeyJson: json}];
}
- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message {
NSDictionary *ack = @{@"seq": @(seq), @"ok": @(ok), @"jsReady": @(jsReady), @"message": message ?: @""};
NSData *data = [NSJSONSerialization dataWithJSONObject:ack options:0 error:nil];
if (!data) return;
[self sendToWatch:kPathAck json:[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]];
}
#pragma mark - Sending
- (void)sendLive:(NSDictionary *)message {
if (!WCSession.isSupported) return;
dispatch_async(self.sendQueue, ^{
WCSession *session = WCSession.defaultSession;
if (session.activationState != WCSessionActivationStateActivated || !session.isReachable) return;
[session sendMessage:message replyHandler:nil errorHandler:^(NSError *error) {
NSLog(@"[FaceclawWatch] send %@ failed: %@", message[kKeyPath], error);
}];
});
}
/** Queue the current state and held event as the application context; only the latest matters. */
- (void)pushContext {
if (!WCSession.isSupported) return;
NSMutableDictionary *context = [NSMutableDictionary new];
if (self.lastState) {
context[kKeyJson] = self.lastState;
context[kKeyUpdatedAt] = @(self.lastStateAt);
}
if (self.heldEvent) {
context[kKeyEvent] = self.heldEvent;
context[kKeyEventAt] = @(self.heldEventAt);
}
BOOL alreadyQueued;
@synchronized (self) {
alreadyQueued = self.queuedContext != nil;
self.queuedContext = context;
}
if (alreadyQueued) return;
dispatch_async(self.sendQueue, ^{
NSDictionary *next;
@synchronized (self) {
next = self.queuedContext;
self.queuedContext = nil;
}
WCSession *session = WCSession.defaultSession;
if (!next || session.activationState != WCSessionActivationStateActivated) return;
if (!session.isPaired || !session.isWatchAppInstalled) return;
NSError *error = nil;
if (![session updateApplicationContext:next error:&error]) {
NSLog(@"[FaceclawWatch] application context update failed: %@", error);
}
});
}
#pragma mark - Receiving
- (void)handleMessage:(NSString *)path json:(NSString *)json {
if (self.messageHandler) {
self.messageHandler(path, json);
return;
}
// A battery report is status and a touch-down only matters live; neither expects an ack.
if ([path isEqualToString:kPathBattery] || [path isEqualToString:kPathPress]) return;
// Gestures and commands are refused rather than replayed: the watch reports them as failed,
// and running them later would act on whatever the glasses happen to show by then.
if ([path isEqualToString:kPathStateRequest]) {
NSDate *now = NSDate.date;
while (self.pending.count > 0 && [now timeIntervalSinceDate:self.pending.firstObject.receivedAt] > kPendingMaxAge) {
[self.pending removeObjectAtIndex:0];
}
if (self.pending.count >= kPendingLimit) [self.pending removeObjectAtIndex:0];
FaceclawWatchPendingMessage *message = [FaceclawWatchPendingMessage new];
message.path = path;
message.json = json;
message.receivedAt = now;
[self.pending addObject:message];
}
[self sendAck:ReadSeq(json) ok:NO jsReady:NO message:@"Faceclaw is not running on the phone. Open it to connect the glasses."];
}
/** The watch leaves its battery in the application context while it can't message us live. */
- (void)deliverBatteryFrom:(NSDictionary<NSString *, id> *)context {
id json = context[kKeyBattery];
if ([json isKindOfClass:NSString.class] && self.messageHandler) self.messageHandler(kPathBattery, json);
}
- (void)notifyConnection {
if (self.connectionHandler) self.connectionHandler();
}
#pragma mark - WCSessionDelegate (any thread)
- (void)session:(WCSession *)session activationDidCompleteWithState:(WCSessionActivationState)state error:(NSError *)error {
if (error) NSLog(@"[FaceclawWatch] activation failed: %@", error);
dispatch_async(dispatch_get_main_queue(), ^{
// Anything published before activation couldn't be delivered.
if (self.lastState || self.heldEvent) [self pushContext];
[self deliverBatteryFrom:session.receivedApplicationContext];
[self notifyConnection];
});
}
- (void)sessionDidBecomeInactive:(WCSession *)session {}
- (void)sessionDidDeactivate:(WCSession *)session {
// The user switched watches: the session must be reactivated for the new one.
[session activateSession];
}
- (void)sessionWatchStateDidChange:(WCSession *)session {
dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; });
}
- (void)sessionReachabilityDidChange:(WCSession *)session {
dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; });
}
- (void)session:(WCSession *)session didReceiveMessage:(NSDictionary<NSString *, id> *)message {
id path = message[kKeyPath], json = message[kKeyJson];
if (![path isKindOfClass:NSString.class]) return;
NSString *body = [json isKindOfClass:NSString.class] ? json : @"{}";
dispatch_async(dispatch_get_main_queue(), ^{ [self handleMessage:path json:body]; });
}
- (void)session:(WCSession *)session didReceiveApplicationContext:(NSDictionary<NSString *, id> *)context {
dispatch_async(dispatch_get_main_queue(), ^{ [self deliverBatteryFrom:context]; });
}
@end
+1
View File
@@ -14,5 +14,6 @@ module FaceclawGraphics {
header "FaceclawLoopbackRedirect.h"
header "FaceclawWebSignIn.h"
header "FaceclawResourceUsage.h"
header "FaceclawWatchBridge.h"
export *
}
+8 -1
View File
@@ -13,7 +13,14 @@ remote that drives Faceclaw over the Wearable Data Layer. Its phone-side
counterparts are FaceclawWearBridge.kt, app/native/wear-bridge.ts and
app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md.
`build.sh` and `build_wear.sh` build the Android and Wear OS apps respectively.
apple-watch/ is the same remote for Apple Watch (Swift + SwiftUI), driving the iOS
app over WatchConnectivity. Its phone-side counterparts are
App_Resources/iOS/src/FaceclawWatchBridge.m, app/native/wear-bridge.ios.ts and the
same app/g2/wear-remote.ts. It only links to a phone app that embeds it
(FACECLAW_EMBED_WATCH=1); see apple-watch/README.md.
`build.sh`, `build_wear.sh` and `build_apple_watch.sh` build the Android, Wear OS
and Apple Watch apps respectively.
`build_and_run.sh` builds the Android app then also installs it on a phone
paired with `adb`. Building may require configuring environment variables
(JAVA_HOME, ANDROID_HOME, PATH, ANDROID_KEYSTORE, ANDROID_KEYSTORE) to point at
+3
View File
@@ -163,6 +163,9 @@ section has the phone-side switches. See [wear/README.md](wear/README.md) for
building and installing it; it is a separate Gradle build, sideloaded over
adb, and needs no Play Store. The phone side requires Google Play services.
The iOS app has the same remote for Apple Watch, with the same pad and screens;
see [apple-watch/README.md](apple-watch/README.md).
## Connecting an external agent (OpenClaw)
Instead of calling an LLM API directly from the phone, the voice assistant
+51 -5
View File
@@ -29,7 +29,8 @@ import { OsEventTypeList } from './events'
import { loadDeviceAddresses } from './device-addresses'
import { updateGlassesPresence } from './glasses-presence'
import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat'
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy'
import { WearRemote, type WearRemoteInputKind } from './wear-remote'
import { deviceAddressError } from './ios-peripheral-identity'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
@@ -167,6 +168,8 @@ export class IosPreviewController {
private continuousPending = false
private continuousCaptureToken = 0
private sessionConnected = false
/** The Apple Watch remote; constructed last so it sees a fully wired shell. */
private wearRemote: WearRemote | null = null
private readonly actions: LayerActions = {
...noopLayerActions,
playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload),
@@ -210,7 +213,7 @@ export class IosPreviewController {
prepareVoiceCapture: () => this.prepareVoiceCapture(),
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() },
openNotificationsWindow: () =>
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
onSwitcherBarHeightChanged: () => {
@@ -227,6 +230,7 @@ export class IosPreviewController {
this.unblankAfterShellRender = on
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
this.requestShellRender()
this.wearRemote?.schedulePublish()
},
})
const launcher = createLauncherWindow({
@@ -275,6 +279,20 @@ export class IosPreviewController {
// Every capture end (drained, failed, replaced) may leave continuous capture to restart.
iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture())
for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app))
this.wearRemote = new WearRemote({
apps: ALL_APPS.filter(app => app.showInLauncher !== false && !iosAppUnavailableReason(app.appId)),
injectInput: kind => this.injectWatchInput(kind),
launchApp: id => this.launchApp(id),
// The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons.
connect: () => { resumeAutoReconnect(); return this.connect(true) },
disconnect: () => { suppressAutoReconnect(); return this.disconnect() },
setGlassesLocked: locked => this.setGlassesLocked(locked),
requestShellRender: () => this.requestShellRender(),
previewInputActive: () => !this.communicator && this.runtimeRunning,
getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked,
glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }),
appendLog: line => this.logBluetooth(line),
})
}
/** The compositor frames go to: the live session, else the headless preview. */
@@ -443,6 +461,7 @@ export class IosPreviewController {
// a session's first snapshot) wakes the screen, to the lock notice if locked.
const putOn = wearing && this.glassesWorn === false
this.glassesWorn = wearing
this.wearRemote?.schedulePublish()
this.syncPresence()
this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`)
if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true)
@@ -451,6 +470,7 @@ export class IosPreviewController {
private setGlassesLocked(locked: boolean): void {
if (locked === this.glassesLocked) return
this.glassesLocked = locked
this.wearRemote?.schedulePublish()
if (locked) {
this.glance.dismiss()
if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true
@@ -599,7 +619,25 @@ export class IosPreviewController {
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
/** Input from the Apple Watch: shares the controller queue with physical input. */
private injectWatchInput(kind: WearRemoteInputKind): Promise<void> {
const task = this.inputQueue.then(async () => {
this.onActivity()
// As on Android, and because the watch may be acting on a stale state snapshot: a dark
// display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing.
if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return
// A real hold releases when the finger lifts; a discrete long-press releases at once.
const type = kind === 'long-press-start' ? 'long-press' : kind
await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload))
if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' }))
console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`)
})
this.inputQueue = task.catch(error => this.fail(error))
return task
}
private async receiveInput(event: InputEvent, headTilt = false): Promise<void> {
// Input can change the foreground app or the display state, and the shell doesn't announce those.
this.wearRemote?.schedulePublish()
if (!acceptInput(event)) return
if (this.glassesLocked) {
// Preserve the locked display's sleep/wake controls without dispatching
@@ -807,6 +845,7 @@ export class IosPreviewController {
this.syncRuntime()
if (this.active) this.onConnectionState({ ...this.state })
this.requestShellRender()
this.wearRemote?.schedulePublish()
}
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
private pushBatteryLevels(): void {
@@ -814,11 +853,16 @@ export class IosPreviewController {
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
}
async connect(): Promise<void> {
if (!this.active || this.connecting || this.stopping) return
/** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */
async connect(fromBackground = false): Promise<void> {
if ((!this.active && !fromBackground) || this.connecting || this.stopping) return
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses)
if (error) { this.onError(error); return }
if (error) {
if (this.active) this.onError(error)
else this.logBluetooth(error)
return
}
this.connecting = true
this.sessionCreated = true
try {
@@ -882,6 +926,7 @@ export class IosPreviewController {
this.pushBatteryLevels()
if (this.active) this.onConnectionState({ ...this.state })
this.requestShellRender()
this.wearRemote?.schedulePublish()
}),
communicator.onCaseState(state => {
if (this.communicator !== communicator) return
@@ -889,6 +934,7 @@ export class IosPreviewController {
// the case's own charging is only known while the glasses are docked.
shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}),
caseCharging: state.docked ? state.charging : null })
this.wearRemote?.schedulePublish()
}),
communicator.onFirmwareInfo(info => {
if (this.communicator !== communicator) return
+47 -13
View File
@@ -1,5 +1,6 @@
/**
* The Wear OS watch as an input device and status mirror for the glasses.
* The watch (Wear OS or Apple Watch) as an input device and status mirror for
* the glasses.
*
* Inbound, the watch sends ring-style gestures (tap, double-tap, hold,
* swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path
@@ -17,11 +18,14 @@
* The watch's own battery flows the other way for the glasses' top-bar and
* Glanceboard indicators: the phone polls it (the watch answers from a
* listener service even while its app is closed) and the open watch app
* pushes changes unprompted. The level lives in shell.getBatteryLevels()
* beside the G2 and R1 ones and is cleared when no watch is reachable.
* pushes changes unprompted. An Apple Watch app can't be woken to answer, so
* it reports when it can instead (see wear/PROTOCOL.md). The level lives in
* shell.getBatteryLevels() beside the G2 and R1 ones and is cleared when no
* watch is reachable or the last report has gone stale.
*
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the
* message shapes are documented in wear/PROTOCOL.md.
* Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge) or
* wear-bridge.ios.ts (FaceclawWatchBridge); the message shapes are documented
* in wear/PROTOCOL.md.
*/
import { flagAnalyticsEvent } from "../analytics/analytics";
import { type AppDefinition } from "../apps/app-definition";
@@ -102,6 +106,10 @@ const WATCH_BATTERY_POLL_INTERVAL_MS = 5 * 60_000;
// glasses connecting) would otherwise trigger back to back.
const WATCH_BATTERY_POLL_MIN_GAP_MS = 20_000;
const WATCH_BATTERY_URGENT_POLL_LIMIT = 3;
// Wear OS answers every poll. A closed Apple Watch app only reports when the
// system gives it background time, which can be hours apart, so a level this
// old says more about the watch being gone than about its charge.
const WATCH_BATTERY_MAX_AGE_MS = 4 * 60 * 60_000;
/** What the dashboard controller lends the watch remote. */
export type WearRemoteHost = {
@@ -113,6 +121,11 @@ export type WearRemoteHost = {
disconnect: () => Promise<void>;
setGlassesLocked: (locked: boolean, reason: string) => void;
requestShellRender: () => void;
/**
* The shell is running on the phone's preview with no glasses involved, so
* gestures have somewhere to go even though nothing is connected.
*/
previewInputActive?: () => boolean;
getState: () => {
phase: string;
status: string;
@@ -136,6 +149,7 @@ export class WearRemote {
private lastAssistantSentAtMs = 0;
private lastWatchBatteryPollAtMs = 0;
private watchBatteryKnown = false;
private watchBatteryReportedAtMs = 0;
// Urgent polls that got no answer; a watch app too old to answer them
// must not be poked every publish, so after a few the cadence drops back.
private watchBatteryPollsUnanswered = 0;
@@ -143,7 +157,7 @@ export class WearRemote {
constructor(private readonly host: WearRemoteHost) {
this.active = wearBridge.isAvailable();
if (!this.active) {
host.appendLog("watch: Google Play services unavailable; watch remote disabled");
host.appendLog(`watch: ${global.isIOS ? "WatchConnectivity" : "Google Play services"} unavailable; watch remote disabled`);
return;
}
wearBridge.onMessage((message) => {
@@ -183,10 +197,25 @@ export class WearRemote {
// Catch-all refresh for state the controller does not announce (cheap:
// identical states are dropped on the Java side).
setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS);
setInterval(() => this.pollWatchBattery(false), WATCH_BATTERY_POLL_INTERVAL_MS);
setInterval(() => {
this.expireWatchBattery();
this.pollWatchBattery(false);
}, WATCH_BATTERY_POLL_INTERVAL_MS);
this.schedulePublish();
}
/** Gestures have a display to act on: the glasses, or the phone's preview without them. */
private takesInput(): boolean {
const phase = this.host.getState().phase;
return phase === "connected" || phase === "charging" || this.host.previewInputActive?.() === true;
}
private expireWatchBattery(): void {
if (this.watchBatteryKnown && Date.now() - this.watchBatteryReportedAtMs > WATCH_BATTERY_MAX_AGE_MS) {
this.setWatchBattery(null, null);
}
}
/**
* Ask the reachable watch for its battery. `urgent` skips the poll gap for
* a level that is not known yet (the glasses just connected, the watch
@@ -257,6 +286,7 @@ export class WearRemote {
phase: state.phase,
status: state.status,
connected: state.phase === "connected" || state.phase === "charging",
preview: this.host.previewInputActive?.() === true,
screenOn: shell.isScreenOn(),
locked: state.glassesLocked,
worn: state.glassesWorn,
@@ -307,7 +337,11 @@ export class WearRemote {
// acked (the watch sends it fire-and-forget, without a seq).
this.watchBatteryPollsUnanswered = 0;
const battery = readNumber(message.payload.battery, -1);
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100;
// A report the watch left behind earlier says when it was taken; a live one is current.
const reportedAtMs = readNumber(message.payload.at, Date.now());
const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100 &&
Date.now() - reportedAtMs <= WATCH_BATTERY_MAX_AGE_MS;
this.watchBatteryReportedAtMs = reportedAtMs;
this.setWatchBattery(valid ? battery : null, valid ? message.payload.charging === true : null);
return;
}
@@ -315,8 +349,7 @@ export class WearRemote {
// Touch-down ahead of the gesture that follows it. Fire-and-forget like
// the battery (no seq, never acked): wherever that gesture would be
// refused, the press is dropped silently and the gesture's ack says why.
const phase = this.host.getState().phase;
if (watchRemoteEnabledSetting.get() && (phase === "connected" || phase === "charging")) {
if (watchRemoteEnabledSetting.get() && this.takesInput()) {
await this.host.injectInput("ring-press");
}
return;
@@ -378,8 +411,7 @@ export class WearRemote {
return;
}
const kind = gesture as WearRemoteInputKind;
const state = this.host.getState();
if (state.phase !== "connected" && state.phase !== "charging") {
if (!this.takesInput()) {
ack(false, "The glasses are not connected.");
return;
}
@@ -422,7 +454,7 @@ export class WearRemote {
default:
break;
}
if (!connected) {
if (!this.takesInput()) {
ack(false, "The glasses are not connected.");
return;
}
@@ -561,6 +593,8 @@ export class WearRemote {
}
function readNumber(value: unknown, fallback: number): number {
// Number(null) is 0: a watch reporting an unknown battery would read as empty.
if (value === null) return fallback;
const numeric = typeof value === "number" ? value : Number(value);
return Number.isFinite(numeric) ? numeric : fallback;
}
+52
View File
@@ -0,0 +1,52 @@
/**
* The watch transport's wire names and shapes, shared by the Wear OS bridge
* (wear-bridge.ts) and the Apple Watch one (wear-bridge.ios.ts). See
* wear/PROTOCOL.md for the messages themselves.
*/
/** Message paths shared with the watch apps (wear/app/.../Protocol.kt, apple-watch/Faceclaw/Protocol.swift). */
export const WEAR_PATHS = {
/** Watch -> phone: a ring-style gesture. */
input: "/faceclaw/input",
/** Watch -> phone: a finger landed on the pad (the ring-press ahead of a gesture); never acked. */
press: "/faceclaw/press",
/** Watch -> phone: a shell command (launch app, wake, lock, ...). */
command: "/faceclaw/command",
/** Watch -> phone: a text query for the assistant. */
assistant: "/faceclaw/assistant",
/** Watch -> phone: text to type into the foreground app. */
text: "/faceclaw/text",
/** Watch -> phone: please re-publish the state item. */
stateRequest: "/faceclaw/state/request",
/** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */
battery: "/faceclaw/battery",
/** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */
batteryRequest: "/faceclaw/battery/request",
/** Phone -> watch: reply to any of the above. */
ack: "/faceclaw/ack",
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
event: "/faceclaw/event",
/** Phone -> watch Data Layer item: the mirrored dashboard state. */
state: "/faceclaw/state",
} as const;
export type WearMessage = {
path: string;
/** Parsed JSON payload ({} when the watch sent none or it was malformed). */
payload: Record<string, unknown>;
nodeId: string;
};
export type WearWatchConnection = {
reachable: boolean;
watchName: string;
};
export function parseWearPayload(json: string): Record<string, unknown> {
try {
const parsed = JSON.parse(json);
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
} catch {
return {};
}
}
+123
View File
@@ -0,0 +1,123 @@
/**
* iOS twin of wear-bridge.ts: the phone side of the Apple Watch remote, over
* WatchConnectivity (App_Resources/iOS/src/FaceclawWatchBridge). It has the
* same surface and carries the same (path, JSON) messages, so
* app/g2/wear-remote.ts gives them their meaning on both platforms.
*
* "Reachable" here means an Apple Watch with the Faceclaw app is paired, not
* that the app is running: iOS can't wake a closed watch app, and it is
* suspended whenever the wrist drops, so WCSession's live reachability flips
* constantly. The native side sends live while it can and otherwise leaves
* the latest state in the application context for the app's next wake.
*/
import { WEAR_PATHS, parseWearPayload, type WearMessage, type WearWatchConnection } from './wear-bridge-types'
export { WEAR_PATHS, type WearMessage, type WearWatchConnection }
declare const FaceclawWatchBridge: any
/** WatchConnectivity doesn't expose the watch's own name. */
const WATCH_NAME = 'Apple Watch'
/** Replies go to the session's one counterpart; this only fills WearMessage.nodeId. */
const WATCH_NODE_ID = 'apple-watch'
type MessageListener = (message: WearMessage) => void
type ConnectionListener = (connection: WearWatchConnection) => void
class WearBridge {
private nativeBridge: any = undefined
private listening = false
private readonly messageListeners = new Set<MessageListener>()
private readonly connectionListeners = new Set<ConnectionListener>()
private native(): any {
if (this.nativeBridge !== undefined) return this.nativeBridge
try {
const bridge = FaceclawWatchBridge.shared()
this.nativeBridge = bridge.isAvailable() ? bridge : null
} catch (error) {
console.warn('watch bridge unavailable', error)
this.nativeBridge = null
}
return this.nativeBridge
}
/** Whether WatchConnectivity works on this phone (it does on every iPhone). */
isAvailable(): boolean {
return this.native() !== null
}
private ensureListener(): void {
if (this.listening) return
const native = this.native()
if (!native) return
this.listening = true
native.setMessageHandlerConnectionHandler((path: string, json: string) => {
const message: WearMessage = { path: String(path), payload: parseWearPayload(String(json)), nodeId: WATCH_NODE_ID }
for (const listener of Array.from(this.messageListeners)) {
try { listener(message) }
catch (error) { console.warn('watch message listener failed', error) }
}
}, () => {
const connection = this.getWatchConnection()
for (const listener of Array.from(this.connectionListeners)) {
try { listener(connection) }
catch (error) { console.warn('watch connection listener failed', error) }
}
})
}
/** Subscribe to watch messages. Registers the native listener on first use. */
onMessage(listener: MessageListener): () => void {
this.messageListeners.add(listener)
this.ensureListener()
return () => { this.messageListeners.delete(listener) }
}
/**
* Subscribe to the watch's availability; the current value is delivered
* immediately. Also fires when the watch app comes to the front with the
* value unchanged, which is what prompts WearRemote to push it fresh state.
*/
onWatchConnection(listener: ConnectionListener): () => void {
this.connectionListeners.add(listener)
this.ensureListener()
listener(this.getWatchConnection())
return () => { this.connectionListeners.delete(listener) }
}
getWatchConnection(): WearWatchConnection {
const reachable = Boolean(this.native()?.isWatchAppAvailable())
return { reachable, watchName: reachable ? WATCH_NAME : '' }
}
/** WatchConnectivity pushes changes; there is nothing to re-query. */
refreshWatchNodes(): void {}
/** Mirror the dashboard state; unchanged states are dropped natively unless `force`. */
publishState(state: Record<string, unknown>, force = false): void {
this.native()?.publishStateForce(JSON.stringify(state), force)
}
/**
* Send an event to the watch. A suspended app misses live messages, so all
* but streaming snapshots also stay behind for its next wake: that is how a
* reply that finished with the wrist down still reaches the Assistant screen.
*/
sendEvent(payload: Record<string, unknown>): void {
const sticky = !(payload.type === 'assistant' && payload.phase === 'streaming')
this.native()?.sendEventSticky(JSON.stringify(payload), sticky)
}
/** Send a live message on any phone -> watch path; dropped if the app isn't in front. */
sendToWatch(path: string, payload: Record<string, unknown>): void {
this.native()?.sendToWatchJson(path, JSON.stringify(payload))
}
/** Acknowledge a watch message; `seq` echoes the watch's sequence number. */
sendAck(_nodeId: string, seq: number, ok: boolean, message = ''): void {
this.native()?.sendAckOkJsReadyMessage(seq, ok, true, message)
}
}
export const wearBridge = new WearBridge()
+4 -48
View File
@@ -9,47 +9,12 @@
* Data Layer is part of it), so the rest of the app never has to care.
*/
import { Utils } from "@nativescript/core";
import { WEAR_PATHS, parseWearPayload, type WearMessage, type WearWatchConnection } from "./wear-bridge-types";
export { WEAR_PATHS, type WearMessage, type WearWatchConnection };
declare const com: any;
/** Message paths shared with the watch app (wear/app/.../Protocol.kt). */
export const WEAR_PATHS = {
/** Watch -> phone: a ring-style gesture. */
input: "/faceclaw/input",
/** Watch -> phone: a finger landed on the pad (the ring-press ahead of a gesture); never acked. */
press: "/faceclaw/press",
/** Watch -> phone: a shell command (launch app, wake, lock, ...). */
command: "/faceclaw/command",
/** Watch -> phone: a text query for the assistant. */
assistant: "/faceclaw/assistant",
/** Watch -> phone: text to type into the foreground app. */
text: "/faceclaw/text",
/** Watch -> phone: please re-publish the state item. */
stateRequest: "/faceclaw/state/request",
/** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */
battery: "/faceclaw/battery",
/** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */
batteryRequest: "/faceclaw/battery/request",
/** Phone -> watch: reply to any of the above. */
ack: "/faceclaw/ack",
/** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */
event: "/faceclaw/event",
/** Phone -> watch Data Layer item: the mirrored dashboard state. */
state: "/faceclaw/state",
} as const;
export type WearMessage = {
path: string;
/** Parsed JSON payload ({} when the watch sent none or it was malformed). */
payload: Record<string, unknown>;
nodeId: string;
};
export type WearWatchConnection = {
reachable: boolean;
watchName: string;
};
type MessageListener = (message: WearMessage) => void;
type ConnectionListener = (connection: WearWatchConnection) => void;
@@ -91,7 +56,7 @@ class WearBridge {
onMessage: (path: string, json: string, nodeId: string) => {
const message: WearMessage = {
path: String(path),
payload: parsePayload(String(json)),
payload: parseWearPayload(String(json)),
nodeId: String(nodeId),
};
for (const listener of Array.from(this.messageListeners)) {
@@ -177,13 +142,4 @@ class WearBridge {
}
}
function parsePayload(json: string): Record<string, unknown> {
try {
const parsed = JSON.parse(json);
return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {};
} catch {
return {};
}
}
export const wearBridge = new WearBridge();
+3 -3
View File
@@ -298,7 +298,7 @@ export const glassesBatteryVisibilitySetting = batteryVisibilitySetting(
export const ringBatteryVisibilitySetting = batteryVisibilitySetting(
"ringBatteryVisibility", "R1", "display.battery.ringVisibility",
);
/** The Wear OS watch; the indicator only exists while a watch is reachable. */
/** The watch; the indicator only exists while a watch is reachable. */
export const watchBatteryVisibilitySetting = batteryVisibilitySetting(
"watchBatteryVisibility", "Watch", "display.battery.watchVisibility",
);
@@ -679,7 +679,7 @@ export const mirrorTouchSetting = new ConfigSettingBoolean({
"Let touches on the phone's mirror act on the glasses UI: tap selects what it lands on, double-tap goes back, a hold opens the menu, swipes navigate.",
});
// Wear OS watch remote (app/g2/wear-remote.ts, wear/). All three are read on
// Watch remote (app/g2/wear-remote.ts, wear/, apple-watch/). All three are read on
// every watch message, so a change applies immediately.
export const watchRemoteEnabledSetting = new ConfigSettingBoolean({
id: "watch-remote-enabled",
@@ -687,7 +687,7 @@ export const watchRemoteEnabledSetting = new ConfigSettingBoolean({
storageKey: "watch.remoteEnabled",
defaultValue: true,
description:
"Accept input from the Faceclaw Wear OS app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.",
"Accept input from the Faceclaw watch app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.",
});
export const watchCanUnlockSetting = new ConfigSettingBoolean({
+16 -12
View File
@@ -209,8 +209,8 @@ function settingsSections(): SettingsSection[] {
},
{
label: "Watch",
// Wear OS remote (wear/); the status line above the items says whether
// a watch running the companion app is currently reachable.
// Watch remote (wear/, apple-watch/); the status line above the items says
// whether a watch running the companion app is currently reachable.
items: [
toggleSettingMenuItem(watchRemoteEnabledSetting),
toggleSettingMenuItem(watchCrownClockwiseNextSetting),
@@ -272,17 +272,12 @@ function settingsSections(): SettingsSection[] {
},
];
if (!global.isIOS) return sections;
const deferred = new Set(["Watch"]);
return sections.map(section => {
if (section.label === "Developer") return { ...section, items: [enumSettingMenuItem(ringConnectionModeSetting),
toggleSettingMenuItem(showBleBandwidthSetting)] };
// No downloadable on-device models: iOS's on-device option is Apple's recognizer.
if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] };
if (deferred.has(section.label)) return { label: section.label, items: [{
label: "Not available on iOS yet", disabled: true, onSelect: () => {},
description: `${section.label} integration has not been ported to iOS.`,
}] };
if (section.label === "Display") return { ...section, items: section.items.filter(item =>
item.label !== screenTimeoutSetting.label) };
return section;
@@ -360,7 +355,7 @@ function batteryIndicatorsMenuItem(): MenuItem {
return {
label: "Battery indicators",
description:
"Top-bar battery indicators for the phone, Wear OS watch, glasses, and ring: their style, and whether each shows always, only when low, or never.",
"Top-bar battery indicators for the phone, watch, glasses, and ring: their style, and whether each shows always, only when low, or never.",
onSelect: (ctx) => {
openSettingsSubMenu(ctx, "Battery indicators", [
enumSettingMenuItem(batteryDisplayModeSetting),
@@ -603,12 +598,21 @@ function renderWatchStatus(args: { image: GrayImage; x: number; y: number; width
const font = getDefaultSmallFont();
let status: string;
if (!wearBridge.isAvailable()) {
status = "Google Play services is unavailable on this phone, so no watch can connect.";
status = global.isIOS
? "This device can't pair with an Apple Watch."
: "Google Play services is unavailable on this phone, so no watch can connect.";
} else {
const connection = wearBridge.getWatchConnection();
status = connection.reachable
? `Connected to ${connection.watchName || "a watch"}.`
: "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone.";
if (global.isIOS) {
// iOS only says whether the app is installed; it may not be running.
status = connection.reachable
? "The Faceclaw app is installed on your Apple Watch."
: "No watch connected. Install the Faceclaw watch app (apple-watch/ in the source tree) on an Apple Watch paired with this iPhone.";
} else {
status = connection.reachable
? `Connected to ${connection.watchName || "a watch"}.`
: "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone.";
}
}
const lines = wrapText(font, status, width);
for (let i = 0; i < lines.length; i++) {
+4 -4
View File
@@ -2,9 +2,9 @@ import type { RingInput } from "../g2/ring-input";
/**
* Where an input came from. The ring and the glasses' arms are the stock
* sources; "watch" is the Wear OS remote, which components may treat with a
* richer scheme (see isWatchInput) — the ring's semantics are never changed
* by it.
* sources; "watch" is the watch remote (Wear OS or Apple Watch), which
* components may treat with a richer scheme (see isWatchInput) — the ring's
* semantics are never changed by it.
*/
export type InputSource = "ring" | "left-arm" | "right-arm" | "watch";
@@ -80,7 +80,7 @@ export function makeInputEvent(payload: InputEventPayload): InputEvent {
return { ...payload, timestampMs: Date.now() };
}
/** True for input from the Wear OS remote (any type that carries a source). */
/** True for input from the watch remote (any type that carries a source). */
export function isWatchInput(event: InputEvent): boolean {
return "source" in event && event.source === "watch";
}
+21
View File
@@ -0,0 +1,21 @@
// Settings shared by every target of the watch project.
//
// The watch app is tied to the phone app by bundle ID: its own must begin with
// the phone app's, and WKCompanionAppBundleIdentifier must equal it, or
// WatchConnectivity has no counterpart. Local.xcconfig (not checked in) overrides
// this and sets the signing team; build_apple_watch.sh writes it from build_paths.sh.
FACECLAW_IOS_BUNDLE_ID = com.faceclaw.app
SDKROOT = watchos
SUPPORTED_PLATFORMS = watchos watchsimulator
TARGETED_DEVICE_FAMILY = 4
// Observation (@Observable) needs watchOS 10: Series 6, SE 2, Ultra and later.
WATCHOS_DEPLOYMENT_TARGET = 10.0
SWIFT_VERSION = 5.0
CODE_SIGN_STYLE = Automatic
CURRENT_PROJECT_VERSION = 1
MARKETING_VERSION = 1.0
ENABLE_USER_SCRIPT_SANDBOXING = YES
ALWAYS_SEARCH_USER_PATHS = NO
#include? "Local.xcconfig"
@@ -0,0 +1,315 @@
// !$*UTF8*$!
{
archiveVersion = 1;
classes = {
};
objectVersion = 77;
objects = {
/* Begin PBXContainerItemProxy section */
FC0000000000000000000041 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = FC0000000000000000000001 /* Project object */;
proxyType = 1;
remoteGlobalIDString = FC0000000000000000000010;
remoteInfo = Faceclaw;
};
/* End PBXContainerItemProxy section */
/* Begin PBXFileReference section */
FC0000000000000000000020 /* Faceclaw.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Faceclaw.app; sourceTree = BUILT_PRODUCTS_DIR; };
FC0000000000000000000021 /* FaceclawUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FaceclawUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
FC0000000000000000000022 /* Base.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Base.xcconfig; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFileSystemSynchronizedRootGroup section */
FC0000000000000000000030 /* Faceclaw */ = {isa = PBXFileSystemSynchronizedRootGroup; path = Faceclaw; sourceTree = "<group>"; };
FC0000000000000000000031 /* FaceclawUITests */ = {isa = PBXFileSystemSynchronizedRootGroup; path = FaceclawUITests; sourceTree = "<group>"; };
/* End PBXFileSystemSynchronizedRootGroup section */
/* Begin PBXFrameworksBuildPhase section */
FC0000000000000000000050 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
FC0000000000000000000051 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXFrameworksBuildPhase section */
/* Begin PBXGroup section */
FC0000000000000000000002 = {
isa = PBXGroup;
children = (
FC0000000000000000000022 /* Base.xcconfig */,
FC0000000000000000000030 /* Faceclaw */,
FC0000000000000000000031 /* FaceclawUITests */,
FC0000000000000000000003 /* Products */,
);
sourceTree = "<group>";
};
FC0000000000000000000003 /* Products */ = {
isa = PBXGroup;
children = (
FC0000000000000000000020 /* Faceclaw.app */,
FC0000000000000000000021 /* FaceclawUITests.xctest */,
);
name = Products;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
FC0000000000000000000010 /* Faceclaw */ = {
isa = PBXNativeTarget;
buildConfigurationList = FC0000000000000000000071 /* Build configuration list for PBXNativeTarget "Faceclaw" */;
buildPhases = (
FC0000000000000000000060 /* Sources */,
FC0000000000000000000050 /* Frameworks */,
FC0000000000000000000062 /* Resources */,
);
buildRules = (
);
dependencies = (
);
fileSystemSynchronizedGroups = (
FC0000000000000000000030 /* Faceclaw */,
);
name = Faceclaw;
productName = Faceclaw;
productReference = FC0000000000000000000020 /* Faceclaw.app */;
productType = "com.apple.product-type.application";
};
FC0000000000000000000011 /* FaceclawUITests */ = {
isa = PBXNativeTarget;
buildConfigurationList = FC0000000000000000000072 /* Build configuration list for PBXNativeTarget "FaceclawUITests" */;
buildPhases = (
FC0000000000000000000061 /* Sources */,
FC0000000000000000000051 /* Frameworks */,
FC0000000000000000000063 /* Resources */,
);
buildRules = (
);
dependencies = (
FC0000000000000000000042 /* PBXTargetDependency */,
);
fileSystemSynchronizedGroups = (
FC0000000000000000000031 /* FaceclawUITests */,
);
name = FaceclawUITests;
productName = FaceclawUITests;
productReference = FC0000000000000000000021 /* FaceclawUITests.xctest */;
productType = "com.apple.product-type.bundle.ui-testing";
};
/* End PBXNativeTarget section */
/* Begin PBXProject section */
FC0000000000000000000001 /* Project object */ = {
isa = PBXProject;
attributes = {
BuildIndependentTargetsInParallel = 1;
LastSwiftUpdateCheck = 2640;
LastUpgradeCheck = 2640;
TargetAttributes = {
FC0000000000000000000010 = {
CreatedOnToolsVersion = 26.4;
};
FC0000000000000000000011 = {
CreatedOnToolsVersion = 26.4;
TestTargetID = FC0000000000000000000010;
};
};
};
buildConfigurationList = FC0000000000000000000070 /* Build configuration list for PBXProject "Faceclaw" */;
developmentRegion = en;
hasScannedForEncodings = 0;
knownRegions = (
en,
Base,
);
mainGroup = FC0000000000000000000002;
minimizedProjectReferenceProxies = 1;
preferredProjectObjectVersion = 77;
productRefGroup = FC0000000000000000000003 /* Products */;
projectDirPath = "";
projectRoot = "";
targets = (
FC0000000000000000000010 /* Faceclaw */,
FC0000000000000000000011 /* FaceclawUITests */,
);
};
/* End PBXProject section */
/* Begin PBXResourcesBuildPhase section */
FC0000000000000000000062 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
FC0000000000000000000063 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXResourcesBuildPhase section */
/* Begin PBXSourcesBuildPhase section */
FC0000000000000000000060 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
FC0000000000000000000061 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXTargetDependency section */
FC0000000000000000000042 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = FC0000000000000000000010 /* Faceclaw */;
targetProxy = FC0000000000000000000041 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
/* Begin XCBuildConfiguration section */
FC0000000000000000000080 /* Debug */ = {
isa = XCBuildConfiguration;
baseConfigurationReference = FC0000000000000000000022 /* Base.xcconfig */;
buildSettings = {
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = dwarf;
ENABLE_TESTABILITY = YES;
GCC_OPTIMIZATION_LEVEL = 0;
ONLY_ACTIVE_ARCH = YES;
SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)";
SWIFT_OPTIMIZATION_LEVEL = "-Onone";
};
name = Debug;
};
FC0000000000000000000081 /* Release */ = {
isa = XCBuildConfiguration;
baseConfigurationReference = FC0000000000000000000022 /* Base.xcconfig */;
buildSettings = {
COPY_PHASE_STRIP = NO;
DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym";
SWIFT_COMPILATION_MODE = wholemodule;
VALIDATE_PRODUCT = YES;
};
name = Release;
};
FC0000000000000000000082 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_CFBundleDisplayName = Faceclaw;
INFOPLIST_KEY_UISupportedInterfaceOrientations = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown";
INFOPLIST_KEY_WKCompanionAppBundleIdentifier = "$(FACECLAW_IOS_BUNDLE_ID)";
INFOPLIST_KEY_WKRunsIndependentlyOfCompanionApp = YES;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp";
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
};
name = Debug;
};
FC0000000000000000000083 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
ENABLE_PREVIEWS = YES;
GENERATE_INFOPLIST_FILE = YES;
INFOPLIST_KEY_CFBundleDisplayName = Faceclaw;
INFOPLIST_KEY_UISupportedInterfaceOrientations = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown";
INFOPLIST_KEY_WKCompanionAppBundleIdentifier = "$(FACECLAW_IOS_BUNDLE_ID)";
INFOPLIST_KEY_WKRunsIndependentlyOfCompanionApp = YES;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp";
PRODUCT_NAME = "$(TARGET_NAME)";
SKIP_INSTALL = YES;
SWIFT_EMIT_LOC_STRINGS = YES;
};
name = Release;
};
FC0000000000000000000084 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
GENERATE_INFOPLIST_FILE = YES;
PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp.uitests";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
TEST_TARGET_NAME = Faceclaw;
};
name = Debug;
};
FC0000000000000000000085 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
GENERATE_INFOPLIST_FILE = YES;
PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp.uitests";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
TEST_TARGET_NAME = Faceclaw;
};
name = Release;
};
/* End XCBuildConfiguration section */
/* Begin XCConfigurationList section */
FC0000000000000000000070 /* Build configuration list for PBXProject "Faceclaw" */ = {
isa = XCConfigurationList;
buildConfigurations = (
FC0000000000000000000080 /* Debug */,
FC0000000000000000000081 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
FC0000000000000000000071 /* Build configuration list for PBXNativeTarget "Faceclaw" */ = {
isa = XCConfigurationList;
buildConfigurations = (
FC0000000000000000000082 /* Debug */,
FC0000000000000000000083 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
FC0000000000000000000072 /* Build configuration list for PBXNativeTarget "FaceclawUITests" */ = {
isa = XCConfigurationList;
buildConfigurations = (
FC0000000000000000000084 /* Debug */,
FC0000000000000000000085 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
/* End XCConfigurationList section */
};
rootObject = FC0000000000000000000001 /* Project object */;
}
@@ -0,0 +1,86 @@
<?xml version="1.0" encoding="UTF-8"?>
<Scheme
LastUpgradeVersion = "2640"
version = "1.3">
<BuildAction
parallelizeBuildables = "YES"
buildImplicitDependencies = "YES">
<BuildActionEntries>
<BuildActionEntry
buildForTesting = "YES"
buildForRunning = "YES"
buildForProfiling = "YES"
buildForArchiving = "YES"
buildForAnalyzing = "YES">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "FC0000000000000000000010"
BuildableName = "Faceclaw.app"
BlueprintName = "Faceclaw"
ReferencedContainer = "container:Faceclaw.xcodeproj">
</BuildableReference>
</BuildActionEntry>
</BuildActionEntries>
</BuildAction>
<TestAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
shouldUseLaunchSchemeArgsEnv = "YES">
<Testables>
<TestableReference
skipped = "NO">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "FC0000000000000000000011"
BuildableName = "FaceclawUITests.xctest"
BlueprintName = "FaceclawUITests"
ReferencedContainer = "container:Faceclaw.xcodeproj">
</BuildableReference>
</TestableReference>
</Testables>
</TestAction>
<LaunchAction
buildConfiguration = "Debug"
selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
launchStyle = "0"
useCustomWorkingDirectory = "NO"
ignoresPersistentStateOnLaunch = "NO"
debugDocumentVersioning = "YES"
debugServiceExtension = "internal"
allowLocationSimulation = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "FC0000000000000000000010"
BuildableName = "Faceclaw.app"
ReferencedContainer = "container:Faceclaw.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
</LaunchAction>
<ProfileAction
buildConfiguration = "Release"
shouldUseLaunchSchemeArgsEnv = "YES"
savedToolIdentifier = ""
useCustomWorkingDirectory = "NO"
debugDocumentVersioning = "YES">
<BuildableProductRunnable
runnableDebuggingMode = "0">
<BuildableReference
BuildableIdentifier = "primary"
BlueprintIdentifier = "FC0000000000000000000010"
BuildableName = "Faceclaw.app"
ReferencedContainer = "container:Faceclaw.xcodeproj">
</BuildableReference>
</BuildableProductRunnable>
</ProfileAction>
<AnalyzeAction
buildConfiguration = "Debug">
</AnalyzeAction>
<ArchiveAction
buildConfiguration = "Release"
revealArchiveInOrganizer = "YES">
</ArchiveAction>
</Scheme>
Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

@@ -0,0 +1 @@
{"images":[{"filename":"AppIcon.png","idiom":"universal","platform":"watchos","size":"1024x1024"}],"info":{"author":"xcode","version":1}}
@@ -0,0 +1 @@
{"info":{"author":"xcode","version":1}}
@@ -0,0 +1,59 @@
import SwiftUI
@main
struct FaceclawWatchApp: App {
private let link: PhoneLink
private let prefs: WatchPrefs
init() {
link = PhoneLink()
prefs = WatchPrefs()
// Here rather than in a view: a background launch never builds one.
link.start()
}
var body: some Scene {
WindowGroup {
RootView()
.environment(link)
.environment(prefs)
}
.backgroundTask(.appRefresh(WatchBattery.refreshIdentifier)) { [link] in
await link.reportBatteryFromBackground()
}
}
}
enum Destination: Hashable {
case assistant, apps, status, settings
}
private struct RootView: View {
@Environment(PhoneLink.self) private var link
@Environment(WatchPrefs.self) private var prefs
@Environment(\.scenePhase) private var scenePhase
@State private var path: [Destination] = []
var body: some View {
NavigationStack(path: $path) {
RemoteScreen(link: link, prefs: prefs, open: { path.append($0) })
.navigationDestination(for: Destination.self) { destination in
switch destination {
case .assistant: AssistantScreen()
case .apps: AppsScreen()
case .status: StatusScreen(openSettings: { path.append(.settings) })
case .settings: SettingsScreen()
}
}
}
.overlay(alignment: .bottom) { NoticeOverlay(notice: link.notice) }
.onChange(of: scenePhase, initial: true) { _, phase in
if phase == .active { link.becameActive() } else { link.resignedActive() }
}
// A refused or unanswered gesture gets a distinct buzz, so the wrist
// knows without looking; the notice pill says why.
.onChange(of: link.notice?.id) { _, _ in
if link.notice?.isError == true { Haptics(prefs: prefs).error() }
}
}
}
+185
View File
@@ -0,0 +1,185 @@
import CoreGraphics
import Foundation
import Observation
import os
/**
What the pad's touches and crown turns mean: the watch-side half of the remote,
the twin of the callbacks in wear/.../RemoteScreen.kt.
*/
@MainActor
@Observable
final class PadController {
/** The action buttons, hidden until a swipe up from the bottom edge. */
var controlsVisible = false
@ObservationIgnored private let log = Logger(subsystem: "com.faceclaw.watch", category: "Pad")
@ObservationIgnored private let link: PhoneLink
@ObservationIgnored private let haptics: Haptics
@ObservationIgnored private let recognizer = PadGestureRecognizer()
@ObservationIgnored private var deadlineTask: Task<Void, Never>?
@ObservationIgnored private var longPressSent = false
@ObservationIgnored private var touchStartedAt: Date?
@ObservationIgnored private var crownRemainder = 0.0
/** Pad height in points, for the edge-reveal zone. */
@ObservationIgnored var padHeight: CGFloat = 200
init(link: PhoneLink, prefs: WatchPrefs) {
self.link = link
self.haptics = Haptics(prefs: prefs)
recognizer.onEvent = { [weak self] event in self?.handle(event) }
}
// MARK: - Touch input
func touchChanged(start: CGPoint, location: CGPoint, time: Date) {
if !recognizer.isTouching {
touchStartedAt = time
recognizer.touchDown(at: start, time: now())
}
recognizer.touchMoved(to: location)
syncDeadline()
}
func touchEnded(location: CGPoint) {
recognizer.touchUp(at: location, time: now())
syncDeadline()
}
/** The touch ended without a touch-up: the system took it, or the wrist dropped. */
func touchCancelled() {
recognizer.cancel()
syncDeadline()
}
/**
SwiftUI says a touch is over. If it's the current one and it hasn't ended, no
touch-up is coming, and a hold still has to be released on the phone.
*/
func touchWentAway(startedAt: Date) {
if recognizer.isTouching && startedAt == touchStartedAt { touchCancelled() }
}
private func now() -> TimeInterval { ProcessInfo.processInfo.systemUptime }
private func syncDeadline() {
deadlineTask?.cancel()
deadlineTask = nil
guard let deadline = recognizer.deadline else { return }
let delay = max(0, deadline - now())
deadlineTask = Task { [weak self] in
try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000))
guard !Task.isCancelled, let self else { return }
self.recognizer.advance(to: self.now())
self.syncDeadline()
}
}
// MARK: - What gestures mean
private var glassesScreenIsOff: Bool {
guard let state = link.state else { return false }
return state.takesInput && !state.screenOn
}
/**
A dark display still takes a double-click (wake), and a tap or a hold (the
phone's Glanceboard). Scrolls, swipes and the menu gesture mean nothing there,
so they're dropped here without a buzz.
*/
private func gestureAllowed(_ gesture: RingGesture) -> Bool {
guard glassesScreenIsOff else { return true }
switch gesture {
case .doubleClick, .click, .longPressStart, .longPressRelease: return true
default: return false
}
}
private func handle(_ event: PadEvent) {
log.debug("pad \(String(describing: event), privacy: .public)")
switch event {
case .press:
link.sendPress()
case .tap:
click()
case .doubleTap:
haptics.doubleClick()
link.sendGesture(.doubleClick)
case .shortThenLongPress:
guard !glassesScreenIsOff else { return }
haptics.heavy()
link.sendGesture(.shortThenLongPress)
case .longPressStart:
longPressSent = gestureAllowed(.longPressStart)
if longPressSent {
haptics.heavy()
link.sendGesture(.longPressStart)
}
case .longPressEnd:
if longPressSent {
longPressSent = false
haptics.click()
link.sendGesture(.longPressRelease)
}
case let .swipe(start, dx, dy):
swipe(start: start, dx: dx, dy: dy)
}
}
/** The system's double-tap hand gesture lands here too. */
func click() {
guard gestureAllowed(.click) else { return }
haptics.click()
link.sendGesture(.click)
}
private func swipe(start: CGPoint, dx: CGFloat, dy: CGFloat) {
let vertical = abs(dy) >= abs(dx)
if vertical && !controlsVisible && dy < 0 && start.y >= padHeight - Self.edgeRevealZone {
controlsVisible = true
haptics.click()
} else if vertical && controlsVisible && dy > 0 && start.y >= padHeight - Self.buttonTrayHeight {
controlsVisible = false
haptics.click()
} else if vertical {
// One step per swipe, regardless of length: the glasses repaint per
// step, and multi-step swipes overshoot lists when the display lags
// the finger. Repeated flicks or the crown cover long distances.
send(dy < 0 ? .swipeUp : .swipeDown, steps: 1)
} else {
send(dx < 0 ? .swipeLeft : .swipeRight, steps: 1)
}
}
private func send(_ gesture: RingGesture, steps: Int) {
guard gestureAllowed(gesture) else { return }
haptics.click()
link.sendGesture(gesture, steps: steps)
}
func dismissControls() {
guard controlsVisible else { return }
controlsVisible = false
haptics.click()
}
// MARK: - Crown
/** A crown turn, in detents; positive is clockwise seen from the crown's end. */
func crownTurned(by delta: Double) {
crownRemainder += delta
let steps = Int(crownRemainder.rounded(.towardZero))
guard steps != 0 else { return }
crownRemainder -= Double(steps)
let clockwiseNext = link.state?.crownClockwiseNext == true
let gesture: RingGesture = (steps > 0) == clockwiseNext ? .scrollDown : .scrollUp
guard gestureAllowed(gesture) else { return }
// Capped low: a fast twirl otherwise outruns the display. The crown's
// own detent haptics are the feedback, so there's no buzz here.
link.sendGesture(gesture, steps: min(abs(steps), Self.maxCrownSteps))
}
static let edgeRevealZone: CGFloat = 28
static let buttonTrayHeight: CGFloat = 56
private static let maxCrownSteps = 2
}
@@ -0,0 +1,143 @@
import CoreGraphics
import Foundation
/** What the remote's touch surface can report. */
enum PadEvent: Equatable {
/** A finger landed (the second tap of a pair included), before it's known what the touch will become. */
case press
case tap
case doubleTap
/** A tap whose immediate second press was held: the G2 tap-then-hold gesture. */
case shortThenLongPress
case longPressStart
case longPressEnd
/** The finger travelled past touch slop; dx/dy are the total displacement. */
case swipe(start: CGPoint, dx: CGFloat, dy: CGFloat)
}
/**
One detector for everything the pad does, so a gesture is reported exactly
once: tap, double-tap (a second tap inside the double-tap timeout), a hold
(reported at the long-press timeout and again on release), tap-then-hold (a
second tap held past the long-press timeout) and a swipe. The same state
machine as wear/.../TouchpadGestures.kt, without its two-finger gestures:
SwiftUI on watchOS only ever sees one finger.
It has no clock of its own. The owner passes the time with each event and calls
`advance(to:)` once `deadline` passes, which is what makes it testable.
Every longPressStart is paired with a longPressEnd, whatever ends the touch:
the phone treats a start without a release as a hold still in progress.
*/
final class PadGestureRecognizer {
struct Timing {
var longPress: TimeInterval = 0.4
var doubleTap: TimeInterval = 0.3
var slop: CGFloat = 8
}
private enum Phase {
case idle
case firstTouch(start: CGPoint, moved: Bool)
case holding
case awaitingSecondTouch
case secondTouch(held: Bool)
}
private var phase = Phase.idle
private let timing: Timing
/** When `advance(to:)` next has something to do; nil when only a touch event can move things on. */
private(set) var deadline: TimeInterval?
var onEvent: (PadEvent) -> Void = { _ in }
init(timing: Timing = Timing()) {
self.timing = timing
}
/** A finger is on the pad, as far as this recognizer knows. */
var isTouching: Bool {
switch phase {
case .firstTouch, .holding, .secondTouch: return true
case .idle, .awaitingSecondTouch: return false
}
}
func touchDown(at location: CGPoint, time: TimeInterval) {
if isTouching { cancel() }
onEvent(.press)
if case .awaitingSecondTouch = phase {
phase = .secondTouch(held: false)
} else {
phase = .firstTouch(start: location, moved: false)
}
deadline = time + timing.longPress
}
func touchMoved(to location: CGPoint) {
guard case let .firstTouch(start, moved) = phase, !moved else { return }
if hypot(location.x - start.x, location.y - start.y) > timing.slop {
phase = .firstTouch(start: start, moved: true)
deadline = nil
}
}
func touchUp(at location: CGPoint, time: TimeInterval) {
switch phase {
case let .firstTouch(start, moved):
if moved {
reset()
onEvent(.swipe(start: start, dx: location.x - start.x, dy: location.y - start.y))
} else {
// A plain tap; give a second tap a chance to make it a double.
phase = .awaitingSecondTouch
deadline = time + timing.doubleTap
}
case .holding:
reset()
onEvent(.longPressEnd)
case let .secondTouch(held):
reset()
if !held { onEvent(.doubleTap) }
case .idle, .awaitingSecondTouch:
break
}
}
func advance(to time: TimeInterval) {
guard let deadline, time >= deadline else { return }
self.deadline = nil
switch phase {
case .firstTouch:
phase = .holding
onEvent(.longPressStart)
case .secondTouch:
phase = .secondTouch(held: true)
onEvent(.shortThenLongPress)
case .awaitingSecondTouch:
reset()
onEvent(.tap)
case .idle, .holding:
break
}
}
/**
The touch is over but no touch-up will come: the system took the gesture or
the wrist dropped. A touch in progress is dropped rather than mistaken for a
tap; a tap already completed and a hold already reported are both honoured.
*/
func cancel() {
let cancelled = phase
reset()
switch cancelled {
case .holding: onEvent(.longPressEnd)
case .awaitingSecondTouch: onEvent(.tap)
case .idle, .firstTouch, .secondTouch: break
}
}
private func reset() {
phase = .idle
deadline = nil
}
}
+77
View File
@@ -0,0 +1,77 @@
import Foundation
/** What we know about the phone and the Faceclaw app on it. */
struct LinkStatus: Equatable {
/** The WatchConnectivity session is up; until then nothing below is known. */
var activated = false
/** The iPhone can be messaged right now. */
var phoneReachable = false
/** That iPhone has Faceclaw installed. */
var phoneAppInstalled = false
/**
Whether the phone app's dashboard (the JS side) answered our last message;
nil until we have heard anything. False means Faceclaw is installed but its
dashboard isn't running, so the user has to open it.
*/
var appRunning: Bool?
}
enum StatusTone { case good, busy, off, problem }
/**
The pad's indicator line, with a tone that picks its colour, and an optional
smaller line under it (the charging case's level while charging).
*/
struct StatusLine: Equatable {
var text: String
var tone: StatusTone
var detail: String?
}
/**
The pad's indicator: what the next gesture lands on when everything is
connected (the foreground app, or the screen/lock state in the way of it),
otherwise the connection problem to fix first.
*/
func padLine(state: PhoneState?, link: LinkStatus) -> StatusLine {
guard link.activated else { return StatusLine(text: "Waiting for phone", tone: .busy) }
guard link.phoneAppInstalled else { return StatusLine(text: "Install Faceclaw on phone", tone: .problem) }
guard link.phoneReachable else { return StatusLine(text: "No phone", tone: .problem) }
if link.appRunning == false { return StatusLine(text: "Open Faceclaw on phone", tone: .problem) }
guard let state else { return StatusLine(text: "Waiting for phone", tone: .busy) }
if !state.remoteEnabled { return StatusLine(text: "Watch control off", tone: .problem) }
if state.connected {
if state.charging || state.phase == "charging" {
return StatusLine(text: chargingLine(state.battery), tone: .busy, detail: caseLine(state))
}
if state.silentMode { return StatusLine(text: "Silent mode", tone: .off) }
if state.locked { return StatusLine(text: "Locked", tone: .off) }
if !state.screenOn { return StatusLine(text: "●● Wake", tone: .off) }
if state.listening { return StatusLine(text: "Listening…", tone: .busy) }
return StatusLine(text: state.foreground?.title ?? "Launcher", tone: .good)
}
switch state.phase {
case "connecting": return StatusLine(text: "Connecting…", tone: .busy)
case "retrying": return StatusLine(text: "Reconnecting…", tone: .busy)
case "disconnecting": return StatusLine(text: "Disconnecting…", tone: .busy)
default: break
}
if state.preview {
if state.locked { return StatusLine(text: "Locked", tone: .off) }
if !state.screenOn { return StatusLine(text: "●● Wake", tone: .off) }
return StatusLine(text: state.foreground?.title ?? "Launcher", tone: .good, detail: "Phone preview")
}
return StatusLine(text: "Disconnected", tone: .off)
}
private func chargingLine(_ battery: Int?) -> String {
battery.map { "Charging · G2 \($0)%" } ?? "Charging"
}
/**
"Case 64%", with ⚡ while the case is on USB; nil when the level is unknown. The
case only reports while the glasses are docked, so this is only drawn while they charge.
*/
private func caseLine(_ state: PhoneState) -> String? {
state.caseBattery.map { "Case \($0)%" + (state.caseCharging == true ? " ⚡" : "") }
}
+376
View File
@@ -0,0 +1,376 @@
import Foundation
import Observation
import WatchConnectivity
import WatchKit
import os
/** A transient message for the UI (ack failures, confirmations). */
struct Notice: Equatable {
var text: String
var isError: Bool
var id: UUID
}
/**
The watch's end of the WatchConnectivity conversation with the phone: sends
gestures, commands and text as messages, receives acks and events, and mirrors
the phone's state. The twin of wear/.../PhoneLink.kt, with these differences:
- Live messages only work while this app is in front. The iPhone app can't wake
us, so the phone also leaves its state and latest event in the application
context, which is delivered whenever we next run.
- `isReachable` lags a moment behind a wrist raise, which is exactly when a
gesture arrives, so messages are held briefly instead of failing at once.
*/
@MainActor
@Observable
final class PhoneLink: NSObject {
private(set) var state: PhoneState?
private(set) var link = LinkStatus()
private(set) var assistant = AssistantMirror()
private(set) var notice: Notice?
@ObservationIgnored private let log = Logger(subsystem: "com.faceclaw.watch", category: "PhoneLink")
@ObservationIgnored private var nextSeq: Int64 = 1
@ObservationIgnored private var pendingAcks: [Int64: Task<Void, Never>] = [:]
@ObservationIgnored private var held: [Outgoing] = []
@ObservationIgnored private var heldExpiry: Task<Void, Never>?
@ObservationIgnored private var noticeTask: Task<Void, Never>?
@ObservationIgnored private var batteryTimer: Task<Void, Never>?
@ObservationIgnored private var lastBatteryReported: WatchBatteryReading?
@ObservationIgnored private var lastEventAt: Int64 = 0
private struct Outgoing {
var path: String
var json: String
var seq: Int64
var quiet: Bool
}
private var session: WCSession { WCSession.default }
/** Activate the session. Called during launch, including a background launch. */
func start() {
guard WCSession.isSupported() else { return }
session.delegate = self
session.activate()
}
// MARK: - App lifecycle
/** The app came to the front: catch up on everything that happened while suspended. */
func becameActive() {
refresh()
batteryTimer?.cancel()
// watchOS has no battery-change notification, and the app is only awake to be polled here.
batteryTimer = Task { [weak self] in
while !Task.isCancelled {
try? await Task.sleep(nanoseconds: Self.batteryPollNanos)
guard !Task.isCancelled else { return }
self?.reportBattery(force: false)
}
}
}
func resignedActive() {
batteryTimer?.cancel()
batteryTimer = nil
WatchBattery.scheduleBackgroundRefresh()
}
/** Re-read the session, the cached state and ask the phone for a fresh one. */
func refresh() {
updateLink()
guard session.activationState == .activated else { return }
applyContext(session.receivedApplicationContext)
if link.phoneAppInstalled {
requestState()
reportBattery(force: true)
}
}
/** The system gave us background time: leave the battery level for the phone to find. */
func reportBatteryFromBackground() async {
for _ in 0..<50 where session.activationState != .activated {
try? await Task.sleep(nanoseconds: 100_000_000)
}
reportBattery(force: true)
WatchBattery.scheduleBackgroundRefresh()
}
// MARK: - Outbound
func sendGesture(_ gesture: RingGesture, steps: Int = 1) {
send(WirePath.input, ["gesture": gesture.rawValue, "steps": steps])
}
/**
A finger landed: the phone delivers it as the ring's touch-down (ring-press)
ahead of whatever gesture the touch becomes. Fire-and-forget and silent, since
the gesture that follows reports any failure; never held, since a late
touch-down would be worse than none.
*/
func sendPress() {
guard let state, state.takesInput, state.screenOn, session.isReachable else { return }
transmit(path: WirePath.press, json: "{}")
}
func sendCommand(_ command: String, extras: [String: String] = [:]) {
var body: [String: Any] = ["command": command]
for (key, value) in extras { body[key] = value }
send(WirePath.command, body)
}
func sendAssistantQuery(_ text: String) {
assistant = AssistantMirror(phase: "thinking", text: "")
send(WirePath.assistant, ["text": text])
}
func sendText(_ text: String) {
send(WirePath.text, ["text": text])
}
func requestState() {
send(WirePath.stateRequest, [:], quiet: true)
}
func clearAssistant() {
assistant = AssistantMirror()
}
private func send(_ path: String, _ body: [String: Any], quiet: Bool = false) {
let seq = nextSeq
nextSeq += 1
var body = body
body["seq"] = seq
guard let json = encodeJSON(body) else { return }
let message = Outgoing(path: path, json: json, seq: seq, quiet: quiet)
if session.activationState == .activated && session.isReachable {
transmit(message)
} else if quiet {
// Only state requests are quiet, and reachability returning sends a fresh one.
} else {
hold(message)
}
}
private func hold(_ message: Outgoing) {
log.debug("hold \(message.path, privacy: .public) seq=\(message.seq) (not reachable)")
held.append(message)
if held.count > Self.heldLimit { held.removeFirst() }
guard heldExpiry == nil else { return }
heldExpiry = Task { [weak self] in
try? await Task.sleep(nanoseconds: Self.holdNanos)
guard !Task.isCancelled else { return }
self?.expireHeld()
}
}
private func flushHeld() {
heldExpiry?.cancel()
heldExpiry = nil
let messages = held
held = []
messages.forEach(transmit)
}
private func expireHeld() {
heldExpiry = nil
guard !held.isEmpty else { return }
held = []
showNotice(link.phoneAppInstalled ? "The phone is out of range." : "Faceclaw isn't installed on the phone.", isError: true)
}
private func transmit(_ message: Outgoing) {
log.debug("send \(message.path, privacy: .public) seq=\(message.seq)")
transmit(path: message.path, json: message.json) { [weak self] error in
self?.sendFailed(message, error)
}
// The phone acks every message; silence means its app process never received it.
pendingAcks[message.seq] = Task { [weak self] in
try? await Task.sleep(nanoseconds: Self.ackTimeoutNanos)
guard !Task.isCancelled, let self, self.pendingAcks.removeValue(forKey: message.seq) != nil else { return }
if self.link.appRunning == nil { self.link.appRunning = false }
if !message.quiet { self.showNotice("No reply from the phone.", isError: true) }
}
}
private func transmit(path: String, json: String, onError: (@MainActor (Error) -> Void)? = nil) {
session.sendMessage([WireKey.path: path, WireKey.json: json], replyHandler: nil) { [log] error in
log.error("send \(path, privacy: .public) failed: \(error.localizedDescription, privacy: .public)")
Task { @MainActor in onError?(error) }
}
}
private func sendFailed(_ message: Outgoing, _ error: Error) {
pendingAcks.removeValue(forKey: message.seq)?.cancel()
updateLink()
guard !message.quiet else { return }
let code = (error as? WCError)?.code
switch code {
case .notReachable: showNotice("The phone is out of range.", isError: true)
case .companionAppNotInstalled: showNotice("Faceclaw isn't installed on the phone.", isError: true)
default: showNotice("Couldn't reach the phone.", isError: true)
}
}
/** Send our battery to the phone (for the glasses' Watch indicator), live and for later. */
private func reportBattery(force: Bool) {
guard session.activationState == .activated else { return }
let reading = WatchBattery.read()
if !force && reading == lastBatteryReported { return }
lastBatteryReported = reading
guard let json = encodeJSON(reading.payload(at: Date())) else { return }
if session.isReachable { transmit(path: WirePath.battery, json: json) }
do {
try session.updateApplicationContext([WireKey.battery: json])
} catch {
log.debug("battery context not sent: \(error.localizedDescription, privacy: .public)")
}
}
// MARK: - Inbound
private func handleMessage(path: String, json: String, updatedAt: Int64, at: Int64) {
switch path {
case WirePath.ack:
if let ack = Ack.parse(json) { handleAck(ack) }
case WirePath.event:
handleEvent(json: json, at: at, fromContext: false)
case WirePath.state:
// A live state comes from the running dashboard, unlike one left in the context.
applyState(json: json, updatedAt: updatedAt)
link.appRunning = true
case WirePath.batteryRequest:
reportBattery(force: true)
default:
break
}
}
private func handleAck(_ ack: Ack) {
pendingAcks.removeValue(forKey: ack.seq)?.cancel()
link.appRunning = ack.jsReady
if !ack.ok {
showNotice(ack.message.isEmpty ? "The phone refused that." : ack.message, isError: true)
} else if !ack.message.isEmpty {
showNotice(ack.message, isError: false)
}
}
private func handleEvent(json: String, at: Int64, fromContext: Bool) {
guard let event = PhoneEvent.parse(json) else { return }
// The same event can arrive both live and in the context.
if at != 0 {
guard at > lastEventAt else { return }
lastEventAt = at
}
let age = at == 0 ? 0 : Date().timeIntervalSince1970 - Double(at) / 1000
switch event {
case let .assistant(phase, text):
if fromContext && age > Self.assistantEventMaxAge { return }
assistant = assistant.applying(phase: phase, text: text)
case let .alert(text):
if fromContext && age > Self.alertEventMaxAge { return }
showNotice(text, isError: false)
}
}
private func applyContext(_ context: [String: Any]) {
if let json = context[WireKey.json] as? String {
applyState(json: json, updatedAt: (context[WireKey.updatedAt] as? NSNumber)?.int64Value ?? 0)
}
if let json = context[WireKey.event] as? String {
handleEvent(json: json, at: (context[WireKey.eventAt] as? NSNumber)?.int64Value ?? 0, fromContext: true)
}
}
private func applyState(json: String, updatedAt: Int64) {
// A live message and the context can carry the same or an older state, in either order.
if let state, updatedAt < state.updatedAt { return }
if let parsed = PhoneState.parse(json, updatedAt: updatedAt) { state = parsed }
}
private func updateLink() {
// The session logs an error for each property read before activation.
guard WCSession.isSupported(), session.activationState == .activated else { return }
link.activated = true
link.phoneReachable = session.isReachable
link.phoneAppInstalled = session.isCompanionAppInstalled
if !link.phoneAppInstalled { link.appRunning = nil }
}
private func reachabilityChanged() {
updateLink()
log.debug("reachable=\(self.link.phoneReachable)")
if session.isReachable {
flushHeld()
requestState()
reportBattery(force: true)
}
}
private func activated() {
updateLink()
applyContext(session.receivedApplicationContext)
reachabilityChanged()
}
private func showNotice(_ text: String, isError: Bool) {
noticeTask?.cancel()
notice = Notice(text: text, isError: isError, id: UUID())
noticeTask = Task { [weak self] in
try? await Task.sleep(nanoseconds: isError ? Self.noticeErrorNanos : Self.noticeInfoNanos)
guard !Task.isCancelled else { return }
self?.notice = nil
}
}
private static let ackTimeoutNanos: UInt64 = 4_000_000_000
/** How long a message waits for reachability, e.g. right after a wrist raise. */
private static let holdNanos: UInt64 = 1_500_000_000
private static let heldLimit = 8
private static let batteryPollNanos: UInt64 = 60_000_000_000
private static let noticeInfoNanos: UInt64 = 1_800_000_000
private static let noticeErrorNanos: UInt64 = 3_200_000_000
/** A reply that finished while we slept is still worth showing for this long. */
private static let assistantEventMaxAge: TimeInterval = 600
private static let alertEventMaxAge: TimeInterval = 60
}
extension PhoneLink: WCSessionDelegate {
nonisolated func session(_ session: WCSession, activationDidCompleteWith state: WCSessionActivationState, error: Error?) {
let log = Logger(subsystem: "com.faceclaw.watch", category: "PhoneLink")
log.info("session activation finished: state \(state.rawValue), error \(error?.localizedDescription ?? "none", privacy: .public)")
Task { @MainActor in self.activated() }
}
nonisolated func sessionReachabilityDidChange(_ session: WCSession) {
Task { @MainActor in self.reachabilityChanged() }
}
nonisolated func sessionCompanionAppInstalledDidChange(_ session: WCSession) {
Task { @MainActor in self.updateLink() }
}
nonisolated func session(_ session: WCSession, didReceiveMessage message: [String: Any]) {
guard let path = message[WireKey.path] as? String else { return }
let json = message[WireKey.json] as? String ?? "{}"
let updatedAt = (message[WireKey.updatedAt] as? NSNumber)?.int64Value ?? 0
let at = (message[WireKey.at] as? NSNumber)?.int64Value ?? 0
Task { @MainActor in self.handleMessage(path: path, json: json, updatedAt: updatedAt, at: at) }
}
nonisolated func session(_ session: WCSession, didReceiveApplicationContext applicationContext: [String: Any]) {
Task { @MainActor in self.refreshFromContext() }
}
@MainActor private func refreshFromContext() {
applyContext(session.receivedApplicationContext)
}
}
private func encodeJSON(_ object: [String: Any]) -> String? {
guard let data = try? JSONSerialization.data(withJSONObject: object) else { return nil }
return String(data: data, encoding: .utf8)
}
+232
View File
@@ -0,0 +1,232 @@
import Foundation
/**
Wire format shared with the phone app. Mirror of app/native/wear-bridge-types.ts
(paths), app/g2/wear-remote.ts (payloads) and App_Resources/iOS/src/FaceclawWatchBridge.m
(WatchConnectivity keys); see wear/PROTOCOL.md.
*/
enum WirePath {
static let input = "/faceclaw/input"
/** Watch -> phone: a finger landed on the pad, ahead of the gesture it becomes; no seq, never acked. */
static let press = "/faceclaw/press"
static let command = "/faceclaw/command"
static let assistant = "/faceclaw/assistant"
static let text = "/faceclaw/text"
static let stateRequest = "/faceclaw/state/request"
/** Watch -> phone: our own battery ({battery, charging, at}); no seq, never acked. */
static let battery = "/faceclaw/battery"
/** Phone -> watch: please send `battery` now. Only reaches an app that's in front. */
static let batteryRequest = "/faceclaw/battery/request"
static let ack = "/faceclaw/ack"
static let event = "/faceclaw/event"
static let state = "/faceclaw/state"
}
/** Keys of the WatchConnectivity dictionaries that carry those messages. */
enum WireKey {
static let path = "path"
static let json = "json"
static let updatedAt = "updatedAt"
/** The time a live event was sent, matching `eventAt` when it's also in the context. */
static let at = "at"
/** Application context: the latest event a suspended app would otherwise have missed. */
static let event = "event"
static let eventAt = "eventAt"
/** Application context, watch -> phone: the battery report, as JSON. */
static let battery = "battery"
}
/** Ring-style gestures, as the phone's synthetic ring input understands them. */
enum RingGesture: String {
case click
case doubleClick = "double-click"
case scrollUp = "scroll-up"
case scrollDown = "scroll-down"
/** Finger down: the hold lasts until longPressRelease. */
case longPressStart = "long-press-start"
case longPressRelease = "long-press-release"
/** Tap then hold: opens the app's context menu on the glasses. */
case shortThenLongPress = "short-then-long-press"
/** The spoken "Hey Even": opens the glasses' own voice dialog. */
case wakeword
/**
Spatial (four-way) input the ring cannot produce. The glasses UI gives it a
spatial meaning where one exists (grid rows and columns, panel columns,
tracks) and otherwise treats up/down as scroll, right as select and left as back.
*/
case swipeUp = "swipe-up"
case swipeDown = "swipe-down"
case swipeLeft = "swipe-left"
case swipeRight = "swipe-right"
}
/** Shell commands; things the ring cannot express. */
enum PhoneCommand {
static let launchApp = "launch-app"
static let focusWindow = "focus-window"
static let closeWindow = "close-window"
static let wake = "wake"
static let sleep = "sleep"
static let lock = "lock"
static let unlock = "unlock"
static let connect = "connect"
static let disconnect = "disconnect"
static let closeAssistant = "close-assistant"
static let sidebar = "sidebar"
}
struct AppEntry: Equatable, Identifiable {
var appId: String
var title: String
var id: String { appId }
}
struct WindowEntry: Equatable, Identifiable {
var windowId: String
var title: String
var appId: String
var focused: Bool
var closeable: Bool
var acceptsText: Bool
var id: String { windowId }
}
/** The phone's mirrored dashboard state (/faceclaw/state). */
struct PhoneState: Equatable {
var version = ""
var phase = "disconnected"
var status = ""
var connected = false
/** The shell is running on the phone's preview with no glasses, so gestures still land. */
var preview = false
var screenOn = false
var locked = false
var worn: Bool?
var listening = false
/** Current or last-known G2 charge percentage. */
var battery: Int?
var charging = false
/** Last-known charging-case percentage; live only while the glasses are docked. */
var caseBattery: Int?
/** The case is on USB; nil while the glasses are out of the case. */
var caseCharging: Bool?
var silentMode = false
var foreground: AppEntry?
var windows: [WindowEntry] = []
var apps: [AppEntry] = []
var remoteEnabled = true
/** False by default: clockwise moves to the previous item. */
var crownClockwiseNext = false
var canUnlock = true
var mirrorAssistant = true
var assistantAvailable = false
var updatedAt: Int64 = 0
/** Gestures have a display to act on, whether that's the glasses or the phone's preview. */
var takesInput: Bool { connected || preview }
static func parse(_ json: String, updatedAt: Int64) -> PhoneState? {
guard let o = parseObject(json) else { return nil }
var state = PhoneState()
state.version = o["version"] as? String ?? ""
state.phase = o["phase"] as? String ?? "disconnected"
state.status = o["status"] as? String ?? ""
state.connected = o["connected"] as? Bool ?? false
state.preview = o["preview"] as? Bool ?? false
state.screenOn = o["screenOn"] as? Bool ?? false
state.locked = o["locked"] as? Bool ?? false
state.worn = o["worn"] as? Bool
state.listening = o["listening"] as? Bool ?? false
state.battery = percent(o["battery"])
state.charging = o["charging"] as? Bool ?? false
state.caseBattery = percent(o["caseBattery"])
state.caseCharging = o["caseCharging"] as? Bool
state.silentMode = o["silentMode"] as? Bool ?? false
if let foreground = o["foreground"] as? [String: Any] { state.foreground = appEntry(foreground) }
state.windows = (o["windows"] as? [[String: Any]] ?? []).map { w in
WindowEntry(
windowId: w["windowId"] as? String ?? "",
title: w["title"] as? String ?? "",
appId: w["appId"] as? String ?? "",
focused: w["focused"] as? Bool ?? false,
closeable: w["closeable"] as? Bool ?? true,
acceptsText: w["acceptsText"] as? Bool ?? false
)
}
state.apps = (o["apps"] as? [[String: Any]] ?? []).map(appEntry)
state.remoteEnabled = o["remoteEnabled"] as? Bool ?? true
state.crownClockwiseNext = o["crownClockwiseNext"] as? Bool ?? false
state.canUnlock = o["canUnlock"] as? Bool ?? true
state.mirrorAssistant = o["mirrorAssistant"] as? Bool ?? true
state.assistantAvailable = o["assistantAvailable"] as? Bool ?? false
state.updatedAt = updatedAt
return state
}
private static func appEntry(_ o: [String: Any]) -> AppEntry {
AppEntry(appId: o["appId"] as? String ?? "", title: o["title"] as? String ?? "")
}
private static func percent(_ value: Any?) -> Int? {
guard let number = value as? NSNumber, (0...100).contains(number.intValue) else { return nil }
return number.intValue
}
}
/** The phone's reply to one of our messages. */
struct Ack: Equatable {
var seq: Int64
var ok: Bool
/** False: the phone's app process answered, but its dashboard isn't up. */
var jsReady: Bool
var message: String
static func parse(_ json: String) -> Ack? {
guard let o = parseObject(json) else { return nil }
return Ack(
seq: (o["seq"] as? NSNumber)?.int64Value ?? 0,
ok: o["ok"] as? Bool ?? false,
jsReady: o["jsReady"] as? Bool ?? true,
message: o["message"] as? String ?? ""
)
}
}
/** Phone -> watch events (/faceclaw/event). */
enum PhoneEvent: Equatable {
/** phase: thinking | streaming | done | error | closed; text is the reply so far. */
case assistant(phase: String, text: String)
case alert(text: String)
static func parse(_ json: String) -> PhoneEvent? {
guard let o = parseObject(json) else { return nil }
switch o["type"] as? String {
case "assistant": return .assistant(phase: o["phase"] as? String ?? "", text: o["text"] as? String ?? "")
case "alert": return .alert(text: o["text"] as? String ?? "")
default: return nil
}
}
}
/** The assistant overlay as mirrored from the glasses. */
struct AssistantMirror: Equatable {
/** idle | thinking | streaming | done | error */
var phase = "idle"
var text = ""
func applying(phase next: String, text nextText: String) -> AssistantMirror {
switch next {
case "thinking": return AssistantMirror(phase: "thinking", text: "")
case "streaming": return AssistantMirror(phase: "streaming", text: nextText)
case "done": return AssistantMirror(phase: "done", text: nextText.isEmpty ? text : nextText)
case "error": return AssistantMirror(phase: "error", text: nextText)
case "closed": return AssistantMirror(phase: "idle", text: text)
default: return self
}
}
}
private func parseObject(_ json: String) -> [String: Any]? {
guard let data = json.data(using: .utf8) else { return nil }
return (try? JSONSerialization.jsonObject(with: data)) as? [String: Any]
}
+210
View File
@@ -0,0 +1,210 @@
import SwiftUI
/**
The touchpad. The whole screen is the pad: tap = select, double-tap = back, hold
= long press (released when the finger lifts), tap then hold = the app's menu,
swipes in four directions = spatial navigation, crown = plain scrolling. An upward
swipe that begins at the bottom edge reveals the otherwise-hidden action buttons.
*/
struct RemoteScreen: View {
let link: PhoneLink
let prefs: WatchPrefs
let open: (Destination) -> Void
@Environment(\.isLuminanceReduced) private var isDimmed
@Environment(\.scenePhase) private var scenePhase
@State private var pad: PadController
@State private var crownValue = 0.0
/** When the touch in progress began; nil once it's over, however it ended. */
@GestureState private var touchStart: Date?
@FocusState private var crownFocused: Bool
init(link: PhoneLink, prefs: WatchPrefs, open: @escaping (Destination) -> Void) {
self.link = link
self.prefs = prefs
self.open = open
_pad = State(initialValue: PadController(link: link, prefs: prefs))
}
var body: some View {
GeometryReader { geometry in
ZStack {
padSurface
primaryActionShortcut
clockAndStatus.allowsHitTesting(false)
if pad.controlsVisible && !isDimmed { tray }
}
.onChange(of: geometry.size.height, initial: true) { _, height in pad.padHeight = height }
}
.ignoresSafeArea()
.toolbar(.hidden, for: .navigationBar)
.animation(.easeOut(duration: 0.18), value: pad.controlsVisible)
// Crown events only reach a focused view, and the focus is lost to every screen pushed on top.
.focusable(true)
.focused($crownFocused)
.focusEffectDisabled()
.digitalCrownRotation(
$crownValue,
from: -Self.crownRange,
through: Self.crownRange,
by: 1,
sensitivity: prefs.crownSensitivity.digitalCrown,
isContinuous: true,
isHapticFeedbackEnabled: prefs.haptics
)
.digitalCrownAccessory(.hidden)
.onChange(of: crownValue) { old, new in
// A jump across the whole range is the value wrapping, not a turn.
let delta = new - old
if abs(delta) < Self.crownRange { pad.crownTurned(by: delta) }
}
// A touch-up also clears this, which arrives late enough to fall inside the next
// touch of a quick double-tap; the pad checks which touch it's being told about.
.onChange(of: touchStart) { old, new in
if new == nil, let old { pad.touchWentAway(startedAt: old) }
}
.onChange(of: scenePhase) { _, phase in
if phase == .active { crownFocused = true } else { pad.touchCancelled() }
}
.onAppear { crownFocused = true }
}
private var padSurface: some View {
Color.black
.contentShape(Rectangle())
.gesture(
DragGesture(minimumDistance: 0, coordinateSpace: .local)
.updating($touchStart) { value, state, _ in
if state == nil { state = value.time }
}
.onChanged { pad.touchChanged(start: $0.startLocation, location: $0.location, time: $0.time) }
.onEnded { pad.touchEnded(location: $0.location) }
)
.accessibilityIdentifier("pad")
}
/**
The double-tap hand gesture (Series 9, Ultra 2, SE 3 and later) selects. It needs a
control to attach to, and the pad surface is a gesture rather than a control,
so this one sits underneath everything as a transparent stand-in.
*/
@ViewBuilder
private var primaryActionShortcut: some View {
if #available(watchOS 11.0, *) {
Button("Select") { pad.click() }
.buttonStyle(.plain)
.handGestureShortcut(.primaryAction)
.frame(width: 1, height: 1)
.opacity(0.01)
.allowsHitTesting(false)
.accessibilityHidden(true)
}
}
private var clockAndStatus: some View {
let status = padLine(state: link.state, link: link.link)
return ZStack {
// A large, straight clock fixed on the vertical centre of the pad.
TimelineView(.everyMinute) { context in
Text(context.date.formatted(.dateTime.hour(.defaultDigits(amPM: .omitted)).minute()))
.font(.system(size: 44, weight: .regular, design: .rounded))
.monospacedDigit()
.foregroundStyle(.white)
.lineLimit(1)
.minimumScaleFactor(0.7)
}
if !pad.controlsVisible {
// Offset rather than stacked, so the clock stays put whatever this says.
VStack(spacing: 0) {
Text(status.text)
.font(.headline)
.foregroundStyle(statusColor(status.tone))
.accessibilityIdentifier("padStatus")
if let detail = status.detail {
Text(detail)
.font(.caption2)
.foregroundStyle(.secondary)
}
}
.lineLimit(1)
.minimumScaleFactor(0.7)
.padding(.horizontal, 12)
.offset(y: 46)
.transition(.opacity)
}
}
}
private func statusColor(_ tone: StatusTone) -> Color {
if isDimmed { return .gray }
return tone == .good ? .white : tone.color
}
private var tray: some View {
VStack {
Spacer()
HStack(spacing: 6) {
TrayButton(symbol: "sparkles", label: "Assistant") { openFromTray(.assistant) }
TextFieldLink(prompt: Text("Type into app")) {
TrayIcon(symbol: "keyboard")
} onSubmit: { text in
send(typed: text)
}
.buttonStyle(.plain)
.accessibilityLabel("Type into app")
.accessibilityIdentifier("Type into app")
TrayButton(symbol: "square.grid.2x2", label: "Apps") { openFromTray(.apps) }
TrayButton(symbol: "ellipsis", label: "Status and settings") { openFromTray(.status) }
}
.padding(.bottom, 10)
// A swipe down that starts on a button dismisses the tray; a tap stays the button's.
.simultaneousGesture(
DragGesture(minimumDistance: 12).onEnded { value in
if value.translation.height > abs(value.translation.width) { pad.dismissControls() }
}
)
}
.transition(.move(edge: .bottom).combined(with: .opacity))
}
/** The tray has done its job: coming back should show the pad's status, not the buttons. */
private func openFromTray(_ destination: Destination) {
pad.controlsVisible = false
open(destination)
}
private func send(typed text: String) {
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
Haptics(prefs: prefs).click()
link.sendText(trimmed)
}
/** Far more detents than a session will ever turn; only the changes matter. */
private static let crownRange = 1_000_000.0
}
private struct TrayButton: View {
let symbol: String
let label: String
let action: () -> Void
var body: some View {
Button(action: action) { TrayIcon(symbol: symbol) }
.buttonStyle(.plain)
.accessibilityLabel(label)
.accessibilityIdentifier(label)
}
}
private struct TrayIcon: View {
let symbol: String
var body: some View {
Image(systemName: symbol)
.font(.system(size: 16, weight: .medium))
.foregroundStyle(.white)
.frame(width: 36, height: 36)
.background(Circle().fill(Color(white: 0.2)))
}
}
+369
View File
@@ -0,0 +1,369 @@
import SwiftUI
extension StatusTone {
var color: Color {
switch self {
case .good: return Color(red: 0.48, green: 0.85, blue: 0.56)
case .busy: return Color(red: 0.95, green: 0.76, blue: 0.31)
case .off: return Color(white: 0.54)
case .problem: return Color(red: 1.0, green: 0.54, blue: 0.5)
}
}
}
/** A transient pill at the bottom of the screen: ack failures and confirmations. */
struct NoticeOverlay: View {
let notice: Notice?
var body: some View {
Group {
if let notice {
Text(notice.text)
.font(.caption2)
.multilineTextAlignment(.center)
.lineLimit(2)
.foregroundStyle(notice.isError ? Color(red: 1, green: 0.71, blue: 0.66) : .white)
.padding(.horizontal, 10)
.padding(.vertical, 5)
.background(
RoundedRectangle(cornerRadius: 12)
.fill(notice.isError ? Color(red: 0.29, green: 0.12, blue: 0.1) : Color(white: 0.16))
)
.padding(.horizontal, 8)
.padding(.bottom, 4)
.transition(.opacity)
.accessibilityIdentifier("notice")
}
}
.animation(.easeOut(duration: 0.15), value: notice?.id)
.allowsHitTesting(false)
}
}
/** A list row: a title with an optional smaller line beneath it. */
private struct RowLabel: View {
let title: String
var detail: String?
var body: some View {
VStack(alignment: .leading, spacing: 1) {
Text(title).lineLimit(1)
if let detail {
Text(detail).font(.caption2).foregroundStyle(.secondary).lineLimit(1)
}
}
.frame(maxWidth: .infinity, alignment: .leading)
}
}
private struct SectionHeader: View {
let title: String
init(_ title: String) { self.title = title }
var body: some View {
Text(title)
.font(.footnote)
.foregroundStyle(.secondary)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.top, 6)
.padding(.leading, 4)
}
}
// MARK: - Apps
/**
The glasses' sidebar and launcher, on the wrist: open windows first (tap to bring
one to the front, × to close it), then every launchable app.
*/
struct AppsScreen: View {
@Environment(PhoneLink.self) private var link
@Environment(WatchPrefs.self) private var prefs
@Environment(\.dismiss) private var dismiss
var body: some View {
let windows = link.state?.windows ?? []
let apps = link.state?.apps ?? []
let onScreen = windows.filter(\.focused)
let others = windows.filter { !$0.focused }
ScrollView {
LazyVStack(spacing: 4) {
if !onScreen.isEmpty {
SectionHeader("On screen")
ForEach(onScreen) { windowRow($0) }
}
if !others.isEmpty {
SectionHeader("Also open")
ForEach(others) { windowRow($0) }
}
SectionHeader(apps.isEmpty ? "No app list yet" : "Launch")
ForEach(apps) { app in
Button {
Haptics(prefs: prefs).click()
link.sendCommand(PhoneCommand.launchApp, extras: ["appId": app.appId])
dismiss()
} label: {
RowLabel(title: app.title)
}
.accessibilityIdentifier("app:\(app.appId)")
}
if apps.isEmpty {
Text("Open Faceclaw on the phone and connect the glasses.")
.font(.caption2)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
}
}
.navigationTitle("Apps")
}
/** One open window: tap to bring it to the front, × to close it. */
private func windowRow(_ window: WindowEntry) -> some View {
HStack(spacing: 4) {
Button {
Haptics(prefs: prefs).click()
link.sendCommand(PhoneCommand.focusWindow, extras: ["windowId": window.windowId])
dismiss()
} label: {
// The launcher's window is titled "Apps" on the glasses; here
// that would read as this very screen.
RowLabel(
title: window.appId == "launcher" ? "Launcher" : window.title,
detail: window.acceptsText ? "takes typed text" : nil
)
}
.tint(window.focused ? StatusTone.good.color : nil)
if window.closeable {
Button {
Haptics(prefs: prefs).click()
link.sendCommand(PhoneCommand.closeWindow, extras: ["windowId": window.windowId])
} label: {
Image(systemName: "xmark").font(.system(size: 12, weight: .semibold))
}
.frame(width: 40)
.accessibilityLabel("Close \(window.title)")
}
}
}
}
// MARK: - Assistant
/**
Ask the assistant from the wrist (the system text sheet, which offers dictation)
and read its reply here as it streams, mirrored from the glasses overlay. "Hey
Even" instead uses the glasses' own microphone and voice dialog.
*/
struct AssistantScreen: View {
@Environment(PhoneLink.self) private var link
@Environment(WatchPrefs.self) private var prefs
var body: some View {
let available = link.state?.assistantAvailable != false
let mirror = link.assistant
let haptics = Haptics(prefs: prefs)
ScrollView {
VStack(spacing: 6) {
TextFieldLink(prompt: Text("Ask the assistant")) {
Label("Ask", systemImage: "mic.fill").frame(maxWidth: .infinity)
} onSubmit: { text in
let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines)
guard !trimmed.isEmpty else { return }
haptics.click()
link.sendAssistantQuery(trimmed)
}
.disabled(!available)
.accessibilityIdentifier("assistantAsk")
Text(phaseLabel(mirror.phase, available: available))
.font(.caption)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
.accessibilityIdentifier("assistantPhase")
if !mirror.text.isEmpty {
Text(mirror.text)
.font(.body)
.frame(maxWidth: .infinity, alignment: .leading)
.padding(.horizontal, 4)
}
Button {
haptics.click()
link.sendGesture(.wakeword)
} label: {
RowLabel(title: "“Hey Even”", detail: "uses the glasses mic")
}
if mirror.phase != "idle" {
Button(mirror.phase == "thinking" || mirror.phase == "streaming" ? "Cancel" : "Dismiss") {
haptics.click()
link.sendCommand(PhoneCommand.closeAssistant)
link.clearAssistant()
}
}
}
}
.navigationTitle("Assistant")
// A finished reply deserves a nudge: the wrist may be down while it streams.
.onChange(of: mirror.phase) { _, phase in
if phase == "done" || phase == "error" { haptics.done() }
}
}
private func phaseLabel(_ phase: String, available: Bool) -> String {
switch phase {
case "thinking": return "Thinking…"
case "streaming": return "Replying…"
case "error": return "Error"
case "done": return "Reply"
default: return available ? "Ask something" : "Set up the assistant in Faceclaw's Settings"
}
}
}
// MARK: - Status
/** Glasses and phone status, with the connection / display / lock switches. */
struct StatusScreen: View {
@Environment(PhoneLink.self) private var link
@Environment(WatchPrefs.self) private var prefs
let openSettings: () -> Void
var body: some View {
let state = link.state
let connected = state?.connected == true
let takesInput = state?.takesInput == true
let haptics = Haptics(prefs: prefs)
ScrollView {
VStack(spacing: 4) {
SectionHeader("Glasses")
Button {
haptics.click()
link.sendCommand(connected ? PhoneCommand.disconnect : PhoneCommand.connect)
} label: {
RowLabel(title: connectionLabel(state), detail: connected ? "tap to disconnect" : "tap to connect")
}
.tint(connected ? StatusTone.good.color : nil)
.disabled(link.link.appRunning == false)
let screenOn = state?.screenOn == true
Button {
haptics.click()
link.sendCommand(screenOn ? PhoneCommand.sleep : PhoneCommand.wake)
} label: {
RowLabel(title: screenOn ? "Display on" : "Display off", detail: screenOn ? "tap to turn off" : "tap to wake")
}
.disabled(!takesInput || state?.locked == true)
let locked = state?.locked == true
Button {
haptics.click()
link.sendCommand(locked ? PhoneCommand.unlock : PhoneCommand.lock)
} label: {
RowLabel(title: locked ? "Locked" : "Unlocked", detail: lockDetail(locked: locked, state: state))
}
.disabled(!takesInput || (locked && state?.canUnlock == false))
if let foreground = state?.foreground {
Button {
haptics.click()
link.sendCommand(PhoneCommand.sidebar)
} label: {
RowLabel(title: "On screen: \(foreground.title)", detail: "tap to open the sidebar")
}
.disabled(!takesInput)
}
SectionHeader("Phone")
Button {
haptics.click()
link.refresh()
} label: {
Label {
RowLabel(title: phoneLabel, detail: phoneDetail(state))
} icon: {
Image(systemName: "arrow.clockwise")
}
}
if state?.remoteEnabled == false {
Text("Watch control is turned off in Faceclaw's Settings > Watch.")
.font(.caption2)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
}
Button(action: openSettings) {
Label("Watch settings", systemImage: "gearshape").frame(maxWidth: .infinity, alignment: .leading)
}
}
}
.navigationTitle("Status")
}
private func connectionLabel(_ state: PhoneState?) -> String {
guard let state else { return "No status yet" }
if state.charging || state.phase == "charging" { return "Charging" }
if state.connected { return "Connected" }
switch state.phase {
case "connecting": return "Connecting…"
case "retrying": return "Reconnecting…"
case "disconnecting": return "Disconnecting…"
default: return state.preview ? "Phone preview" : "Disconnected"
}
}
private func lockDetail(locked: Bool, state: PhoneState?) -> String {
if locked && state?.canUnlock == false { return "unlock from the phone" }
return locked ? "tap to unlock" : "tap to lock"
}
private var phoneLabel: String {
let status = link.link
if !status.phoneAppInstalled { return "Faceclaw not installed" }
if !status.phoneReachable { return "No phone connected" }
if status.appRunning == false { return "Faceclaw not running" }
return "iPhone"
}
private func phoneDetail(_ state: PhoneState?) -> String {
let status = link.link
if !status.phoneAppInstalled { return "install it on the phone" }
if !status.phoneReachable { return "check Bluetooth" }
if status.appRunning == false { return "open it on the phone" }
if let version = state?.version, !version.isEmpty { return "Faceclaw v\(version)" }
return "tap to refresh"
}
}
// MARK: - Settings
/** Watch-local preferences: feel of the pad and crown. */
struct SettingsScreen: View {
@Environment(WatchPrefs.self) private var prefs
var body: some View {
@Bindable var prefs = prefs
ScrollView {
VStack(spacing: 4) {
Toggle("Vibrate on input", isOn: $prefs.haptics)
Button {
prefs.crownSensitivity = prefs.crownSensitivity.next
} label: {
RowLabel(title: "Crown sensitivity", detail: prefs.crownSensitivity.rawValue)
}
Text("To keep the remote open between glances, set Settings > General > Return to Clock > Faceclaw on the watch.")
.font(.caption2)
.foregroundStyle(.secondary)
.multilineTextAlignment(.center)
.padding(.top, 6)
Text("Faceclaw watch v\(Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "?")")
.font(.caption2)
.foregroundStyle(.secondary)
}
}
.navigationTitle("Settings")
}
}
+51
View File
@@ -0,0 +1,51 @@
import Foundation
import WatchKit
/** The watch's own charge level and charging state, for the glasses' battery indicators. */
struct WatchBatteryReading: Equatable {
var battery: Int?
var charging: Bool
/** `at` lets the phone discount a report it finds long after it was left in the application context. */
func payload(at date: Date) -> [String: Any] {
[
"battery": battery ?? NSNull(),
"charging": charging,
"at": Int64(date.timeIntervalSince1970 * 1000),
]
}
}
enum WatchBattery {
/** The `.backgroundTask(.appRefresh(...))` identifier. */
static let refreshIdentifier = "battery"
private static let refreshInterval: TimeInterval = 30 * 60
@MainActor
static func read() -> WatchBatteryReading {
let device = WKInterfaceDevice.current()
device.isBatteryMonitoringEnabled = true
// Unknown (the simulator, or just after enabling monitoring) is not a reading of 0%.
let level = device.batteryLevel
let known = device.batteryState != .unknown && level >= 0
let percent = known ? min(100, Int((level * 100).rounded())) : nil
return WatchBatteryReading(
battery: percent,
charging: device.batteryState == .charging || device.batteryState == .full
)
}
/**
A closed app can't answer the phone's battery polls, so it asks to be woken
now and then to leave a report behind. The system decides when, and may
decline; the phone discards the level once it goes stale.
*/
@MainActor
static func scheduleBackgroundRefresh() {
WKApplication.shared().scheduleBackgroundRefresh(
withPreferredDate: Date(timeIntervalSinceNow: refreshInterval),
userInfo: refreshIdentifier as NSString
) { _ in }
}
}
+71
View File
@@ -0,0 +1,71 @@
import Foundation
import Observation
import SwiftUI
import WatchKit
enum CrownSensitivity: String, CaseIterable {
case low = "Low"
case normal = "Normal"
case high = "High"
var next: CrownSensitivity {
let all = Self.allCases
return all[(all.firstIndex(of: self)! + 1) % all.count]
}
var digitalCrown: DigitalCrownRotationalSensitivity {
switch self {
case .low: return .low
case .normal: return .medium
case .high: return .high
}
}
}
/** Watch-local preferences (nothing here is synced to the phone). */
@MainActor
@Observable
final class WatchPrefs {
/** Buzz on every gesture sent, and tick on crown detents. */
var haptics: Bool {
didSet { store.set(haptics, forKey: Keys.haptics) }
}
var crownSensitivity: CrownSensitivity {
didSet { store.set(crownSensitivity.rawValue, forKey: Keys.crown) }
}
@ObservationIgnored private let store = UserDefaults.standard
private enum Keys {
static let haptics = "haptics"
static let crown = "crownSensitivity"
}
init() {
haptics = store.object(forKey: Keys.haptics) as? Bool ?? true
crownSensitivity = store.string(forKey: Keys.crown).flatMap(CrownSensitivity.init(rawValue:)) ?? .normal
}
}
/** The wrist's confirmation of what was sent. */
@MainActor
struct Haptics {
let prefs: WatchPrefs
/** A sent click, swipe or command. */
func click() { play(.click) }
/** A sent double-click: back. */
func doubleClick() { play(.directionDown) }
/** The start of a hold. */
func heavy() { play(.start) }
/** A reply finished arriving. */
func done() { play(.success) }
/** The phone refused or did not answer; always plays. */
func error() { WKInterfaceDevice.current().play(.failure) }
private func play(_ type: WKHapticType) {
if prefs.haptics { WKInterfaceDevice.current().play(type) }
}
}
@@ -0,0 +1,168 @@
import XCTest
/**
Drives the real app on a watch simulator. The screens' own behaviour needs only
the watch; the tests that talk to the phone need Faceclaw running on the paired
iPhone simulator, and skip themselves otherwise. See apple-watch/README.md.
*/
@MainActor
final class PadUITests: XCTestCase {
private var app: XCUIApplication!
override func setUp() async throws {
continueAfterFailure = false
app = XCUIApplication()
app.launch()
XCTAssertTrue(pad.waitForExistence(timeout: 10))
}
private var pad: XCUIElement { app.descendants(matching: .any)["pad"] }
private var status: XCUIElement { app.staticTexts["padStatus"] }
/**
The middle of the pad. An element's own tap() lands on its accessibility activation
point, near the top-left corner, and holding up there is the system's gesture for
pulling down Notification Center.
*/
private var padCenter: XCUICoordinate {
pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
}
private func swipe(_ dx: CGFloat, _ dy: CGFloat) {
padCenter.press(forDuration: 0.02, thenDragTo: pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5 + dx, dy: 0.5 + dy)))
}
private func revealTray() {
// From the very bottom edge, which is what makes it the reveal gesture and not a scroll.
let start = pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.97))
let end = pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5))
start.press(forDuration: 0.02, thenDragTo: end)
}
private func requirePhone() throws {
let deadline = Date().addingTimeInterval(15)
while Date() < deadline {
let text = status.exists ? status.label : ""
if !text.isEmpty && !text.contains("phone") && !text.contains("Install") { return }
RunLoop.current.run(until: Date().addingTimeInterval(0.25))
}
throw XCTSkip("Faceclaw isn't answering on a paired iPhone: \(status.label)")
}
func testTrayRevealsAndOpensEveryScreen() {
revealTray()
let apps = app.buttons["Apps"]
XCTAssertTrue(apps.waitForExistence(timeout: 3), "the tray should reveal from the bottom edge")
XCTAssertTrue(app.buttons["Assistant"].exists)
XCTAssertTrue(app.buttons["Status and settings"].exists)
XCTAssertTrue(app.descendants(matching: .any)["Type into app"].exists)
apps.tap()
XCTAssertTrue(app.staticTexts["Apps"].waitForExistence(timeout: 3))
}
func testStatusScreenReachesSettings() {
revealTray()
let more = app.buttons["Status and settings"]
XCTAssertTrue(more.waitForExistence(timeout: 3))
more.tap()
let settings = app.buttons["Watch settings"]
XCTAssertTrue(settings.waitForExistence(timeout: 3))
settings.tap()
XCTAssertTrue(app.switches["Vibrate on input"].waitForExistence(timeout: 3))
}
/** The whole path: pad gesture -> phone -> shell -> mirrored state -> pad status line. */
func testGesturesReachThePhoneAndComeBackAsState() throws {
try requirePhone()
let launcher = status.label
XCTAssertFalse(launcher.isEmpty)
// Open an app through the remote and watch the foreground title follow it.
revealTray()
let appsButton = app.buttons["Apps"]
XCTAssertTrue(appsButton.waitForExistence(timeout: 3))
appsButton.tap()
let calculator = app.buttons["app:calculator"]
XCTAssertTrue(calculator.waitForExistence(timeout: 5), "the phone's app list should be mirrored")
calculator.tap()
XCTAssertTrue(status.waitForExistence(timeout: 5))
let opened = NSPredicate(format: "label == %@", "Calculator")
expectation(for: opened, evaluatedWith: status)
waitForExpectations(timeout: 10)
// Every other gesture goes the same way, and the phone answers each. A silent phone
// would raise "No reply from the phone" within seconds; the gestures themselves are
// checked on the phone's side by apple-watch/README.md's log command.
padCenter.tap()
RunLoop.current.run(until: Date().addingTimeInterval(0.6))
padCenter.doubleTap()
RunLoop.current.run(until: Date().addingTimeInterval(0.6))
padCenter.press(forDuration: 0.8)
RunLoop.current.run(until: Date().addingTimeInterval(0.6))
// Tap then hold: the first touch's end must not cancel the second, however close they are.
padCenter.tap()
padCenter.press(forDuration: 0.8)
RunLoop.current.run(until: Date().addingTimeInterval(0.6))
swipe(-0.35, 0)
swipe(0.35, 0)
swipe(0, -0.35)
swipe(0, 0.35)
XCUIDevice.shared.rotateDigitalCrown(delta: 1)
RunLoop.current.run(until: Date().addingTimeInterval(5))
let notice = app.descendants(matching: .any)["notice"]
XCTAssertFalse(notice.exists && notice.label.contains("reply"), "the phone should answer every gesture")
}
/** The system text sheet: the same path dictation takes, typed instead. */
private func enterText(_ text: String) {
let field = app.textViews.firstMatch
XCTAssertTrue(field.waitForExistence(timeout: 5))
field.typeText(text)
app.buttons["Done"].tap()
}
func testTypingReachesTheForegroundApp() throws {
try requirePhone()
revealTray()
let apps = app.buttons["Apps"]
XCTAssertTrue(apps.waitForExistence(timeout: 3))
apps.tap()
let calculator = app.buttons["app:calculator"]
XCTAssertTrue(calculator.waitForExistence(timeout: 5))
calculator.tap()
XCTAssertTrue(status.waitForExistence(timeout: 5))
expectation(for: NSPredicate(format: "label == %@", "Calculator"), evaluatedWith: status)
waitForExpectations(timeout: 10)
revealTray()
let type = app.descendants(matching: .any)["Type into app"]
XCTAssertTrue(type.waitForExistence(timeout: 3))
type.tap()
enterText("1234")
// A refusal or a dropped message would show an error pill within the ack timeout.
RunLoop.current.run(until: Date().addingTimeInterval(6))
let notice = app.descendants(matching: .any)["notice"]
XCTAssertFalse(notice.exists, "typing should be accepted: \(notice.exists ? notice.label : "")")
}
/** A query goes out, and whatever the assistant does comes back as events. */
func testAssistantRepliesAreMirrored() throws {
try requirePhone()
revealTray()
let assistant = app.buttons["Assistant"]
XCTAssertTrue(assistant.waitForExistence(timeout: 3))
assistant.tap()
let phase = app.staticTexts["assistantPhase"]
XCTAssertTrue(phase.waitForExistence(timeout: 3))
if phase.label.contains("Set up") { throw XCTSkip("No assistant on the phone") }
app.descendants(matching: .any)["assistantAsk"].tap()
enterText("hello")
// The phone's first event replaces the watch's optimistic "Thinking…" with its own, then
// a real reply or, with no model on a simulator, an error: both prove the channel.
expectation(for: NSPredicate(format: "label == %@ OR label == %@", "Reply", "Error"), evaluatedWith: phase)
waitForExpectations(timeout: 60)
}
}
+165
View File
@@ -0,0 +1,165 @@
# Faceclaw for Apple Watch
A watch remote for Faceclaw on iOS: everything the Even R1 ring can do, plus the
things it can't. It's the same remote as the [Wear OS app](../wear/README.md), and
speaks the [same protocol](../wear/PROTOCOL.md), so the glasses treat them alike.
Needs watchOS 10 or later (Series 6, SE 2, Ultra and newer) and is a remote, not a
standalone app: without Faceclaw on the iPhone it only tells you to install it.
## What it does
**The pad** (the home screen; the whole display is the touch surface):
| On the watch | On the glasses |
| --- | --- |
| tap | select |
| double-tap | **back**, everywhere: closes menus, leaves a settings column, leaves a folder or the launcher grid, returns to the sidebar (and from the sidebar turns the display off, as the ring does) |
| press and hold | the system menu: Focus app switcher, Voice input, Close window (held until you lift your finger; games that use a long press as a move get it instead) |
| tap, then press and hold | the app's context menu |
| swipe ↑ ↓ ← → | **spatial navigation**: the launcher moves one app by a row or a column, the settings panel moves between its columns and down its lists, the sidebar moves up and down, Music skips tracks. Right goes *into* things and left comes back *out*. Anywhere without a spatial meaning, up/down scroll, right selects and left goes back. One step per swipe. |
| turn the crown | previous / next. Clockwise is previous by default; reverse it in the on-glasses Settings > Watch section. Sensitivity is in the watch's own settings. |
| the Double Tap hand gesture (Series 9, Ultra 2, SE 3 and later, watchOS 11) | select |
Swipe up from the bottom edge to reveal the buttons, and down from them to hide them:
- **Assistant** — ask from the wrist using the system text sheet, which offers
dictation, and read the reply as it streams. The watch buzzes when it finishes. A
reply that finishes with your wrist down is waiting when you raise it. "Hey Even"
instead opens the glasses' own voice dialog.
- **Type into app** — dictate or type into the foreground app, such as the terminal.
- **Apps** — the open windows (tap to switch, × to close) and the full launcher,
including installed EvenHub apps.
- **Status** — connection, display on/off, lock/unlock, whether Faceclaw is
answering on the iPhone, and the watch's settings.
Phone-side switches live in Faceclaw's Settings > Watch. While the glasses display
is off, a double-tap wakes it and a tap or hold shows the Glanceboard; other gestures
do nothing. The watch buzzes differently when the phone refuses or doesn't answer a
gesture, and the pill at the bottom says why.
With no glasses connected, the pad drives the iPhone's on-screen preview instead, so
all of this can be tried without them.
### What's different from Wear OS
These come from what watchOS allows apps to do:
- **No two-finger gestures.** SwiftUI on watchOS only reports one finger.
- **No side buttons.** The crown press and side button belong to the system.
- **The display can't be kept on.** To keep the remote open between glances, set
Settings > General > Return to Clock > Faceclaw on the watch to a longer time.
- **The tap that wakes the display does nothing.** The system consumes it, and an
app can't tell it from a wrist raise, so it can't stand in for a first tap.
- **Holding near the top edge opens Notification Center.** The system takes that
touch after about 300 ms, and watchOS has no way to defer it. Hold lower down.
- **The system's small clock stays in the corner**, beside the pad's large one. An
app can't hide it.
- **The glasses' Watch battery indicator can lag.** A closed app can't be asked for
its level, so it reports when open and when the system grants background time. The
phone hides a level more than 4 hours old.
There's no tip-tap detector like Wear OS's experimental one; Double Tap is the
system's reliable version of the same idea.
## Building and installing
The watch app is its own Xcode project, so it can be opened and previewed alone:
```bash
./build_apple_watch.sh # simulator build, no signing
./build_apple_watch.sh --device # signed build for a real watch
./build_apple_watch.sh --install # as --device, then installs on the paired watch
```
**It only works with a phone app that embeds it.** The system links the two apps
through the copy in the phone app's `Watch/` folder; install them separately and each
reports the other as not installed. Embedding is off by default, because signing it
needs a watch registered to your team and would break device builds for everyone
without one. To turn it on, in `build_paths.sh`:
```bash
export IOS_TEAM_ID=... # as for any iOS device build
export FACECLAW_EMBED_WATCH=1
```
Then, the first time:
1. Connect the iPhone, and enable Developer Mode on the watch (Settings > Privacy &
Security; it appears after Xcode has seen the watch in Window > Devices and
Simulators). After it restarts, accept any "Trust this computer" prompt.
2. `./build_apple_watch.sh --install`, which registers the watch with your team and
installs the app on it.
3. `npm run run:ios -- --device <iphone-id> --no-hmr` builds and installs the iPhone
app with the watch app inside.
Xcode has to be new enough for the watch's OS: Xcode 26.4 can't prepare a watch on
watchOS 27 for development ("Could not support development", "doesn't have a known
architecture"), and that also blocks registering it and installing directly. On an
older Xcode you can still add the watch's UDID by hand at developer.apple.com and let
the iPhone's Watch app install the embedded copy.
To see what the watch shows without looking at it (Xcode 27):
```bash
xcrun devicectl device capture screenshot --device <watch-udid> --destination watch.png
```
A hook (`scripts/apple-watch-embed.cjs`) adds the watch project to the generated iOS
project after each prepare, and takes it back out if the setting is removed. The
bundle ID follows `IOS_BUNDLE_ID` (the watch app's is `<that>.watchkitapp`) and the
version follows `app/version.ts`.
## Testing
```bash
node tests/apple-watch-native.cjs # gestures, wire format, status line, on the Mac
node --test tests/ios-watch-remote.test.cjs # the phone side
```
The UI tests drive the real app with synthesized touches on a watch simulator. Those
that talk to the phone need Faceclaw running on the paired iPhone simulator, and skip
themselves otherwise:
```bash
# Once: a watchOS simulator runtime and a pair (ids from `xcrun simctl list devices`).
xcodebuild -downloadPlatform watchOS
xcrun simctl pair <watch> <iphone>
xcrun simctl boot <iphone>; xcrun simctl boot <watch>
# The phone app, embedding the watch app.
FACECLAW_EMBED_WATCH=1 scripts/ios.sh build ios
xcrun simctl install <iphone> platforms/ios/build/Debug-iphonesimulator/faceclaw.app
xcrun simctl spawn <iphone> defaults write com.faceclaw.app onboarding.complete -bool YES
xcrun simctl spawn <iphone> defaults write com.faceclaw.app onboarding.previewOnly -bool YES
xcrun simctl launch <iphone> com.faceclaw.app
# The tests, which install the watch app themselves.
xcodebuild test -project apple-watch/Faceclaw.xcodeproj -scheme Faceclaw \
-destination "platform=watchOS Simulator,id=<watch>" \
-derivedDataPath apple-watch/build CODE_SIGNING_ALLOWED=NO
# What the phone made of each gesture.
xcrun simctl spawn <iphone> log show --last 5m --style compact \
--predicate 'process == "faceclaw" AND eventMessage CONTAINS "apple-watch"'
```
The simulator delivers messages in bursts about a second apart, which real hardware
isn't expected to do; don't judge gesture latency from it. `XCUIElement.tap()` lands
near an element's top-left corner, so tests that hold must use a coordinate.
## Troubleshooting
- *`Timed out while attempting to establish tunnel`, or the watch stays `connecting`
in `devicectl`* — check the watch's Wi-Fi network. It has to reach this Mac directly,
and a watch that joined a different network on the same router fails like this, even
though it still looks paired. It also powers down Wi-Fi when idle, so raise your
wrist; the first connection can take a minute or two.
- *"No phone"* — the watch can't reach the iPhone right now.
- *"Install Faceclaw on phone"* — the system doesn't see a linked phone app: it isn't
installed, or it was built without `FACECLAW_EMBED_WATCH=1`, or the bundle IDs differ.
- *"Open Faceclaw on phone"* — the phone app woke for our message but its dashboard
isn't up (for instance, onboarding isn't finished). Open it once.
- *"The glasses are not connected"* — tap Status > Connect.
- Nothing happens and no message: check Settings > Watch on the phone.
Protocol details: [wear/PROTOCOL.md](../wear/PROTOCOL.md).
+60
View File
@@ -0,0 +1,60 @@
#!/bin/bash
# Builds the Apple Watch app (apple-watch/).
#
# ./build_apple_watch.sh Simulator build; no signing needed.
# ./build_apple_watch.sh --device Signed build for a real watch.
# ./build_apple_watch.sh --install As --device, then installs on the paired watch.
#
# Signing uses IOS_TEAM_ID from build_paths.sh. The bundle ID follows IOS_BUNDLE_ID,
# because the watch app is tied to the phone app by it. To pick among several
# watches, set APPLE_WATCH_DEVICE_ID to its UDID there (see `xcrun devicectl list devices`).
#
# This is for working on the watch app alone. It can only talk to a phone app that
# was built with FACECLAW_EMBED_WATCH=1 (see apple-watch/README.md).
cd "$(dirname "$0")"; source scripts/before_build.sh
set -euo pipefail
mode="${1:-simulator}"
project=apple-watch/Faceclaw.xcodeproj
derived=apple-watch/build
node scripts/apple-watch-embed.cjs config
case "$mode" in
simulator|--simulator)
xcodebuild -quiet -project "$project" -scheme Faceclaw -configuration Debug \
-destination 'generic/platform=watchOS Simulator' -derivedDataPath "$derived" \
CODE_SIGNING_ALLOWED=NO build
echo "Built $derived/Build/Products/Debug-watchsimulator/Faceclaw.app"
;;
--device|--install)
if [[ -z "${IOS_TEAM_ID:-}" ]]; then
echo "Set IOS_TEAM_ID in build_paths.sh to sign for a real watch." >&2
exit 1
fi
destination='generic/platform=watchOS'
if [[ "$mode" == --install ]]; then
watch="${APPLE_WATCH_DEVICE_ID:-$(scripts/apple-watch-device.sh)}"
if [[ -z "$watch" ]]; then
echo "No Apple Watch found. Enable Developer Mode on it, with its iPhone connected to this Mac." >&2
exit 1
fi
# Naming the watch is what gets it registered with the team and into the
# profile; a generic destination signs for whatever is registered already.
destination="platform=watchOS,id=$watch"
fi
# A watch's Wi-Fi link can take minutes to wake, far past xcodebuild's default wait.
xcodebuild -quiet -project "$project" -scheme Faceclaw -configuration Debug \
-destination "$destination" -destination-timeout 300 -derivedDataPath "$derived" \
-allowProvisioningUpdates -allowProvisioningDeviceRegistration build
product="$derived/Build/Products/Debug-watchos/Faceclaw.app"
echo "Built $product"
if [[ "$mode" == --install ]]; then
xcrun devicectl device install app --device "$watch" "$product"
fi
;;
*)
echo "Usage: $0 [--device|--install]" >&2
exit 2
;;
esac
+5
View File
@@ -11,4 +11,9 @@
# iOS bundle ID; set one your signing team owns to sideload alongside the
# official build (default com.faceclaw.app). Also used by the iOS config scripts.
#export IOS_BUNDLE_ID=
# Embed the Apple Watch app in the iOS app. Its link to the phone needs this, and
# signing it needs a watch registered to your team, so it's off by default.
#export FACECLAW_EMBED_WATCH=1
# Which Apple Watch build_apple_watch.sh --install targets (default: the first found)
#export APPLE_WATCH_DEVICE_ID=
+15
View File
@@ -0,0 +1,15 @@
#!/bin/bash
# Prints the UDID of the first real Apple Watch devicectl can see, if any. A UDID
# works both as a devicectl --device and as an xcodebuild destination id.
set -euo pipefail
output="$(mktemp)"
trap 'rm -f "$output"' EXIT
xcrun devicectl list devices --quiet --json-output "$output" >/dev/null
node -e '
const devices = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).result.devices;
// Xcode 27 lists simulators too and moved these fields into `properties`. `reality`
// is absent on a real device until it has connected, so only "simulated" rules one out.
const hardware = device => device.properties?.hardware ?? device.hardwareProperties ?? {};
const watch = devices.map(hardware).find(h => h.platform === "watchOS" && h.reality !== "simulated");
if (watch) console.log(watch.udid);
' "$output"
+69
View File
@@ -0,0 +1,69 @@
// Ties the Apple Watch app (apple-watch/) to the iOS build. WatchConnectivity
// only links the two apps when the phone app's bundle carries the watch app in
// Watch/, so a phone build that wants a working watch must embed it.
//
// Embedding is opt-in (FACECLAW_EMBED_WATCH=1 in build_paths.sh): signing the
// result needs a provisioning profile for the watch app, which can't be made
// for a team with no watch registered, so it would break everyone else's builds.
//
// node scripts/apple-watch-embed.cjs config Write apple-watch/Local.xcconfig.
// node scripts/apple-watch-embed.cjs sync Make platforms/ios match the setting.
const { execFileSync } = require('node:child_process');
const fs = require('node:fs');
const path = require('node:path');
const root = path.resolve(__dirname, '..');
const pbxproj = path.join(root, 'platforms/ios/faceclaw.xcodeproj/project.pbxproj');
// The CLI drops `-sdk iphoneos` from xcodebuild, which would otherwise force
// the watch target onto the iOS SDK, only if this directory exists. Its own watch
// support (the legacy two-target layout) stays inert without a `watchextension`.
const cliWatchMarker = path.join(root, 'App_Resources/iOS/watchapp');
function embedEnabled() {
return process.env.FACECLAW_EMBED_WATCH === '1';
}
/** Xcode's own builds of the watch project read this too, so they sign the same way. */
function writeLocalConfig() {
const version = /FACECLAW_VERSION = "(.*)"/.exec(fs.readFileSync(path.join(root, 'app/version.ts'), 'utf8'))[1];
const lines = [
`FACECLAW_IOS_BUNDLE_ID = ${process.env.IOS_BUNDLE_ID || 'com.faceclaw.app'}`,
// An embedded watch app has to carry the phone app's version.
`MARKETING_VERSION = ${version}`,
];
if (process.env.IOS_TEAM_ID) lines.push(`DEVELOPMENT_TEAM = ${process.env.IOS_TEAM_ID}`);
fs.writeFileSync(path.join(root, 'apple-watch/Local.xcconfig'), `${lines.join('\n')}\n`);
}
function syncEmbed() {
const enabled = embedEnabled();
if (enabled) writeLocalConfig();
// Without the setting there's normally nothing to undo, and no need for Ruby.
const embedded = fs.existsSync(pbxproj) && fs.readFileSync(pbxproj, 'utf8').includes('Embed Watch Content');
if (enabled !== embedded && fs.existsSync(pbxproj)) {
execFileSync('ruby', [path.join(__dirname, 'embed-apple-watch.rb'), enabled ? 'embed' : 'remove'], { cwd: root, stdio: 'inherit' });
}
// Last: a project that still embeds the watch app can't build without this.
if (enabled) {
fs.mkdirSync(cliWatchMarker, { recursive: true });
} else {
fs.rmSync(cliWatchMarker, { recursive: true, force: true });
// Xcode leaves what it copied in earlier builds, which would still embed the old app.
const buildDir = path.join(root, 'platforms/ios/build');
for (const config of fs.existsSync(buildDir) ? fs.readdirSync(buildDir) : []) {
fs.rmSync(path.join(buildDir, config, 'faceclaw.app/Watch'), { recursive: true, force: true });
}
}
}
module.exports = { embedEnabled, syncEmbed, writeLocalConfig };
if (require.main === module) {
const command = process.argv[2];
if (command === 'config') writeLocalConfig();
else if (command === 'sync') syncEmbed();
else {
console.error('usage: apple-watch-embed.cjs config|sync');
process.exit(2);
}
}
+50
View File
@@ -0,0 +1,50 @@
#!/usr/bin/env ruby
# Adds apple-watch/Faceclaw.xcodeproj to the generated iOS project as a subproject
# and embeds its app in the phone app's Watch/ folder, or takes all that back out.
# See apple-watch-embed.cjs, which decides when to run this.
require 'xcodeproj'
PHASE_NAME = 'Embed Watch Content'
WATCH_TARGET = 'Faceclaw'
root = File.expand_path('..', __dir__)
watch_project_path = File.join(root, 'apple-watch/Faceclaw.xcodeproj')
project = Xcodeproj::Project.open(File.join(root, 'platforms/ios/faceclaw.xcodeproj'))
app = project.targets.find { |target| target.name == 'faceclaw' } or abort 'No faceclaw target in the iOS project'
def remove_embed(project, app, watch_project_path)
app.copy_files_build_phases.select { |phase| phase.name == PHASE_NAME }.each do |phase|
phase.files.to_a.each(&:remove_from_project)
phase.remove_from_project
end
app.dependencies.select { |dependency| dependency.display_name == WATCH_TARGET }.each(&:remove_from_project)
reference = project.reference_for_path(watch_project_path)
return unless reference
# Saving fails on any proxy still pointing at the subproject.
project.objects.select { |object| object.isa == 'PBXContainerItemProxy' && object.container_portal == reference.uuid }
.each(&:remove_from_project)
entries = project.root_object.project_references
entries.select { |entry| entry[:project_ref] == reference }.each do |entry|
entry[:product_group].remove_from_project
entries.delete(entry)
end
reference.remove_from_project
end
remove_embed(project, app, watch_project_path)
if ARGV[0] == 'embed'
reference = project.main_group.new_file(watch_project_path)
watch_target = Xcodeproj::Project.open(watch_project_path).targets.find { |target| target.name == WATCH_TARGET }
app.add_dependency(watch_target)
entry = project.root_object.project_references.find { |candidate| candidate[:project_ref] == reference }
product = entry[:product_group].children.find { |child| child.path == "#{WATCH_TARGET}.app" } or abort 'The watch project has no app product'
phase = app.new_copy_files_build_phase(PHASE_NAME)
phase.symbol_dst_subfolder_spec = :products_directory
phase.dst_path = '$(CONTENTS_FOLDER_PATH)/Watch'
phase.add_file_reference(product).settings = { 'ATTRIBUTES' => ['RemoveHeadersOnCopy'] }
end
project.save
@@ -0,0 +1,6 @@
const { syncEmbed } = require('../../scripts/apple-watch-embed.cjs');
// After prepare, because that's when the iOS project and its pods are finished.
module.exports = function (hookArgs) {
if (hookArgs.prepareData.platform === 'ios') syncEmbed();
};
+1
View File
@@ -8,6 +8,7 @@ const path = require('node:path');
const root = path.resolve(__dirname, '..');
const hooks = [
{ source: 'scripts/hooks/faceclaw-kotlin.before-prepare.js', target: 'hooks/before-prepare/faceclaw-kotlin.js' },
{ source: 'scripts/hooks/faceclaw-watch.after-prepare.js', target: 'hooks/after-prepare/faceclaw-watch.js' },
];
function installHooks() {
+10
View File
@@ -0,0 +1,10 @@
// Compile the watch app's platform-independent Swift for the Mac and run its checks.
const { execFileSync } = require('node:child_process');
const { mkdtempSync } = require('node:fs');
const { tmpdir } = require('node:os');
const { join, resolve } = require('node:path');
const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-apple-watch-'));
const src = 'apple-watch/Faceclaw';
execFileSync('xcrun', ['swiftc', '-O', `${src}/Protocol.swift`, `${src}/PadStatus.swift`, `${src}/PadGestureRecognizer.swift`,
'tests/native/apple-watch/main.swift', '-o', join(dir, 'apple-watch-test')], { cwd: root, stdio: 'inherit' });
execFileSync(join(dir, 'apple-watch-test'), [], { stdio: 'inherit' });
+1
View File
@@ -157,6 +157,7 @@ function fixture() {
'./lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) },
'./glasses-presence': load('app/g2/glasses-presence.ts', {}),
'./device-addresses': { loadDeviceAddresses: () => ({ right: 'AA', left: 'BB', ring: '' }) }, './ios-peripheral-identity': { deviceAddressError: () => null },
'./wear-remote': { WearRemote: class { schedulePublish() {} } },
'./firmware-compat': load('app/g2/firmware-compat.ts', {}), './reconnect-policy': load('app/g2/reconnect-policy.ts', {}),
'../apps/launcher': { launcherEntries: () => [] },
'../apps/evenhub/installed-apps': {}, '../apps/evenhub/manager': { onPhoneShownChanged: () => () => {}, phoneUiButton: () => null },
+215
View File
@@ -0,0 +1,215 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const vm = require('node:vm');
const ts = require('typescript');
const path = require('node:path');
function load(file, context = {}) {
const sandbox = { exports: {}, console, ...context };
sandbox.global = sandbox;
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, sandbox);
return sandbox.exports;
}
const types = () => load('app/native/wear-bridge-types.ts');
/** What FaceclawWatchBridge.m does, as far as JS can tell. */
function fakeNative({ available = true, watchApp = true } = {}) {
return {
calls: [], message: null, connection: null, watchApp,
isAvailable() { return available; },
isWatchAppAvailable() { return this.watchApp; },
setMessageHandlerConnectionHandler(message, connection) { this.message = message; this.connection = connection; },
publishStateForce(json, force) { this.calls.push(['state', JSON.parse(json), force]); },
sendEventSticky(json, sticky) { this.calls.push(['event', JSON.parse(json), sticky]); },
sendToWatchJson(pathName, json) { this.calls.push(['send', pathName, JSON.parse(json)]); },
sendAckOkJsReadyMessage(seq, ok, jsReady, message) { this.calls.push(['ack', seq, ok, jsReady, message]); },
};
}
function loadBridge(native) {
return load('app/native/wear-bridge.ios.ts', {
FaceclawWatchBridge: { shared: () => native },
require: () => types(),
}).wearBridge;
}
test('iOS bridge carries the same messages as the Android one', () => {
const native = fakeNative();
const bridge = loadBridge(native);
assert.equal(bridge.isAvailable(), true);
const received = [];
bridge.onMessage(message => received.push(message));
native.message('/faceclaw/input', '{"gesture":"click","seq":3}');
native.message('/faceclaw/press', 'not json');
assert.equal(received[0].path, '/faceclaw/input');
assert.equal(received[0].payload.gesture, 'click');
assert.equal(Object.keys(received[1].payload).length, 0, 'a malformed payload is empty, not a crash');
bridge.publishState({ phase: 'connected' }, true);
bridge.sendToWatch('/faceclaw/battery/request', {});
bridge.sendAck(received[0].nodeId, 3, false, 'refused');
assert.deepEqual(JSON.parse(JSON.stringify(native.calls)), [
['state', { phase: 'connected' }, true],
['send', '/faceclaw/battery/request', {}],
['ack', 3, false, true, 'refused'],
]);
});
test('only streaming assistant snapshots skip the sticky application context', () => {
const native = fakeNative();
const bridge = loadBridge(native);
bridge.sendEvent({ type: 'assistant', phase: 'streaming', text: 'par' });
bridge.sendEvent({ type: 'assistant', phase: 'done', text: 'partial' });
bridge.sendEvent({ type: 'alert', text: 'hi' });
assert.deepEqual(native.calls.map(call => call[2]), [false, true, true]);
});
test('availability follows the watch app, and a change reaches listeners', () => {
const native = fakeNative({ watchApp: false });
const bridge = loadBridge(native);
const seen = [];
bridge.onWatchConnection(connection => seen.push(connection.reachable));
assert.deepEqual(seen, [false], 'the current value is delivered immediately');
native.watchApp = true;
native.connection();
assert.deepEqual(seen, [false, true]);
assert.equal(bridge.getWatchConnection().watchName, 'Apple Watch');
});
test('a phone that cannot use WatchConnectivity turns the bridge into a no-op', () => {
const native = fakeNative({ available: false });
const bridge = loadBridge(native);
assert.equal(bridge.isAvailable(), false);
bridge.publishState({});
bridge.sendEvent({ type: 'alert', text: 'x' });
assert.equal(bridge.getWatchConnection().reachable, false);
assert.equal(native.calls.length, 0);
});
/** A WearRemote over a fake bridge and shell. */
function loadRemote({ phase = 'disconnected', preview = false, now = 1_800_000_000_000 } = {}) {
const bridge = {
messageListener: null, connectionListener: null, sent: [],
isAvailable: () => true,
onMessage(listener) { this.messageListener = listener; return () => {}; },
onWatchConnection(listener) { this.connectionListener = listener; listener({ reachable: true, watchName: 'Apple Watch' }); return () => {}; },
publishState() {},
sendEvent() {},
sendToWatch() {},
sendAck(_node, seq, ok, message) { this.sent.push({ seq, ok, message }); },
};
const levels = { watch: null, watchCharging: null };
const shell = {
onAssistantActivity() {}, onAlertShown() {},
getBatteryLevels: () => levels,
setBatteryLevels(update) { Object.assign(levels, update); },
foregroundWindow: () => null, getForegroundApp: () => null, getWindows: () => [],
isScreenOn: () => true, wake: () => true, isAssistantAvailable: () => false,
};
const setting = value => ({ get: () => value });
const intervals = [];
const realDate = Date;
const clock = { now };
class FakeDate extends realDate { static now() { return clock.now; } }
const inputs = [];
const state = { phase, status: '', glassesLocked: false, glassesWorn: null, silentMode: false, lastHeadsetBattery: null };
const { WearRemote } = load('app/g2/wear-remote.ts', {
Date: FakeDate,
setTimeout: () => 0, clearTimeout() {},
setInterval: fn => { intervals.push(fn); return intervals.length; },
Number, Math, JSON,
require: id => ({
'../analytics/analytics': { flagAnalyticsEvent() {} },
'../apps/evenhub/installed-apps': { getInstalledEvenHubApps: () => [], installedEvenHubAppId: id => id },
'../native/wear-bridge': { wearBridge: bridge, WEAR_PATHS: types().WEAR_PATHS },
'../ui/dashboard-settings': {
onAnySettingChanged() {}, watchCanUnlockSetting: setting(true), watchCrownClockwiseNextSetting: setting(false),
watchMirrorAssistantSetting: setting(true), watchRemoteEnabledSetting: setting(true),
},
'../ui/shell/shell': { shell },
'../ui/shell/voice-activity': { voiceActivity: { subscribe() {}, isActive: () => false } },
'../version': { FACECLAW_VERSION: 'test' },
})[id] ?? {},
});
const remote = new WearRemote({
apps: [{ appId: 'timer', title: 'Timer' }],
injectInput: async kind => { inputs.push(kind); },
launchApp: async () => {}, connect: async () => {}, disconnect: async () => {},
setGlassesLocked() {}, requestShellRender() {},
previewInputActive: () => preview,
getState: () => state,
appendLog() {},
});
const deliver = (pathName, payload) => bridge.messageListener({ path: pathName, payload, nodeId: 'apple-watch' });
return { remote, bridge, shell, levels, inputs, intervals, clock, deliver };
}
const settle = () => new Promise(resolve => setImmediate(resolve));
test('gestures are refused with no display, and land on the preview without glasses', async () => {
const none = loadRemote();
none.deliver('/faceclaw/input', { gesture: 'click', seq: 1 });
none.deliver('/faceclaw/press', {});
await settle();
assert.deepEqual(none.inputs, []);
assert.equal(none.bridge.sent[0].ok, false);
assert.match(none.bridge.sent[0].message, /not connected/);
const preview = loadRemote({ preview: true });
preview.deliver('/faceclaw/press', {});
preview.deliver('/faceclaw/input', { gesture: 'click', seq: 2 });
await settle();
assert.deepEqual(preview.inputs, ['ring-press', 'click']);
assert.equal(preview.bridge.sent[0].ok, true);
const glasses = loadRemote({ phase: 'connected' });
glasses.deliver('/faceclaw/input', { gesture: 'swipe-left', seq: 3 });
await settle();
assert.deepEqual(glasses.inputs, ['swipe-left']);
});
test('a watch battery report is shown, kept while fresh, and dropped once stale', async () => {
const { levels, intervals, clock, deliver, bridge } = loadRemote({ phase: 'connected' });
deliver('/faceclaw/battery', { battery: 64, charging: true, at: clock.now });
await settle();
assert.equal(levels.watch, 64);
assert.equal(levels.watchCharging, true);
assert.equal(bridge.sent.length, 0, 'battery reports are never acked');
const tick = () => intervals.forEach(fn => fn());
clock.now += 3 * 60 * 60_000;
tick();
assert.equal(levels.watch, 64, 'a closed watch app reports rarely; three hours is still shown');
clock.now += 2 * 60 * 60_000;
tick();
assert.equal(levels.watch, null, 'five hours says the watch is gone');
});
test('a battery report found long after it was left is not shown as current', async () => {
const { levels, clock, deliver } = loadRemote({ phase: 'connected' });
deliver('/faceclaw/battery', { battery: 90, charging: false, at: clock.now - 5 * 60 * 60_000 });
await settle();
assert.equal(levels.watch, null);
// A live report without a timestamp is current by definition, as Wear OS's always are.
deliver('/faceclaw/battery', { battery: 80, charging: false });
await settle();
assert.equal(levels.watch, 80);
});
test('a watch that does not know its battery shows no level, not an empty one', async () => {
const { levels, deliver } = loadRemote({ phase: 'connected' });
deliver('/faceclaw/battery', { battery: 50, charging: false });
await settle();
assert.equal(levels.watch, 50);
deliver('/faceclaw/battery', { battery: null, charging: false });
await settle();
assert.equal(levels.watch, null);
});
test('the mirrored state says when gestures are landing on the preview', () => {
const state = loadRemote({ preview: true }).remote.buildState();
assert.equal(state.preview, true);
assert.equal(state.connected, false);
assert.equal(loadRemote({ phase: 'connected' }).remote.buildState().connected, true);
});
+221
View File
@@ -0,0 +1,221 @@
// Exercises the watch app's platform-independent logic on the Mac: the gesture
// state machine, the wire format and the pad's status line.
import Foundation
var failures = 0
func check(_ condition: @autoclosure () -> Bool, _ message: String, line: Int = #line) {
if !condition() {
failures += 1
print("FAILED line \(line): \(message)")
}
}
// MARK: - Gestures
final class Harness {
let recognizer = PadGestureRecognizer()
var events: [PadEvent] = []
var time: TimeInterval = 100
init() { recognizer.onEvent = { [unowned self] in self.events.append($0) } }
func down(_ point: CGPoint = .zero) { recognizer.touchDown(at: point, time: time) }
func move(_ point: CGPoint) { recognizer.touchMoved(to: point) }
func up(_ point: CGPoint = .zero) { recognizer.touchUp(at: point, time: time) }
func wait(_ seconds: TimeInterval) { time += seconds; recognizer.advance(to: time) }
}
do {
// A lone tap is only known to be one once the double-tap window closes.
let h = Harness()
h.down(); h.up()
check(h.events == [.press], "a tap waits out the double-tap window: \(h.events)")
h.wait(0.2)
check(h.events == [.press], "still inside the window")
h.wait(0.2)
check(h.events == [.press, .tap], "tap after the window: \(h.events)")
check(h.recognizer.deadline == nil, "nothing left to wait for")
}
do {
let h = Harness()
h.down(); h.up(); h.wait(0.1); h.down(); h.wait(0.05); h.up()
check(h.events == [.press, .press, .doubleTap], "double-tap reports both presses: \(h.events)")
h.wait(1)
check(h.events.count == 3, "no late tap after a double-tap")
}
do {
// Hold: reported at the timeout and again on release, however long it lasts.
let h = Harness()
h.down(); h.wait(0.39)
check(h.events == [.press], "not yet a hold")
h.wait(0.02)
check(h.events == [.press, .longPressStart], "hold starts at the timeout: \(h.events)")
h.wait(5); h.up()
check(h.events == [.press, .longPressStart, .longPressEnd], "hold ends on release: \(h.events)")
}
do {
// Movement during a hold doesn't turn it into a swipe.
let h = Harness()
h.down(); h.wait(0.5); h.move(CGPoint(x: 50, y: 50)); h.up(CGPoint(x: 50, y: 50))
check(h.events == [.press, .longPressStart, .longPressEnd], "a hold stays a hold: \(h.events)")
}
do {
let h = Harness()
h.down(); h.up(); h.wait(0.1); h.down(); h.wait(0.5)
check(h.events == [.press, .press, .shortThenLongPress], "tap then hold: \(h.events)")
h.up(); h.wait(1)
check(h.events.count == 3, "no double-tap or release after tap-then-hold: \(h.events)")
}
do {
let h = Harness()
let start = CGPoint(x: 100, y: 180), end = CGPoint(x: 104, y: 60)
h.down(start); h.move(end); h.wait(1); h.up(end)
check(h.events == [.press, .swipe(start: start, dx: 4, dy: -120)], "a swipe, never a hold: \(h.events)")
}
do {
// A small wobble under the slop is still a tap.
let h = Harness()
h.down(.zero); h.move(CGPoint(x: 3, y: 3)); h.up(CGPoint(x: 3, y: 3)); h.wait(0.4)
check(h.events == [.press, .tap], "wobble is a tap: \(h.events)")
}
do {
// Whatever ends a hold, the phone must hear a release or it sees a stuck long-press.
let h = Harness()
h.down(); h.wait(0.5); h.recognizer.cancel()
check(h.events == [.press, .longPressStart, .longPressEnd], "a cancelled hold is released: \(h.events)")
check(!h.recognizer.isTouching, "idle after cancel")
}
do {
let h = Harness()
h.down(); h.recognizer.cancel(); h.wait(1)
check(h.events == [.press], "a cancelled touch is not a tap: \(h.events)")
}
do {
// The wrist dropped after a complete tap: it was still a tap, and there's no touch left to wait for.
let h = Harness()
h.down(); h.up(); h.recognizer.cancel()
check(h.events == [.press, .tap], "a finished tap survives cancel: \(h.events)")
h.wait(1)
check(h.events.count == 2, "and isn't reported twice")
}
do {
// A touch whose end was never seen must not wedge the next one.
let h = Harness()
h.down(); h.wait(0.5); h.down()
check(h.events == [.press, .longPressStart, .longPressEnd, .press], "a lost touch-up is recovered: \(h.events)")
}
// MARK: - Wire format
do {
let json = """
{"protocol":1,"version":"0.8.6","phase":"connected","status":"ok","connected":true,"preview":false,
"screenOn":true,"locked":false,"worn":null,"listening":false,"battery":78,"charging":false,"silentMode":false,
"caseBattery":64,"caseCharging":null,"foreground":{"appId":"music","title":"Music"},
"windows":[{"windowId":"w1","title":"Music","appId":"music","focused":true,"closeable":false,"acceptsText":true}],
"apps":[{"appId":"timer","title":"Timer"}],"remoteEnabled":true,"crownClockwiseNext":true,"canUnlock":false,
"mirrorAssistant":true,"assistantAvailable":true}
"""
let state = PhoneState.parse(json, updatedAt: 42)
check(state != nil, "state parses")
check(state?.connected == true && state?.takesInput == true, "connected")
check(state?.battery == 78 && state?.caseBattery == 64, "batteries")
check(state?.worn == nil && state?.caseCharging == nil, "JSON null stays nil")
check(state?.foreground == AppEntry(appId: "music", title: "Music"), "foreground")
check(state?.windows.first?.acceptsText == true && state?.windows.first?.closeable == false, "window flags")
check(state?.apps.map(\.appId) == ["timer"], "apps")
check(state?.crownClockwiseNext == true && state?.canUnlock == false, "settings")
check(state?.updatedAt == 42, "updatedAt")
}
do {
// A state from an older or newer phone app parses with defaults rather than failing.
let state = PhoneState.parse("{\"battery\":140,\"unknownField\":[1,2]}", updatedAt: 0)
check(state?.battery == nil, "out-of-range percentage is unknown")
check(state?.remoteEnabled == true && state?.canUnlock == true, "permissive defaults")
check(state?.phase == "disconnected" && state?.takesInput == false, "disconnected default")
check(PhoneState.parse("not json", updatedAt: 0) == nil, "garbage is rejected")
check(PhoneState.parse("[1]", updatedAt: 0) == nil, "a non-object is rejected")
}
do {
let ack = Ack.parse("{\"seq\":7,\"ok\":false,\"jsReady\":false,\"message\":\"nope\"}")
check(ack == Ack(seq: 7, ok: false, jsReady: false, message: "nope"), "ack: \(String(describing: ack))")
check(Ack.parse("{\"seq\":7,\"ok\":true}")?.jsReady == true, "jsReady defaults to true")
}
do {
check(PhoneEvent.parse("{\"type\":\"assistant\",\"phase\":\"done\",\"text\":\"hi\"}") == .assistant(phase: "done", text: "hi"), "assistant event")
check(PhoneEvent.parse("{\"type\":\"alert\",\"text\":\"x\"}") == .alert(text: "x"), "alert event")
check(PhoneEvent.parse("{\"type\":\"other\"}") == nil, "unknown event")
}
do {
var mirror = AssistantMirror()
mirror = mirror.applying(phase: "thinking", text: "")
check(mirror == AssistantMirror(phase: "thinking", text: ""), "thinking")
mirror = mirror.applying(phase: "streaming", text: "Hel")
mirror = mirror.applying(phase: "done", text: "")
check(mirror == AssistantMirror(phase: "done", text: "Hel"), "an empty done keeps the streamed text")
mirror = mirror.applying(phase: "closed", text: "")
check(mirror == AssistantMirror(phase: "idle", text: "Hel"), "closed keeps the text to read")
check(mirror.applying(phase: "bogus", text: "x") == mirror, "unknown phases change nothing")
}
// MARK: - Status line
do {
var up = LinkStatus(activated: true, phoneReachable: true, phoneAppInstalled: true, appRunning: true)
var state = PhoneState()
state.connected = true
state.screenOn = true
state.foreground = AppEntry(appId: "music", title: "Music")
check(padLine(state: nil, link: LinkStatus()).text == "Waiting for phone", "before activation")
check(padLine(state: state, link: LinkStatus(activated: true, phoneAppInstalled: false)).text == "Install Faceclaw on phone", "no phone app")
check(padLine(state: state, link: LinkStatus(activated: true, phoneAppInstalled: true)).text == "No phone", "unreachable")
up.appRunning = false
check(padLine(state: state, link: up).text == "Open Faceclaw on phone", "dashboard not up")
up.appRunning = true
check(padLine(state: nil, link: up).text == "Waiting for phone", "no state yet")
check(padLine(state: state, link: up) == StatusLine(text: "Music", tone: .good), "foreground app")
state.screenOn = false
check(padLine(state: state, link: up).text == "●● Wake", "display off")
state.locked = true
check(padLine(state: state, link: up).text == "Locked", "locked beats display off")
state.charging = true
state.battery = 50
state.caseBattery = 64
state.caseCharging = true
check(padLine(state: state, link: up) == StatusLine(text: "Charging · G2 50%", tone: .busy, detail: "Case 64% ⚡"), "charging")
var off = PhoneState()
check(padLine(state: off, link: up).text == "Disconnected", "disconnected")
off.phase = "retrying"
check(padLine(state: off, link: up).text == "Reconnecting…", "iOS's retrying phase")
off.phase = "disconnected"
off.preview = true
off.screenOn = true
check(padLine(state: off, link: up) == StatusLine(text: "Launcher", tone: .good, detail: "Phone preview"), "phone preview")
off.screenOn = false
check(padLine(state: off, link: up).text == "●● Wake", "preview display off")
off.remoteEnabled = false
check(padLine(state: off, link: up).text == "Watch control off", "remote disabled")
}
if failures > 0 {
print("\(failures) check(s) failed")
exit(1)
}
print("apple-watch logic: all checks passed")
+40 -1
View File
@@ -1,5 +1,10 @@
# Watch ⇄ phone protocol
The Wear OS and Apple Watch apps speak the same messages over different
transports, and `app/g2/wear-remote.ts` gives them their meaning on both phones.
This document describes the messages and the Wear OS transport; see "Apple Watch
transport" at the end for what differs there.
Transport: the Wearable Data Layer (Google Play services). Both apps share the
`com.faceclaw.app` application id and, for release builds, the signing key —
the Data Layer routes only between matching apps. Payloads are UTF-8 JSON
@@ -55,7 +60,7 @@ dashboard state, re-sent whenever it changes (and at least every 30 s):
```json
{
"protocol": 1, "version": "…",
"phase": "connected", "status": "…", "connected": true,
"phase": "connected", "status": "…", "connected": true, "preview": false,
"screenOn": true, "locked": false, "worn": true, "listening": false,
"battery": 78, "charging": false, "silentMode": false,
"caseBattery": 64, "caseCharging": true,
@@ -85,3 +90,37 @@ rotation sends previous/up, and when true it sends next/down.
`apps` is the launcher grid plus installed EvenHub packages, i.e. exactly what
the assistant's `apps.launch` tool accepts.
`preview` says gestures are landing on the phone's own display preview, with no
glasses involved: `connected` stays false, but input isn't refused. Only the iOS
app sets it for now.
## Apple Watch transport
Transport: WatchConnectivity. Phone side:
`App_Resources/iOS/src/FaceclawWatchBridge.m` (transport),
`app/native/wear-bridge.ios.ts` (JS wrapper). Watch side:
`apple-watch/Faceclaw/PhoneLink.swift`, `Protocol.swift`.
The system only links the two apps when the watch app's
`WKCompanionAppBundleIdentifier` names the phone app, its bundle ID begins with
the phone app's, **and the phone app's bundle embeds it in `Watch/`**; installing
each app separately leaves both sides reporting the other as not installed. See
`apple-watch/README.md`.
A message is a dictionary `{path, json}`, where `json` is the JSON payload above;
every path and payload is unchanged. `sendMessage` is used with no reply handler,
and acks are ordinary `/faceclaw/ack` messages. What differs follows from one fact:
**the phone can't wake the watch app, and the app is suspended whenever the wrist
drops.** The watch can wake the phone app, and does so with any message.
| Wear OS | Apple Watch |
| --- | --- |
| `/faceclaw/state` is a Data Layer item | The application context, `{json, updatedAt}`, for the app's next wake. While the app is in front it's also sent live as `{path: "/faceclaw/state", json, updatedAt}`, since context delivery can lag. Whichever `updatedAt` is newer wins. |
| `/faceclaw/event` is lost if the app is closed | Also sent live, with `at`. All but `streaming` snapshots also stay in the context as `{event, eventAt}`, so a reply that finished with the wrist down is there on the next raise. The watch drops anything not newer than the last `at`, and ignores a stale context event (assistant after 10 minutes, alert after 1). |
| `/faceclaw/battery/request` wakes a listener service | Only reaches an app in front. The app reports on open, every minute while open, and when the system grants it background time (asked for every 30 minutes), leaving `{battery: <json>}` in its own context for the phone to find. The payload adds `at` (ms), and the phone treats a level over 4 hours old as unknown. |
| "Reachable" is a capability query | `isPaired && isWatchAppInstalled`, whether or not the app is running. Live reachability changes also fire the connection callback, which is what pushes fresh state to an app coming to the front. |
| `nodeId` routes acks | Unused: a session has one counterpart. |
`isReachable` lags a moment behind a wrist raise, so the watch holds gestures for up
to 1.5 s rather than failing them. Touch-downs (`/faceclaw/press`) are never held.