diff --git a/.gitignore b/.gitignore index 6659b26a..326fbf1f 100644 --- a/.gitignore +++ b/.gitignore @@ -70,3 +70,9 @@ tests/kotlin/build/ # tools/health-preview.cjs output (rendered PNGs, regenerate with the tool) preview/ + +# Apple Watch app: build output, and the settings build_apple_watch.sh writes from build_paths.sh +apple-watch/build/ +apple-watch/Local.xcconfig +App_Resources/iOS/watchapp/ +xcuserdata/ diff --git a/App_Resources/iOS/Info.plist b/App_Resources/iOS/Info.plist index 77b3edd1..a22eb26f 100644 --- a/App_Resources/iOS/Info.plist +++ b/App_Resources/iOS/Info.plist @@ -57,6 +57,26 @@ Faceclaw connects to the terminal servers and developer app URLs you configure on your local network. UILaunchScreen + + UIApplicationSceneManifest + + UIApplicationSupportsMultipleScenes + + UISceneConfigurations + + UIWindowSceneSessionRoleApplication + + + UISceneConfigurationName + Default Configuration + UISceneDelegateClassName + SceneDelegate + + + + UISupportedInterfaceOrientations UIInterfaceOrientationPortrait diff --git a/App_Resources/iOS/build.xcconfig b/App_Resources/iOS/build.xcconfig index 331a8be9..e8254dfe 100644 --- a/App_Resources/iOS/build.xcconfig +++ b/App_Resources/iOS/build.xcconfig @@ -5,7 +5,7 @@ CODE_SIGN_STYLE = Automatic; // Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team. // FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+, // so it is weak-linked: the app still launches on older iOS and reports the model unavailable. -OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels +OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -framework WatchConnectivity -weak_framework FoundationModels ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; diff --git a/App_Resources/iOS/src/FaceclawWatchBridge.h b/App_Resources/iOS/src/FaceclawWatchBridge.h new file mode 100644 index 00000000..1cde6ba8 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawWatchBridge.h @@ -0,0 +1,43 @@ +#import + +NS_ASSUME_NONNULL_BEGIN + +/** + * Phone side of the Apple Watch remote: the single place that talks to + * WatchConnectivity, so JS only ever sees path + JSON strings. The twin of the + * Android FaceclawWearBridge; the messages are in wear/PROTOCOL.md. + * + * The session activates during launch, ahead of JS, because the watch can wake + * this app in the background and messages arriving before a delegate exists + * are lost. Until JS registers its handlers, messages are answered with + * jsReady:NO so the watch can say "open Faceclaw on the phone". + * + * The public methods and both handlers belong to the main thread. + */ +@interface FaceclawWatchBridge : NSObject ++ (instancetype)shared; +/** WCSession works on this device (every iPhone; no iPad). */ +- (BOOL)isAvailable; +/** A paired Apple Watch has the Faceclaw app. It needn't be running. */ +- (BOOL)isWatchAppAvailable; +/** + * Register JS. messageHandler gets every watch message; connectionHandler fires + * when isWatchAppAvailable or the watch app's live reachability changes. + */ +- (void)setMessageHandler:(void (^)(NSString *path, NSString *json))messageHandler + connectionHandler:(void (^)(void))connectionHandler; +/** + * Mirror the dashboard state: kept in the application context for the app's + * next launch, and sent live too while it's in front. An unchanged state is + * dropped unless `force`. + */ +- (void)publishState:(NSString *)json force:(BOOL)force; +/** Send an event live. A sticky one also stays in the application context for a suspended app. */ +- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky; +/** A live message on any phone -> watch path; dropped if the watch app isn't in front. */ +- (void)sendToWatch:(NSString *)path json:(NSString *)json; +/** Reply to a watch message; `seq` echoes the watch's sequence number. */ +- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message; +@end + +NS_ASSUME_NONNULL_END diff --git a/App_Resources/iOS/src/FaceclawWatchBridge.m b/App_Resources/iOS/src/FaceclawWatchBridge.m new file mode 100644 index 00000000..a20d74ac --- /dev/null +++ b/App_Resources/iOS/src/FaceclawWatchBridge.m @@ -0,0 +1,272 @@ +#import "FaceclawWatchBridge.h" +#import +#import + +// Live messages are {path, json, ...}; the watch app's Protocol.swift mirrors these keys. +static NSString *const kKeyPath = @"path"; +static NSString *const kKeyJson = @"json"; +static NSString *const kKeyUpdatedAt = @"updatedAt"; +static NSString *const kKeyEvent = @"event"; +static NSString *const kKeyEventAt = @"eventAt"; +static NSString *const kKeyAt = @"at"; +static NSString *const kKeyBattery = @"battery"; + +static NSString *const kPathState = @"/faceclaw/state"; +static NSString *const kPathStateRequest = @"/faceclaw/state/request"; +static NSString *const kPathBattery = @"/faceclaw/battery"; +static NSString *const kPathPress = @"/faceclaw/press"; +static NSString *const kPathAck = @"/faceclaw/ack"; +static NSString *const kPathEvent = @"/faceclaw/event"; + +/** State requests kept for a JS handler that isn't registered yet. */ +static const NSUInteger kPendingLimit = 16; +static const NSTimeInterval kPendingMaxAge = 30; + +static long long NowMs(void) { + return (long long)(NSDate.date.timeIntervalSince1970 * 1000); +} + +static double ReadSeq(NSString *json) { + NSData *data = [json dataUsingEncoding:NSUTF8StringEncoding]; + id parsed = data ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil; + id seq = [parsed isKindOfClass:NSDictionary.class] ? parsed[@"seq"] : nil; + return [seq isKindOfClass:NSNumber.class] ? [seq doubleValue] : 0; +} + +@interface FaceclawWatchPendingMessage : NSObject +@property(nonatomic, copy) NSString *path; +@property(nonatomic, copy) NSString *json; +@property(nonatomic, strong) NSDate *receivedAt; +@end +@implementation FaceclawWatchPendingMessage +@end + +@interface FaceclawWatchBridge () +@property(nonatomic, copy, nullable) void (^messageHandler)(NSString *, NSString *); +@property(nonatomic, copy, nullable) void (^connectionHandler)(void); +@property(nonatomic, strong) NSMutableArray *pending; +@property(nonatomic, copy, nullable) NSString *lastState; +@property(nonatomic) long long lastStateAt; +@property(nonatomic, copy, nullable) NSString *heldEvent; +@property(nonatomic) long long heldEventAt; +/** Sends and context updates run here: they cost IPC, and the main thread also runs BLE. */ +@property(nonatomic, strong) dispatch_queue_t sendQueue; +/** Latest context waiting for sendQueue; nil when none is queued. Guarded by @synchronized(self). */ +@property(nonatomic, copy, nullable) NSDictionary *queuedContext; +@end + +@implementation FaceclawWatchBridge + ++ (void)load { + // Registering here is what puts the delegate in place during launch, ahead of JS. + [NSNotificationCenter.defaultCenter addObserverForName:UIApplicationDidFinishLaunchingNotification + object:nil + queue:NSOperationQueue.mainQueue + usingBlock:^(NSNotification *note) { + [[FaceclawWatchBridge shared] activate]; + }]; +} + ++ (instancetype)shared { + static FaceclawWatchBridge *instance; + static dispatch_once_t once; + dispatch_once(&once, ^{ instance = [FaceclawWatchBridge new]; }); + return instance; +} + +- (instancetype)init { + if ((self = [super init])) { + _pending = [NSMutableArray new]; + _sendQueue = dispatch_queue_create("com.faceclaw.watch.send", DISPATCH_QUEUE_SERIAL); + } + return self; +} + +- (void)activate { + if (!WCSession.isSupported) return; + WCSession *session = WCSession.defaultSession; + if (session.delegate == self) return; + session.delegate = self; + [session activateSession]; +} + +#pragma mark - Public API (main thread) + +- (BOOL)isAvailable { + return WCSession.isSupported; +} + +- (BOOL)isWatchAppAvailable { + if (!WCSession.isSupported) return NO; + WCSession *session = WCSession.defaultSession; + if (session.activationState != WCSessionActivationStateActivated) return NO; + return session.isReachable || (session.isPaired && session.isWatchAppInstalled); +} + +- (void)setMessageHandler:(void (^)(NSString *, NSString *))messageHandler + connectionHandler:(void (^)(void))connectionHandler { + self.messageHandler = messageHandler; + self.connectionHandler = connectionHandler; + [self activate]; + NSArray *replay = [self.pending copy]; + [self.pending removeAllObjects]; + for (FaceclawWatchPendingMessage *message in replay) { + if (-message.receivedAt.timeIntervalSinceNow <= kPendingMaxAge) messageHandler(message.path, message.json); + } + // The watch's last battery report may have arrived while no one was listening. + if (WCSession.isSupported && WCSession.defaultSession.activationState == WCSessionActivationStateActivated) { + [self deliverBatteryFrom:WCSession.defaultSession.receivedApplicationContext]; + } +} + +- (void)publishState:(NSString *)json force:(BOOL)force { + if (!force && [json isEqualToString:self.lastState]) return; + self.lastState = json; + self.lastStateAt = NowMs(); + [self pushContext]; + [self sendLive:@{kKeyPath: kPathState, kKeyJson: json, kKeyUpdatedAt: @(self.lastStateAt)}]; +} + +- (void)sendEvent:(NSString *)json sticky:(BOOL)sticky { + long long at = NowMs(); + if (sticky) { + self.heldEvent = json; + self.heldEventAt = at; + [self pushContext]; + } + [self sendLive:@{kKeyPath: kPathEvent, kKeyJson: json, kKeyAt: @(at)}]; +} + +- (void)sendToWatch:(NSString *)path json:(NSString *)json { + [self sendLive:@{kKeyPath: path, kKeyJson: json}]; +} + +- (void)sendAck:(double)seq ok:(BOOL)ok jsReady:(BOOL)jsReady message:(NSString *)message { + NSDictionary *ack = @{@"seq": @(seq), @"ok": @(ok), @"jsReady": @(jsReady), @"message": message ?: @""}; + NSData *data = [NSJSONSerialization dataWithJSONObject:ack options:0 error:nil]; + if (!data) return; + [self sendToWatch:kPathAck json:[[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]]; +} + +#pragma mark - Sending + +- (void)sendLive:(NSDictionary *)message { + if (!WCSession.isSupported) return; + dispatch_async(self.sendQueue, ^{ + WCSession *session = WCSession.defaultSession; + if (session.activationState != WCSessionActivationStateActivated || !session.isReachable) return; + [session sendMessage:message replyHandler:nil errorHandler:^(NSError *error) { + NSLog(@"[FaceclawWatch] send %@ failed: %@", message[kKeyPath], error); + }]; + }); +} + +/** Queue the current state and held event as the application context; only the latest matters. */ +- (void)pushContext { + if (!WCSession.isSupported) return; + NSMutableDictionary *context = [NSMutableDictionary new]; + if (self.lastState) { + context[kKeyJson] = self.lastState; + context[kKeyUpdatedAt] = @(self.lastStateAt); + } + if (self.heldEvent) { + context[kKeyEvent] = self.heldEvent; + context[kKeyEventAt] = @(self.heldEventAt); + } + BOOL alreadyQueued; + @synchronized (self) { + alreadyQueued = self.queuedContext != nil; + self.queuedContext = context; + } + if (alreadyQueued) return; + dispatch_async(self.sendQueue, ^{ + NSDictionary *next; + @synchronized (self) { + next = self.queuedContext; + self.queuedContext = nil; + } + WCSession *session = WCSession.defaultSession; + if (!next || session.activationState != WCSessionActivationStateActivated) return; + if (!session.isPaired || !session.isWatchAppInstalled) return; + NSError *error = nil; + if (![session updateApplicationContext:next error:&error]) { + NSLog(@"[FaceclawWatch] application context update failed: %@", error); + } + }); +} + +#pragma mark - Receiving + +- (void)handleMessage:(NSString *)path json:(NSString *)json { + if (self.messageHandler) { + self.messageHandler(path, json); + return; + } + // A battery report is status and a touch-down only matters live; neither expects an ack. + if ([path isEqualToString:kPathBattery] || [path isEqualToString:kPathPress]) return; + // Gestures and commands are refused rather than replayed: the watch reports them as failed, + // and running them later would act on whatever the glasses happen to show by then. + if ([path isEqualToString:kPathStateRequest]) { + NSDate *now = NSDate.date; + while (self.pending.count > 0 && [now timeIntervalSinceDate:self.pending.firstObject.receivedAt] > kPendingMaxAge) { + [self.pending removeObjectAtIndex:0]; + } + if (self.pending.count >= kPendingLimit) [self.pending removeObjectAtIndex:0]; + FaceclawWatchPendingMessage *message = [FaceclawWatchPendingMessage new]; + message.path = path; + message.json = json; + message.receivedAt = now; + [self.pending addObject:message]; + } + [self sendAck:ReadSeq(json) ok:NO jsReady:NO message:@"Faceclaw is not running on the phone. Open it to connect the glasses."]; +} + +/** The watch leaves its battery in the application context while it can't message us live. */ +- (void)deliverBatteryFrom:(NSDictionary *)context { + id json = context[kKeyBattery]; + if ([json isKindOfClass:NSString.class] && self.messageHandler) self.messageHandler(kPathBattery, json); +} + +- (void)notifyConnection { + if (self.connectionHandler) self.connectionHandler(); +} + +#pragma mark - WCSessionDelegate (any thread) + +- (void)session:(WCSession *)session activationDidCompleteWithState:(WCSessionActivationState)state error:(NSError *)error { + if (error) NSLog(@"[FaceclawWatch] activation failed: %@", error); + dispatch_async(dispatch_get_main_queue(), ^{ + // Anything published before activation couldn't be delivered. + if (self.lastState || self.heldEvent) [self pushContext]; + [self deliverBatteryFrom:session.receivedApplicationContext]; + [self notifyConnection]; + }); +} + +- (void)sessionDidBecomeInactive:(WCSession *)session {} + +- (void)sessionDidDeactivate:(WCSession *)session { + // The user switched watches: the session must be reactivated for the new one. + [session activateSession]; +} + +- (void)sessionWatchStateDidChange:(WCSession *)session { + dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; }); +} + +- (void)sessionReachabilityDidChange:(WCSession *)session { + dispatch_async(dispatch_get_main_queue(), ^{ [self notifyConnection]; }); +} + +- (void)session:(WCSession *)session didReceiveMessage:(NSDictionary *)message { + id path = message[kKeyPath], json = message[kKeyJson]; + if (![path isKindOfClass:NSString.class]) return; + NSString *body = [json isKindOfClass:NSString.class] ? json : @"{}"; + dispatch_async(dispatch_get_main_queue(), ^{ [self handleMessage:path json:body]; }); +} + +- (void)session:(WCSession *)session didReceiveApplicationContext:(NSDictionary *)context { + dispatch_async(dispatch_get_main_queue(), ^{ [self deliverBatteryFrom:context]; }); +} + +@end diff --git a/App_Resources/iOS/src/module.modulemap b/App_Resources/iOS/src/module.modulemap index 2ef469f1..9bc01ad2 100644 --- a/App_Resources/iOS/src/module.modulemap +++ b/App_Resources/iOS/src/module.modulemap @@ -14,5 +14,6 @@ module FaceclawGraphics { header "FaceclawLoopbackRedirect.h" header "FaceclawWebSignIn.h" header "FaceclawResourceUsage.h" + header "FaceclawWatchBridge.h" export * } diff --git a/CLAUDE.md b/CLAUDE.md index 128f6379..041eb486 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,7 +13,14 @@ remote that drives Faceclaw over the Wearable Data Layer. Its phone-side counterparts are FaceclawWearBridge.kt, app/native/wear-bridge.ts and app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md. -`build.sh` and `build_wear.sh` build the Android and Wear OS apps respectively. +apple-watch/ is the same remote for Apple Watch (Swift + SwiftUI), driving the iOS +app over WatchConnectivity. Its phone-side counterparts are +App_Resources/iOS/src/FaceclawWatchBridge.m, app/native/wear-bridge.ios.ts and the +same app/g2/wear-remote.ts. It only links to a phone app that embeds it +(FACECLAW_EMBED_WATCH=1); see apple-watch/README.md. + +`build.sh`, `build_wear.sh` and `build_apple_watch.sh` build the Android, Wear OS +and Apple Watch apps respectively. `build_and_run.sh` builds the Android app then also installs it on a phone paired with `adb`. Building may require configuring environment variables (JAVA_HOME, ANDROID_HOME, PATH, ANDROID_KEYSTORE, ANDROID_KEYSTORE) to point at diff --git a/README.md b/README.md index 39655e62..6952f2dc 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,9 @@ section has the phone-side switches. See [wear/README.md](wear/README.md) for building and installing it; it is a separate Gradle build, sideloaded over adb, and needs no Play Store. The phone side requires Google Play services. +The iOS app has the same remote for Apple Watch, with the same pad and screens; +see [apple-watch/README.md](apple-watch/README.md). + ## Connecting an external agent (OpenClaw) Instead of calling an LLM API directly from the phone, the voice assistant diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 2442b87c..247d8f1d 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -29,7 +29,8 @@ import { OsEventTypeList } from './events' import { loadDeviceAddresses } from './device-addresses' import { updateGlassesPresence } from './glasses-presence' import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat' -import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy' +import { isHaltedSessionPhase, resumeAutoReconnect, suppressAutoReconnect } from './reconnect-policy' +import { WearRemote, type WearRemoteInputKind } from './wear-remote' import { deviceAddressError } from './ios-peripheral-identity' import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app' import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app' @@ -167,6 +168,8 @@ export class IosPreviewController { private continuousPending = false private continuousCaptureToken = 0 private sessionConnected = false + /** The Apple Watch remote; constructed last so it sees a fully wired shell. */ + private wearRemote: WearRemote | null = null private readonly actions: LayerActions = { ...noopLayerActions, playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload), @@ -210,7 +213,7 @@ export class IosPreviewController { prepareVoiceCapture: () => this.prepareVoiceCapture(), getScreenTimeoutMs: () => null, requestShellRender: () => this.requestShellRender(), - onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() }, + onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi(); this.wearRemote?.schedulePublish() }, openNotificationsWindow: () => this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow), onSwitcherBarHeightChanged: () => { @@ -227,6 +230,7 @@ export class IosPreviewController { this.unblankAfterShellRender = on if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error)) this.requestShellRender() + this.wearRemote?.schedulePublish() }, }) const launcher = createLauncherWindow({ @@ -275,6 +279,20 @@ export class IosPreviewController { // Every capture end (drained, failed, replaced) may leave continuous capture to restart. iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture()) for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app)) + this.wearRemote = new WearRemote({ + apps: ALL_APPS.filter(app => app.showInLauncher !== false && !iosAppUnavailableReason(app.appId)), + injectInput: kind => this.injectWatchInput(kind), + launchApp: id => this.launchApp(id), + // The watch's Connect/Disconnect are explicit user choices, like the phone's own buttons. + connect: () => { resumeAutoReconnect(); return this.connect(true) }, + disconnect: () => { suppressAutoReconnect(); return this.disconnect() }, + setGlassesLocked: locked => this.setGlassesLocked(locked), + requestShellRender: () => this.requestShellRender(), + previewInputActive: () => !this.communicator && this.runtimeRunning, + getState: () => ({ phase: this.state.phase, status: this.state.status, glassesLocked: this.glassesLocked, + glassesWorn: this.glassesWorn, silentMode: false, lastHeadsetBattery: this.state.battery }), + appendLog: line => this.logBluetooth(line), + }) } /** The compositor frames go to: the live session, else the headless preview. */ @@ -443,6 +461,7 @@ export class IosPreviewController { // a session's first snapshot) wakes the screen, to the lock notice if locked. const putOn = wearing && this.glassesWorn === false this.glassesWorn = wearing + this.wearRemote?.schedulePublish() this.syncPresence() this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`) if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true) @@ -451,6 +470,7 @@ export class IosPreviewController { private setGlassesLocked(locked: boolean): void { if (locked === this.glassesLocked) return this.glassesLocked = locked + this.wearRemote?.schedulePublish() if (locked) { this.glance.dismiss() if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true @@ -599,7 +619,25 @@ export class IosPreviewController { console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`) }).catch(error => this.fail(error)) } + /** Input from the Apple Watch: shares the controller queue with physical input. */ + private injectWatchInput(kind: WearRemoteInputKind): Promise { + const task = this.inputQueue.then(async () => { + this.onActivity() + // As on Android, and because the watch may be acting on a stale state snapshot: a dark + // display takes a double-click (wake) or a tap/hold (Glanceboard), and scrolls mean nothing. + if (!shell.isScreenOn() && (kind === 'scroll-up' || kind === 'scroll-down' || kind === 'short-then-long-press')) return + // A real hold releases when the finger lifts; a discrete long-press releases at once. + const type = kind === 'long-press-start' ? 'long-press' : kind + await this.receiveInput(makeInputEvent({ type, source: 'watch' } as InputEventPayload)) + if (kind === 'long-press') await this.receiveInput(makeInputEvent({ type: 'long-press-release', source: 'watch' })) + console.log(`[ios-preview] apple-watch ${kind}: ${shell.describeInputTarget()}`) + }) + this.inputQueue = task.catch(error => this.fail(error)) + return task + } private async receiveInput(event: InputEvent, headTilt = false): Promise { + // Input can change the foreground app or the display state, and the shell doesn't announce those. + this.wearRemote?.schedulePublish() if (!acceptInput(event)) return if (this.glassesLocked) { // Preserve the locked display's sleep/wake controls without dispatching @@ -807,6 +845,7 @@ export class IosPreviewController { this.syncRuntime() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() + this.wearRemote?.schedulePublish() } /** The ring level arrives through the glasses, so like Android it is shown only while connected. */ private pushBatteryLevels(): void { @@ -814,11 +853,16 @@ export class IosPreviewController { shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging, ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null }) } - async connect(): Promise { - if (!this.active || this.connecting || this.stopping) return + /** `fromBackground`: the watch asked, and the phone may be in a pocket with the app suspended. */ + async connect(fromBackground = false): Promise { + if ((!this.active && !fromBackground) || this.connecting || this.stopping) return if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return const addresses = loadDeviceAddresses(), error = deviceAddressError(addresses) - if (error) { this.onError(error); return } + if (error) { + if (this.active) this.onError(error) + else this.logBluetooth(error) + return + } this.connecting = true this.sessionCreated = true try { @@ -882,6 +926,7 @@ export class IosPreviewController { this.pushBatteryLevels() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() + this.wearRemote?.schedulePublish() }), communicator.onCaseState(state => { if (this.communicator !== communicator) return @@ -889,6 +934,7 @@ export class IosPreviewController { // the case's own charging is only known while the glasses are docked. shell.setBatteryLevels({ ...(state.battery !== null ? { case: state.battery } : {}), caseCharging: state.docked ? state.charging : null }) + this.wearRemote?.schedulePublish() }), communicator.onFirmwareInfo(info => { if (this.communicator !== communicator) return diff --git a/app/g2/wear-remote.ts b/app/g2/wear-remote.ts index 92a62902..c2e2b597 100644 --- a/app/g2/wear-remote.ts +++ b/app/g2/wear-remote.ts @@ -1,5 +1,6 @@ /** - * The Wear OS watch as an input device and status mirror for the glasses. + * The watch (Wear OS or Apple Watch) as an input device and status mirror for + * the glasses. * * Inbound, the watch sends ring-style gestures (tap, double-tap, hold, * swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path @@ -17,11 +18,14 @@ * The watch's own battery flows the other way for the glasses' top-bar and * Glanceboard indicators: the phone polls it (the watch answers from a * listener service even while its app is closed) and the open watch app - * pushes changes unprompted. The level lives in shell.getBatteryLevels() - * beside the G2 and R1 ones and is cleared when no watch is reachable. + * pushes changes unprompted. An Apple Watch app can't be woken to answer, so + * it reports when it can instead (see wear/PROTOCOL.md). The level lives in + * shell.getBatteryLevels() beside the G2 and R1 ones and is cleared when no + * watch is reachable or the last report has gone stale. * - * Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the - * message shapes are documented in wear/PROTOCOL.md. + * Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge) or + * wear-bridge.ios.ts (FaceclawWatchBridge); the message shapes are documented + * in wear/PROTOCOL.md. */ import { flagAnalyticsEvent } from "../analytics/analytics"; import { type AppDefinition } from "../apps/app-definition"; @@ -33,6 +37,8 @@ import { watchCrownClockwiseNextSetting, watchMirrorAssistantSetting, watchRemoteEnabledSetting, + watchSoundEffectsSetting, + watchVibrateOnInputSetting, } from "../ui/dashboard-settings"; import { shell, type AssistantActivityEvent } from "../ui/shell/shell"; import { voiceActivity } from "../ui/shell/voice-activity"; @@ -102,6 +108,10 @@ const WATCH_BATTERY_POLL_INTERVAL_MS = 5 * 60_000; // glasses connecting) would otherwise trigger back to back. const WATCH_BATTERY_POLL_MIN_GAP_MS = 20_000; const WATCH_BATTERY_URGENT_POLL_LIMIT = 3; +// Wear OS answers every poll. A closed Apple Watch app only reports when the +// system gives it background time, which can be hours apart, so a level this +// old says more about the watch being gone than about its charge. +const WATCH_BATTERY_MAX_AGE_MS = 4 * 60 * 60_000; /** What the dashboard controller lends the watch remote. */ export type WearRemoteHost = { @@ -113,6 +123,11 @@ export type WearRemoteHost = { disconnect: () => Promise; setGlassesLocked: (locked: boolean, reason: string) => void; requestShellRender: () => void; + /** + * The shell is running on the phone's preview with no glasses involved, so + * gestures have somewhere to go even though nothing is connected. + */ + previewInputActive?: () => boolean; getState: () => { phase: string; status: string; @@ -136,6 +151,7 @@ export class WearRemote { private lastAssistantSentAtMs = 0; private lastWatchBatteryPollAtMs = 0; private watchBatteryKnown = false; + private watchBatteryReportedAtMs = 0; // Urgent polls that got no answer; a watch app too old to answer them // must not be poked every publish, so after a few the cadence drops back. private watchBatteryPollsUnanswered = 0; @@ -143,7 +159,7 @@ export class WearRemote { constructor(private readonly host: WearRemoteHost) { this.active = wearBridge.isAvailable(); if (!this.active) { - host.appendLog("watch: Google Play services unavailable; watch remote disabled"); + host.appendLog(`watch: ${global.isIOS ? "WatchConnectivity" : "Google Play services"} unavailable; watch remote disabled`); return; } wearBridge.onMessage((message) => { @@ -183,10 +199,25 @@ export class WearRemote { // Catch-all refresh for state the controller does not announce (cheap: // identical states are dropped on the Java side). setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS); - setInterval(() => this.pollWatchBattery(false), WATCH_BATTERY_POLL_INTERVAL_MS); + setInterval(() => { + this.expireWatchBattery(); + this.pollWatchBattery(false); + }, WATCH_BATTERY_POLL_INTERVAL_MS); this.schedulePublish(); } + /** Gestures have a display to act on: the glasses, or the phone's preview without them. */ + private takesInput(): boolean { + const phase = this.host.getState().phase; + return phase === "connected" || phase === "charging" || this.host.previewInputActive?.() === true; + } + + private expireWatchBattery(): void { + if (this.watchBatteryKnown && Date.now() - this.watchBatteryReportedAtMs > WATCH_BATTERY_MAX_AGE_MS) { + this.setWatchBattery(null, null); + } + } + /** * Ask the reachable watch for its battery. `urgent` skips the poll gap for * a level that is not known yet (the glasses just connected, the watch @@ -257,6 +288,7 @@ export class WearRemote { phase: state.phase, status: state.status, connected: state.phase === "connected" || state.phase === "charging", + preview: this.host.previewInputActive?.() === true, screenOn: shell.isScreenOn(), locked: state.glassesLocked, worn: state.glassesWorn, @@ -278,6 +310,8 @@ export class WearRemote { apps: this.listApps(), remoteEnabled: watchRemoteEnabledSetting.get(), crownClockwiseNext: watchCrownClockwiseNextSetting.get(), + vibrateOnInput: watchVibrateOnInputSetting.get(), + soundEffects: watchSoundEffectsSetting.get(), canUnlock: watchCanUnlockSetting.get(), mirrorAssistant: watchMirrorAssistantSetting.get(), assistantAvailable: shell.isAssistantAvailable(), @@ -307,7 +341,11 @@ export class WearRemote { // acked (the watch sends it fire-and-forget, without a seq). this.watchBatteryPollsUnanswered = 0; const battery = readNumber(message.payload.battery, -1); - const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100; + // A report the watch left behind earlier says when it was taken; a live one is current. + const reportedAtMs = readNumber(message.payload.at, Date.now()); + const valid = Number.isInteger(battery) && battery >= 0 && battery <= 100 && + Date.now() - reportedAtMs <= WATCH_BATTERY_MAX_AGE_MS; + this.watchBatteryReportedAtMs = reportedAtMs; this.setWatchBattery(valid ? battery : null, valid ? message.payload.charging === true : null); return; } @@ -315,8 +353,7 @@ export class WearRemote { // Touch-down ahead of the gesture that follows it. Fire-and-forget like // the battery (no seq, never acked): wherever that gesture would be // refused, the press is dropped silently and the gesture's ack says why. - const phase = this.host.getState().phase; - if (watchRemoteEnabledSetting.get() && (phase === "connected" || phase === "charging")) { + if (watchRemoteEnabledSetting.get() && this.takesInput()) { await this.host.injectInput("ring-press"); } return; @@ -378,8 +415,7 @@ export class WearRemote { return; } const kind = gesture as WearRemoteInputKind; - const state = this.host.getState(); - if (state.phase !== "connected" && state.phase !== "charging") { + if (!this.takesInput()) { ack(false, "The glasses are not connected."); return; } @@ -422,7 +458,7 @@ export class WearRemote { default: break; } - if (!connected) { + if (!this.takesInput()) { ack(false, "The glasses are not connected."); return; } @@ -561,6 +597,8 @@ export class WearRemote { } function readNumber(value: unknown, fallback: number): number { + // Number(null) is 0: a watch reporting an unknown battery would read as empty. + if (value === null) return fallback; const numeric = typeof value === "number" ? value : Number(value); return Number.isFinite(numeric) ? numeric : fallback; } diff --git a/app/native/wear-bridge-types.ts b/app/native/wear-bridge-types.ts new file mode 100644 index 00000000..39241d62 --- /dev/null +++ b/app/native/wear-bridge-types.ts @@ -0,0 +1,52 @@ +/** + * The watch transport's wire names and shapes, shared by the Wear OS bridge + * (wear-bridge.ts) and the Apple Watch one (wear-bridge.ios.ts). See + * wear/PROTOCOL.md for the messages themselves. + */ + +/** Message paths shared with the watch apps (wear/app/.../Protocol.kt, apple-watch/Faceclaw/Protocol.swift). */ +export const WEAR_PATHS = { + /** Watch -> phone: a ring-style gesture. */ + input: "/faceclaw/input", + /** Watch -> phone: a finger landed on the pad (the ring-press ahead of a gesture); never acked. */ + press: "/faceclaw/press", + /** Watch -> phone: a shell command (launch app, wake, lock, ...). */ + command: "/faceclaw/command", + /** Watch -> phone: a text query for the assistant. */ + assistant: "/faceclaw/assistant", + /** Watch -> phone: text to type into the foreground app. */ + text: "/faceclaw/text", + /** Watch -> phone: please re-publish the state item. */ + stateRequest: "/faceclaw/state/request", + /** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */ + battery: "/faceclaw/battery", + /** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */ + batteryRequest: "/faceclaw/battery/request", + /** Phone -> watch: reply to any of the above. */ + ack: "/faceclaw/ack", + /** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */ + event: "/faceclaw/event", + /** Phone -> watch Data Layer item: the mirrored dashboard state. */ + state: "/faceclaw/state", +} as const; + +export type WearMessage = { + path: string; + /** Parsed JSON payload ({} when the watch sent none or it was malformed). */ + payload: Record; + nodeId: string; +}; + +export type WearWatchConnection = { + reachable: boolean; + watchName: string; +}; + +export function parseWearPayload(json: string): Record { + try { + const parsed = JSON.parse(json); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; + } catch { + return {}; + } +} diff --git a/app/native/wear-bridge.ios.ts b/app/native/wear-bridge.ios.ts new file mode 100644 index 00000000..d498e136 --- /dev/null +++ b/app/native/wear-bridge.ios.ts @@ -0,0 +1,123 @@ +/** + * iOS twin of wear-bridge.ts: the phone side of the Apple Watch remote, over + * WatchConnectivity (App_Resources/iOS/src/FaceclawWatchBridge). It has the + * same surface and carries the same (path, JSON) messages, so + * app/g2/wear-remote.ts gives them their meaning on both platforms. + * + * "Reachable" here means an Apple Watch with the Faceclaw app is paired, not + * that the app is running: iOS can't wake a closed watch app, and it is + * suspended whenever the wrist drops, so WCSession's live reachability flips + * constantly. The native side sends live while it can and otherwise leaves + * the latest state in the application context for the app's next wake. + */ +import { WEAR_PATHS, parseWearPayload, type WearMessage, type WearWatchConnection } from './wear-bridge-types' + +export { WEAR_PATHS, type WearMessage, type WearWatchConnection } + +declare const FaceclawWatchBridge: any + +/** WatchConnectivity doesn't expose the watch's own name. */ +const WATCH_NAME = 'Apple Watch' +/** Replies go to the session's one counterpart; this only fills WearMessage.nodeId. */ +const WATCH_NODE_ID = 'apple-watch' + +type MessageListener = (message: WearMessage) => void +type ConnectionListener = (connection: WearWatchConnection) => void + +class WearBridge { + private nativeBridge: any = undefined + private listening = false + private readonly messageListeners = new Set() + private readonly connectionListeners = new Set() + + private native(): any { + if (this.nativeBridge !== undefined) return this.nativeBridge + try { + const bridge = FaceclawWatchBridge.shared() + this.nativeBridge = bridge.isAvailable() ? bridge : null + } catch (error) { + console.warn('watch bridge unavailable', error) + this.nativeBridge = null + } + return this.nativeBridge + } + + /** Whether WatchConnectivity works on this phone (it does on every iPhone). */ + isAvailable(): boolean { + return this.native() !== null + } + + private ensureListener(): void { + if (this.listening) return + const native = this.native() + if (!native) return + this.listening = true + native.setMessageHandlerConnectionHandler((path: string, json: string) => { + const message: WearMessage = { path: String(path), payload: parseWearPayload(String(json)), nodeId: WATCH_NODE_ID } + for (const listener of Array.from(this.messageListeners)) { + try { listener(message) } + catch (error) { console.warn('watch message listener failed', error) } + } + }, () => { + const connection = this.getWatchConnection() + for (const listener of Array.from(this.connectionListeners)) { + try { listener(connection) } + catch (error) { console.warn('watch connection listener failed', error) } + } + }) + } + + /** Subscribe to watch messages. Registers the native listener on first use. */ + onMessage(listener: MessageListener): () => void { + this.messageListeners.add(listener) + this.ensureListener() + return () => { this.messageListeners.delete(listener) } + } + + /** + * Subscribe to the watch's availability; the current value is delivered + * immediately. Also fires when the watch app comes to the front with the + * value unchanged, which is what prompts WearRemote to push it fresh state. + */ + onWatchConnection(listener: ConnectionListener): () => void { + this.connectionListeners.add(listener) + this.ensureListener() + listener(this.getWatchConnection()) + return () => { this.connectionListeners.delete(listener) } + } + + getWatchConnection(): WearWatchConnection { + const reachable = Boolean(this.native()?.isWatchAppAvailable()) + return { reachable, watchName: reachable ? WATCH_NAME : '' } + } + + /** WatchConnectivity pushes changes; there is nothing to re-query. */ + refreshWatchNodes(): void {} + + /** Mirror the dashboard state; unchanged states are dropped natively unless `force`. */ + publishState(state: Record, force = false): void { + this.native()?.publishStateForce(JSON.stringify(state), force) + } + + /** + * Send an event to the watch. A suspended app misses live messages, so all + * but streaming snapshots also stay behind for its next wake: that is how a + * reply that finished with the wrist down still reaches the Assistant screen. + */ + sendEvent(payload: Record): void { + const sticky = !(payload.type === 'assistant' && payload.phase === 'streaming') + this.native()?.sendEventSticky(JSON.stringify(payload), sticky) + } + + /** Send a live message on any phone -> watch path; dropped if the app isn't in front. */ + sendToWatch(path: string, payload: Record): void { + this.native()?.sendToWatchJson(path, JSON.stringify(payload)) + } + + /** Acknowledge a watch message; `seq` echoes the watch's sequence number. */ + sendAck(_nodeId: string, seq: number, ok: boolean, message = ''): void { + this.native()?.sendAckOkJsReadyMessage(seq, ok, true, message) + } +} + +export const wearBridge = new WearBridge() diff --git a/app/native/wear-bridge.ts b/app/native/wear-bridge.ts index a2871218..ede435ef 100644 --- a/app/native/wear-bridge.ts +++ b/app/native/wear-bridge.ts @@ -9,47 +9,12 @@ * Data Layer is part of it), so the rest of the app never has to care. */ import { Utils } from "@nativescript/core"; +import { WEAR_PATHS, parseWearPayload, type WearMessage, type WearWatchConnection } from "./wear-bridge-types"; + +export { WEAR_PATHS, type WearMessage, type WearWatchConnection }; declare const com: any; -/** Message paths shared with the watch app (wear/app/.../Protocol.kt). */ -export const WEAR_PATHS = { - /** Watch -> phone: a ring-style gesture. */ - input: "/faceclaw/input", - /** Watch -> phone: a finger landed on the pad (the ring-press ahead of a gesture); never acked. */ - press: "/faceclaw/press", - /** Watch -> phone: a shell command (launch app, wake, lock, ...). */ - command: "/faceclaw/command", - /** Watch -> phone: a text query for the assistant. */ - assistant: "/faceclaw/assistant", - /** Watch -> phone: text to type into the foreground app. */ - text: "/faceclaw/text", - /** Watch -> phone: please re-publish the state item. */ - stateRequest: "/faceclaw/state/request", - /** Watch -> phone: the watch's own battery ({battery, charging}); never acked. */ - battery: "/faceclaw/battery", - /** Phone -> watch: please report the watch battery (answered even while the watch app is closed). */ - batteryRequest: "/faceclaw/battery/request", - /** Phone -> watch: reply to any of the above. */ - ack: "/faceclaw/ack", - /** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */ - event: "/faceclaw/event", - /** Phone -> watch Data Layer item: the mirrored dashboard state. */ - state: "/faceclaw/state", -} as const; - -export type WearMessage = { - path: string; - /** Parsed JSON payload ({} when the watch sent none or it was malformed). */ - payload: Record; - nodeId: string; -}; - -export type WearWatchConnection = { - reachable: boolean; - watchName: string; -}; - type MessageListener = (message: WearMessage) => void; type ConnectionListener = (connection: WearWatchConnection) => void; @@ -91,7 +56,7 @@ class WearBridge { onMessage: (path: string, json: string, nodeId: string) => { const message: WearMessage = { path: String(path), - payload: parsePayload(String(json)), + payload: parseWearPayload(String(json)), nodeId: String(nodeId), }; for (const listener of Array.from(this.messageListeners)) { @@ -177,13 +142,4 @@ class WearBridge { } } -function parsePayload(json: string): Record { - try { - const parsed = JSON.parse(json); - return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; - } catch { - return {}; - } -} - export const wearBridge = new WearBridge(); diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 80168597..2c6670c5 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -307,7 +307,7 @@ export const glassesBatteryVisibilitySetting = batteryVisibilitySetting( export const ringBatteryVisibilitySetting = batteryVisibilitySetting( "ringBatteryVisibility", "R1", "display.battery.ringVisibility", ); -/** The Wear OS watch; the indicator only exists while a watch is reachable. */ +/** The watch; the indicator only exists while a watch is reachable. */ export const watchBatteryVisibilitySetting = batteryVisibilitySetting( "watchBatteryVisibility", "Watch", "display.battery.watchVisibility", ); @@ -702,7 +702,7 @@ export const mirrorTouchSetting = new ConfigSettingBoolean({ "Let touches on the phone's mirror act on the glasses UI: tap selects what it lands on, double-tap goes back, a hold opens the menu, swipes navigate.", }); -// Wear OS watch remote (app/g2/wear-remote.ts, wear/). All three are read on +// Watch remote (app/g2/wear-remote.ts, wear/, apple-watch/). All three are read on // every watch message, so a change applies immediately. export const watchRemoteEnabledSetting = new ConfigSettingBoolean({ id: "watch-remote-enabled", @@ -710,7 +710,7 @@ export const watchRemoteEnabledSetting = new ConfigSettingBoolean({ storageKey: "watch.remoteEnabled", defaultValue: true, description: - "Accept input from the Faceclaw Wear OS app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.", + "Accept input from the Faceclaw watch app: spatial swipes, taps, holds, crown, voice queries and app commands. The ring's scheme is unaffected. Turn off to ignore the watch.", }); export const watchCanUnlockSetting = new ConfigSettingBoolean({ @@ -722,6 +722,7 @@ export const watchCanUnlockSetting = new ConfigSettingBoolean({ "Let the watch unlock the glasses' lock screen (which otherwise waits for the phone to be unlocked). Your watch is on your wrist; turn this off if you would rather it stay a phone-only unlock.", }); +// Wear OS only: the Apple Watch's crown doesn't reach the app, so iOS doesn't offer this. export const watchCrownClockwiseNextSetting = new ConfigSettingBoolean({ id: "watch-crown-clockwise-next", label: "Clockwise crown = next", @@ -731,6 +732,26 @@ export const watchCrownClockwiseNextSetting = new ConfigSettingBoolean({ "Choose which crown direction moves to the next item. Off: clockwise moves to the previous item. On: clockwise moves to the next item.", }); +// Apple Watch only: Wear OS has no sounds, and keeps its vibration setting on the watch. +// The watch reads these from the mirrored state (apple-watch/Faceclaw/WatchPrefs.swift). +export const watchVibrateOnInputSetting = new ConfigSettingBoolean({ + id: "watch-vibrate-on-input", + label: "Vibrate on input", + storageKey: "watch.vibrateOnInput", + defaultValue: true, + description: + "Adds a slight vibration with each watch input", +}); + +export const watchSoundEffectsSetting = new ConfigSettingBoolean({ + id: "watch-sound-effects", + label: "Sound effects", + storageKey: "watch.soundEffects", + defaultValue: false, + description: + "On: each input gets its own watchOS feedback, a sound with a vibration (watchOS can't play one alone, so this vibrates even with Vibrate on input off). Off: a light tap for every input. Only Silent Mode on the watch guarantees no sound.", +}); + export const watchMirrorAssistantSetting = new ConfigSettingBoolean({ id: "watch-mirror-assistant", label: "Mirror assistant to watch", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index d4cc0719..41e3ceeb 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -95,6 +95,8 @@ import { watchCrownClockwiseNextSetting, watchMirrorAssistantSetting, watchRemoteEnabledSetting, + watchSoundEffectsSetting, + watchVibrateOnInputSetting, } from "../dashboard-settings"; import { wearBridge } from "../../native/wear-bridge"; import { openSettingsSubMenu, SettingsPanelLayer, type SettingsSection } from "./settings-panel"; @@ -210,11 +212,16 @@ function settingsSections(): SettingsSection[] { }, { label: "Watch", - // Wear OS remote (wear/); the status line above the items says whether - // a watch running the companion app is currently reachable. + // Watch remote (wear/, apple-watch/); the status line above the items says + // whether a watch running the companion app is currently reachable. items: [ toggleSettingMenuItem(watchRemoteEnabledSetting), - toggleSettingMenuItem(watchCrownClockwiseNextSetting), + ...(global.isIOS ? [ + toggleSettingMenuItem(watchVibrateOnInputSetting), + toggleSettingMenuItem(watchSoundEffectsSetting), + ] : [ + toggleSettingMenuItem(watchCrownClockwiseNextSetting), + ]), toggleSettingMenuItem(watchCanUnlockSetting), toggleSettingMenuItem(watchMirrorAssistantSetting), ], @@ -273,17 +280,12 @@ function settingsSections(): SettingsSection[] { }, ]; if (!global.isIOS) return sections; - const deferred = new Set(["Watch"]); return sections.map(section => { if (section.label === "Developer") return { ...section, items: [enumSettingMenuItem(ringConnectionModeSetting), toggleSettingMenuItem(showBleBandwidthSetting)] }; // No downloadable on-device models: iOS's on-device option is Apple's recognizer. if (section.label === "Voice") return { label: "Voice", items: [enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), toggleSettingMenuItem(saveTranscribeAudioSetting), ...whisperServerMenuItems()] }; - if (deferred.has(section.label)) return { label: section.label, items: [{ - label: "Not available on iOS yet", disabled: true, onSelect: () => {}, - description: `${section.label} integration has not been ported to iOS.`, - }] }; if (section.label === "Display") return { ...section, items: section.items.filter(item => item.label !== screenTimeoutSetting.label) }; return section; @@ -361,7 +363,7 @@ function batteryIndicatorsMenuItem(): MenuItem { return { label: "Battery indicators", description: - "Top-bar battery indicators for the phone, Wear OS watch, glasses, and ring: their style, and whether each shows always, only when low, or never.", + "Top-bar battery indicators for the phone, watch, glasses, and ring: their style, and whether each shows always, only when low, or never.", onSelect: (ctx) => { openSettingsSubMenu(ctx, "Battery indicators", [ enumSettingMenuItem(batteryDisplayModeSetting), @@ -622,12 +624,21 @@ function renderWatchStatus(args: { image: GrayImage; x: number; y: number; width const font = getDefaultSmallFont(); let status: string; if (!wearBridge.isAvailable()) { - status = "Google Play services is unavailable on this phone, so no watch can connect."; + status = global.isIOS + ? "This device can't pair with an Apple Watch." + : "Google Play services is unavailable on this phone, so no watch can connect."; } else { const connection = wearBridge.getWatchConnection(); - status = connection.reachable - ? `Connected to ${connection.watchName || "a watch"}.` - : "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone."; + if (global.isIOS) { + // iOS only says whether the app is installed; it may not be running. + status = connection.reachable + ? "The Faceclaw app is installed on your Apple Watch." + : "No watch connected. Install the Faceclaw watch app (apple-watch/ in the source tree) on an Apple Watch paired with this iPhone."; + } else { + status = connection.reachable + ? `Connected to ${connection.watchName || "a watch"}.` + : "No watch connected. Install the Faceclaw watch app (wear/ in the source tree) on a Wear OS watch paired with this phone."; + } } const lines = wrapText(font, status, width); for (let i = 0; i < lines.length; i++) { diff --git a/app/ui/gestures.ts b/app/ui/gestures.ts index 047ebc1b..8b3f3eb8 100644 --- a/app/ui/gestures.ts +++ b/app/ui/gestures.ts @@ -2,9 +2,9 @@ import type { RingInput } from "../g2/ring-input"; /** * Where an input came from. The ring and the glasses' arms are the stock - * sources; "watch" is the Wear OS remote, which components may treat with a - * richer scheme (see isWatchInput) — the ring's semantics are never changed - * by it. + * sources; "watch" is the watch remote (Wear OS or Apple Watch), which + * components may treat with a richer scheme (see isWatchInput) — the ring's + * semantics are never changed by it. */ export type InputSource = "ring" | "left-arm" | "right-arm" | "watch"; @@ -80,7 +80,7 @@ export function makeInputEvent(payload: InputEventPayload): InputEvent { return { ...payload, timestampMs: Date.now() }; } -/** True for input from the Wear OS remote (any type that carries a source). */ +/** True for input from the watch remote (any type that carries a source). */ export function isWatchInput(event: InputEvent): boolean { return "source" in event && event.source === "watch"; } diff --git a/apple-watch/Base.xcconfig b/apple-watch/Base.xcconfig new file mode 100644 index 00000000..02a3592f --- /dev/null +++ b/apple-watch/Base.xcconfig @@ -0,0 +1,21 @@ +// Settings shared by every target of the watch project. +// +// The watch app is tied to the phone app by bundle ID: its own must begin with +// the phone app's, and WKCompanionAppBundleIdentifier must equal it, or +// WatchConnectivity has no counterpart. Local.xcconfig (not checked in) overrides +// this and sets the signing team; build_apple_watch.sh writes it from build_paths.sh. +FACECLAW_IOS_BUNDLE_ID = com.faceclaw.app + +SDKROOT = watchos +SUPPORTED_PLATFORMS = watchos watchsimulator +TARGETED_DEVICE_FAMILY = 4 +// Observation (@Observable) needs watchOS 10: Series 6, SE 2, Ultra and later. +WATCHOS_DEPLOYMENT_TARGET = 10.0 +SWIFT_VERSION = 5.0 +CODE_SIGN_STYLE = Automatic +CURRENT_PROJECT_VERSION = 1 +MARKETING_VERSION = 1.0 +ENABLE_USER_SCRIPT_SANDBOXING = YES +ALWAYS_SEARCH_USER_PATHS = NO + +#include? "Local.xcconfig" diff --git a/apple-watch/Faceclaw.xcodeproj/project.pbxproj b/apple-watch/Faceclaw.xcodeproj/project.pbxproj new file mode 100644 index 00000000..492f5d64 --- /dev/null +++ b/apple-watch/Faceclaw.xcodeproj/project.pbxproj @@ -0,0 +1,315 @@ +// !$*UTF8*$! +{ + archiveVersion = 1; + classes = { + }; + objectVersion = 77; + objects = { + +/* Begin PBXContainerItemProxy section */ + FC0000000000000000000041 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = FC0000000000000000000001 /* Project object */; + proxyType = 1; + remoteGlobalIDString = FC0000000000000000000010; + remoteInfo = Faceclaw; + }; +/* End PBXContainerItemProxy section */ + +/* Begin PBXFileReference section */ + FC0000000000000000000020 /* Faceclaw.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = Faceclaw.app; sourceTree = BUILT_PRODUCTS_DIR; }; + FC0000000000000000000021 /* FaceclawUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = FaceclawUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + FC0000000000000000000022 /* Base.xcconfig */ = {isa = PBXFileReference; lastKnownFileType = text.xcconfig; path = Base.xcconfig; sourceTree = ""; }; +/* End PBXFileReference section */ + +/* Begin PBXFileSystemSynchronizedRootGroup section */ + FC0000000000000000000030 /* Faceclaw */ = {isa = PBXFileSystemSynchronizedRootGroup; path = Faceclaw; sourceTree = ""; }; + FC0000000000000000000031 /* FaceclawUITests */ = {isa = PBXFileSystemSynchronizedRootGroup; path = FaceclawUITests; sourceTree = ""; }; +/* End PBXFileSystemSynchronizedRootGroup section */ + +/* Begin PBXFrameworksBuildPhase section */ + FC0000000000000000000050 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FC0000000000000000000051 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXFrameworksBuildPhase section */ + +/* Begin PBXGroup section */ + FC0000000000000000000002 = { + isa = PBXGroup; + children = ( + FC0000000000000000000022 /* Base.xcconfig */, + FC0000000000000000000030 /* Faceclaw */, + FC0000000000000000000031 /* FaceclawUITests */, + FC0000000000000000000003 /* Products */, + ); + sourceTree = ""; + }; + FC0000000000000000000003 /* Products */ = { + isa = PBXGroup; + children = ( + FC0000000000000000000020 /* Faceclaw.app */, + FC0000000000000000000021 /* FaceclawUITests.xctest */, + ); + name = Products; + sourceTree = ""; + }; +/* End PBXGroup section */ + +/* Begin PBXNativeTarget section */ + FC0000000000000000000010 /* Faceclaw */ = { + isa = PBXNativeTarget; + buildConfigurationList = FC0000000000000000000071 /* Build configuration list for PBXNativeTarget "Faceclaw" */; + buildPhases = ( + FC0000000000000000000060 /* Sources */, + FC0000000000000000000050 /* Frameworks */, + FC0000000000000000000062 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + FC0000000000000000000030 /* Faceclaw */, + ); + name = Faceclaw; + productName = Faceclaw; + productReference = FC0000000000000000000020 /* Faceclaw.app */; + productType = "com.apple.product-type.application"; + }; + FC0000000000000000000011 /* FaceclawUITests */ = { + isa = PBXNativeTarget; + buildConfigurationList = FC0000000000000000000072 /* Build configuration list for PBXNativeTarget "FaceclawUITests" */; + buildPhases = ( + FC0000000000000000000061 /* Sources */, + FC0000000000000000000051 /* Frameworks */, + FC0000000000000000000063 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + FC0000000000000000000042 /* PBXTargetDependency */, + ); + fileSystemSynchronizedGroups = ( + FC0000000000000000000031 /* FaceclawUITests */, + ); + name = FaceclawUITests; + productName = FaceclawUITests; + productReference = FC0000000000000000000021 /* FaceclawUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; + }; +/* End PBXNativeTarget section */ + +/* Begin PBXProject section */ + FC0000000000000000000001 /* Project object */ = { + isa = PBXProject; + attributes = { + BuildIndependentTargetsInParallel = 1; + LastSwiftUpdateCheck = 2640; + LastUpgradeCheck = 2640; + TargetAttributes = { + FC0000000000000000000010 = { + CreatedOnToolsVersion = 26.4; + }; + FC0000000000000000000011 = { + CreatedOnToolsVersion = 26.4; + TestTargetID = FC0000000000000000000010; + }; + }; + }; + buildConfigurationList = FC0000000000000000000070 /* Build configuration list for PBXProject "Faceclaw" */; + developmentRegion = en; + hasScannedForEncodings = 0; + knownRegions = ( + en, + Base, + ); + mainGroup = FC0000000000000000000002; + minimizedProjectReferenceProxies = 1; + preferredProjectObjectVersion = 77; + productRefGroup = FC0000000000000000000003 /* Products */; + projectDirPath = ""; + projectRoot = ""; + targets = ( + FC0000000000000000000010 /* Faceclaw */, + FC0000000000000000000011 /* FaceclawUITests */, + ); + }; +/* End PBXProject section */ + +/* Begin PBXResourcesBuildPhase section */ + FC0000000000000000000062 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FC0000000000000000000063 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXResourcesBuildPhase section */ + +/* Begin PBXSourcesBuildPhase section */ + FC0000000000000000000060 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; + FC0000000000000000000061 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; +/* End PBXSourcesBuildPhase section */ + +/* Begin PBXTargetDependency section */ + FC0000000000000000000042 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = FC0000000000000000000010 /* Faceclaw */; + targetProxy = FC0000000000000000000041 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + +/* Begin XCBuildConfiguration section */ + FC0000000000000000000080 /* Debug */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = FC0000000000000000000022 /* Base.xcconfig */; + buildSettings = { + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = dwarf; + ENABLE_TESTABILITY = YES; + GCC_OPTIMIZATION_LEVEL = 0; + ONLY_ACTIVE_ARCH = YES; + SWIFT_ACTIVE_COMPILATION_CONDITIONS = "DEBUG $(inherited)"; + SWIFT_OPTIMIZATION_LEVEL = "-Onone"; + }; + name = Debug; + }; + FC0000000000000000000081 /* Release */ = { + isa = XCBuildConfiguration; + baseConfigurationReference = FC0000000000000000000022 /* Base.xcconfig */; + buildSettings = { + COPY_PHASE_STRIP = NO; + DEBUG_INFORMATION_FORMAT = "dwarf-with-dsym"; + SWIFT_COMPILATION_MODE = wholemodule; + VALIDATE_PRODUCT = YES; + }; + name = Release; + }; + FC0000000000000000000082 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = Faceclaw; + INFOPLIST_KEY_UISupportedInterfaceOrientations = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown"; + INFOPLIST_KEY_WKCompanionAppBundleIdentifier = "$(FACECLAW_IOS_BUNDLE_ID)"; + INFOPLIST_KEY_WKRunsIndependentlyOfCompanionApp = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + }; + name = Debug; + }; + FC0000000000000000000083 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; + ENABLE_PREVIEWS = YES; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_KEY_CFBundleDisplayName = Faceclaw; + INFOPLIST_KEY_UISupportedInterfaceOrientations = "UIInterfaceOrientationPortrait UIInterfaceOrientationPortraitUpsideDown"; + INFOPLIST_KEY_WKCompanionAppBundleIdentifier = "$(FACECLAW_IOS_BUNDLE_ID)"; + INFOPLIST_KEY_WKRunsIndependentlyOfCompanionApp = YES; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SKIP_INSTALL = YES; + SWIFT_EMIT_LOC_STRINGS = YES; + }; + name = Release; + }; + FC0000000000000000000084 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + GENERATE_INFOPLIST_FILE = YES; + PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp.uitests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TEST_TARGET_NAME = Faceclaw; + }; + name = Debug; + }; + FC0000000000000000000085 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + GENERATE_INFOPLIST_FILE = YES; + PRODUCT_BUNDLE_IDENTIFIER = "$(FACECLAW_IOS_BUNDLE_ID).watchkitapp.uitests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; + TEST_TARGET_NAME = Faceclaw; + }; + name = Release; + }; +/* End XCBuildConfiguration section */ + +/* Begin XCConfigurationList section */ + FC0000000000000000000070 /* Build configuration list for PBXProject "Faceclaw" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FC0000000000000000000080 /* Debug */, + FC0000000000000000000081 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + FC0000000000000000000071 /* Build configuration list for PBXNativeTarget "Faceclaw" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FC0000000000000000000082 /* Debug */, + FC0000000000000000000083 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + FC0000000000000000000072 /* Build configuration list for PBXNativeTarget "FaceclawUITests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + FC0000000000000000000084 /* Debug */, + FC0000000000000000000085 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; +/* End XCConfigurationList section */ + }; + rootObject = FC0000000000000000000001 /* Project object */; +} diff --git a/apple-watch/Faceclaw.xcodeproj/xcshareddata/xcschemes/Faceclaw.xcscheme b/apple-watch/Faceclaw.xcodeproj/xcshareddata/xcschemes/Faceclaw.xcscheme new file mode 100644 index 00000000..c23e7568 --- /dev/null +++ b/apple-watch/Faceclaw.xcodeproj/xcshareddata/xcschemes/Faceclaw.xcscheme @@ -0,0 +1,86 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/AppIcon.png b/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/AppIcon.png new file mode 100644 index 00000000..42d0c7b7 Binary files /dev/null and b/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/AppIcon.png differ diff --git a/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/Contents.json b/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/Contents.json new file mode 100644 index 00000000..11bdb4dd --- /dev/null +++ b/apple-watch/Faceclaw/Assets.xcassets/AppIcon.appiconset/Contents.json @@ -0,0 +1 @@ +{"images":[{"filename":"AppIcon.png","idiom":"universal","platform":"watchos","size":"1024x1024"}],"info":{"author":"xcode","version":1}} diff --git a/apple-watch/Faceclaw/Assets.xcassets/Contents.json b/apple-watch/Faceclaw/Assets.xcassets/Contents.json new file mode 100644 index 00000000..f223229d --- /dev/null +++ b/apple-watch/Faceclaw/Assets.xcassets/Contents.json @@ -0,0 +1 @@ +{"info":{"author":"xcode","version":1}} diff --git a/apple-watch/Faceclaw/FaceclawWatchApp.swift b/apple-watch/Faceclaw/FaceclawWatchApp.swift new file mode 100644 index 00000000..becf911a --- /dev/null +++ b/apple-watch/Faceclaw/FaceclawWatchApp.swift @@ -0,0 +1,59 @@ +import SwiftUI + +@main +struct FaceclawWatchApp: App { + private let link: PhoneLink + private let prefs: WatchPrefs + + init() { + link = PhoneLink() + prefs = WatchPrefs(link: link) + // Here rather than in a view: a background launch never builds one. + link.start() + } + + var body: some Scene { + WindowGroup { + RootView() + .environment(link) + .environment(prefs) + } + .backgroundTask(.appRefresh(WatchBattery.refreshIdentifier)) { [link] in + await link.reportBatteryFromBackground() + } + } +} + +enum Destination: Hashable { + case assistant, apps, status, settings +} + +private struct RootView: View { + @Environment(PhoneLink.self) private var link + @Environment(WatchPrefs.self) private var prefs + @Environment(\.scenePhase) private var scenePhase + @State private var path: [Destination] = [] + + var body: some View { + NavigationStack(path: $path) { + RemoteScreen(link: link, prefs: prefs, open: { path.append($0) }) + .navigationDestination(for: Destination.self) { destination in + switch destination { + case .assistant: AssistantScreen() + case .apps: AppsScreen() + case .status: StatusScreen(openSettings: { path.append(.settings) }) + case .settings: SettingsScreen() + } + } + } + .overlay(alignment: .bottom) { NoticeOverlay(notice: link.notice) } + .onChange(of: scenePhase, initial: true) { _, phase in + if phase == .active { link.becameActive() } else { link.resignedActive() } + } + // A refused or unanswered gesture gets a distinct buzz, so the wrist + // knows without looking; the notice pill says why. + .onChange(of: link.notice?.id) { _, _ in + if link.notice?.isError == true { Haptics(prefs: prefs).error() } + } + } +} diff --git a/apple-watch/Faceclaw/PadController.swift b/apple-watch/Faceclaw/PadController.swift new file mode 100644 index 00000000..6c57401d --- /dev/null +++ b/apple-watch/Faceclaw/PadController.swift @@ -0,0 +1,185 @@ +import CoreGraphics +import Foundation +import Observation +import os + +/** + What the pad's touches and crown turns mean: the watch-side half of the remote, + the twin of the callbacks in wear/.../RemoteScreen.kt. + */ +@MainActor +@Observable +final class PadController { + /** The action buttons, hidden until a swipe up from the bottom edge. */ + var controlsVisible = false + + @ObservationIgnored private let log = Logger(subsystem: "com.faceclaw.watch", category: "Pad") + @ObservationIgnored private let link: PhoneLink + @ObservationIgnored private let haptics: Haptics + @ObservationIgnored private let recognizer = PadGestureRecognizer() + @ObservationIgnored private var deadlineTask: Task? + @ObservationIgnored private var longPressSent = false + @ObservationIgnored private var touchStartedAt: Date? + @ObservationIgnored private var crownRemainder = 0.0 + /** Pad height in points, for the edge-reveal zone. */ + @ObservationIgnored var padHeight: CGFloat = 200 + + init(link: PhoneLink, prefs: WatchPrefs) { + self.link = link + self.haptics = Haptics(prefs: prefs) + recognizer.onEvent = { [weak self] event in self?.handle(event) } + } + + // MARK: - Touch input + + func touchChanged(start: CGPoint, location: CGPoint, time: Date) { + if !recognizer.isTouching { + touchStartedAt = time + recognizer.touchDown(at: start, time: now()) + } + recognizer.touchMoved(to: location) + syncDeadline() + } + + func touchEnded(location: CGPoint) { + recognizer.touchUp(at: location, time: now()) + syncDeadline() + } + + /** The touch ended without a touch-up: the system took it, or the wrist dropped. */ + func touchCancelled() { + recognizer.cancel() + syncDeadline() + } + + /** + SwiftUI says a touch is over. If it's the current one and it hasn't ended, no + touch-up is coming, and a hold still has to be released on the phone. + */ + func touchWentAway(startedAt: Date) { + if recognizer.isTouching && startedAt == touchStartedAt { touchCancelled() } + } + + private func now() -> TimeInterval { ProcessInfo.processInfo.systemUptime } + + private func syncDeadline() { + deadlineTask?.cancel() + deadlineTask = nil + guard let deadline = recognizer.deadline else { return } + let delay = max(0, deadline - now()) + deadlineTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: UInt64(delay * 1_000_000_000)) + guard !Task.isCancelled, let self else { return } + self.recognizer.advance(to: self.now()) + self.syncDeadline() + } + } + + // MARK: - What gestures mean + + private var glassesScreenIsOff: Bool { + guard let state = link.state else { return false } + return state.takesInput && !state.screenOn + } + + /** + A dark display still takes a double-click (wake), and a tap or a hold (the + phone's Glanceboard). Scrolls, swipes and the menu gesture mean nothing there, + so they're dropped here without a buzz. + */ + private func gestureAllowed(_ gesture: RingGesture) -> Bool { + guard glassesScreenIsOff else { return true } + switch gesture { + case .doubleClick, .click, .longPressStart, .longPressRelease: return true + default: return false + } + } + + private func handle(_ event: PadEvent) { + log.debug("pad \(String(describing: event), privacy: .public)") + switch event { + case .press: + link.sendPress() + case .tap: + click() + case .doubleTap: + haptics.doubleClick() + link.sendGesture(.doubleClick) + case .shortThenLongPress: + guard !glassesScreenIsOff else { return } + haptics.heavy() + link.sendGesture(.shortThenLongPress) + case .longPressStart: + longPressSent = gestureAllowed(.longPressStart) + if longPressSent { + haptics.heavy() + link.sendGesture(.longPressStart) + } + case .longPressEnd: + if longPressSent { + longPressSent = false + haptics.click() + link.sendGesture(.longPressRelease) + } + case let .swipe(start, dx, dy): + swipe(start: start, dx: dx, dy: dy) + } + } + + /** The system's double-tap hand gesture lands here too. */ + func click() { + guard gestureAllowed(.click) else { return } + haptics.click() + link.sendGesture(.click) + } + + private func swipe(start: CGPoint, dx: CGFloat, dy: CGFloat) { + let vertical = abs(dy) >= abs(dx) + if vertical && !controlsVisible && dy < 0 && start.y >= padHeight - Self.edgeRevealZone { + controlsVisible = true + haptics.click() + } else if vertical && controlsVisible && dy > 0 && start.y >= padHeight - Self.buttonTrayHeight { + controlsVisible = false + haptics.click() + } else if vertical { + // One step per swipe, regardless of length: the glasses repaint per + // step, and multi-step swipes overshoot lists when the display lags + // the finger. Repeated flicks or the crown cover long distances. + send(dy < 0 ? .swipeUp : .swipeDown, steps: 1) + } else { + send(dx < 0 ? .swipeLeft : .swipeRight, steps: 1) + } + } + + private func send(_ gesture: RingGesture, steps: Int) { + guard gestureAllowed(gesture) else { return } + haptics.click() + link.sendGesture(gesture, steps: steps) + } + + func dismissControls() { + guard controlsVisible else { return } + controlsVisible = false + haptics.click() + } + + // MARK: - Crown + + /** A crown turn, in detents; positive is clockwise seen from the crown's end. */ + func crownTurned(by delta: Double) { + crownRemainder += delta + let steps = Int(crownRemainder.rounded(.towardZero)) + guard steps != 0 else { return } + crownRemainder -= Double(steps) + // Clockwise moves to the previous item. + let gesture: RingGesture = steps > 0 ? .scrollUp : .scrollDown + guard gestureAllowed(gesture) else { return } + // Capped low: a fast twirl otherwise outruns the display. The crown's + // own detent haptics are the feedback, so there's no buzz here. + link.sendGesture(gesture, steps: min(abs(steps), Self.maxCrownSteps)) + } + + static let edgeRevealZone: CGFloat = 28 + static let buttonTrayHeight: CGFloat = 56 + private static let maxCrownSteps = 2 +} diff --git a/apple-watch/Faceclaw/PadGestureRecognizer.swift b/apple-watch/Faceclaw/PadGestureRecognizer.swift new file mode 100644 index 00000000..6f8c8d62 --- /dev/null +++ b/apple-watch/Faceclaw/PadGestureRecognizer.swift @@ -0,0 +1,143 @@ +import CoreGraphics +import Foundation + +/** What the remote's touch surface can report. */ +enum PadEvent: Equatable { + /** A finger landed (the second tap of a pair included), before it's known what the touch will become. */ + case press + case tap + case doubleTap + /** A tap whose immediate second press was held: the G2 tap-then-hold gesture. */ + case shortThenLongPress + case longPressStart + case longPressEnd + /** The finger travelled past touch slop; dx/dy are the total displacement. */ + case swipe(start: CGPoint, dx: CGFloat, dy: CGFloat) +} + +/** + One detector for everything the pad does, so a gesture is reported exactly + once: tap, double-tap (a second tap inside the double-tap timeout), a hold + (reported at the long-press timeout and again on release), tap-then-hold (a + second tap held past the long-press timeout) and a swipe. The same state + machine as wear/.../TouchpadGestures.kt, without its two-finger gestures: + SwiftUI on watchOS only ever sees one finger. + + It has no clock of its own. The owner passes the time with each event and calls + `advance(to:)` once `deadline` passes, which is what makes it testable. + Every longPressStart is paired with a longPressEnd, whatever ends the touch: + the phone treats a start without a release as a hold still in progress. + */ +final class PadGestureRecognizer { + struct Timing { + var longPress: TimeInterval = 0.4 + var doubleTap: TimeInterval = 0.3 + var slop: CGFloat = 8 + } + + private enum Phase { + case idle + case firstTouch(start: CGPoint, moved: Bool) + case holding + case awaitingSecondTouch + case secondTouch(held: Bool) + } + + private var phase = Phase.idle + private let timing: Timing + + /** When `advance(to:)` next has something to do; nil when only a touch event can move things on. */ + private(set) var deadline: TimeInterval? + var onEvent: (PadEvent) -> Void = { _ in } + + init(timing: Timing = Timing()) { + self.timing = timing + } + + /** A finger is on the pad, as far as this recognizer knows. */ + var isTouching: Bool { + switch phase { + case .firstTouch, .holding, .secondTouch: return true + case .idle, .awaitingSecondTouch: return false + } + } + + func touchDown(at location: CGPoint, time: TimeInterval) { + if isTouching { cancel() } + onEvent(.press) + if case .awaitingSecondTouch = phase { + phase = .secondTouch(held: false) + } else { + phase = .firstTouch(start: location, moved: false) + } + deadline = time + timing.longPress + } + + func touchMoved(to location: CGPoint) { + guard case let .firstTouch(start, moved) = phase, !moved else { return } + if hypot(location.x - start.x, location.y - start.y) > timing.slop { + phase = .firstTouch(start: start, moved: true) + deadline = nil + } + } + + func touchUp(at location: CGPoint, time: TimeInterval) { + switch phase { + case let .firstTouch(start, moved): + if moved { + reset() + onEvent(.swipe(start: start, dx: location.x - start.x, dy: location.y - start.y)) + } else { + // A plain tap; give a second tap a chance to make it a double. + phase = .awaitingSecondTouch + deadline = time + timing.doubleTap + } + case .holding: + reset() + onEvent(.longPressEnd) + case let .secondTouch(held): + reset() + if !held { onEvent(.doubleTap) } + case .idle, .awaitingSecondTouch: + break + } + } + + func advance(to time: TimeInterval) { + guard let deadline, time >= deadline else { return } + self.deadline = nil + switch phase { + case .firstTouch: + phase = .holding + onEvent(.longPressStart) + case .secondTouch: + phase = .secondTouch(held: true) + onEvent(.shortThenLongPress) + case .awaitingSecondTouch: + reset() + onEvent(.tap) + case .idle, .holding: + break + } + } + + /** + The touch is over but no touch-up will come: the system took the gesture or + the wrist dropped. A touch in progress is dropped rather than mistaken for a + tap; a tap already completed and a hold already reported are both honoured. + */ + func cancel() { + let cancelled = phase + reset() + switch cancelled { + case .holding: onEvent(.longPressEnd) + case .awaitingSecondTouch: onEvent(.tap) + case .idle, .firstTouch, .secondTouch: break + } + } + + private func reset() { + phase = .idle + deadline = nil + } +} diff --git a/apple-watch/Faceclaw/PadStatus.swift b/apple-watch/Faceclaw/PadStatus.swift new file mode 100644 index 00000000..5007efa0 --- /dev/null +++ b/apple-watch/Faceclaw/PadStatus.swift @@ -0,0 +1,77 @@ +import Foundation + +/** What we know about the phone and the Faceclaw app on it. */ +struct LinkStatus: Equatable { + /** The WatchConnectivity session is up; until then nothing below is known. */ + var activated = false + /** The iPhone can be messaged right now. */ + var phoneReachable = false + /** That iPhone has Faceclaw installed. */ + var phoneAppInstalled = false + /** + Whether the phone app's dashboard (the JS side) answered our last message; + nil until we have heard anything. False means Faceclaw is installed but its + dashboard isn't running, so the user has to open it. + */ + var appRunning: Bool? +} + +enum StatusTone { case good, busy, off, problem } + +/** + The pad's indicator line, with a tone that picks its colour, and an optional + smaller line under it (the charging case's level while charging). + */ +struct StatusLine: Equatable { + var text: String + var tone: StatusTone + var detail: String? +} + +/** + The pad's indicator: what the next gesture lands on when everything is + connected (the foreground app, or the screen/lock state in the way of it), + otherwise the connection problem to fix first. + */ +func padLine(state: PhoneState?, link: LinkStatus) -> StatusLine { + guard link.activated else { return StatusLine(text: "Waiting for phone", tone: .busy) } + guard link.phoneAppInstalled else { return StatusLine(text: "Install Faceclaw on phone", tone: .problem) } + guard link.phoneReachable else { return StatusLine(text: "No phone", tone: .problem) } + if link.appRunning == false { return StatusLine(text: "Open Faceclaw on phone", tone: .problem) } + guard let state else { return StatusLine(text: "Waiting for phone", tone: .busy) } + if !state.remoteEnabled { return StatusLine(text: "Watch control off", tone: .problem) } + if state.connected { + if state.charging || state.phase == "charging" { + return StatusLine(text: chargingLine(state.battery), tone: .busy, detail: caseLine(state)) + } + if state.silentMode { return StatusLine(text: "Silent mode", tone: .off) } + if state.locked { return StatusLine(text: "Locked", tone: .off) } + if !state.screenOn { return StatusLine(text: "●● Wake", tone: .off) } + if state.listening { return StatusLine(text: "Listening…", tone: .busy) } + return StatusLine(text: state.foreground?.title ?? "Launcher", tone: .good) + } + switch state.phase { + case "connecting": return StatusLine(text: "Connecting…", tone: .busy) + case "retrying": return StatusLine(text: "Reconnecting…", tone: .busy) + case "disconnecting": return StatusLine(text: "Disconnecting…", tone: .busy) + default: break + } + if state.preview { + if state.locked { return StatusLine(text: "Locked", tone: .off) } + if !state.screenOn { return StatusLine(text: "●● Wake", tone: .off) } + return StatusLine(text: state.foreground?.title ?? "Launcher", tone: .good, detail: "Phone preview") + } + return StatusLine(text: "Disconnected", tone: .off) +} + +private func chargingLine(_ battery: Int?) -> String { + battery.map { "Charging · G2 \($0)%" } ?? "Charging" +} + +/** + "Case 64%", with ⚡ while the case is on USB; nil when the level is unknown. The + case only reports while the glasses are docked, so this is only drawn while they charge. + */ +private func caseLine(_ state: PhoneState) -> String? { + state.caseBattery.map { "Case \($0)%" + (state.caseCharging == true ? " ⚡" : "") } +} diff --git a/apple-watch/Faceclaw/PhoneLink.swift b/apple-watch/Faceclaw/PhoneLink.swift new file mode 100644 index 00000000..7bec5523 --- /dev/null +++ b/apple-watch/Faceclaw/PhoneLink.swift @@ -0,0 +1,376 @@ +import Foundation +import Observation +import WatchConnectivity +import WatchKit +import os + +/** A transient message for the UI (ack failures, confirmations). */ +struct Notice: Equatable { + var text: String + var isError: Bool + var id: UUID +} + +/** + The watch's end of the WatchConnectivity conversation with the phone: sends + gestures, commands and text as messages, receives acks and events, and mirrors + the phone's state. The twin of wear/.../PhoneLink.kt, with these differences: + + - Live messages only work while this app is in front. The iPhone app can't wake + us, so the phone also leaves its state and latest event in the application + context, which is delivered whenever we next run. + - `isReachable` lags a moment behind a wrist raise, which is exactly when a + gesture arrives, so messages are held briefly instead of failing at once. + */ +@MainActor +@Observable +final class PhoneLink: NSObject { + private(set) var state: PhoneState? + private(set) var link = LinkStatus() + private(set) var assistant = AssistantMirror() + private(set) var notice: Notice? + + @ObservationIgnored private let log = Logger(subsystem: "com.faceclaw.watch", category: "PhoneLink") + @ObservationIgnored private var nextSeq: Int64 = 1 + @ObservationIgnored private var pendingAcks: [Int64: Task] = [:] + @ObservationIgnored private var held: [Outgoing] = [] + @ObservationIgnored private var heldExpiry: Task? + @ObservationIgnored private var noticeTask: Task? + @ObservationIgnored private var batteryTimer: Task? + @ObservationIgnored private var lastBatteryReported: WatchBatteryReading? + @ObservationIgnored private var lastEventAt: Int64 = 0 + + private struct Outgoing { + var path: String + var json: String + var seq: Int64 + var quiet: Bool + } + + private var session: WCSession { WCSession.default } + + /** Activate the session. Called during launch, including a background launch. */ + func start() { + guard WCSession.isSupported() else { return } + session.delegate = self + session.activate() + } + + // MARK: - App lifecycle + + /** The app came to the front: catch up on everything that happened while suspended. */ + func becameActive() { + refresh() + batteryTimer?.cancel() + // watchOS has no battery-change notification, and the app is only awake to be polled here. + batteryTimer = Task { [weak self] in + while !Task.isCancelled { + try? await Task.sleep(nanoseconds: Self.batteryPollNanos) + guard !Task.isCancelled else { return } + self?.reportBattery(force: false) + } + } + } + + func resignedActive() { + batteryTimer?.cancel() + batteryTimer = nil + WatchBattery.scheduleBackgroundRefresh() + } + + /** Re-read the session, the cached state and ask the phone for a fresh one. */ + func refresh() { + updateLink() + guard session.activationState == .activated else { return } + applyContext(session.receivedApplicationContext) + if link.phoneAppInstalled { + requestState() + reportBattery(force: true) + } + } + + /** The system gave us background time: leave the battery level for the phone to find. */ + func reportBatteryFromBackground() async { + for _ in 0..<50 where session.activationState != .activated { + try? await Task.sleep(nanoseconds: 100_000_000) + } + reportBattery(force: true) + WatchBattery.scheduleBackgroundRefresh() + } + + // MARK: - Outbound + + func sendGesture(_ gesture: RingGesture, steps: Int = 1) { + send(WirePath.input, ["gesture": gesture.rawValue, "steps": steps]) + } + + /** + A finger landed: the phone delivers it as the ring's touch-down (ring-press) + ahead of whatever gesture the touch becomes. Fire-and-forget and silent, since + the gesture that follows reports any failure; never held, since a late + touch-down would be worse than none. + */ + func sendPress() { + guard let state, state.takesInput, state.screenOn, session.isReachable else { return } + transmit(path: WirePath.press, json: "{}") + } + + func sendCommand(_ command: String, extras: [String: String] = [:]) { + var body: [String: Any] = ["command": command] + for (key, value) in extras { body[key] = value } + send(WirePath.command, body) + } + + func sendAssistantQuery(_ text: String) { + assistant = AssistantMirror(phase: "thinking", text: "") + send(WirePath.assistant, ["text": text]) + } + + func sendText(_ text: String) { + send(WirePath.text, ["text": text]) + } + + func requestState() { + send(WirePath.stateRequest, [:], quiet: true) + } + + func clearAssistant() { + assistant = AssistantMirror() + } + + private func send(_ path: String, _ body: [String: Any], quiet: Bool = false) { + let seq = nextSeq + nextSeq += 1 + var body = body + body["seq"] = seq + guard let json = encodeJSON(body) else { return } + let message = Outgoing(path: path, json: json, seq: seq, quiet: quiet) + if session.activationState == .activated && session.isReachable { + transmit(message) + } else if quiet { + // Only state requests are quiet, and reachability returning sends a fresh one. + } else { + hold(message) + } + } + + private func hold(_ message: Outgoing) { + log.debug("hold \(message.path, privacy: .public) seq=\(message.seq) (not reachable)") + held.append(message) + if held.count > Self.heldLimit { held.removeFirst() } + guard heldExpiry == nil else { return } + heldExpiry = Task { [weak self] in + try? await Task.sleep(nanoseconds: Self.holdNanos) + guard !Task.isCancelled else { return } + self?.expireHeld() + } + } + + private func flushHeld() { + heldExpiry?.cancel() + heldExpiry = nil + let messages = held + held = [] + messages.forEach(transmit) + } + + private func expireHeld() { + heldExpiry = nil + guard !held.isEmpty else { return } + held = [] + showNotice(link.phoneAppInstalled ? "The phone is out of range." : "Faceclaw isn't installed on the phone.", isError: true) + } + + private func transmit(_ message: Outgoing) { + log.debug("send \(message.path, privacy: .public) seq=\(message.seq)") + transmit(path: message.path, json: message.json) { [weak self] error in + self?.sendFailed(message, error) + } + // The phone acks every message; silence means its app process never received it. + pendingAcks[message.seq] = Task { [weak self] in + try? await Task.sleep(nanoseconds: Self.ackTimeoutNanos) + guard !Task.isCancelled, let self, self.pendingAcks.removeValue(forKey: message.seq) != nil else { return } + if self.link.appRunning == nil { self.link.appRunning = false } + if !message.quiet { self.showNotice("No reply from the phone.", isError: true) } + } + } + + private func transmit(path: String, json: String, onError: (@MainActor (Error) -> Void)? = nil) { + session.sendMessage([WireKey.path: path, WireKey.json: json], replyHandler: nil) { [log] error in + log.error("send \(path, privacy: .public) failed: \(error.localizedDescription, privacy: .public)") + Task { @MainActor in onError?(error) } + } + } + + private func sendFailed(_ message: Outgoing, _ error: Error) { + pendingAcks.removeValue(forKey: message.seq)?.cancel() + updateLink() + guard !message.quiet else { return } + let code = (error as? WCError)?.code + switch code { + case .notReachable: showNotice("The phone is out of range.", isError: true) + case .companionAppNotInstalled: showNotice("Faceclaw isn't installed on the phone.", isError: true) + default: showNotice("Couldn't reach the phone.", isError: true) + } + } + + /** Send our battery to the phone (for the glasses' Watch indicator), live and for later. */ + private func reportBattery(force: Bool) { + guard session.activationState == .activated else { return } + let reading = WatchBattery.read() + if !force && reading == lastBatteryReported { return } + lastBatteryReported = reading + guard let json = encodeJSON(reading.payload(at: Date())) else { return } + if session.isReachable { transmit(path: WirePath.battery, json: json) } + do { + try session.updateApplicationContext([WireKey.battery: json]) + } catch { + log.debug("battery context not sent: \(error.localizedDescription, privacy: .public)") + } + } + + // MARK: - Inbound + + private func handleMessage(path: String, json: String, updatedAt: Int64, at: Int64) { + switch path { + case WirePath.ack: + if let ack = Ack.parse(json) { handleAck(ack) } + case WirePath.event: + handleEvent(json: json, at: at, fromContext: false) + case WirePath.state: + // A live state comes from the running dashboard, unlike one left in the context. + applyState(json: json, updatedAt: updatedAt) + link.appRunning = true + case WirePath.batteryRequest: + reportBattery(force: true) + default: + break + } + } + + private func handleAck(_ ack: Ack) { + pendingAcks.removeValue(forKey: ack.seq)?.cancel() + link.appRunning = ack.jsReady + if !ack.ok { + showNotice(ack.message.isEmpty ? "The phone refused that." : ack.message, isError: true) + } else if !ack.message.isEmpty { + showNotice(ack.message, isError: false) + } + } + + private func handleEvent(json: String, at: Int64, fromContext: Bool) { + guard let event = PhoneEvent.parse(json) else { return } + // The same event can arrive both live and in the context. + if at != 0 { + guard at > lastEventAt else { return } + lastEventAt = at + } + let age = at == 0 ? 0 : Date().timeIntervalSince1970 - Double(at) / 1000 + switch event { + case let .assistant(phase, text): + if fromContext && age > Self.assistantEventMaxAge { return } + assistant = assistant.applying(phase: phase, text: text) + case let .alert(text): + if fromContext && age > Self.alertEventMaxAge { return } + showNotice(text, isError: false) + } + } + + private func applyContext(_ context: [String: Any]) { + if let json = context[WireKey.json] as? String { + applyState(json: json, updatedAt: (context[WireKey.updatedAt] as? NSNumber)?.int64Value ?? 0) + } + if let json = context[WireKey.event] as? String { + handleEvent(json: json, at: (context[WireKey.eventAt] as? NSNumber)?.int64Value ?? 0, fromContext: true) + } + } + + private func applyState(json: String, updatedAt: Int64) { + // A live message and the context can carry the same or an older state, in either order. + if let state, updatedAt < state.updatedAt { return } + if let parsed = PhoneState.parse(json, updatedAt: updatedAt) { state = parsed } + } + + private func updateLink() { + // The session logs an error for each property read before activation. + guard WCSession.isSupported(), session.activationState == .activated else { return } + link.activated = true + link.phoneReachable = session.isReachable + link.phoneAppInstalled = session.isCompanionAppInstalled + if !link.phoneAppInstalled { link.appRunning = nil } + } + + private func reachabilityChanged() { + updateLink() + log.debug("reachable=\(self.link.phoneReachable)") + if session.isReachable { + flushHeld() + requestState() + reportBattery(force: true) + } + } + + private func activated() { + updateLink() + applyContext(session.receivedApplicationContext) + reachabilityChanged() + } + + private func showNotice(_ text: String, isError: Bool) { + noticeTask?.cancel() + notice = Notice(text: text, isError: isError, id: UUID()) + noticeTask = Task { [weak self] in + try? await Task.sleep(nanoseconds: isError ? Self.noticeErrorNanos : Self.noticeInfoNanos) + guard !Task.isCancelled else { return } + self?.notice = nil + } + } + + private static let ackTimeoutNanos: UInt64 = 4_000_000_000 + /** How long a message waits for reachability, e.g. right after a wrist raise. */ + private static let holdNanos: UInt64 = 1_500_000_000 + private static let heldLimit = 8 + private static let batteryPollNanos: UInt64 = 60_000_000_000 + private static let noticeInfoNanos: UInt64 = 1_800_000_000 + private static let noticeErrorNanos: UInt64 = 3_200_000_000 + /** A reply that finished while we slept is still worth showing for this long. */ + private static let assistantEventMaxAge: TimeInterval = 600 + private static let alertEventMaxAge: TimeInterval = 60 +} + +extension PhoneLink: WCSessionDelegate { + nonisolated func session(_ session: WCSession, activationDidCompleteWith state: WCSessionActivationState, error: Error?) { + let log = Logger(subsystem: "com.faceclaw.watch", category: "PhoneLink") + log.info("session activation finished: state \(state.rawValue), error \(error?.localizedDescription ?? "none", privacy: .public)") + Task { @MainActor in self.activated() } + } + + nonisolated func sessionReachabilityDidChange(_ session: WCSession) { + Task { @MainActor in self.reachabilityChanged() } + } + + nonisolated func sessionCompanionAppInstalledDidChange(_ session: WCSession) { + Task { @MainActor in self.updateLink() } + } + + nonisolated func session(_ session: WCSession, didReceiveMessage message: [String: Any]) { + guard let path = message[WireKey.path] as? String else { return } + let json = message[WireKey.json] as? String ?? "{}" + let updatedAt = (message[WireKey.updatedAt] as? NSNumber)?.int64Value ?? 0 + let at = (message[WireKey.at] as? NSNumber)?.int64Value ?? 0 + Task { @MainActor in self.handleMessage(path: path, json: json, updatedAt: updatedAt, at: at) } + } + + nonisolated func session(_ session: WCSession, didReceiveApplicationContext applicationContext: [String: Any]) { + Task { @MainActor in self.refreshFromContext() } + } + + @MainActor private func refreshFromContext() { + applyContext(session.receivedApplicationContext) + } +} + +private func encodeJSON(_ object: [String: Any]) -> String? { + guard let data = try? JSONSerialization.data(withJSONObject: object) else { return nil } + return String(data: data, encoding: .utf8) +} diff --git a/apple-watch/Faceclaw/Protocol.swift b/apple-watch/Faceclaw/Protocol.swift new file mode 100644 index 00000000..b7574d69 --- /dev/null +++ b/apple-watch/Faceclaw/Protocol.swift @@ -0,0 +1,234 @@ +import Foundation + +/** + Wire format shared with the phone app. Mirror of app/native/wear-bridge-types.ts + (paths), app/g2/wear-remote.ts (payloads) and App_Resources/iOS/src/FaceclawWatchBridge.m + (WatchConnectivity keys); see wear/PROTOCOL.md. + */ +enum WirePath { + static let input = "/faceclaw/input" + /** Watch -> phone: a finger landed on the pad, ahead of the gesture it becomes; no seq, never acked. */ + static let press = "/faceclaw/press" + static let command = "/faceclaw/command" + static let assistant = "/faceclaw/assistant" + static let text = "/faceclaw/text" + static let stateRequest = "/faceclaw/state/request" + /** Watch -> phone: our own battery ({battery, charging, at}); no seq, never acked. */ + static let battery = "/faceclaw/battery" + /** Phone -> watch: please send `battery` now. Only reaches an app that's in front. */ + static let batteryRequest = "/faceclaw/battery/request" + static let ack = "/faceclaw/ack" + static let event = "/faceclaw/event" + static let state = "/faceclaw/state" +} + +/** Keys of the WatchConnectivity dictionaries that carry those messages. */ +enum WireKey { + static let path = "path" + static let json = "json" + static let updatedAt = "updatedAt" + /** The time a live event was sent, matching `eventAt` when it's also in the context. */ + static let at = "at" + /** Application context: the latest event a suspended app would otherwise have missed. */ + static let event = "event" + static let eventAt = "eventAt" + /** Application context, watch -> phone: the battery report, as JSON. */ + static let battery = "battery" +} + +/** Ring-style gestures, as the phone's synthetic ring input understands them. */ +enum RingGesture: String { + case click + case doubleClick = "double-click" + case scrollUp = "scroll-up" + case scrollDown = "scroll-down" + /** Finger down: the hold lasts until longPressRelease. */ + case longPressStart = "long-press-start" + case longPressRelease = "long-press-release" + /** Tap then hold: opens the app's context menu on the glasses. */ + case shortThenLongPress = "short-then-long-press" + /** The spoken "Hey Even": opens the glasses' own voice dialog. */ + case wakeword + /** + Spatial (four-way) input the ring cannot produce. The glasses UI gives it a + spatial meaning where one exists (grid rows and columns, panel columns, + tracks) and otherwise treats up/down as scroll, right as select and left as back. + */ + case swipeUp = "swipe-up" + case swipeDown = "swipe-down" + case swipeLeft = "swipe-left" + case swipeRight = "swipe-right" +} + +/** Shell commands; things the ring cannot express. */ +enum PhoneCommand { + static let launchApp = "launch-app" + static let focusWindow = "focus-window" + static let closeWindow = "close-window" + static let wake = "wake" + static let sleep = "sleep" + static let lock = "lock" + static let unlock = "unlock" + static let connect = "connect" + static let disconnect = "disconnect" + static let closeAssistant = "close-assistant" + static let sidebar = "sidebar" +} + +struct AppEntry: Equatable, Identifiable { + var appId: String + var title: String + var id: String { appId } +} + +struct WindowEntry: Equatable, Identifiable { + var windowId: String + var title: String + var appId: String + var focused: Bool + var closeable: Bool + var acceptsText: Bool + var id: String { windowId } +} + +/** The phone's mirrored dashboard state (/faceclaw/state). */ +struct PhoneState: Equatable { + var version = "" + var phase = "disconnected" + var status = "" + var connected = false + /** The shell is running on the phone's preview with no glasses, so gestures still land. */ + var preview = false + var screenOn = false + var locked = false + var worn: Bool? + var listening = false + /** Current or last-known G2 charge percentage. */ + var battery: Int? + var charging = false + /** Last-known charging-case percentage; live only while the glasses are docked. */ + var caseBattery: Int? + /** The case is on USB; nil while the glasses are out of the case. */ + var caseCharging: Bool? + var silentMode = false + var foreground: AppEntry? + var windows: [WindowEntry] = [] + var apps: [AppEntry] = [] + var remoteEnabled = true + /** Settings > Watch on the glasses; WatchPrefs and Haptics say what they do. */ + var vibrateOnInput = true + var soundEffects = false + var canUnlock = true + var mirrorAssistant = true + var assistantAvailable = false + var updatedAt: Int64 = 0 + + /** Gestures have a display to act on, whether that's the glasses or the phone's preview. */ + var takesInput: Bool { connected || preview } + + static func parse(_ json: String, updatedAt: Int64) -> PhoneState? { + guard let o = parseObject(json) else { return nil } + var state = PhoneState() + state.version = o["version"] as? String ?? "" + state.phase = o["phase"] as? String ?? "disconnected" + state.status = o["status"] as? String ?? "" + state.connected = o["connected"] as? Bool ?? false + state.preview = o["preview"] as? Bool ?? false + state.screenOn = o["screenOn"] as? Bool ?? false + state.locked = o["locked"] as? Bool ?? false + state.worn = o["worn"] as? Bool + state.listening = o["listening"] as? Bool ?? false + state.battery = percent(o["battery"]) + state.charging = o["charging"] as? Bool ?? false + state.caseBattery = percent(o["caseBattery"]) + state.caseCharging = o["caseCharging"] as? Bool + state.silentMode = o["silentMode"] as? Bool ?? false + if let foreground = o["foreground"] as? [String: Any] { state.foreground = appEntry(foreground) } + state.windows = (o["windows"] as? [[String: Any]] ?? []).map { w in + WindowEntry( + windowId: w["windowId"] as? String ?? "", + title: w["title"] as? String ?? "", + appId: w["appId"] as? String ?? "", + focused: w["focused"] as? Bool ?? false, + closeable: w["closeable"] as? Bool ?? true, + acceptsText: w["acceptsText"] as? Bool ?? false + ) + } + state.apps = (o["apps"] as? [[String: Any]] ?? []).map(appEntry) + state.remoteEnabled = o["remoteEnabled"] as? Bool ?? true + state.vibrateOnInput = o["vibrateOnInput"] as? Bool ?? true + state.soundEffects = o["soundEffects"] as? Bool ?? false + state.canUnlock = o["canUnlock"] as? Bool ?? true + state.mirrorAssistant = o["mirrorAssistant"] as? Bool ?? true + state.assistantAvailable = o["assistantAvailable"] as? Bool ?? false + state.updatedAt = updatedAt + return state + } + + private static func appEntry(_ o: [String: Any]) -> AppEntry { + AppEntry(appId: o["appId"] as? String ?? "", title: o["title"] as? String ?? "") + } + + private static func percent(_ value: Any?) -> Int? { + guard let number = value as? NSNumber, (0...100).contains(number.intValue) else { return nil } + return number.intValue + } +} + +/** The phone's reply to one of our messages. */ +struct Ack: Equatable { + var seq: Int64 + var ok: Bool + /** False: the phone's app process answered, but its dashboard isn't up. */ + var jsReady: Bool + var message: String + + static func parse(_ json: String) -> Ack? { + guard let o = parseObject(json) else { return nil } + return Ack( + seq: (o["seq"] as? NSNumber)?.int64Value ?? 0, + ok: o["ok"] as? Bool ?? false, + jsReady: o["jsReady"] as? Bool ?? true, + message: o["message"] as? String ?? "" + ) + } +} + +/** Phone -> watch events (/faceclaw/event). */ +enum PhoneEvent: Equatable { + /** phase: thinking | streaming | done | error | closed; text is the reply so far. */ + case assistant(phase: String, text: String) + case alert(text: String) + + static func parse(_ json: String) -> PhoneEvent? { + guard let o = parseObject(json) else { return nil } + switch o["type"] as? String { + case "assistant": return .assistant(phase: o["phase"] as? String ?? "", text: o["text"] as? String ?? "") + case "alert": return .alert(text: o["text"] as? String ?? "") + default: return nil + } + } +} + +/** The assistant overlay as mirrored from the glasses. */ +struct AssistantMirror: Equatable { + /** idle | thinking | streaming | done | error */ + var phase = "idle" + var text = "" + + func applying(phase next: String, text nextText: String) -> AssistantMirror { + switch next { + case "thinking": return AssistantMirror(phase: "thinking", text: "") + case "streaming": return AssistantMirror(phase: "streaming", text: nextText) + case "done": return AssistantMirror(phase: "done", text: nextText.isEmpty ? text : nextText) + case "error": return AssistantMirror(phase: "error", text: nextText) + case "closed": return AssistantMirror(phase: "idle", text: text) + default: return self + } + } +} + +private func parseObject(_ json: String) -> [String: Any]? { + guard let data = json.data(using: .utf8) else { return nil } + return (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] +} diff --git a/apple-watch/Faceclaw/RemoteScreen.swift b/apple-watch/Faceclaw/RemoteScreen.swift new file mode 100644 index 00000000..7f04ada5 --- /dev/null +++ b/apple-watch/Faceclaw/RemoteScreen.swift @@ -0,0 +1,210 @@ +import SwiftUI + +/** + The touchpad. The whole screen is the pad: tap = select, double-tap = back, hold + = long press (released when the finger lifts), tap then hold = the app's menu, + swipes in four directions = spatial navigation, crown = plain scrolling. An upward + swipe that begins at the bottom edge reveals the otherwise-hidden action buttons. + */ +struct RemoteScreen: View { + let link: PhoneLink + let prefs: WatchPrefs + let open: (Destination) -> Void + + @Environment(\.isLuminanceReduced) private var isDimmed + @Environment(\.scenePhase) private var scenePhase + @State private var pad: PadController + @State private var crownValue = 0.0 + /** When the touch in progress began; nil once it's over, however it ended. */ + @GestureState private var touchStart: Date? + @FocusState private var crownFocused: Bool + + init(link: PhoneLink, prefs: WatchPrefs, open: @escaping (Destination) -> Void) { + self.link = link + self.prefs = prefs + self.open = open + _pad = State(initialValue: PadController(link: link, prefs: prefs)) + } + + var body: some View { + GeometryReader { geometry in + ZStack { + padSurface + primaryActionShortcut + clockAndStatus.allowsHitTesting(false) + if pad.controlsVisible && !isDimmed { tray } + } + .onChange(of: geometry.size.height, initial: true) { _, height in pad.padHeight = height } + } + .ignoresSafeArea() + .toolbar(.hidden, for: .navigationBar) + .animation(.easeOut(duration: 0.18), value: pad.controlsVisible) + // Crown events only reach a focused view, and the focus is lost to every screen pushed on top. + .focusable(true) + .focused($crownFocused) + .focusEffectDisabled() + .digitalCrownRotation( + $crownValue, + from: -Self.crownRange, + through: Self.crownRange, + by: 1, + sensitivity: prefs.crownSensitivity.digitalCrown, + isContinuous: true, + isHapticFeedbackEnabled: prefs.haptics + ) + .digitalCrownAccessory(.hidden) + .onChange(of: crownValue) { old, new in + // A jump across the whole range is the value wrapping, not a turn. + let delta = new - old + if abs(delta) < Self.crownRange { pad.crownTurned(by: delta) } + } + // A touch-up also clears this, which arrives late enough to fall inside the next + // touch of a quick double-tap; the pad checks which touch it's being told about. + .onChange(of: touchStart) { old, new in + if new == nil, let old { pad.touchWentAway(startedAt: old) } + } + .onChange(of: scenePhase) { _, phase in + if phase == .active { crownFocused = true } else { pad.touchCancelled() } + } + .onAppear { crownFocused = true } + } + + private var padSurface: some View { + Color.black + .contentShape(Rectangle()) + .gesture( + DragGesture(minimumDistance: 0, coordinateSpace: .local) + .updating($touchStart) { value, state, _ in + if state == nil { state = value.time } + } + .onChanged { pad.touchChanged(start: $0.startLocation, location: $0.location, time: $0.time) } + .onEnded { pad.touchEnded(location: $0.location) } + ) + .accessibilityIdentifier("pad") + } + + /** + The double-tap hand gesture (Series 9, Ultra 2, SE 3 and later) selects. It needs a + control to attach to, and the pad surface is a gesture rather than a control, + so this one sits underneath everything as a transparent stand-in. + */ + @ViewBuilder + private var primaryActionShortcut: some View { + if #available(watchOS 11.0, *) { + Button("Select") { pad.click() } + .buttonStyle(.plain) + .handGestureShortcut(.primaryAction) + .frame(width: 1, height: 1) + .opacity(0.01) + .allowsHitTesting(false) + .accessibilityHidden(true) + } + } + + private var clockAndStatus: some View { + let status = padLine(state: link.state, link: link.link) + return ZStack { + // A large, straight clock fixed on the vertical centre of the pad. + TimelineView(.everyMinute) { context in + Text(context.date.formatted(.dateTime.hour(.defaultDigits(amPM: .omitted)).minute())) + .font(.system(size: 44, weight: .regular, design: .rounded)) + .monospacedDigit() + .foregroundStyle(.white) + .lineLimit(1) + .minimumScaleFactor(0.7) + } + if !pad.controlsVisible { + // Offset rather than stacked, so the clock stays put whatever this says. + VStack(spacing: 0) { + Text(status.text) + .font(.headline) + .foregroundStyle(statusColor(status.tone)) + .accessibilityIdentifier("padStatus") + if let detail = status.detail { + Text(detail) + .font(.caption2) + .foregroundStyle(.secondary) + } + } + .lineLimit(1) + .minimumScaleFactor(0.7) + .padding(.horizontal, 12) + .offset(y: 46) + .transition(.opacity) + } + } + } + + private func statusColor(_ tone: StatusTone) -> Color { + if isDimmed { return .gray } + return tone == .good ? .white : tone.color + } + + private var tray: some View { + VStack { + Spacer() + HStack(spacing: 6) { + TrayButton(symbol: "sparkles", label: "Assistant") { openFromTray(.assistant) } + TextFieldLink(prompt: Text("Type into app")) { + TrayIcon(symbol: "keyboard") + } onSubmit: { text in + send(typed: text) + } + .buttonStyle(.plain) + .accessibilityLabel("Type into app") + .accessibilityIdentifier("Type into app") + TrayButton(symbol: "square.grid.2x2", label: "Apps") { openFromTray(.apps) } + TrayButton(symbol: "ellipsis", label: "Status and settings") { openFromTray(.status) } + } + .padding(.bottom, 10) + // A swipe down that starts on a button dismisses the tray; a tap stays the button's. + .simultaneousGesture( + DragGesture(minimumDistance: 12).onEnded { value in + if value.translation.height > abs(value.translation.width) { pad.dismissControls() } + } + ) + } + .transition(.move(edge: .bottom).combined(with: .opacity)) + } + + /** The tray has done its job: coming back should show the pad's status, not the buttons. */ + private func openFromTray(_ destination: Destination) { + pad.controlsVisible = false + open(destination) + } + + private func send(typed text: String) { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + Haptics(prefs: prefs).click() + link.sendText(trimmed) + } + + /** Far more detents than a session will ever turn; only the changes matter. */ + private static let crownRange = 1_000_000.0 +} + +private struct TrayButton: View { + let symbol: String + let label: String + let action: () -> Void + + var body: some View { + Button(action: action) { TrayIcon(symbol: symbol) } + .buttonStyle(.plain) + .accessibilityLabel(label) + .accessibilityIdentifier(label) + } +} + +private struct TrayIcon: View { + let symbol: String + + var body: some View { + Image(systemName: symbol) + .font(.system(size: 16, weight: .medium)) + .foregroundStyle(.white) + .frame(width: 36, height: 36) + .background(Circle().fill(Color(white: 0.2))) + } +} diff --git a/apple-watch/Faceclaw/Screens.swift b/apple-watch/Faceclaw/Screens.swift new file mode 100644 index 00000000..bfd7a903 --- /dev/null +++ b/apple-watch/Faceclaw/Screens.swift @@ -0,0 +1,372 @@ +import SwiftUI + +extension StatusTone { + var color: Color { + switch self { + case .good: return Color(red: 0.48, green: 0.85, blue: 0.56) + case .busy: return Color(red: 0.95, green: 0.76, blue: 0.31) + case .off: return Color(white: 0.54) + case .problem: return Color(red: 1.0, green: 0.54, blue: 0.5) + } + } +} + +/** A transient pill at the bottom of the screen: ack failures and confirmations. */ +struct NoticeOverlay: View { + let notice: Notice? + + var body: some View { + Group { + if let notice { + Text(notice.text) + .font(.caption2) + .multilineTextAlignment(.center) + .lineLimit(2) + .foregroundStyle(notice.isError ? Color(red: 1, green: 0.71, blue: 0.66) : .white) + .padding(.horizontal, 10) + .padding(.vertical, 5) + .background( + RoundedRectangle(cornerRadius: 12) + .fill(notice.isError ? Color(red: 0.29, green: 0.12, blue: 0.1) : Color(white: 0.16)) + ) + .padding(.horizontal, 8) + .padding(.bottom, 4) + .transition(.opacity) + .accessibilityIdentifier("notice") + } + } + .animation(.easeOut(duration: 0.15), value: notice?.id) + .allowsHitTesting(false) + } +} + +/** A list row: a title with an optional smaller line beneath it. */ +private struct RowLabel: View { + let title: String + var detail: String? + + var body: some View { + VStack(alignment: .leading, spacing: 1) { + Text(title).lineLimit(1) + if let detail { + Text(detail).font(.caption2).foregroundStyle(.secondary).lineLimit(1) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + } +} + +private struct SectionHeader: View { + let title: String + init(_ title: String) { self.title = title } + + var body: some View { + Text(title) + .font(.footnote) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.top, 6) + .padding(.leading, 4) + } +} + +// MARK: - Apps + +/** + The glasses' sidebar and launcher, on the wrist: open windows first (tap to bring + one to the front, × to close it), then every launchable app. + */ +struct AppsScreen: View { + @Environment(PhoneLink.self) private var link + @Environment(WatchPrefs.self) private var prefs + @Environment(\.dismiss) private var dismiss + + var body: some View { + let windows = link.state?.windows ?? [] + let apps = link.state?.apps ?? [] + let onScreen = windows.filter(\.focused) + let others = windows.filter { !$0.focused } + ScrollView { + LazyVStack(spacing: 4) { + if !onScreen.isEmpty { + SectionHeader("On screen") + ForEach(onScreen) { windowRow($0) } + } + if !others.isEmpty { + SectionHeader("Also open") + ForEach(others) { windowRow($0) } + } + SectionHeader(apps.isEmpty ? "No app list yet" : "Launch") + ForEach(apps) { app in + Button { + Haptics(prefs: prefs).click() + link.sendCommand(PhoneCommand.launchApp, extras: ["appId": app.appId]) + dismiss() + } label: { + RowLabel(title: app.title) + } + .accessibilityIdentifier("app:\(app.appId)") + } + if apps.isEmpty { + Text("Open Faceclaw on the phone and connect the glasses.") + .font(.caption2) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + } + } + } + .navigationTitle("Apps") + } + + /** One open window: tap to bring it to the front, × to close it. */ + private func windowRow(_ window: WindowEntry) -> some View { + HStack(spacing: 4) { + Button { + Haptics(prefs: prefs).click() + link.sendCommand(PhoneCommand.focusWindow, extras: ["windowId": window.windowId]) + dismiss() + } label: { + // The launcher's window is titled "Apps" on the glasses; here + // that would read as this very screen. + RowLabel( + title: window.appId == "launcher" ? "Launcher" : window.title, + detail: window.acceptsText ? "takes typed text" : nil + ) + } + .tint(window.focused ? StatusTone.good.color : nil) + if window.closeable { + Button { + Haptics(prefs: prefs).click() + link.sendCommand(PhoneCommand.closeWindow, extras: ["windowId": window.windowId]) + } label: { + Image(systemName: "xmark").font(.system(size: 12, weight: .semibold)) + } + .frame(width: 40) + .accessibilityLabel("Close \(window.title)") + } + } + } +} + +// MARK: - Assistant + +/** + Ask the assistant from the wrist (the system text sheet, which offers dictation) + and read its reply here as it streams, mirrored from the glasses overlay. "Hey + Even" instead uses the glasses' own microphone and voice dialog. + */ +struct AssistantScreen: View { + @Environment(PhoneLink.self) private var link + @Environment(WatchPrefs.self) private var prefs + + var body: some View { + let available = link.state?.assistantAvailable != false + let mirror = link.assistant + let haptics = Haptics(prefs: prefs) + ScrollView { + VStack(spacing: 6) { + TextFieldLink(prompt: Text("Ask the assistant")) { + Label("Ask", systemImage: "mic.fill").frame(maxWidth: .infinity) + } onSubmit: { text in + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return } + haptics.click() + link.sendAssistantQuery(trimmed) + } + .disabled(!available) + .accessibilityIdentifier("assistantAsk") + + Text(phaseLabel(mirror.phase, available: available)) + .font(.caption) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .accessibilityIdentifier("assistantPhase") + + if !mirror.text.isEmpty { + Text(mirror.text) + .font(.body) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal, 4) + } + + Button { + haptics.click() + link.sendGesture(.wakeword) + } label: { + RowLabel(title: "“Hey Even”", detail: "uses the glasses mic") + } + + if mirror.phase != "idle" { + Button(mirror.phase == "thinking" || mirror.phase == "streaming" ? "Cancel" : "Dismiss") { + haptics.click() + link.sendCommand(PhoneCommand.closeAssistant) + link.clearAssistant() + } + } + } + } + .navigationTitle("Assistant") + // A finished reply deserves a nudge: the wrist may be down while it streams. + .onChange(of: mirror.phase) { _, phase in + if phase == "done" || phase == "error" { haptics.done() } + } + } + + private func phaseLabel(_ phase: String, available: Bool) -> String { + switch phase { + case "thinking": return "Thinking…" + case "streaming": return "Replying…" + case "error": return "Error" + case "done": return "Reply" + default: return available ? "Ask something" : "Set up the assistant in Faceclaw's Settings" + } + } +} + +// MARK: - Status + +/** Glasses and phone status, with the connection / display / lock switches. */ +struct StatusScreen: View { + @Environment(PhoneLink.self) private var link + @Environment(WatchPrefs.self) private var prefs + let openSettings: () -> Void + + var body: some View { + let state = link.state + let connected = state?.connected == true + let takesInput = state?.takesInput == true + let haptics = Haptics(prefs: prefs) + ScrollView { + VStack(spacing: 4) { + SectionHeader("Glasses") + Button { + haptics.click() + link.sendCommand(connected ? PhoneCommand.disconnect : PhoneCommand.connect) + } label: { + RowLabel(title: connectionLabel(state), detail: connected ? "tap to disconnect" : "tap to connect") + } + .tint(connected ? StatusTone.good.color : nil) + .disabled(link.link.appRunning == false) + + let screenOn = state?.screenOn == true + Button { + haptics.click() + link.sendCommand(screenOn ? PhoneCommand.sleep : PhoneCommand.wake) + } label: { + RowLabel(title: screenOn ? "Display on" : "Display off", detail: screenOn ? "tap to turn off" : "tap to wake") + } + .disabled(!takesInput || state?.locked == true) + + let locked = state?.locked == true + Button { + haptics.click() + link.sendCommand(locked ? PhoneCommand.unlock : PhoneCommand.lock) + } label: { + RowLabel(title: locked ? "Locked" : "Unlocked", detail: lockDetail(locked: locked, state: state)) + } + .disabled(!takesInput || (locked && state?.canUnlock == false)) + + if let foreground = state?.foreground { + Button { + haptics.click() + link.sendCommand(PhoneCommand.sidebar) + } label: { + RowLabel(title: "On screen: \(foreground.title)", detail: "tap to open the sidebar") + } + .disabled(!takesInput) + } + + SectionHeader("Phone") + Button { + haptics.click() + link.refresh() + } label: { + Label { + RowLabel(title: phoneLabel, detail: phoneDetail(state)) + } icon: { + Image(systemName: "arrow.clockwise") + } + } + + if state?.remoteEnabled == false { + Text("Watch control is turned off in Faceclaw's Settings > Watch.") + .font(.caption2) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + } + + Button(action: openSettings) { + Label("Watch settings", systemImage: "gearshape").frame(maxWidth: .infinity, alignment: .leading) + } + } + } + .navigationTitle("Status") + } + + private func connectionLabel(_ state: PhoneState?) -> String { + guard let state else { return "No status yet" } + if state.charging || state.phase == "charging" { return "Charging" } + if state.connected { return "Connected" } + switch state.phase { + case "connecting": return "Connecting…" + case "retrying": return "Reconnecting…" + case "disconnecting": return "Disconnecting…" + default: return state.preview ? "Phone preview" : "Disconnected" + } + } + + private func lockDetail(locked: Bool, state: PhoneState?) -> String { + if locked && state?.canUnlock == false { return "unlock from the phone" } + return locked ? "tap to unlock" : "tap to lock" + } + + private var phoneLabel: String { + let status = link.link + if !status.phoneAppInstalled { return "Faceclaw not installed" } + if !status.phoneReachable { return "No phone connected" } + if status.appRunning == false { return "Faceclaw not running" } + return "iPhone" + } + + private func phoneDetail(_ state: PhoneState?) -> String { + let status = link.link + if !status.phoneAppInstalled { return "install it on the phone" } + if !status.phoneReachable { return "check Bluetooth" } + if status.appRunning == false { return "open it on the phone" } + if let version = state?.version, !version.isEmpty { return "Faceclaw v\(version)" } + return "tap to refresh" + } +} + +// MARK: - Settings + +/** Watch-local preferences: feel of the crown. */ +struct SettingsScreen: View { + @Environment(WatchPrefs.self) private var prefs + + var body: some View { + ScrollView { + VStack(spacing: 4) { + Button { + prefs.crownSensitivity = prefs.crownSensitivity.next + } label: { + RowLabel(title: "Crown sensitivity", detail: prefs.crownSensitivity.rawValue) + } + Text("Vibration and sound effects are in Settings > Watch on the glasses.") + .font(.caption2) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .padding(.top, 6) + Text("To keep the remote open between glances, set Settings > General > Return to Clock > Faceclaw on the watch.") + .font(.caption2) + .foregroundStyle(.secondary) + .multilineTextAlignment(.center) + .padding(.top, 6) + Text("Faceclaw watch v\(Bundle.main.object(forInfoDictionaryKey: "CFBundleShortVersionString") as? String ?? "?")") + .font(.caption2) + .foregroundStyle(.secondary) + } + } + .navigationTitle("Settings") + } +} diff --git a/apple-watch/Faceclaw/WatchBattery.swift b/apple-watch/Faceclaw/WatchBattery.swift new file mode 100644 index 00000000..ef515772 --- /dev/null +++ b/apple-watch/Faceclaw/WatchBattery.swift @@ -0,0 +1,51 @@ +import Foundation +import WatchKit + +/** The watch's own charge level and charging state, for the glasses' battery indicators. */ +struct WatchBatteryReading: Equatable { + var battery: Int? + var charging: Bool + + /** `at` lets the phone discount a report it finds long after it was left in the application context. */ + func payload(at date: Date) -> [String: Any] { + [ + "battery": battery ?? NSNull(), + "charging": charging, + "at": Int64(date.timeIntervalSince1970 * 1000), + ] + } +} + +enum WatchBattery { + /** The `.backgroundTask(.appRefresh(...))` identifier. */ + static let refreshIdentifier = "battery" + + private static let refreshInterval: TimeInterval = 30 * 60 + + @MainActor + static func read() -> WatchBatteryReading { + let device = WKInterfaceDevice.current() + device.isBatteryMonitoringEnabled = true + // Unknown (the simulator, or just after enabling monitoring) is not a reading of 0%. + let level = device.batteryLevel + let known = device.batteryState != .unknown && level >= 0 + let percent = known ? min(100, Int((level * 100).rounded())) : nil + return WatchBatteryReading( + battery: percent, + charging: device.batteryState == .charging || device.batteryState == .full + ) + } + + /** + A closed app can't answer the phone's battery polls, so it asks to be woken + now and then to leave a report behind. The system decides when, and may + decline; the phone discards the level once it goes stale. + */ + @MainActor + static func scheduleBackgroundRefresh() { + WKApplication.shared().scheduleBackgroundRefresh( + withPreferredDate: Date(timeIntervalSinceNow: refreshInterval), + userInfo: refreshIdentifier as NSString + ) { _ in } + } +} diff --git a/apple-watch/Faceclaw/WatchPrefs.swift b/apple-watch/Faceclaw/WatchPrefs.swift new file mode 100644 index 00000000..e35c02d0 --- /dev/null +++ b/apple-watch/Faceclaw/WatchPrefs.swift @@ -0,0 +1,100 @@ +import Foundation +import Observation +import SwiftUI +import WatchKit + +enum CrownSensitivity: String, CaseIterable { + case low = "Low" + case normal = "Normal" + case high = "High" + + var next: CrownSensitivity { + let all = Self.allCases + return all[(all.firstIndex(of: self)! + 1) % all.count] + } + + var digitalCrown: DigitalCrownRotationalSensitivity { + switch self { + case .low: return .low + case .normal: return .medium + case .high: return .high + } + } +} + +/** + Crown sensitivity is the watch's own. Vibration and sound follow Settings > Watch on + the glasses, through the mirrored state, which the application context keeps for + when the phone is out of reach. + */ +@MainActor +@Observable +final class WatchPrefs { + /** Confirm each gesture sent, and tick on crown detents. */ + var haptics: Bool { link.state?.vibrateOnInput ?? true } + + /** Give each gesture its own cue, sound included; see Haptics. */ + var soundEffects: Bool { link.state?.soundEffects ?? false } + + var crownSensitivity: CrownSensitivity { + didSet { store.set(crownSensitivity.rawValue, forKey: Keys.crown) } + } + + @ObservationIgnored private let link: PhoneLink + @ObservationIgnored private let store = UserDefaults.standard + + private enum Keys { + static let crown = "crownSensitivity" + } + + init(link: PhoneLink) { + self.link = link + crownSensitivity = store.string(forKey: Keys.crown).flatMap(CrownSensitivity.init(rawValue:)) ?? .normal + } +} + +/** + The wrist's confirmation of what was sent. watchOS has no call that plays only a + vibration or only a sound: every WKHapticType is both, and only the watch's own Silent + Mode mutes the sound. So sound effects on means the full set of cues, which vibrate + whatever the vibrate setting says, and off means `.click` for everything. Whether + `.click` is truly silent is unconfirmed; see notes/apple-watch-followups.md. + */ +@MainActor +struct Haptics { + let prefs: WatchPrefs + + /** A sent click, swipe or command. */ + func click() { play(.click) } + /** A sent double-click: back. */ + func doubleClick() { play(.directionDown) } + /** The start of a hold. */ + func heavy() { play(.start) } + /** A reply finished arriving. */ + func done() { play(.success) } + + /** The phone refused or did not answer; always plays, and never feels like a sent gesture. */ + func error() { + let device = WKInterfaceDevice.current() + if prefs.soundEffects { + device.play(.failure) + return + } + device.play(.click) + Task { + try? await Task.sleep(nanoseconds: Self.errorTapGapNanos) + device.play(.click) + } + } + + private func play(_ cue: WKHapticType) { + if prefs.soundEffects { + WKInterfaceDevice.current().play(cue) + } else if prefs.haptics { + WKInterfaceDevice.current().play(.click) + } + } + + /** Apple's documentation asks for at least 100 ms between haptics. */ + private static let errorTapGapNanos: UInt64 = 150_000_000 +} diff --git a/apple-watch/FaceclawUITests/PadUITests.swift b/apple-watch/FaceclawUITests/PadUITests.swift new file mode 100644 index 00000000..4815f396 --- /dev/null +++ b/apple-watch/FaceclawUITests/PadUITests.swift @@ -0,0 +1,168 @@ +import XCTest + +/** + Drives the real app on a watch simulator. The screens' own behaviour needs only + the watch; the tests that talk to the phone need Faceclaw running on the paired + iPhone simulator, and skip themselves otherwise. See apple-watch/README.md. + */ +@MainActor +final class PadUITests: XCTestCase { + private var app: XCUIApplication! + + override func setUp() async throws { + continueAfterFailure = false + app = XCUIApplication() + app.launch() + XCTAssertTrue(pad.waitForExistence(timeout: 10)) + } + + private var pad: XCUIElement { app.descendants(matching: .any)["pad"] } + private var status: XCUIElement { app.staticTexts["padStatus"] } + + /** + The middle of the pad. An element's own tap() lands on its accessibility activation + point, near the top-left corner, and holding up there is the system's gesture for + pulling down Notification Center. + */ + private var padCenter: XCUICoordinate { + pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + } + + private func swipe(_ dx: CGFloat, _ dy: CGFloat) { + padCenter.press(forDuration: 0.02, thenDragTo: pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5 + dx, dy: 0.5 + dy))) + } + + private func revealTray() { + // From the very bottom edge, which is what makes it the reveal gesture and not a scroll. + let start = pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.97)) + let end = pad.coordinate(withNormalizedOffset: CGVector(dx: 0.5, dy: 0.5)) + start.press(forDuration: 0.02, thenDragTo: end) + } + + private func requirePhone() throws { + let deadline = Date().addingTimeInterval(15) + while Date() < deadline { + let text = status.exists ? status.label : "" + if !text.isEmpty && !text.contains("phone") && !text.contains("Install") { return } + RunLoop.current.run(until: Date().addingTimeInterval(0.25)) + } + throw XCTSkip("Faceclaw isn't answering on a paired iPhone: \(status.label)") + } + + func testTrayRevealsAndOpensEveryScreen() { + revealTray() + let apps = app.buttons["Apps"] + XCTAssertTrue(apps.waitForExistence(timeout: 3), "the tray should reveal from the bottom edge") + XCTAssertTrue(app.buttons["Assistant"].exists) + XCTAssertTrue(app.buttons["Status and settings"].exists) + XCTAssertTrue(app.descendants(matching: .any)["Type into app"].exists) + + apps.tap() + XCTAssertTrue(app.staticTexts["Apps"].waitForExistence(timeout: 3)) + } + + func testStatusScreenReachesSettings() { + revealTray() + let more = app.buttons["Status and settings"] + XCTAssertTrue(more.waitForExistence(timeout: 3)) + more.tap() + let settings = app.buttons["Watch settings"] + XCTAssertTrue(settings.waitForExistence(timeout: 3)) + settings.tap() + XCTAssertTrue(app.switches["Vibrate on input"].waitForExistence(timeout: 3)) + } + + /** The whole path: pad gesture -> phone -> shell -> mirrored state -> pad status line. */ + func testGesturesReachThePhoneAndComeBackAsState() throws { + try requirePhone() + let launcher = status.label + XCTAssertFalse(launcher.isEmpty) + + // Open an app through the remote and watch the foreground title follow it. + revealTray() + let appsButton = app.buttons["Apps"] + XCTAssertTrue(appsButton.waitForExistence(timeout: 3)) + appsButton.tap() + let calculator = app.buttons["app:calculator"] + XCTAssertTrue(calculator.waitForExistence(timeout: 5), "the phone's app list should be mirrored") + calculator.tap() + + XCTAssertTrue(status.waitForExistence(timeout: 5)) + let opened = NSPredicate(format: "label == %@", "Calculator") + expectation(for: opened, evaluatedWith: status) + waitForExpectations(timeout: 10) + + // Every other gesture goes the same way, and the phone answers each. A silent phone + // would raise "No reply from the phone" within seconds; the gestures themselves are + // checked on the phone's side by apple-watch/README.md's log command. + padCenter.tap() + RunLoop.current.run(until: Date().addingTimeInterval(0.6)) + padCenter.doubleTap() + RunLoop.current.run(until: Date().addingTimeInterval(0.6)) + padCenter.press(forDuration: 0.8) + RunLoop.current.run(until: Date().addingTimeInterval(0.6)) + // Tap then hold: the first touch's end must not cancel the second, however close they are. + padCenter.tap() + padCenter.press(forDuration: 0.8) + RunLoop.current.run(until: Date().addingTimeInterval(0.6)) + swipe(-0.35, 0) + swipe(0.35, 0) + swipe(0, -0.35) + swipe(0, 0.35) + XCUIDevice.shared.rotateDigitalCrown(delta: 1) + RunLoop.current.run(until: Date().addingTimeInterval(5)) + let notice = app.descendants(matching: .any)["notice"] + XCTAssertFalse(notice.exists && notice.label.contains("reply"), "the phone should answer every gesture") + } + + /** The system text sheet: the same path dictation takes, typed instead. */ + private func enterText(_ text: String) { + let field = app.textViews.firstMatch + XCTAssertTrue(field.waitForExistence(timeout: 5)) + field.typeText(text) + app.buttons["Done"].tap() + } + + func testTypingReachesTheForegroundApp() throws { + try requirePhone() + revealTray() + let apps = app.buttons["Apps"] + XCTAssertTrue(apps.waitForExistence(timeout: 3)) + apps.tap() + let calculator = app.buttons["app:calculator"] + XCTAssertTrue(calculator.waitForExistence(timeout: 5)) + calculator.tap() + XCTAssertTrue(status.waitForExistence(timeout: 5)) + expectation(for: NSPredicate(format: "label == %@", "Calculator"), evaluatedWith: status) + waitForExpectations(timeout: 10) + + revealTray() + let type = app.descendants(matching: .any)["Type into app"] + XCTAssertTrue(type.waitForExistence(timeout: 3)) + type.tap() + enterText("1234") + + // A refusal or a dropped message would show an error pill within the ack timeout. + RunLoop.current.run(until: Date().addingTimeInterval(6)) + let notice = app.descendants(matching: .any)["notice"] + XCTAssertFalse(notice.exists, "typing should be accepted: \(notice.exists ? notice.label : "")") + } + + /** A query goes out, and whatever the assistant does comes back as events. */ + func testAssistantRepliesAreMirrored() throws { + try requirePhone() + revealTray() + let assistant = app.buttons["Assistant"] + XCTAssertTrue(assistant.waitForExistence(timeout: 3)) + assistant.tap() + let phase = app.staticTexts["assistantPhase"] + XCTAssertTrue(phase.waitForExistence(timeout: 3)) + if phase.label.contains("Set up") { throw XCTSkip("No assistant on the phone") } + app.descendants(matching: .any)["assistantAsk"].tap() + enterText("hello") + // The phone's first event replaces the watch's optimistic "Thinking…" with its own, then + // a real reply or, with no model on a simulator, an error: both prove the channel. + expectation(for: NSPredicate(format: "label == %@ OR label == %@", "Reply", "Error"), evaluatedWith: phase) + waitForExpectations(timeout: 60) + } +} diff --git a/apple-watch/README.md b/apple-watch/README.md new file mode 100644 index 00000000..ca107eab --- /dev/null +++ b/apple-watch/README.md @@ -0,0 +1,171 @@ +# Faceclaw for Apple Watch + +A watch remote for Faceclaw on iOS: everything the Even R1 ring can do, plus the +things it can't. It's the same remote as the [Wear OS app](../wear/README.md), and +speaks the [same protocol](../wear/PROTOCOL.md), so the glasses treat them alike. +Needs watchOS 10 or later (Series 6, SE 2, Ultra and newer) and is a remote, not a +standalone app: without Faceclaw on the iPhone it only tells you to install it. + +## What it does + +**The pad** (the home screen; the whole display is the touch surface): + +| On the watch | On the glasses | +| --- | --- | +| tap | select | +| double-tap | **back**, everywhere: closes menus, leaves a settings column, leaves a folder or the launcher grid, returns to the sidebar (and from the sidebar turns the display off, as the ring does) | +| press and hold | the system menu: Focus app switcher, Voice input, Close window (held until you lift your finger; games that use a long press as a move get it instead) | +| tap, then press and hold | the app's context menu | +| swipe ↑ ↓ ← → | **spatial navigation**: the launcher moves one app by a row or a column, the settings panel moves between its columns and down its lists, the sidebar moves up and down, Music skips tracks. Right goes *into* things and left comes back *out*. Anywhere without a spatial meaning, up/down scroll, right selects and left goes back. One step per swipe. | +| turn the crown | previous / next; clockwise is previous. Don't count on it: on a real watch the crown has triggered watchOS's own actions instead of reaching the app. Sensitivity is in the watch's own settings. | +| the Double Tap hand gesture (Series 9, Ultra 2, SE 3 and later, watchOS 11) | select | + +Swipe up from the bottom edge to reveal the buttons, and down from them to hide them: + +- **Assistant** — ask from the wrist using the system text sheet, which offers + dictation, and read the reply as it streams. The watch buzzes when it finishes. A + reply that finishes with your wrist down is waiting when you raise it. "Hey Even" + instead opens the glasses' own voice dialog. +- **Type into app** — dictate or type into the foreground app, such as the terminal. +- **Apps** — the open windows (tap to switch, × to close) and the full launcher, + including installed EvenHub apps. +- **Status** — connection, display on/off, lock/unlock, whether Faceclaw is + answering on the iPhone, and the watch's settings. + +Phone-side switches live in Faceclaw's Settings > Watch, including vibration and +sound effects. While the glasses display +is off, a double-tap wakes it and a tap or hold shows the Glanceboard; other gestures +do nothing. The watch buzzes differently when the phone refuses or doesn't answer a +gesture, and the pill at the bottom says why. + +With no glasses connected, the pad drives the iPhone's on-screen preview instead, so +all of this can be tried without them. + +### What's different from Wear OS + +These come from what watchOS allows apps to do: + +- **No two-finger gestures.** SwiftUI on watchOS only reports one finger. +- **No side buttons.** The crown press and side button belong to the system. +- **Vibration and sound come as a pair.** watchOS has no call that plays only one, and + only the watch's own Silent Mode mutes the sound. With sound effects off, every input + gets a light tap rather than its own cue; with them on, inputs vibrate even if + Vibrate on input is off. +- **The display can't be kept on.** To keep the remote open between glances, set + Settings > General > Return to Clock > Faceclaw on the watch to a longer time. +- **The tap that wakes the display does nothing.** The system consumes it, and an + app can't tell it from a wrist raise, so it can't stand in for a first tap. +- **Holding near the top edge opens Notification Center.** The system takes that + touch after about 300 ms, and watchOS has no way to defer it. Hold lower down. +- **The system's small clock stays in the corner**, beside the pad's large one. An + app can't hide it. +- **The glasses' Watch battery indicator can lag.** A closed app can't be asked for + its level, so it reports when open and when the system grants background time. The + phone hides a level more than 4 hours old. + +There's no tip-tap detector like Wear OS's experimental one; Double Tap is the +system's reliable version of the same idea. + +## Building and installing + +The watch app is its own Xcode project, so it can be opened and previewed alone: + +```bash +./build_apple_watch.sh # simulator build, no signing +./build_apple_watch.sh --device # signed build for a real watch +./build_apple_watch.sh --install # as --device, then installs on the paired watch +npm run install:apple-watch # the same as --install +``` + +**It only works with a phone app that embeds it.** The system links the two apps +through the copy in the phone app's `Watch/` folder; install them separately and each +reports the other as not installed. Embedding is off by default, because signing it +needs a watch registered to your team and would break device builds for everyone +without one. To turn it on, in `build_paths.sh`: + +```bash +export IOS_TEAM_ID=... # as for any iOS device build +export FACECLAW_EMBED_WATCH=1 +``` + +Then, the first time: + +1. Connect the iPhone, and enable Developer Mode on the watch (Settings > Privacy & + Security; it appears after Xcode has seen the watch in Window > Devices and + Simulators). After it restarts, accept any "Trust this computer" prompt. +2. `./build_apple_watch.sh --install`, which registers the watch with your team and + installs the app on it. +3. `npm run run:ios -- --device --no-hmr` builds and installs the iPhone + app with the watch app inside. + +Xcode has to be new enough for the watch's OS: Xcode 26.4 can't prepare a watch on +watchOS 27 for development ("Could not support development", "doesn't have a known +architecture"), and that also blocks registering it and installing directly. On an +older Xcode you can still add the watch's UDID by hand at developer.apple.com and let +the iPhone's Watch app install the embedded copy. + +To see what the watch shows without looking at it (Xcode 27): + +```bash +xcrun devicectl device capture screenshot --device --destination watch.png +``` + +A hook (`scripts/apple-watch-embed.cjs`) adds the watch project to the generated iOS +project after each prepare, and takes it back out if the setting is removed. The +bundle ID follows `IOS_BUNDLE_ID` (the watch app's is `.watchkitapp`) and the +version follows `app/version.ts`. + +## Testing + +```bash +node tests/apple-watch-native.cjs # gestures, wire format, status line, on the Mac +node --test tests/ios-watch-remote.test.cjs # the phone side +``` + +The UI tests drive the real app with synthesized touches on a watch simulator. Those +that talk to the phone need Faceclaw running on the paired iPhone simulator, and skip +themselves otherwise: + +```bash +# Once: a watchOS simulator runtime and a pair (ids from `xcrun simctl list devices`). +xcodebuild -downloadPlatform watchOS +xcrun simctl pair +xcrun simctl boot ; xcrun simctl boot + +# The phone app, embedding the watch app. +FACECLAW_EMBED_WATCH=1 scripts/ios.sh build ios +xcrun simctl install platforms/ios/build/Debug-iphonesimulator/faceclaw.app +xcrun simctl spawn defaults write com.faceclaw.app onboarding.complete -bool YES +xcrun simctl spawn defaults write com.faceclaw.app onboarding.previewOnly -bool YES +xcrun simctl launch com.faceclaw.app + +# The tests, which install the watch app themselves. +xcodebuild test -project apple-watch/Faceclaw.xcodeproj -scheme Faceclaw \ + -destination "platform=watchOS Simulator,id=" \ + -derivedDataPath apple-watch/build CODE_SIGNING_ALLOWED=NO + +# What the phone made of each gesture. +xcrun simctl spawn log show --last 5m --style compact \ + --predicate 'process == "faceclaw" AND eventMessage CONTAINS "apple-watch"' +``` + +The simulator delivers messages in bursts about a second apart, which real hardware +isn't expected to do; don't judge gesture latency from it. `XCUIElement.tap()` lands +near an element's top-left corner, so tests that hold must use a coordinate. + +## Troubleshooting + +- *`Timed out while attempting to establish tunnel`, or the watch stays `connecting` + in `devicectl`* — check the watch's Wi-Fi network. It has to reach this Mac directly, + and a watch that joined a different network on the same router fails like this, even + though it still looks paired. It also powers down Wi-Fi when idle, so raise your + wrist; the first connection can take a minute or two. +- *"No phone"* — the watch can't reach the iPhone right now. +- *"Install Faceclaw on phone"* — the system doesn't see a linked phone app: it isn't + installed, or it was built without `FACECLAW_EMBED_WATCH=1`, or the bundle IDs differ. +- *"Open Faceclaw on phone"* — the phone app woke for our message but its dashboard + isn't up (for instance, onboarding isn't finished). Open it once. +- *"The glasses are not connected"* — tap Status > Connect. +- Nothing happens and no message: check Settings > Watch on the phone. + +Protocol details: [wear/PROTOCOL.md](../wear/PROTOCOL.md). diff --git a/build_apple_watch.sh b/build_apple_watch.sh new file mode 100755 index 00000000..6306c70a --- /dev/null +++ b/build_apple_watch.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Builds the Apple Watch app (apple-watch/). +# +# ./build_apple_watch.sh Simulator build; no signing needed. +# ./build_apple_watch.sh --device Signed build for a real watch. +# ./build_apple_watch.sh --install As --device, then installs on the paired watch. +# +# Signing uses IOS_TEAM_ID from build_paths.sh. The bundle ID follows IOS_BUNDLE_ID, +# because the watch app is tied to the phone app by it. To pick among several +# watches, set APPLE_WATCH_DEVICE_ID to its UDID there (see `xcrun devicectl list devices`). +# +# This is for working on the watch app alone. It can only talk to a phone app that +# was built with FACECLAW_EMBED_WATCH=1 (see apple-watch/README.md). +cd "$(dirname "$0")"; source scripts/before_build.sh +set -euo pipefail + +mode="${1:-simulator}" +project=apple-watch/Faceclaw.xcodeproj +derived=apple-watch/build + +node scripts/apple-watch-embed.cjs config + +case "$mode" in + simulator|--simulator) + xcodebuild -quiet -project "$project" -scheme Faceclaw -configuration Debug \ + -destination 'generic/platform=watchOS Simulator' -derivedDataPath "$derived" \ + CODE_SIGNING_ALLOWED=NO build + echo "Built $derived/Build/Products/Debug-watchsimulator/Faceclaw.app" + ;; + --device|--install) + if [[ -z "${IOS_TEAM_ID:-}" ]]; then + echo "Set IOS_TEAM_ID in build_paths.sh to sign for a real watch." >&2 + exit 1 + fi + destination='generic/platform=watchOS' + if [[ "$mode" == --install ]]; then + watch="${APPLE_WATCH_DEVICE_ID:-$(scripts/apple-watch-device.sh)}" + if [[ -z "$watch" ]]; then + echo "No Apple Watch found. Enable Developer Mode on it, with its iPhone connected to this Mac." >&2 + exit 1 + fi + # Naming the watch is what gets it registered with the team and into the + # profile; a generic destination signs for whatever is registered already. + destination="platform=watchOS,id=$watch" + fi + # A watch's Wi-Fi link can take minutes to wake, far past xcodebuild's default wait. + xcodebuild -quiet -project "$project" -scheme Faceclaw -configuration Debug \ + -destination "$destination" -destination-timeout 300 -derivedDataPath "$derived" \ + -allowProvisioningUpdates -allowProvisioningDeviceRegistration build + product="$derived/Build/Products/Debug-watchos/Faceclaw.app" + echo "Built $product" + if [[ "$mode" == --install ]]; then + xcrun devicectl device install app --device "$watch" "$product" + fi + ;; + *) + echo "Usage: $0 [--device|--install]" >&2 + exit 2 + ;; +esac diff --git a/build_paths.sh.template b/build_paths.sh.template index 55181ee9..5cc81e1c 100644 --- a/build_paths.sh.template +++ b/build_paths.sh.template @@ -11,4 +11,9 @@ # iOS bundle ID; set one your signing team owns to sideload alongside the # official build (default com.faceclaw.app). Also used by the iOS config scripts. #export IOS_BUNDLE_ID= +# Embed the Apple Watch app in the iOS app. Its link to the phone needs this, and +# signing it needs a watch registered to your team, so it's off by default. +#export FACECLAW_EMBED_WATCH=1 +# Which Apple Watch build_apple_watch.sh --install targets (default: the first found) +#export APPLE_WATCH_DEVICE_ID= diff --git a/package.json b/package.json index bea34dac..75bc1f7e 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "run:ios": "scripts/ios.sh run ios", "prepare:ios": "scripts/ios.sh prepare ios", "doctor:ios": "scripts/ios.sh doctor ios", + "install:apple-watch": "./build_apple_watch.sh --install", "build": "./build.sh", "lint": "./lint.sh", "run": "./build_and_run.sh", diff --git a/scripts/apple-watch-device.sh b/scripts/apple-watch-device.sh new file mode 100755 index 00000000..97204d51 --- /dev/null +++ b/scripts/apple-watch-device.sh @@ -0,0 +1,15 @@ +#!/bin/bash +# Prints the UDID of the first real Apple Watch devicectl can see, if any. A UDID +# works both as a devicectl --device and as an xcodebuild destination id. +set -euo pipefail +output="$(mktemp)" +trap 'rm -f "$output"' EXIT +xcrun devicectl list devices --quiet --json-output "$output" >/dev/null +node -e ' + const devices = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8")).result.devices; + // Xcode 27 lists simulators too and moved these fields into `properties`. `reality` + // is absent on a real device until it has connected, so only "simulated" rules one out. + const hardware = device => device.properties?.hardware ?? device.hardwareProperties ?? {}; + const watch = devices.map(hardware).find(h => h.platform === "watchOS" && h.reality !== "simulated"); + if (watch) console.log(watch.udid); +' "$output" diff --git a/scripts/apple-watch-embed.cjs b/scripts/apple-watch-embed.cjs new file mode 100644 index 00000000..2c2dc392 --- /dev/null +++ b/scripts/apple-watch-embed.cjs @@ -0,0 +1,69 @@ +// Ties the Apple Watch app (apple-watch/) to the iOS build. WatchConnectivity +// only links the two apps when the phone app's bundle carries the watch app in +// Watch/, so a phone build that wants a working watch must embed it. +// +// Embedding is opt-in (FACECLAW_EMBED_WATCH=1 in build_paths.sh): signing the +// result needs a provisioning profile for the watch app, which can't be made +// for a team with no watch registered, so it would break everyone else's builds. +// +// node scripts/apple-watch-embed.cjs config Write apple-watch/Local.xcconfig. +// node scripts/apple-watch-embed.cjs sync Make platforms/ios match the setting. +const { execFileSync } = require('node:child_process'); +const fs = require('node:fs'); +const path = require('node:path'); + +const root = path.resolve(__dirname, '..'); +const pbxproj = path.join(root, 'platforms/ios/faceclaw.xcodeproj/project.pbxproj'); +// The CLI drops `-sdk iphoneos` from xcodebuild, which would otherwise force +// the watch target onto the iOS SDK, only if this directory exists. Its own watch +// support (the legacy two-target layout) stays inert without a `watchextension`. +const cliWatchMarker = path.join(root, 'App_Resources/iOS/watchapp'); + +function embedEnabled() { + return process.env.FACECLAW_EMBED_WATCH === '1'; +} + +/** Xcode's own builds of the watch project read this too, so they sign the same way. */ +function writeLocalConfig() { + const version = /FACECLAW_VERSION = "(.*)"/.exec(fs.readFileSync(path.join(root, 'app/version.ts'), 'utf8'))[1]; + const lines = [ + `FACECLAW_IOS_BUNDLE_ID = ${process.env.IOS_BUNDLE_ID || 'com.faceclaw.app'}`, + // An embedded watch app has to carry the phone app's version. + `MARKETING_VERSION = ${version}`, + ]; + if (process.env.IOS_TEAM_ID) lines.push(`DEVELOPMENT_TEAM = ${process.env.IOS_TEAM_ID}`); + fs.writeFileSync(path.join(root, 'apple-watch/Local.xcconfig'), `${lines.join('\n')}\n`); +} + +function syncEmbed() { + const enabled = embedEnabled(); + if (enabled) writeLocalConfig(); + // Without the setting there's normally nothing to undo, and no need for Ruby. + const embedded = fs.existsSync(pbxproj) && fs.readFileSync(pbxproj, 'utf8').includes('Embed Watch Content'); + if (enabled !== embedded && fs.existsSync(pbxproj)) { + execFileSync('ruby', [path.join(__dirname, 'embed-apple-watch.rb'), enabled ? 'embed' : 'remove'], { cwd: root, stdio: 'inherit' }); + } + // Last: a project that still embeds the watch app can't build without this. + if (enabled) { + fs.mkdirSync(cliWatchMarker, { recursive: true }); + } else { + fs.rmSync(cliWatchMarker, { recursive: true, force: true }); + // Xcode leaves what it copied in earlier builds, which would still embed the old app. + const buildDir = path.join(root, 'platforms/ios/build'); + for (const config of fs.existsSync(buildDir) ? fs.readdirSync(buildDir) : []) { + fs.rmSync(path.join(buildDir, config, 'faceclaw.app/Watch'), { recursive: true, force: true }); + } + } +} + +module.exports = { embedEnabled, syncEmbed, writeLocalConfig }; + +if (require.main === module) { + const command = process.argv[2]; + if (command === 'config') writeLocalConfig(); + else if (command === 'sync') syncEmbed(); + else { + console.error('usage: apple-watch-embed.cjs config|sync'); + process.exit(2); + } +} diff --git a/scripts/embed-apple-watch.rb b/scripts/embed-apple-watch.rb new file mode 100644 index 00000000..2bb32f51 --- /dev/null +++ b/scripts/embed-apple-watch.rb @@ -0,0 +1,50 @@ +#!/usr/bin/env ruby +# Adds apple-watch/Faceclaw.xcodeproj to the generated iOS project as a subproject +# and embeds its app in the phone app's Watch/ folder, or takes all that back out. +# See apple-watch-embed.cjs, which decides when to run this. +require 'xcodeproj' + +PHASE_NAME = 'Embed Watch Content' +WATCH_TARGET = 'Faceclaw' + +root = File.expand_path('..', __dir__) +watch_project_path = File.join(root, 'apple-watch/Faceclaw.xcodeproj') +project = Xcodeproj::Project.open(File.join(root, 'platforms/ios/faceclaw.xcodeproj')) +app = project.targets.find { |target| target.name == 'faceclaw' } or abort 'No faceclaw target in the iOS project' + +def remove_embed(project, app, watch_project_path) + app.copy_files_build_phases.select { |phase| phase.name == PHASE_NAME }.each do |phase| + phase.files.to_a.each(&:remove_from_project) + phase.remove_from_project + end + app.dependencies.select { |dependency| dependency.display_name == WATCH_TARGET }.each(&:remove_from_project) + reference = project.reference_for_path(watch_project_path) + return unless reference + + # Saving fails on any proxy still pointing at the subproject. + project.objects.select { |object| object.isa == 'PBXContainerItemProxy' && object.container_portal == reference.uuid } + .each(&:remove_from_project) + entries = project.root_object.project_references + entries.select { |entry| entry[:project_ref] == reference }.each do |entry| + entry[:product_group].remove_from_project + entries.delete(entry) + end + reference.remove_from_project +end + +remove_embed(project, app, watch_project_path) + +if ARGV[0] == 'embed' + reference = project.main_group.new_file(watch_project_path) + watch_target = Xcodeproj::Project.open(watch_project_path).targets.find { |target| target.name == WATCH_TARGET } + app.add_dependency(watch_target) + + entry = project.root_object.project_references.find { |candidate| candidate[:project_ref] == reference } + product = entry[:product_group].children.find { |child| child.path == "#{WATCH_TARGET}.app" } or abort 'The watch project has no app product' + phase = app.new_copy_files_build_phase(PHASE_NAME) + phase.symbol_dst_subfolder_spec = :products_directory + phase.dst_path = '$(CONTENTS_FOLDER_PATH)/Watch' + phase.add_file_reference(product).settings = { 'ATTRIBUTES' => ['RemoveHeadersOnCopy'] } +end + +project.save diff --git a/scripts/hooks/faceclaw-watch.after-prepare.js b/scripts/hooks/faceclaw-watch.after-prepare.js new file mode 100644 index 00000000..3bc61731 --- /dev/null +++ b/scripts/hooks/faceclaw-watch.after-prepare.js @@ -0,0 +1,6 @@ +const { syncEmbed } = require('../../scripts/apple-watch-embed.cjs'); + +// After prepare, because that's when the iOS project and its pods are finished. +module.exports = function (hookArgs) { + if (hookArgs.prepareData.platform === 'ios') syncEmbed(); +}; diff --git a/scripts/install-hooks.cjs b/scripts/install-hooks.cjs index a0aebec6..682ad18c 100644 --- a/scripts/install-hooks.cjs +++ b/scripts/install-hooks.cjs @@ -8,6 +8,7 @@ const path = require('node:path'); const root = path.resolve(__dirname, '..'); const hooks = [ { source: 'scripts/hooks/faceclaw-kotlin.before-prepare.js', target: 'hooks/before-prepare/faceclaw-kotlin.js' }, + { source: 'scripts/hooks/faceclaw-watch.after-prepare.js', target: 'hooks/after-prepare/faceclaw-watch.js' }, ]; function installHooks() { diff --git a/tests/apple-watch-native.cjs b/tests/apple-watch-native.cjs new file mode 100644 index 00000000..a08ae63f --- /dev/null +++ b/tests/apple-watch-native.cjs @@ -0,0 +1,10 @@ +// Compile the watch app's platform-independent Swift for the Mac and run its checks. +const { execFileSync } = require('node:child_process'); +const { mkdtempSync } = require('node:fs'); +const { tmpdir } = require('node:os'); +const { join, resolve } = require('node:path'); +const root = resolve(__dirname, '..'), dir = mkdtempSync(join(tmpdir(), 'faceclaw-apple-watch-')); +const src = 'apple-watch/Faceclaw'; +execFileSync('xcrun', ['swiftc', '-O', `${src}/Protocol.swift`, `${src}/PadStatus.swift`, `${src}/PadGestureRecognizer.swift`, + 'tests/native/apple-watch/main.swift', '-o', join(dir, 'apple-watch-test')], { cwd: root, stdio: 'inherit' }); +execFileSync(join(dir, 'apple-watch-test'), [], { stdio: 'inherit' }); diff --git a/tests/ios-glanceboard.test.cjs b/tests/ios-glanceboard.test.cjs index ae2ab387..073ebf05 100644 --- a/tests/ios-glanceboard.test.cjs +++ b/tests/ios-glanceboard.test.cjs @@ -157,6 +157,7 @@ function fixture() { './lock-screen': { LOCK_SCREEN_SURFACE_ID: 'lock-screen', createLockScreenImage: () => new images.GrayImage(640, 480, 123) }, './glasses-presence': load('app/g2/glasses-presence.ts', {}), './device-addresses': { loadDeviceAddresses: () => ({ right: 'AA', left: 'BB', ring: '' }) }, './ios-peripheral-identity': { deviceAddressError: () => null }, + './wear-remote': { WearRemote: class { schedulePublish() {} } }, './firmware-compat': load('app/g2/firmware-compat.ts', {}), './reconnect-policy': load('app/g2/reconnect-policy.ts', {}), '../apps/launcher': { launcherEntries: () => [] }, '../apps/evenhub/installed-apps': {}, '../apps/evenhub/manager': { onPhoneShownChanged: () => () => {}, phoneUiButton: () => null }, diff --git a/tests/ios-watch-remote.test.cjs b/tests/ios-watch-remote.test.cjs new file mode 100644 index 00000000..316b854b --- /dev/null +++ b/tests/ios-watch-remote.test.cjs @@ -0,0 +1,222 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const vm = require('node:vm'); +const ts = require('typescript'); +const path = require('node:path'); + +function load(file, context = {}) { + const sandbox = { exports: {}, console, ...context }; + sandbox.global = sandbox; + vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText, sandbox); + return sandbox.exports; +} +const types = () => load('app/native/wear-bridge-types.ts'); + +/** What FaceclawWatchBridge.m does, as far as JS can tell. */ +function fakeNative({ available = true, watchApp = true } = {}) { + return { + calls: [], message: null, connection: null, watchApp, + isAvailable() { return available; }, + isWatchAppAvailable() { return this.watchApp; }, + setMessageHandlerConnectionHandler(message, connection) { this.message = message; this.connection = connection; }, + publishStateForce(json, force) { this.calls.push(['state', JSON.parse(json), force]); }, + sendEventSticky(json, sticky) { this.calls.push(['event', JSON.parse(json), sticky]); }, + sendToWatchJson(pathName, json) { this.calls.push(['send', pathName, JSON.parse(json)]); }, + sendAckOkJsReadyMessage(seq, ok, jsReady, message) { this.calls.push(['ack', seq, ok, jsReady, message]); }, + }; +} +function loadBridge(native) { + return load('app/native/wear-bridge.ios.ts', { + FaceclawWatchBridge: { shared: () => native }, + require: () => types(), + }).wearBridge; +} + +test('iOS bridge carries the same messages as the Android one', () => { + const native = fakeNative(); + const bridge = loadBridge(native); + assert.equal(bridge.isAvailable(), true); + const received = []; + bridge.onMessage(message => received.push(message)); + native.message('/faceclaw/input', '{"gesture":"click","seq":3}'); + native.message('/faceclaw/press', 'not json'); + assert.equal(received[0].path, '/faceclaw/input'); + assert.equal(received[0].payload.gesture, 'click'); + assert.equal(Object.keys(received[1].payload).length, 0, 'a malformed payload is empty, not a crash'); + + bridge.publishState({ phase: 'connected' }, true); + bridge.sendToWatch('/faceclaw/battery/request', {}); + bridge.sendAck(received[0].nodeId, 3, false, 'refused'); + assert.deepEqual(JSON.parse(JSON.stringify(native.calls)), [ + ['state', { phase: 'connected' }, true], + ['send', '/faceclaw/battery/request', {}], + ['ack', 3, false, true, 'refused'], + ]); +}); + +test('only streaming assistant snapshots skip the sticky application context', () => { + const native = fakeNative(); + const bridge = loadBridge(native); + bridge.sendEvent({ type: 'assistant', phase: 'streaming', text: 'par' }); + bridge.sendEvent({ type: 'assistant', phase: 'done', text: 'partial' }); + bridge.sendEvent({ type: 'alert', text: 'hi' }); + assert.deepEqual(native.calls.map(call => call[2]), [false, true, true]); +}); + +test('availability follows the watch app, and a change reaches listeners', () => { + const native = fakeNative({ watchApp: false }); + const bridge = loadBridge(native); + const seen = []; + bridge.onWatchConnection(connection => seen.push(connection.reachable)); + assert.deepEqual(seen, [false], 'the current value is delivered immediately'); + native.watchApp = true; + native.connection(); + assert.deepEqual(seen, [false, true]); + assert.equal(bridge.getWatchConnection().watchName, 'Apple Watch'); +}); + +test('a phone that cannot use WatchConnectivity turns the bridge into a no-op', () => { + const native = fakeNative({ available: false }); + const bridge = loadBridge(native); + assert.equal(bridge.isAvailable(), false); + bridge.publishState({}); + bridge.sendEvent({ type: 'alert', text: 'x' }); + assert.equal(bridge.getWatchConnection().reachable, false); + assert.equal(native.calls.length, 0); +}); + +/** A WearRemote over a fake bridge and shell. */ +function loadRemote({ phase = 'disconnected', preview = false, now = 1_800_000_000_000 } = {}) { + const bridge = { + messageListener: null, connectionListener: null, sent: [], + isAvailable: () => true, + onMessage(listener) { this.messageListener = listener; return () => {}; }, + onWatchConnection(listener) { this.connectionListener = listener; listener({ reachable: true, watchName: 'Apple Watch' }); return () => {}; }, + publishState() {}, + sendEvent() {}, + sendToWatch() {}, + sendAck(_node, seq, ok, message) { this.sent.push({ seq, ok, message }); }, + }; + const levels = { watch: null, watchCharging: null }; + const shell = { + onAssistantActivity() {}, onAlertShown() {}, + getBatteryLevels: () => levels, + setBatteryLevels(update) { Object.assign(levels, update); }, + foregroundWindow: () => null, getForegroundApp: () => null, getWindows: () => [], + isScreenOn: () => true, wake: () => true, isAssistantAvailable: () => false, + }; + const setting = value => ({ get: () => value }); + const intervals = []; + const realDate = Date; + const clock = { now }; + class FakeDate extends realDate { static now() { return clock.now; } } + const inputs = []; + const state = { phase, status: '', glassesLocked: false, glassesWorn: null, silentMode: false, lastHeadsetBattery: null }; + const { WearRemote } = load('app/g2/wear-remote.ts', { + Date: FakeDate, + setTimeout: () => 0, clearTimeout() {}, + setInterval: fn => { intervals.push(fn); return intervals.length; }, + Number, Math, JSON, + require: id => ({ + '../analytics/analytics': { flagAnalyticsEvent() {} }, + '../apps/evenhub/installed-apps': { getInstalledEvenHubApps: () => [], installedEvenHubAppId: id => id }, + '../native/wear-bridge': { wearBridge: bridge, WEAR_PATHS: types().WEAR_PATHS }, + '../ui/dashboard-settings': { + onAnySettingChanged() {}, watchCanUnlockSetting: setting(true), watchCrownClockwiseNextSetting: setting(false), + watchMirrorAssistantSetting: setting(true), watchRemoteEnabledSetting: setting(true), + watchVibrateOnInputSetting: setting(true), watchSoundEffectsSetting: setting(false), + }, + '../ui/shell/shell': { shell }, + '../ui/shell/voice-activity': { voiceActivity: { subscribe() {}, isActive: () => false } }, + '../version': { FACECLAW_VERSION: 'test' }, + })[id] ?? {}, + }); + const remote = new WearRemote({ + apps: [{ appId: 'timer', title: 'Timer' }], + injectInput: async kind => { inputs.push(kind); }, + launchApp: async () => {}, connect: async () => {}, disconnect: async () => {}, + setGlassesLocked() {}, requestShellRender() {}, + previewInputActive: () => preview, + getState: () => state, + appendLog() {}, + }); + const deliver = (pathName, payload) => bridge.messageListener({ path: pathName, payload, nodeId: 'apple-watch' }); + return { remote, bridge, shell, levels, inputs, intervals, clock, deliver }; +} +const settle = () => new Promise(resolve => setImmediate(resolve)); + +test('gestures are refused with no display, and land on the preview without glasses', async () => { + const none = loadRemote(); + none.deliver('/faceclaw/input', { gesture: 'click', seq: 1 }); + none.deliver('/faceclaw/press', {}); + await settle(); + assert.deepEqual(none.inputs, []); + assert.equal(none.bridge.sent[0].ok, false); + assert.match(none.bridge.sent[0].message, /not connected/); + + const preview = loadRemote({ preview: true }); + preview.deliver('/faceclaw/press', {}); + preview.deliver('/faceclaw/input', { gesture: 'click', seq: 2 }); + await settle(); + assert.deepEqual(preview.inputs, ['ring-press', 'click']); + assert.equal(preview.bridge.sent[0].ok, true); + + const glasses = loadRemote({ phase: 'connected' }); + glasses.deliver('/faceclaw/input', { gesture: 'swipe-left', seq: 3 }); + await settle(); + assert.deepEqual(glasses.inputs, ['swipe-left']); +}); + +test('a watch battery report is shown, kept while fresh, and dropped once stale', async () => { + const { levels, intervals, clock, deliver, bridge } = loadRemote({ phase: 'connected' }); + deliver('/faceclaw/battery', { battery: 64, charging: true, at: clock.now }); + await settle(); + assert.equal(levels.watch, 64); + assert.equal(levels.watchCharging, true); + assert.equal(bridge.sent.length, 0, 'battery reports are never acked'); + + const tick = () => intervals.forEach(fn => fn()); + clock.now += 3 * 60 * 60_000; + tick(); + assert.equal(levels.watch, 64, 'a closed watch app reports rarely; three hours is still shown'); + clock.now += 2 * 60 * 60_000; + tick(); + assert.equal(levels.watch, null, 'five hours says the watch is gone'); +}); + +test('a battery report found long after it was left is not shown as current', async () => { + const { levels, clock, deliver } = loadRemote({ phase: 'connected' }); + deliver('/faceclaw/battery', { battery: 90, charging: false, at: clock.now - 5 * 60 * 60_000 }); + await settle(); + assert.equal(levels.watch, null); + // A live report without a timestamp is current by definition, as Wear OS's always are. + deliver('/faceclaw/battery', { battery: 80, charging: false }); + await settle(); + assert.equal(levels.watch, 80); +}); + +test('a watch that does not know its battery shows no level, not an empty one', async () => { + const { levels, deliver } = loadRemote({ phase: 'connected' }); + deliver('/faceclaw/battery', { battery: 50, charging: false }); + await settle(); + assert.equal(levels.watch, 50); + deliver('/faceclaw/battery', { battery: null, charging: false }); + await settle(); + assert.equal(levels.watch, null); +}); + +test('the mirrored state carries the Apple Watch feedback settings', () => { + const state = loadRemote({ preview: true }).remote.buildState(); + assert.equal(state.vibrateOnInput, true); + assert.equal(state.soundEffects, false); +}); + +test('the mirrored state says when gestures are landing on the preview', () => { + const state = loadRemote({ preview: true }).remote.buildState(); + assert.equal(state.preview, true); + assert.equal(state.connected, false); + assert.equal(loadRemote({ phase: 'connected' }).remote.buildState().connected, true); +}); diff --git a/tests/native/apple-watch/main.swift b/tests/native/apple-watch/main.swift new file mode 100644 index 00000000..f9121be1 --- /dev/null +++ b/tests/native/apple-watch/main.swift @@ -0,0 +1,222 @@ +// Exercises the watch app's platform-independent logic on the Mac: the gesture +// state machine, the wire format and the pad's status line. +import Foundation + +var failures = 0 +func check(_ condition: @autoclosure () -> Bool, _ message: String, line: Int = #line) { + if !condition() { + failures += 1 + print("FAILED line \(line): \(message)") + } +} + +// MARK: - Gestures + +final class Harness { + let recognizer = PadGestureRecognizer() + var events: [PadEvent] = [] + var time: TimeInterval = 100 + + init() { recognizer.onEvent = { [unowned self] in self.events.append($0) } } + + func down(_ point: CGPoint = .zero) { recognizer.touchDown(at: point, time: time) } + func move(_ point: CGPoint) { recognizer.touchMoved(to: point) } + func up(_ point: CGPoint = .zero) { recognizer.touchUp(at: point, time: time) } + func wait(_ seconds: TimeInterval) { time += seconds; recognizer.advance(to: time) } +} + +do { + // A lone tap is only known to be one once the double-tap window closes. + let h = Harness() + h.down(); h.up() + check(h.events == [.press], "a tap waits out the double-tap window: \(h.events)") + h.wait(0.2) + check(h.events == [.press], "still inside the window") + h.wait(0.2) + check(h.events == [.press, .tap], "tap after the window: \(h.events)") + check(h.recognizer.deadline == nil, "nothing left to wait for") +} + +do { + let h = Harness() + h.down(); h.up(); h.wait(0.1); h.down(); h.wait(0.05); h.up() + check(h.events == [.press, .press, .doubleTap], "double-tap reports both presses: \(h.events)") + h.wait(1) + check(h.events.count == 3, "no late tap after a double-tap") +} + +do { + // Hold: reported at the timeout and again on release, however long it lasts. + let h = Harness() + h.down(); h.wait(0.39) + check(h.events == [.press], "not yet a hold") + h.wait(0.02) + check(h.events == [.press, .longPressStart], "hold starts at the timeout: \(h.events)") + h.wait(5); h.up() + check(h.events == [.press, .longPressStart, .longPressEnd], "hold ends on release: \(h.events)") +} + +do { + // Movement during a hold doesn't turn it into a swipe. + let h = Harness() + h.down(); h.wait(0.5); h.move(CGPoint(x: 50, y: 50)); h.up(CGPoint(x: 50, y: 50)) + check(h.events == [.press, .longPressStart, .longPressEnd], "a hold stays a hold: \(h.events)") +} + +do { + let h = Harness() + h.down(); h.up(); h.wait(0.1); h.down(); h.wait(0.5) + check(h.events == [.press, .press, .shortThenLongPress], "tap then hold: \(h.events)") + h.up(); h.wait(1) + check(h.events.count == 3, "no double-tap or release after tap-then-hold: \(h.events)") +} + +do { + let h = Harness() + let start = CGPoint(x: 100, y: 180), end = CGPoint(x: 104, y: 60) + h.down(start); h.move(end); h.wait(1); h.up(end) + check(h.events == [.press, .swipe(start: start, dx: 4, dy: -120)], "a swipe, never a hold: \(h.events)") +} + +do { + // A small wobble under the slop is still a tap. + let h = Harness() + h.down(.zero); h.move(CGPoint(x: 3, y: 3)); h.up(CGPoint(x: 3, y: 3)); h.wait(0.4) + check(h.events == [.press, .tap], "wobble is a tap: \(h.events)") +} + +do { + // Whatever ends a hold, the phone must hear a release or it sees a stuck long-press. + let h = Harness() + h.down(); h.wait(0.5); h.recognizer.cancel() + check(h.events == [.press, .longPressStart, .longPressEnd], "a cancelled hold is released: \(h.events)") + check(!h.recognizer.isTouching, "idle after cancel") +} + +do { + let h = Harness() + h.down(); h.recognizer.cancel(); h.wait(1) + check(h.events == [.press], "a cancelled touch is not a tap: \(h.events)") +} + +do { + // The wrist dropped after a complete tap: it was still a tap, and there's no touch left to wait for. + let h = Harness() + h.down(); h.up(); h.recognizer.cancel() + check(h.events == [.press, .tap], "a finished tap survives cancel: \(h.events)") + h.wait(1) + check(h.events.count == 2, "and isn't reported twice") +} + +do { + // A touch whose end was never seen must not wedge the next one. + let h = Harness() + h.down(); h.wait(0.5); h.down() + check(h.events == [.press, .longPressStart, .longPressEnd, .press], "a lost touch-up is recovered: \(h.events)") +} + +// MARK: - Wire format + +do { + let json = """ + {"protocol":1,"version":"0.8.6","phase":"connected","status":"ok","connected":true,"preview":false, + "screenOn":true,"locked":false,"worn":null,"listening":false,"battery":78,"charging":false,"silentMode":false, + "caseBattery":64,"caseCharging":null,"foreground":{"appId":"music","title":"Music"}, + "windows":[{"windowId":"w1","title":"Music","appId":"music","focused":true,"closeable":false,"acceptsText":true}], + "apps":[{"appId":"timer","title":"Timer"}],"remoteEnabled":true,"vibrateOnInput":false,"soundEffects":true,"canUnlock":false, + "mirrorAssistant":true,"assistantAvailable":true} + """ + let state = PhoneState.parse(json, updatedAt: 42) + check(state != nil, "state parses") + check(state?.connected == true && state?.takesInput == true, "connected") + check(state?.battery == 78 && state?.caseBattery == 64, "batteries") + check(state?.worn == nil && state?.caseCharging == nil, "JSON null stays nil") + check(state?.foreground == AppEntry(appId: "music", title: "Music"), "foreground") + check(state?.windows.first?.acceptsText == true && state?.windows.first?.closeable == false, "window flags") + check(state?.apps.map(\.appId) == ["timer"], "apps") + check(state?.vibrateOnInput == false && state?.soundEffects == true && state?.canUnlock == false, "settings") + check(state?.updatedAt == 42, "updatedAt") +} + +do { + // A state from an older or newer phone app parses with defaults rather than failing. + let state = PhoneState.parse("{\"battery\":140,\"unknownField\":[1,2]}", updatedAt: 0) + check(state?.battery == nil, "out-of-range percentage is unknown") + check(state?.remoteEnabled == true && state?.canUnlock == true, "permissive defaults") + check(state?.vibrateOnInput == true && state?.soundEffects == false, "feedback defaults") + check(state?.phase == "disconnected" && state?.takesInput == false, "disconnected default") + check(PhoneState.parse("not json", updatedAt: 0) == nil, "garbage is rejected") + check(PhoneState.parse("[1]", updatedAt: 0) == nil, "a non-object is rejected") +} + +do { + let ack = Ack.parse("{\"seq\":7,\"ok\":false,\"jsReady\":false,\"message\":\"nope\"}") + check(ack == Ack(seq: 7, ok: false, jsReady: false, message: "nope"), "ack: \(String(describing: ack))") + check(Ack.parse("{\"seq\":7,\"ok\":true}")?.jsReady == true, "jsReady defaults to true") +} + +do { + check(PhoneEvent.parse("{\"type\":\"assistant\",\"phase\":\"done\",\"text\":\"hi\"}") == .assistant(phase: "done", text: "hi"), "assistant event") + check(PhoneEvent.parse("{\"type\":\"alert\",\"text\":\"x\"}") == .alert(text: "x"), "alert event") + check(PhoneEvent.parse("{\"type\":\"other\"}") == nil, "unknown event") +} + +do { + var mirror = AssistantMirror() + mirror = mirror.applying(phase: "thinking", text: "") + check(mirror == AssistantMirror(phase: "thinking", text: ""), "thinking") + mirror = mirror.applying(phase: "streaming", text: "Hel") + mirror = mirror.applying(phase: "done", text: "") + check(mirror == AssistantMirror(phase: "done", text: "Hel"), "an empty done keeps the streamed text") + mirror = mirror.applying(phase: "closed", text: "") + check(mirror == AssistantMirror(phase: "idle", text: "Hel"), "closed keeps the text to read") + check(mirror.applying(phase: "bogus", text: "x") == mirror, "unknown phases change nothing") +} + +// MARK: - Status line + +do { + var up = LinkStatus(activated: true, phoneReachable: true, phoneAppInstalled: true, appRunning: true) + var state = PhoneState() + state.connected = true + state.screenOn = true + state.foreground = AppEntry(appId: "music", title: "Music") + + check(padLine(state: nil, link: LinkStatus()).text == "Waiting for phone", "before activation") + check(padLine(state: state, link: LinkStatus(activated: true, phoneAppInstalled: false)).text == "Install Faceclaw on phone", "no phone app") + check(padLine(state: state, link: LinkStatus(activated: true, phoneAppInstalled: true)).text == "No phone", "unreachable") + up.appRunning = false + check(padLine(state: state, link: up).text == "Open Faceclaw on phone", "dashboard not up") + up.appRunning = true + check(padLine(state: nil, link: up).text == "Waiting for phone", "no state yet") + check(padLine(state: state, link: up) == StatusLine(text: "Music", tone: .good), "foreground app") + + state.screenOn = false + check(padLine(state: state, link: up).text == "●● Wake", "display off") + state.locked = true + check(padLine(state: state, link: up).text == "Locked", "locked beats display off") + state.charging = true + state.battery = 50 + state.caseBattery = 64 + state.caseCharging = true + check(padLine(state: state, link: up) == StatusLine(text: "Charging · G2 50%", tone: .busy, detail: "Case 64% ⚡"), "charging") + + var off = PhoneState() + check(padLine(state: off, link: up).text == "Disconnected", "disconnected") + off.phase = "retrying" + check(padLine(state: off, link: up).text == "Reconnecting…", "iOS's retrying phase") + off.phase = "disconnected" + off.preview = true + off.screenOn = true + check(padLine(state: off, link: up) == StatusLine(text: "Launcher", tone: .good, detail: "Phone preview"), "phone preview") + off.screenOn = false + check(padLine(state: off, link: up).text == "●● Wake", "preview display off") + off.remoteEnabled = false + check(padLine(state: off, link: up).text == "Watch control off", "remote disabled") +} + +if failures > 0 { + print("\(failures) check(s) failed") + exit(1) +} +print("apple-watch logic: all checks passed") diff --git a/wear/PROTOCOL.md b/wear/PROTOCOL.md index 91fee823..02316c92 100644 --- a/wear/PROTOCOL.md +++ b/wear/PROTOCOL.md @@ -1,5 +1,10 @@ # Watch ⇄ phone protocol +The Wear OS and Apple Watch apps speak the same messages over different +transports, and `app/g2/wear-remote.ts` gives them their meaning on both phones. +This document describes the messages and the Wear OS transport; see "Apple Watch +transport" at the end for what differs there. + Transport: the Wearable Data Layer (Google Play services). Both apps share the `com.faceclaw.app` application id and, for release builds, the signing key — the Data Layer routes only between matching apps. Payloads are UTF-8 JSON @@ -55,7 +60,7 @@ dashboard state, re-sent whenever it changes (and at least every 30 s): ```json { "protocol": 1, "version": "…", - "phase": "connected", "status": "…", "connected": true, + "phase": "connected", "status": "…", "connected": true, "preview": false, "screenOn": true, "locked": false, "worn": true, "listening": false, "battery": 78, "charging": false, "silentMode": false, "caseBattery": 64, "caseCharging": true, @@ -63,6 +68,7 @@ dashboard state, re-sent whenever it changes (and at least every 30 s): "windows": [{"windowId": "…", "title": "…", "appId": "…", "focused": true, "closeable": true, "acceptsText": false}], "apps": [{"appId": "timer", "title": "Timer"}], "remoteEnabled": true, "crownClockwiseNext": false, + "vibrateOnInput": true, "soundEffects": false, "canUnlock": true, "mirrorAssistant": true, "assistantAvailable": true } @@ -81,7 +87,47 @@ The watch shows the case level under the charging line. `protocol` is reserved for incompatible revisions of this format (nothing checks it yet). `crownClockwiseNext` mirrors the on-glasses Settings > Watch toggle; when false (the default), clockwise crown -rotation sends previous/up, and when true it sends next/down. +rotation sends previous/up, and when true it sends next/down. Only Wear OS reads +it, and the toggle only appears on Android; on Apple Watch, clockwise is always +previous/up. + +`vibrateOnInput` and `soundEffects` mirror the Settings > Watch toggles that appear +on iOS, and only the Apple Watch app reads them. Wear OS keeps its vibration setting +on the watch and has no sounds. `apps` is the launcher grid plus installed EvenHub packages, i.e. exactly what the assistant's `apps.launch` tool accepts. + +`preview` says gestures are landing on the phone's own display preview, with no +glasses involved: `connected` stays false, but input isn't refused. Only the iOS +app sets it for now. + +## Apple Watch transport + +Transport: WatchConnectivity. Phone side: +`App_Resources/iOS/src/FaceclawWatchBridge.m` (transport), +`app/native/wear-bridge.ios.ts` (JS wrapper). Watch side: +`apple-watch/Faceclaw/PhoneLink.swift`, `Protocol.swift`. + +The system only links the two apps when the watch app's +`WKCompanionAppBundleIdentifier` names the phone app, its bundle ID begins with +the phone app's, **and the phone app's bundle embeds it in `Watch/`**; installing +each app separately leaves both sides reporting the other as not installed. See +`apple-watch/README.md`. + +A message is a dictionary `{path, json}`, where `json` is the JSON payload above; +every path and payload is unchanged. `sendMessage` is used with no reply handler, +and acks are ordinary `/faceclaw/ack` messages. What differs follows from one fact: +**the phone can't wake the watch app, and the app is suspended whenever the wrist +drops.** The watch can wake the phone app, and does so with any message. + +| Wear OS | Apple Watch | +| --- | --- | +| `/faceclaw/state` is a Data Layer item | The application context, `{json, updatedAt}`, for the app's next wake. While the app is in front it's also sent live as `{path: "/faceclaw/state", json, updatedAt}`, since context delivery can lag. Whichever `updatedAt` is newer wins. | +| `/faceclaw/event` is lost if the app is closed | Also sent live, with `at`. All but `streaming` snapshots also stay in the context as `{event, eventAt}`, so a reply that finished with the wrist down is there on the next raise. The watch drops anything not newer than the last `at`, and ignores a stale context event (assistant after 10 minutes, alert after 1). | +| `/faceclaw/battery/request` wakes a listener service | Only reaches an app in front. The app reports on open, every minute while open, and when the system grants it background time (asked for every 30 minutes), leaving `{battery: }` in its own context for the phone to find. The payload adds `at` (ms), and the phone treats a level over 4 hours old as unknown. | +| "Reachable" is a capability query | `isPaired && isWatchAppInstalled`, whether or not the app is running. Live reachability changes also fire the connection callback, which is what pushes fresh state to an app coming to the front. | +| `nodeId` routes acks | Unused: a session has one counterpart. | + +`isReachable` lags a moment behind a wrist raise, so the watch holds gestures for up +to 1.5 s rather than failing them. Touch-downs (`/faceclaw/press`) are never held.