From 8fa04d1b46902e59cee4f141dfec52bd6b90b2a8 Mon Sep 17 00:00:00 2001 From: Kalani Helekunihi Date: Sun, 23 Aug 2026 15:02:35 -0500 Subject: [PATCH] Add Wear OS companion app and remote controls Add a Wearable Data Layer bridge for syncing dashboard state and commands. Introduce a standalone watch app with assistant, app launcher, touchpad remote, settings, and connectivity UI. --- App_Resources/Android/app.gradle | 3 + .../Android/src/main/AndroidManifest.xml | 16 + .../com/faceclaw/app/FaceclawWearBridge.java | 378 ++++++++++++++ .../faceclaw/app/FaceclawWearListener.java | 19 + .../app/FaceclawWearListenerService.java | 26 + .../Android/src/main/res/values/wear.xml | 9 + CLAUDE.md | 7 + PRIVACY | 15 + README.md | 32 +- app/app.css | 75 +++ app/apps/evenhub/evenhub-window.ts | 10 + app/apps/evenhub/session.ts | 11 +- app/apps/launcher/launcher-app.ts | 130 ++++- app/apps/music/music-app.ts | 26 +- app/g2/dashboard-controller.ts | 288 +++++++++- app/g2/events.ts | 25 + app/g2/wear-remote.ts | 490 ++++++++++++++++++ app/graphics/image.ts | 14 + app/native/faceclaw-communicator.ts | 12 + app/native/wear-bridge.ts | 178 +++++++ app/phone-ui/main-page.xml | 28 +- app/phone-ui/main-view-model.ts | 260 +++++++++- app/phone-ui/remote-controls.xml | 47 ++ app/ui/dashboard-settings.ts | 67 +++ app/ui/dashboard/settings-menus.ts | 39 ++ app/ui/dashboard/settings-panel.ts | 13 + app/ui/layers.ts | 95 +++- app/ui/shell/chrome-layer.ts | 40 +- app/ui/shell/geometry.ts | 40 +- app/ui/shell/in-process-window.ts | 22 + app/ui/shell/shell.ts | 177 ++++++- wear/.gitignore | 6 + wear/PROTOCOL.md | 62 +++ wear/README.md | 109 ++++ wear/app/build.gradle.kts | 67 +++ wear/app/proguard-rules.pro | 1 + wear/app/src/main/AndroidManifest.xml | 35 ++ .../com/faceclaw/wear/FingerTapDetector.kt | 179 +++++++ .../main/kotlin/com/faceclaw/wear/Haptics.kt | 51 ++ .../kotlin/com/faceclaw/wear/MainActivity.kt | 116 +++++ .../kotlin/com/faceclaw/wear/OnBodyMonitor.kt | 55 ++ .../kotlin/com/faceclaw/wear/PhoneLink.kt | 335 ++++++++++++ .../main/kotlin/com/faceclaw/wear/Protocol.kt | 205 ++++++++ .../kotlin/com/faceclaw/wear/WatchPrefs.kt | 119 +++++ .../com/faceclaw/wear/WristTwistDetector.kt | 137 +++++ .../kotlin/com/faceclaw/wear/ui/AppsScreen.kt | 128 +++++ .../com/faceclaw/wear/ui/AssistantScreen.kt | 144 +++++ .../kotlin/com/faceclaw/wear/ui/Common.kt | 109 ++++ .../com/faceclaw/wear/ui/FaceclawWearApp.kt | 65 +++ .../com/faceclaw/wear/ui/RemoteScreen.kt | 384 ++++++++++++++ .../com/faceclaw/wear/ui/SettingsScreen.kt | 170 ++++++ .../com/faceclaw/wear/ui/StatusScreen.kt | 171 ++++++ .../kotlin/com/faceclaw/wear/ui/TextEntry.kt | 82 +++ .../com/faceclaw/wear/ui/TouchpadGestures.kt | 105 ++++ wear/app/src/main/res/drawable/ic_apps.xml | 9 + .../src/main/res/drawable/ic_assistant.xml | 9 + wear/app/src/main/res/drawable/ic_close.xml | 9 + .../app/src/main/res/drawable/ic_keyboard.xml | 9 + .../res/drawable/ic_launcher_foreground.xml | 15 + wear/app/src/main/res/drawable/ic_mic.xml | 9 + wear/app/src/main/res/drawable/ic_more.xml | 9 + wear/app/src/main/res/drawable/ic_refresh.xml | 9 + .../app/src/main/res/drawable/ic_settings.xml | 9 + .../res/mipmap-anydpi-v26/ic_launcher.xml | 5 + .../src/main/res/mipmap-hdpi/ic_launcher.png | Bin 0 -> 9832 bytes .../mipmap-hdpi/ic_launcher_foreground.png | Bin 0 -> 21524 bytes .../src/main/res/mipmap-mdpi/ic_launcher.png | Bin 0 -> 4908 bytes .../mipmap-mdpi/ic_launcher_foreground.png | Bin 0 -> 11065 bytes .../src/main/res/mipmap-xhdpi/ic_launcher.png | Bin 0 -> 16215 bytes .../mipmap-xhdpi/ic_launcher_foreground.png | Bin 0 -> 34493 bytes .../main/res/mipmap-xxhdpi/ic_launcher.png | Bin 0 -> 32856 bytes .../mipmap-xxhdpi/ic_launcher_foreground.png | Bin 0 -> 66599 bytes .../main/res/mipmap-xxxhdpi/ic_launcher.png | Bin 0 -> 54445 bytes .../mipmap-xxxhdpi/ic_launcher_foreground.png | Bin 0 -> 108177 bytes .../res/values/ic_launcher_background.xml | 4 + wear/app/src/main/res/values/strings.xml | 8 + wear/app/src/main/res/values/themes.xml | 9 + wear/app/src/main/res/values/wear.xml | 8 + wear/build.gradle.kts | 9 + wear/gradle.properties | 4 + wear/gradle/wrapper/gradle-wrapper.jar | Bin 0 -> 60756 bytes wear/gradle/wrapper/gradle-wrapper.properties | 7 + wear/gradlew | 240 +++++++++ wear/gradlew.bat | 91 ++++ wear/settings.gradle.kts | 16 + 85 files changed, 5813 insertions(+), 102 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java create mode 100644 App_Resources/Android/src/main/res/values/wear.xml create mode 100644 app/g2/wear-remote.ts create mode 100644 app/native/wear-bridge.ts create mode 100644 app/phone-ui/remote-controls.xml create mode 100644 wear/.gitignore create mode 100644 wear/PROTOCOL.md create mode 100644 wear/README.md create mode 100644 wear/app/build.gradle.kts create mode 100644 wear/app/proguard-rules.pro create mode 100644 wear/app/src/main/AndroidManifest.xml create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/FingerTapDetector.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/Haptics.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/MainActivity.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/OnBodyMonitor.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/PhoneLink.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/Protocol.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/WatchPrefs.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/WristTwistDetector.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/AppsScreen.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/AssistantScreen.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/Common.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/FaceclawWearApp.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/RemoteScreen.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/SettingsScreen.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/StatusScreen.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/TextEntry.kt create mode 100644 wear/app/src/main/kotlin/com/faceclaw/wear/ui/TouchpadGestures.kt create mode 100644 wear/app/src/main/res/drawable/ic_apps.xml create mode 100644 wear/app/src/main/res/drawable/ic_assistant.xml create mode 100644 wear/app/src/main/res/drawable/ic_close.xml create mode 100644 wear/app/src/main/res/drawable/ic_keyboard.xml create mode 100644 wear/app/src/main/res/drawable/ic_launcher_foreground.xml create mode 100644 wear/app/src/main/res/drawable/ic_mic.xml create mode 100644 wear/app/src/main/res/drawable/ic_more.xml create mode 100644 wear/app/src/main/res/drawable/ic_refresh.xml create mode 100644 wear/app/src/main/res/drawable/ic_settings.xml create mode 100644 wear/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml create mode 100644 wear/app/src/main/res/mipmap-hdpi/ic_launcher.png create mode 100644 wear/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.png create mode 100644 wear/app/src/main/res/mipmap-mdpi/ic_launcher.png create mode 100644 wear/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.png create mode 100644 wear/app/src/main/res/mipmap-xhdpi/ic_launcher.png create mode 100644 wear/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.png create mode 100644 wear/app/src/main/res/mipmap-xxhdpi/ic_launcher.png create mode 100644 wear/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.png create mode 100644 wear/app/src/main/res/mipmap-xxxhdpi/ic_launcher.png create mode 100644 wear/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.png create mode 100644 wear/app/src/main/res/values/ic_launcher_background.xml create mode 100644 wear/app/src/main/res/values/strings.xml create mode 100644 wear/app/src/main/res/values/themes.xml create mode 100644 wear/app/src/main/res/values/wear.xml create mode 100644 wear/build.gradle.kts create mode 100644 wear/gradle.properties create mode 100644 wear/gradle/wrapper/gradle-wrapper.jar create mode 100644 wear/gradle/wrapper/gradle-wrapper.properties create mode 100755 wear/gradlew create mode 100644 wear/gradlew.bat create mode 100644 wear/settings.gradle.kts diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 7569cf33..00b6d984 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -12,6 +12,9 @@ dependencies { // code" entry. Scanner UI and camera live in Play Services, so Faceclaw // needs no camera permission and no third-party scanner app. implementation 'com.google.android.gms:play-services-code-scanner:16.1.0' + // Wearable Data Layer (MessageClient / DataClient / CapabilityClient) for + // the Wear OS watch remote; see FaceclawWearBridge and wear/. + implementation 'com.google.android.gms:play-services-wearable:19.0.0' } def faceclawSherpaOnnxVersion = "1.13.0" diff --git a/App_Resources/Android/src/main/AndroidManifest.xml b/App_Resources/Android/src/main/AndroidManifest.xml index 48dda00b..5074e8ea 100644 --- a/App_Resources/Android/src/main/AndroidManifest.xml +++ b/App_Resources/Android/src/main/AndroidManifest.xml @@ -110,6 +110,22 @@ + + + + + + + + + + + pending = new ArrayDeque<>(); + private final List watchNodes = new ArrayList<>(); + private String lastPublishedState = null; + + private static final class PendingMessage { + final String path; + final String json; + final String nodeId; + final long receivedAt; + + PendingMessage(String path, String json, String nodeId, long receivedAt) { + this.path = path; + this.json = json; + this.nodeId = nodeId; + this.receivedAt = receivedAt; + } + } + + private FaceclawWearBridge(Context context) { + this.context = context.getApplicationContext(); + boolean playServices; + try { + playServices = GoogleApiAvailability.getInstance() + .isGooglePlayServicesAvailable(this.context) == ConnectionResult.SUCCESS; + } catch (Throwable error) { + Log.w(TAG, "Play services availability check failed", error); + playServices = false; + } + this.available = playServices; + if (!available) { + Log.i(TAG, "Google Play services unavailable; watch integration disabled"); + } + } + + /** Initialize (idempotent) and return the singleton. */ + public static FaceclawWearBridge getInstance(Context context) { + if (instance == null) { + synchronized (FaceclawWearBridge.class) { + if (instance == null) { + instance = new FaceclawWearBridge(context); + } + } + } + return instance; + } + + /** Whether the Wearable Data Layer can be used on this phone at all. */ + public boolean isAvailable() { + return available; + } + + /** + * Register the JS listener. Must be called from the thread whose isolate + * owns it; that thread's Looper is captured for dispatch. Replays any + * recently queued watch messages and refreshes the reachable-watch set. + */ + public void setListener(FaceclawWearListener newListener) { + Looper looper = Looper.myLooper(); + if (looper == null) { + Log.w(TAG, "wear listener registered from a Looper-less thread; it will never be notified"); + } + List replay; + synchronized (lock) { + listener = newListener; + listenerHandler = looper != null ? new Handler(looper) : null; + replay = new ArrayList<>(pending); + pending.clear(); + } + long now = System.currentTimeMillis(); + for (PendingMessage message : replay) { + if (now - message.receivedAt <= PENDING_MAX_AGE_MS) { + dispatchMessage(message.path, message.json, message.nodeId); + } + } + refreshWatchNodes(); + } + + public void clearListener() { + synchronized (lock) { + listener = null; + listenerHandler = null; + } + } + + public boolean isWatchReachable() { + synchronized (lock) { + return !watchNodes.isEmpty(); + } + } + + /** Display name of a reachable watch ("" if none). */ + public String getWatchName() { + synchronized (lock) { + return watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + } + + /** Re-query which watches advertise the Faceclaw watch-app capability. */ + public void refreshWatchNodes() { + if (!available) return; + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(this::updateWatchNodes) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + } + + /** + * Mirror the dashboard state to the watch. Identical consecutive states + * are not re-sent (the Data Layer would drop them anyway). + */ + public void publishState(String json) { + publishState(json, false); + } + + /** As {@link #publishState(String)}; `force` re-sends an unchanged state. */ + public void publishState(String json, boolean force) { + if (!available || json == null) return; + synchronized (lock) { + if (!force && json.equals(lastPublishedState)) return; + lastPublishedState = json; + } + try { + PutDataMapRequest request = PutDataMapRequest.create(PATH_STATE); + DataMap map = request.getDataMap(); + map.putString(STATE_KEY_JSON, json); + map.putLong(STATE_KEY_UPDATED_AT, System.currentTimeMillis()); + PutDataRequest put = request.asPutDataRequest().setUrgent(); + Wearable.getDataClient(context) + .putDataItem(put) + .addOnFailureListener(error -> Log.w(TAG, "state publish failed", error)); + } catch (Throwable error) { + Log.w(TAG, "state publish failed", error); + } + } + + /** Send a message to every reachable watch running the Faceclaw watch app. */ + public void sendToWatch(String path, String json) { + if (!available) return; + List targets; + synchronized (lock) { + targets = new ArrayList<>(watchNodes); + } + if (targets.isEmpty()) { + // Nothing known yet (first message after boot): look the watch up, then send. + try { + Wearable.getCapabilityClient(context) + .getCapability(CAPABILITY_WATCH, CapabilityClient.FILTER_REACHABLE) + .addOnSuccessListener(info -> { + updateWatchNodes(info); + for (Node node : info.getNodes()) { + sendToNode(node.getId(), path, json); + } + }) + .addOnFailureListener(error -> Log.w(TAG, "capability query failed", error)); + } catch (Throwable error) { + Log.w(TAG, "capability query failed", error); + } + return; + } + for (Node node : targets) { + sendToNode(node.getId(), path, json); + } + } + + /** Send a message to one specific node (e.g. an ack back to the sender). */ + public void sendToNode(String nodeId, String path, String json) { + if (!available || nodeId == null || nodeId.isEmpty()) return; + try { + byte[] payload = (json != null ? json : "{}").getBytes(StandardCharsets.UTF_8); + Wearable.getMessageClient(context) + .sendMessage(nodeId, path, payload) + .addOnFailureListener(error -> Log.w(TAG, "send " + path + " to " + nodeId + " failed", error)); + } catch (Throwable error) { + Log.w(TAG, "send " + path + " failed", error); + } + } + + /** Reply to a watch message; `seq` echoes the watch's sequence number. */ + public void sendAck(String nodeId, long seq, boolean ok, boolean jsReady, String message) { + JSONObject ack = new JSONObject(); + try { + ack.put("seq", seq); + ack.put("ok", ok); + ack.put("jsReady", jsReady); + ack.put("message", message != null ? message : ""); + } catch (JSONException ignored) { + // The keys are constants; this cannot happen. + } + sendToNode(nodeId, PATH_ACK, ack.toString()); + } + + /** Entry point for FaceclawWearListenerService; may run on any thread. */ + void handleMessage(MessageEvent event) { + if (event == null) return; + String path = event.getPath(); + if (path == null || !path.startsWith(PATH_PREFIX)) return; + byte[] data = event.getData(); + String json = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "{}"; + String nodeId = event.getSourceNodeId(); + Log.d(TAG, "recv " + path + " seq=" + readSeq(json)); + + boolean delivered = dispatchMessage(path, json, nodeId); + if (delivered) return; + + // No JS listener yet: keep the message briefly, and tell the watch so + // it can explain why nothing is happening. + synchronized (lock) { + long now = System.currentTimeMillis(); + while (!pending.isEmpty() && now - pending.peekFirst().receivedAt > PENDING_MAX_AGE_MS) { + pending.pollFirst(); + } + if (pending.size() >= PENDING_LIMIT) { + pending.pollFirst(); + } + pending.addLast(new PendingMessage(path, json, nodeId, now)); + } + sendAck(nodeId, readSeq(json), false, false, "Faceclaw is not running on the phone. Open it to connect the glasses."); + } + + /** Entry point for FaceclawWearListenerService's capability callback. */ + void handleCapabilityChanged(CapabilityInfo info) { + if (info == null || !CAPABILITY_WATCH.equals(info.getName())) return; + updateWatchNodes(info); + } + + private void updateWatchNodes(CapabilityInfo info) { + Set nodes = info != null ? info.getNodes() : Collections.emptySet(); + // Every reachable watch gets messages; a nearby (Bluetooth-linked) + // one is listed first so it is the name the phone UI shows. + List reachable = new ArrayList<>(); + for (Node node : nodes) { + if (node.isNearby()) reachable.add(0, node); else reachable.add(node); + } + boolean changed; + String name; + synchronized (lock) { + changed = !sameNodeIds(watchNodes, reachable); + watchNodes.clear(); + watchNodes.addAll(reachable); + name = watchNodes.isEmpty() ? "" : watchNodes.get(0).getDisplayName(); + } + if (changed) { + Log.i(TAG, reachable.isEmpty() ? "no Faceclaw watch reachable" : "watch reachable: " + name); + boolean reachableNow = !reachable.isEmpty(); + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target != null && handler != null) { + handler.post(() -> { + try { + target.onWatchConnection(reachableNow, name); + } catch (Exception error) { + Log.w(TAG, "watch connection listener failed", error); + } + }); + } + } + } + + private static boolean sameNodeIds(List a, List b) { + if (a.size() != b.size()) return false; + for (int i = 0; i < a.size(); i++) { + if (!a.get(i).getId().equals(b.get(i).getId())) return false; + } + return true; + } + + /** Post to the JS listener; false when there is none to post to. */ + private boolean dispatchMessage(String path, String json, String nodeId) { + FaceclawWearListener target; + Handler handler; + synchronized (lock) { + target = listener; + handler = listenerHandler; + } + if (target == null || handler == null) return false; + handler.post(() -> { + try { + target.onMessage(path, json, nodeId); + } catch (Exception error) { + Log.w(TAG, "wear message listener failed for " + path, error); + } + }); + return true; + } + + private static long readSeq(String json) { + try { + return new JSONObject(json).optLong("seq", 0); + } catch (JSONException error) { + return 0; + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java new file mode 100644 index 00000000..590c87d6 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListener.java @@ -0,0 +1,19 @@ +package com.faceclaw.app; + +/** + * Callbacks from the Android Wear (Wear OS) bridge; implemented in JS. Every + * call is delivered on the thread that registered the listener (see + * FaceclawWearBridge.setListener). + */ +public interface FaceclawWearListener { + /** + * A message from the watch app. `path` is the Data Layer message path + * (e.g. "/faceclaw/input"); `json` is the UTF-8 payload (always a JSON + * object, "{}" when the watch sent none); `nodeId` identifies the watch + * so a reply can be addressed to it. + */ + void onMessage(String path, String json, String nodeId); + + /** The set of reachable watches running the Faceclaw watch app changed. */ + void onWatchConnection(boolean reachable, String watchName); +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java new file mode 100644 index 00000000..7520d780 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearListenerService.java @@ -0,0 +1,26 @@ +package com.faceclaw.app; + +import android.util.Log; + +import com.google.android.gms.wearable.CapabilityInfo; +import com.google.android.gms.wearable.MessageEvent; +import com.google.android.gms.wearable.WearableListenerService; + +/** + * Play services starts this service (in the app process) for every Data + * Layer message under /faceclaw and for capability changes, whether or not + * the app is running. It only forwards to FaceclawWearBridge, which owns the + * hand-off to the JS side. + */ +public class FaceclawWearListenerService extends WearableListenerService { + @Override + public void onMessageReceived(MessageEvent event) { + Log.d("FaceclawWear", "service onMessageReceived " + event.getPath() + " from " + event.getSourceNodeId()); + FaceclawWearBridge.getInstance(this).handleMessage(event); + } + + @Override + public void onCapabilityChanged(CapabilityInfo info) { + FaceclawWearBridge.getInstance(this).handleCapabilityChanged(info); + } +} diff --git a/App_Resources/Android/src/main/res/values/wear.xml b/App_Resources/Android/src/main/res/values/wear.xml new file mode 100644 index 00000000..049457ee --- /dev/null +++ b/App_Resources/Android/src/main/res/values/wear.xml @@ -0,0 +1,9 @@ + + + + + faceclaw_phone + + diff --git a/CLAUDE.md b/CLAUDE.md index 2fe8b769..6433edb7 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -5,6 +5,13 @@ for interfacing with Android SDK). Typescript parts are in app/. Java parts are in App_Resources/Android/src/main/java/com/faceclaw/app/. +wear/ is a separate Gradle project (Kotlin + Compose for Wear OS): the watch +remote that drives Faceclaw over the Wearable Data Layer. Its phone-side +counterparts are FaceclawWearBridge.java, app/native/wear-bridge.ts and +app/g2/wear-remote.ts; the message format is in wear/PROTOCOL.md. Build it +with `cd wear && ./gradlew :app:assembleDebug` (needs JAVA_HOME and +ANDROID_HOME); it is not part of `npm run build`. + If build.sh exists, it's a machine-specific wrapper that sets some environment variables to point to where tools are installed, then runs `npm run build`. If build_and_run.sh exists, it does everything build.sh does plus diff --git a/PRIVACY b/PRIVACY index 18e29a55..8b9251c7 100644 --- a/PRIVACY +++ b/PRIVACY @@ -74,3 +74,18 @@ https://support.evenrealities.com/hc/en-us/articles/14127464826511-Privacy-Polic Apps downloaded from the EvenHub app store come with their own privacy policies. + +## Wear OS Watch + +The optional Faceclaw watch app (wear/) talks to the phone over Google Play +services' Wearable Data Layer, the same channel the watch uses for +notifications and other companion apps. Gestures, commands, the watch's own +speech-recognition results, and the mirrored glasses status and assistant +replies travel between your watch and your phone over their Bluetooth (or, +when paired that way, Wi-Fi) link; Faceclaw does not send any of it to a +server. Speech recognized on the watch uses the watch's system speech +recognizer, which may send audio to Google under Google's privacy policy: + + Google: https://policies.google.com/privacy + +The phone does nothing watch-related on phones without Google Play services. diff --git a/README.md b/README.md index 96270b26..d102e885 100644 --- a/README.md +++ b/README.md @@ -65,11 +65,41 @@ To install, run battery life similar to the stock Even app. * Connection management with auto-reconnect, and autodetection of conflict with the official Even Realities app - * On-phone screen mirroring and simulated ring input + * On-phone screen mirroring with touch control: tap what you see on the + mirror (sidebar icons, launcher cells), or use the phone's own touchpad, + d-pad and Back/Menu buttons — the same spatial scheme as the watch — plus + a compact ring simulator. A display-mode picker (576×288 band, 576×480 + tall, or the full 640×480 panel with an auto-hiding sidebar) and a + brightness slider with an Auto toggle sit beside the mirror. + * A Wear OS watch app that replaces (and outdoes) the R1 ring: tap, swipe, + hold and crown gestures, side buttons, app launching and window switching, + voice or keyboard queries to the assistant with the reply on your wrist, + typing into apps, and glasses status/lock/display control (see "Watch + remote" below) * Dual-language NativeScript architecture with Java for the multithreaded Android API and bluetooth stack bits, Typescript for the bits you want to hack on +## Watch remote (Wear OS) + +The `wear/` directory holds a companion app for Wear OS 3+ watches that turns +the watch into a remote for the glasses. The whole watch face is a touchpad +that mirrors the ring (tap, double-tap, hold, swipe, crown scrolling, side +buttons), and beyond the ring it can launch apps, switch and close windows, +wake/sleep/lock the display, send spoken or typed queries straight to the +assistant (using the watch's own microphone) and show the reply on the wrist, +and type text into apps such as the terminal. Its swipes are a spatial +control scheme the ring never had, with no mode to enter first: the +launcher highlights one app and moves it by rows and columns, right goes +into things (sidebar → window, settings section → items, next track in +Music), left backs out, tap selects, hold opens the context menu, double-tap +or a two-finger tap is back everywhere, and two wrist twists go back; up/down +scroll and right/left select/back everywhere else. The ring's own scheme is +unchanged — watch input is tagged as a separate source. Faceclaw's Settings > Watch +section has the phone-side switches. See [wear/README.md](wear/README.md) for +building and installing it; it is a separate Gradle build, sideloaded over +adb, and needs no Play Store. The phone side requires Google Play services. + ## Connecting an external agent (OpenClaw) Instead of calling an LLM API directly from the phone, the voice assistant diff --git a/app/app.css b/app/app.css index 62ad97c2..a3665e0d 100644 --- a/app/app.css +++ b/app/app.css @@ -55,3 +55,78 @@ Button.-primary { border-width: 1; border-radius: 8; } + +/* The phone's controller for the glasses (phone-ui/remote-controls.xml). */ +.remote-controls { + margin: 0 8 0 8; +} +.display-controls { + margin-bottom: 6; +} +.mode-button { + font-size: 13; + padding: 6 10; + margin-right: 6; +} +.brightness-icon { + font-size: 16; + margin: 0 4 0 4; + color: #444444; +} +.auto-toggle Label { + font-size: 13; + margin-right: 2; +} +.touchpad { + height: 230; + background-color: #1f1f1f; + border-radius: 14; + border-width: 1; + border-color: #3a3a3a; + color: #f0f0f0; + margin-top: 2; +} +.pad-focus { + font-size: 20; + color: #ffffff; +} +.pad-dpad-button { + width: 46; + height: 46; + margin: 6; + padding: 0; + font-size: 16; + border-radius: 23; + background-color: #333333; + color: #e8e8e8; +} +.pad-dpad-down { + margin-bottom: 30; +} +.pad-legend { + font-size: 11; + color: #8a8a8a; + margin-bottom: 8; +} +.action-row { + margin-top: 6; +} +.action-button { + margin: 2; + font-size: 14; +} +.ring-row { + margin-top: 4; +} +.ring-label { + font-size: 12; + color: #666666; + margin: 0 6 0 4; +} +.ring-button { + width: 44; + height: 36; + margin: 1; + padding: 0; + font-size: 13; +} diff --git a/app/apps/evenhub/evenhub-window.ts b/app/apps/evenhub/evenhub-window.ts index 3b549980..bd45271b 100644 --- a/app/apps/evenhub/evenhub-window.ts +++ b/app/apps/evenhub/evenhub-window.ts @@ -4,6 +4,7 @@ * makes the content area exactly the 576x288 surface EvenHub apps expect. */ import { GrayImage } from "../../graphics/image"; +import { EVENHUB_SCREEN_WIDTH } from "./compositor"; import { type DashboardInputEvent, type Layer, type LayerContext } from "../../ui/layers"; import { createInProcessWindow, @@ -20,6 +21,15 @@ class EvenHubAppLayer implements Layer { paint(ctx: LayerContext): GrayImage { const size = ctx.stack.getBaseSize(); + // EvenHub apps lay out for the stock 576-wide surface. In the full-panel + // display mode the window is 640 wide; paint at stock width and centre + // it rather than leaving all the slack on the right. + if (size.width > EVENHUB_SCREEN_WIDTH) { + const content = this.session.paint({ width: EVENHUB_SCREEN_WIDTH, height: size.height }, ctx.stack.isFocused()); + const image = new GrayImage(size.width, size.height, 0); + content.composeInto(image, Math.floor((size.width - EVENHUB_SCREEN_WIDTH) / 2), 0); + return image; + } return this.session.paint(size, ctx.stack.isFocused()); } diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 88f1ac6a..a7c80c4d 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -20,7 +20,7 @@ import { ApplicationSettings } from "@nativescript/core"; import { GrayImage } from "../../graphics/image"; import { EvenHubFont } from "../../graphics/evenhub-font"; -import { type DashboardInputEvent, type Layer } from "../../ui/layers"; +import { type DashboardInputEvent, type InputSource, type Layer } from "../../ui/layers"; import { anthropicApiKeySetting, elevenLabsApiKeySetting, @@ -1346,12 +1346,17 @@ function luminance(r: number, g: number, b: number): number { return Math.round(0.2126 * r + 0.7152 * g + 0.0722 * b); } -/** ring/left-arm/right-arm -> PB EventSourceType. */ -function gestureSource(source: "ring" | "left-arm" | "right-arm"): number { +/** + * ring/left-arm/right-arm -> PB EventSourceType. The watch is reported to + * EvenHub apps as the ring: they only know stock hardware, and the watch's + * ring-vocabulary gestures mean the same thing. + */ +function gestureSource(source: InputSource): number { switch (source) { case "right-arm": return 1; case "ring": + case "watch": return 2; case "left-arm": return 3; diff --git a/app/apps/launcher/launcher-app.ts b/app/apps/launcher/launcher-app.ts index 4b9520af..6fe15796 100644 --- a/app/apps/launcher/launcher-app.ts +++ b/app/apps/launcher/launcher-app.ts @@ -5,7 +5,7 @@ import { GrayImage } from "../../graphics/image"; import { type Plane } from "../../graphics/plane"; import { renderIcon, type IconName } from "../../graphics/icons"; import { clamp } from "../../util/numeric-util"; -import { DashboardInputEvent, Layer, LayerActions, LayerContext } from "../../ui/layers"; +import { DashboardInputEvent, isWatchInput, Layer, LayerActions, LayerContext } from "../../ui/layers"; import { drawSelectionHighlight, MenuLayer, scrollToKeepSelectionVisible, type MenuItem } from "../../ui/menu"; import { iconGridMinRowHeight } from "../../ui/metrics"; import { WINDOW_MENU_LAYOUT } from "../../ui/window-menu"; @@ -64,12 +64,19 @@ type LauncherGridEntry = * picks the item); a tap launches an app or opens a folder (the same grid, * restricted to that folder's apps). Double-click backs out one level (item → * row, folder → top grid), and from the top level yields to the sidebar. + * That is the ring's scheme; the watch (see handleWatchInput) skips row mode + * and moves one cell at a time in four directions. * * The folder grouping lives in the settings store (see launcher-folders.ts) * and is re-read every paint, so assistant folder tools take effect without * the layer holding any copy of the state. */ class LauncherGridLayer implements Layer { + // Watch swipes are spatial: up/down move between rows, left/right between + // columns. From row mode, right goes into the row (left backs out, like + // double-click); from the first column, left returns to row mode, which + // keeps "left" pointing toward the sidebar all the way out. + readonly acceptsDirectional = true; private mode: LauncherMode = "row"; private selectedRow = 0; private selectedCol = 1; @@ -307,6 +314,10 @@ class LauncherGridLayer implements Layer { } async handleInput(event: DashboardInputEvent, ctx: LayerContext): Promise { + if (isWatchInput(event)) { + await this.handleWatchInput(event, ctx); + return; + } const entries = this.entries(); const rows = this.rowCount(entries.length); switch (event.type) { @@ -341,20 +352,7 @@ class LauncherGridLayer implements Layer { this.itemsInRow(entries.length, this.selectedRow) - 1, ); } else { - const entry = entries[this.selectedRow * COLS + this.selectedCol]; - if (!entry) return; - if (entry.kind === "folder") { - this.currentFolder = entry.name; - this.mode = "row"; - this.selectedRow = 0; - this.scrollRow = 0; - } else { - // Return to the top grid before launching, so the launcher never - // shows (even briefly) stale folder contents when re-entered. - if (this.currentFolder !== null) this.exitFolder(); - this.mode = "row"; - await this.options.launchApp(entry.appId); - } + await this.openSelected(entries); } return; } @@ -371,6 +369,108 @@ class LauncherGridLayer implements Layer { return; } } + + /** + * The watch's scheme: there is no row mode to enter first. The selection is + * always one cell; up/down/left/right (and the crown) move it spatially, + * select opens it, back leaves the folder or the grid, and stepping left + * past the first column leaves for the sidebar, which is where "left" goes. + * Row mode is restored on the way out so a ring user finds the grid as + * they left it. + */ + private async handleWatchInput(event: DashboardInputEvent, ctx: LayerContext): Promise { + const entries = this.entries(); + const rows = this.rowCount(entries.length); + if (this.mode === "row") { + this.mode = "item"; + this.selectedCol = clamp(this.selectedCol, 0, this.itemsInRow(entries.length, this.selectedRow) - 1); + } + const leave = () => { + this.mode = "row"; + shell.yieldFocusToSidebar(); + }; + switch (event.type) { + case "swipe-up": + case "swipe-down": + case "scroll-up": + case "scroll-down": { + const delta = event.type === "swipe-down" || event.type === "scroll-down" ? 1 : -1; + this.selectedRow = clamp(this.selectedRow + delta, 0, rows - 1); + // Keep the column; a shorter last row clamps it. + this.selectedCol = clamp(this.selectedCol, 0, this.itemsInRow(entries.length, this.selectedRow) - 1); + return; + } + case "swipe-right": + this.selectedCol = clamp(this.selectedCol + 1, 0, this.itemsInRow(entries.length, this.selectedRow) - 1); + return; + case "swipe-left": + if (this.selectedCol > 0) { + this.selectedCol--; + } else if (this.currentFolder !== null) { + this.exitFolder(); + this.mode = "item"; + } else { + leave(); + } + return; + case "click": + await this.openSelected(entries); + return; + case "double-click": + if (this.currentFolder !== null) { + this.exitFolder(); + this.mode = "item"; + } else { + leave(); + } + return; + default: + return; + } + } + + /** + * A touch on the phone's mirror: the cell under it becomes the selection + * and opens (the same as a watch select on it). Uses the geometry paint + * lays the grid out with, so it lands on what the mirror showed. + */ + async hitTest(x: number, y: number, ctx: LayerContext): Promise { + const font = getDefaultSmallFont(); + const { width, height } = ctx.stack.getBaseSize(); + const entries = this.entries(); + const gridTop = this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : GRID_TOP; + const gridBottom = height - 4; + if (y < gridTop || y >= gridBottom || x < 0 || x >= width) return false; + const rowH = iconGridMinRowHeight(font, ICON_SIZE, LABEL_GAP); + const colW = width / COLS; + const row = this.scrollRow + Math.floor((y - gridTop) / rowH); + const col = Math.floor(x / colW); + if (row < 0 || row >= this.rowCount(entries.length)) return false; + if (col < 0 || col >= this.itemsInRow(entries.length, row)) return false; + this.mode = "item"; + this.selectedRow = row; + this.selectedCol = col; + await this.openSelected(entries); + return true; + } + + /** Open the selected cell: enter a folder, or launch the app. */ + private async openSelected(entries: readonly LauncherGridEntry[]): Promise { + const entry = entries[this.selectedRow * COLS + this.selectedCol]; + if (!entry) return; + if (entry.kind === "folder") { + this.currentFolder = entry.name; + this.mode = "row"; + this.selectedRow = 0; + this.scrollRow = 0; + } else { + // Return to the top grid before launching, so the launcher never + // shows (even briefly) stale folder contents when re-entered. + if (this.currentFolder !== null) this.exitFolder(); + this.mode = "row"; + await this.options.launchApp(entry.appId); + } + } } /** diff --git a/app/apps/music/music-app.ts b/app/apps/music/music-app.ts index 1f48186b..c67a9452 100644 --- a/app/apps/music/music-app.ts +++ b/app/apps/music/music-app.ts @@ -17,7 +17,14 @@ import { } from "../../native/media-controller"; import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser"; import { MediaBrowseLayer } from "./media-browse"; -import { Layer, type DashboardInputEvent, type LayerContext, type PaintBelow } from "../../ui/layers"; +import { + directionalFallback, + isDirectionalInput, + Layer, + type DashboardInputEvent, + type LayerContext, + type PaintBelow, +} from "../../ui/layers"; import { createInProcessWindow, YieldAtRootLayer, @@ -59,6 +66,8 @@ type FocusColumn = "actions" | "playlist"; * player's queue when it exposes one (scroll to a track, click to jump). */ class MusicAppLayer implements Layer { + // Watch swipes skip tracks: right = next, left = previous. + readonly acceptsDirectional = true; private focusColumn: FocusColumn = "actions"; private selectedActionIndex = 0; private selectedQueueIndex = 0; @@ -215,6 +224,10 @@ class MusicAppLayer implements Layer { return; } const media = mediaControllerBridge.snapshot(); + if ((!media.accessEnabled || !media.available) && isDirectionalInput(event)) { + // Nothing to skip between yet: the setup prompts take select / back. + return this.handleInput(directionalFallback(event), ctx); + } if (!media.accessEnabled) { if (event.type === "click") { mediaControllerBridge.openNotificationAccessSettings(); @@ -233,6 +246,16 @@ class MusicAppLayer implements Layer { const actions = this.buildActions(media, queue); this.reconcileSelection(actions, queue); switch (event.type) { + case "swipe-right": + if (media.canSkipNext) await mediaControllerBridge.skipNext(); + return; + case "swipe-left": + if (media.canSkipPrevious) await mediaControllerBridge.skipPrevious(); + return; + case "swipe-up": + return this.handleInput({ type: "scroll-up" }, ctx); + case "swipe-down": + return this.handleInput({ type: "scroll-down" }, ctx); case "scroll-up": if (this.focusColumn === "playlist") { this.selectedQueueIndex = Math.max(0, this.selectedQueueIndex - 1); @@ -437,6 +460,7 @@ class VolumeModalLayer implements Layer { /** Let back leave the playlist column before the root wrapper yields to the shell. */ class MusicRootLayer implements Layer { + readonly acceptsDirectional = true; private readonly yieldAtRoot: YieldAtRootLayer; constructor(private readonly music: MusicAppLayer) { diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 26735a88..f7e917e5 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -1,5 +1,5 @@ import { Application, ImageSource } from "@nativescript/core"; -import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName } from "./events"; +import { EvenAIStatus, EvenAIStatusName, EventSourceType, EventSourceTypeName, OsEventTypeList, OsEventTypeName, WatchGestureType, WatchGestureTypeName } from "./events"; import { loadDeviceAddresses } from "./device-addresses"; import { ensureBlePermissions, ensureVoicePermissions } from "./android-permissions"; import { FaceclawCommunicatorBridge, type RawInputEvent } from "../native/faceclaw-communicator"; @@ -12,6 +12,29 @@ import { openEvenAppSettings, readEvenAppNotificationState } from "../native/eve import { grayImageToPreviewSource } from "../native/gray-image-preview"; import { firmwareIncompatibilityMessage } from "./firmware-compat"; import { resumeAutoReconnect, suppressAutoReconnect } from "./reconnect-policy"; +import { WearRemote, type WearRemoteInputKind } from "./wear-remote"; + +/** Who a synthetic (non-firmware) input stands for. */ +type SyntheticInputOrigin = "ring" | "watch"; + +/** Gestures the phone's mirror view reports (see handleMirrorTouch). */ +export type MirrorTouchKind = + | "tap" + | "double-tap" + | "long-press" + | "swipe-up" + | "swipe-down" + | "swipe-left" + | "swipe-right"; + +const MIRROR_TOUCH_GESTURES: Record, WearRemoteInputKind> = { + "double-tap": "double-click", + "long-press": "long-press", + "swipe-up": "swipe-up", + "swipe-down": "swipe-down", + "swipe-left": "swipe-left", + "swipe-right": "swipe-right", +}; import { findSoundEffect, playSoundEffect } from "../ui/sound-effects"; import { isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; @@ -33,9 +56,9 @@ import { ALL_APPS } from "../apps/all-apps"; import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; -import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry"; +import { appViewportRect, SIDEBAR_WIDTH, sidebarWidth, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; -import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, displayModeSetting, watchWakesDisplaySetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -84,13 +107,20 @@ const LOCK_SCREEN_MESSAGE = "Glasses locked; unlock the phone to unlock the glas // Top-bar clock refresh; the phone-side preview polls the Java composite so // it reflects every app (including worker apps the TS side never renders). const SHELL_REFRESH_INTERVAL_MS = 60_000; +// Safety-net poll only; the preview is event-driven (every composited frame +// schedules a refresh), so this just covers anything that slips through. const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; const EVENHUB_SCREEN_OFF_SUSPEND_DELAY_MS = 5_000; const EVENHUB_WAKE_READY_TIMEOUT_MS = 4_500; const FOREGROUND_NOTIFICATION_MIN_UPDATE_MS = 30_000; const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 6_000; -const CONNECTED_PREVIEW_MIN_UPDATE_MS = 1_000; +// Preview refresh floor. Refreshes are scheduled per composited frame with a +// trailing update, so the mirror tracks the glasses within this bound instead +// of the old 1s poll (which lagged up to two polls behind). +const CONNECTED_PREVIEW_MIN_UPDATE_MS = 150; +// The GIF recorder keeps its old cadence; per-frame captures would balloon it. +const RECORDING_MIN_CAPTURE_MS = 1_000; // Below this, a disconnect is more likely a flat battery than a BLE problem. const LOW_BATTERY_PERCENT = 5; const EVEN_APP_DETECTED_MESSAGE = @@ -138,6 +168,9 @@ function eventLabel(kind: string, eventType: number): string { if (kind === "even-ai") { return EvenAIStatusName[eventType] ?? `EVEN_AI_UNKNOWN_${eventType}`; } + if (kind === "watch-gesture") { + return WatchGestureTypeName[eventType] ?? `WATCH_UNKNOWN_${eventType}`; + } return eventName(eventType); } @@ -182,6 +215,8 @@ class DashboardController { private glassesWorn: boolean | null = null; private phoneLocked = false; private glassesLocked = false; + // Wear OS watch remote; constructed last so it sees a fully wired shell. + private wearRemote: WearRemote | null = null; private lockSurfaceConfigured = false; private lastLockScreenEnabled = lockScreenEnabledSetting.get(); private offState: (() => void) | null = null; @@ -265,10 +300,14 @@ class DashboardController { }, getScreenTimeoutMs: () => screenTimeoutSettingToMs(screenTimeoutSetting.get()), requestShellRender: () => this.requestShellRender(), - onWindowsChanged: () => this.persistOpenApps(), + onWindowsChanged: () => { + this.persistOpenApps(); + this.wearRemote?.schedulePublish(); + }, onScreenStateChanged: (on) => { this.handleScreenStateChanged(on); if (on) this.requestShellRender(); + this.wearRemote?.schedulePublish(); }, }); // Boot hooks register windows that exist from startup (the launcher, @@ -290,6 +329,7 @@ class DashboardController { this.pushBrightness(); this.syncEvenHubScreenOffSetting(); this.applyVerticalPositionIfChanged(); + this.applyDisplayModeIfChanged(); this.syncAssistantBridgeIfChanged(); this.syncLockScreenSettingIfChanged(); }); @@ -297,6 +337,24 @@ class DashboardController { // connection stays up (with re-dial) so proactive tool calls work // outside voice turns. this.syncAssistantBridge(); + // The watch drives the same synthetic-input path as the phone UI's test + // buttons, plus app/window/lock commands; it mirrors the state below. + this.wearRemote = new WearRemote({ + apps: LAUNCHABLE_APPS, + injectInput: (kind) => this.injectSyntheticRingInput(kind, "watch"), + launchApp: (appId) => this.launchApp(appId), + connect: () => this.connect(), + disconnect: () => this.disconnect(), + setGlassesLocked: (locked, reason) => this.setGlassesLocked(locked, reason), + requestShellRender: () => this.requestShellRender(), + getState: () => ({ + phase: this.phase, + status: this.status, + glassesLocked: this.glassesLocked, + glassesWorn: this.glassesWorn, + }), + appendLog: (line) => this.appendLog(line), + }); } // Bridge settings changes re-dial the connection; unrelated setting changes @@ -339,6 +397,48 @@ class DashboardController { // don't trigger a full reposition. private lastVerticalPosition = verticalPositionSetting.get(); + private lastDisplayMode = displayModeSetting.get(); + + /** + * Display mode changed (Settings > Display, or the phone page's picker): + * every window's viewport size changes. In-process windows re-measure in + * place; worker windows get their canvas once at open, so they are closed + * and launched again at the new size. + */ + private applyDisplayModeIfChanged(): void { + const mode = displayModeSetting.get(); + if (mode === this.lastDisplayMode) return; + this.lastDisplayMode = mode; + this.appendLog(`display mode: ${mode}`); + const foregroundWindowId = shell.foregroundWindow()?.windowId; + void (async () => { + const relaunch: string[] = []; + for (const window of Array.from(shell.getWindows())) { + if (window.relayout) { + window.relayout(); + await this.configureWindowSurface( + window.surfaceId, + window.windowId === foregroundWindowId, + window.heightMode, + ); + } else if (window.closeable) { + relaunch.push(window.appId); + shell.closeWindow(window.windowId); + } + } + for (const appId of relaunch) { + await this.launchApp(appId); + } + if (foregroundWindowId && shell.getWindows().some((w) => w.windowId === foregroundWindowId)) { + shell.focusWindow(foregroundWindowId); + } + shell.foregroundWindow()?.requestRender(); + this.requestShellRender(); + })().catch((error) => { + this.appendLog(`display mode change failed: ${this.formatError(error)}`); + }); + } + private applyVerticalPositionIfChanged(): void { const position = verticalPositionSetting.get(); if (position === this.lastVerticalPosition) return; @@ -400,6 +500,7 @@ class DashboardController { private handleWearState(wearing: boolean): void { this.glassesWorn = wearing; this.appendLog(wearing ? "glasses wear state: ON_HEAD" : "glasses wear state: OFF_HEAD"); + this.wearRemote?.schedulePublish(); if (!wearing && this.phoneLocked && lockScreenEnabledSetting.get()) { this.setGlassesLocked(true, "glasses removed while phone locked"); } @@ -420,6 +521,7 @@ class DashboardController { if (locked === this.glassesLocked) return; this.glassesLocked = locked; this.appendLog(`glasses ${locked ? "locked" : "unlocked"}: ${reason}`); + this.wearRemote?.schedulePublish(); void this.syncLockSurface().catch((error) => { this.appendLog(`lock screen update failed: ${this.formatError(error)}`); }); @@ -947,6 +1049,7 @@ class DashboardController { // Repaint the top bar (battery indicators live in the shell chrome). this.requestShellRender(); } + this.wearRemote?.schedulePublish(); }); this.offEvenAppConflict = communicator.onEvenAppConflict((message) => { this.refreshEvenAppStatus(); @@ -959,6 +1062,10 @@ class DashboardController { this.emit(); }); this.offFrameMetrics = communicator.onFrameMetrics(() => { + // Every composited frame refreshes the phone-side mirror (bounded by + // CONNECTED_PREVIEW_MIN_UPDATE_MS) so it tracks the glasses instead + // of trailing the 1s safety-net poll. + this.schedulePreviewUpdate(); if (this.phase === "connected") { this.setStatus("Connected."); // A rendered frame means the session is warmed up (fixedLayoutCreated), @@ -1255,20 +1362,94 @@ class DashboardController { } } - async injectSyntheticRingInput( - kind: "click" | "double-click" | "scroll-up" | "scroll-down" | "long-press" | "wakeword", - ): Promise { + /** + * Feed a ring gesture from somewhere other than the ring (the phone UI's + * test buttons, the watch). "long-press" is a complete short hold; + * "long-press-start" / "long-press-release" let a source with a real + * finger-down/finger-up (the watch) hold for as long as the user does. + */ + async injectSyntheticRingInput(kind: WearRemoteInputKind, origin: SyntheticInputOrigin = "ring"): Promise { + // Watch-scheme input (the Wear app, the phone's pad/d-pad, mirror touches) + // wakes a dark display and then lands normally, resuming the previous + // focus. The ring's own scheme keeps its double-tap-only wake. + if ( + origin === "watch" && + !shell.isScreenOn() && + !this.glassesLocked && + (this.phase === "connected" || this.phase === "charging") && + watchWakesDisplaySetting.get() && + kind !== "double-click" && + kind !== "wakeword" && + kind !== "long-press-release" + ) { + shell.wake(shell.getFocus()); + this.requestShellRender(); + this.appendLog(`${kind} (watch scheme) woke the display`); + } if (kind === "long-press") { // Hardware delivers a press event and then a release when the finger // lifts; emulate a short hold so the escape-menu countdown never fires. - await this.handleInputEvent(this.buildSyntheticRingInput("long-press")); - await this.handleInputEvent(this.buildSyntheticRingInput("long-press-release")); + await this.handleInputEvent(this.buildSyntheticRingInput("long-press", origin)); + await this.handleInputEvent(this.buildSyntheticRingInput("long-press-release", origin)); return; } - const event = this.buildSyntheticRingInput(kind); + const event = this.buildSyntheticRingInput(kind === "long-press-start" ? "long-press" : kind, origin); await this.handleInputEvent(event); } + /** + * A touch on the phone's mirror of the glasses display. Gestures other than + * a tap are the watch scheme (swipes navigate, double-tap is back, a hold + * is the menu). A tap lands on what the mirror shows: a sidebar icon + * switches to that window, a launcher cell opens, anything else selects. + * Coordinates are fractions of the mirror (0..1). + */ + async handleMirrorTouch(kind: MirrorTouchKind, nx: number, ny: number): Promise { + if (this.phase !== "connected" && this.phase !== "charging") return; + if (this.glassesLocked) { + // Same as the ring: only a double-tap reaches a locked display. + if (kind === "double-tap") await this.injectSyntheticRingInput("double-click", "watch"); + return; + } + if (!shell.isScreenOn()) { + shell.wake(shell.getFocus()); + this.requestShellRender(); + } + if (kind !== "tap") { + await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch"); + return; + } + const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH); + const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT); + const stripShown = sidebarWidth() > 0 || shell.getFocus() === "sidebar"; + this.appendLog(`mirror tap at ${x},${y}`); + if (!shell.hasOverlay() && stripShown && x < SIDEBAR_WIDTH) { + const target = shell.windowAtSidebarPoint(x, y); + if (target) { + this.appendLog(`mirror tap: sidebar -> ${target.title}`); + shell.focusWindow(target.windowId); + shell.foregroundWindow()?.requestRender(); + this.requestShellRender(); + } + return; + } + const window = shell.foregroundWindow(); + if (window && !shell.hasOverlay()) { + const rect = appViewportRect(window.heightMode); + const inside = x >= rect.x && x < rect.x + rect.width && y >= rect.y && y < rect.y + rect.height; + if (shell.getFocus() !== "window") { + shell.focusWindow(window.windowId); + this.requestShellRender(); + } + if (inside && window.hitTest && (await window.hitTest(x - rect.x, y - rect.y))) { + this.appendLog(`mirror tap: ${window.title} at ${x - rect.x},${y - rect.y}`); + this.requestShellRender(); + return; + } + } + await this.injectSyntheticRingInput("click", "watch"); + } + /** A document arrived via Android's Share intent: open it as a new window. */ async openSharedTextDocument(text: string): Promise { this.appendLog(`shared text document received (${text.length} chars)`); @@ -1410,7 +1591,8 @@ class DashboardController { if (this.glassesLocked) { const ringDoubleTap = inputEvent.type === "double-click" && - event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING; + (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING || + event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH); const suspendedDisplayWake = inputEvent.type === "display-wake"; if (ringDoubleTap || suspendedDisplayWake) { if (shell.isScreenOn()) { @@ -1463,6 +1645,9 @@ class DashboardController { outcome = { ...outcome, shell: true }; } + if (event.kind === "watch-gesture") { + this.appendLog(`watch-gesture ${eventLabel(event.kind, event.eventType)}`); + } if (event.kind === "sys-event") { this.lastSys = `${sourceName(event.eventSource)}/${eventName(event.eventType)}`; this.appendLog(`sys-event ${this.lastSys}`); @@ -1473,7 +1658,10 @@ class DashboardController { ) { this.appendLog("display state invalidated by firmware exit event"); } - if (event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING) { + if ( + event.eventSource === EventSourceType.TOUCH_EVENT_FROM_RING || + event.eventSource === EventSourceType.TOUCH_EVENT_FROM_WATCH + ) { this.lastInput = eventName(event.eventType); } } @@ -1813,7 +2001,7 @@ class DashboardController { `still unsent after ${FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS}ms; shell render loop released`, ); } - this.updateCompositePreview(); + this.schedulePreviewUpdate(); } private async handleWakeWord(keyword: string): Promise { @@ -1878,6 +2066,10 @@ class DashboardController { clearInterval(this.previewTimer); this.previewTimer = null; } + if (this.previewTrailingTimer) { + clearTimeout(this.previewTrailingTimer); + this.previewTrailingTimer = null; + } if (this.screenTimeoutTimer) { clearInterval(this.screenTimeoutTimer); this.screenTimeoutTimer = null; @@ -1928,6 +2120,27 @@ class DashboardController { * foreground, including worker apps the TS side never renders). Throttled * to avoid rebuilding the bitmap faster than the phone UI needs it. */ + private previewTrailingTimer: ReturnType | null = null; + private lastRecordCaptureAtMs = 0; + + /** + * Refresh the preview now if the floor allows, otherwise once the floor + * expires — so the last frame of a burst always reaches the mirror instead + * of waiting for the safety-net poll. + */ + private schedulePreviewUpdate(): void { + if (this.previewTrailingTimer) return; + const wait = this.lastConnectedPreviewUpdateAtMs + CONNECTED_PREVIEW_MIN_UPDATE_MS - Date.now(); + if (wait <= 0) { + this.updateCompositePreview(); + return; + } + this.previewTrailingTimer = setTimeout(() => { + this.previewTrailingTimer = null; + this.updateCompositePreview(); + }, wait); + } + private updateCompositePreview(): void { if (!this.communicator) return; // The connected foreground service intentionally keeps this controller @@ -1947,9 +2160,8 @@ class DashboardController { return; } this.lastConnectedPreviewUpdateAtMs = now; - // The recorder captures at the same cadence the phone-side preview is - // refreshed, so the GIF matches what the phone display showed. - if (this.screenRecordingActive) { + if (this.screenRecordingActive && now - this.lastRecordCaptureAtMs >= RECORDING_MIN_CAPTURE_MS) { + this.lastRecordCaptureAtMs = now; this.communicator.recordScreenFrame(); } const preview = this.communicator.getCompositePreview(); @@ -1973,10 +2185,37 @@ class DashboardController { | "scroll-down" | "long-press" | "long-press-release" - | "wakeword", + | "wakeword" + | "swipe-left" + | "swipe-right" + | "swipe-up" + | "swipe-down", + origin: SyntheticInputOrigin = "ring", ): RawInputEvent { const frameId = frameTimings.startFrame(`input:synthetic:${kind}`); + // The phone UI's test buttons stand in for the ring; the watch is its own + // source so the UI can give it a richer scheme (see InputSource). + const eventSource = + origin === "watch" ? EventSourceType.TOUCH_EVENT_FROM_WATCH : EventSourceType.TOUCH_EVENT_FROM_RING; switch (kind) { + case "swipe-left": + case "swipe-right": + case "swipe-up": + case "swipe-down": + // Watch-only directional input; see WatchGestureType. + return { + kind: "watch-gesture", + containerName: "", + eventType: { + "swipe-left": WatchGestureType.SWIPE_LEFT, + "swipe-right": WatchGestureType.SWIPE_RIGHT, + "swipe-up": WatchGestureType.SWIPE_UP, + "swipe-down": WatchGestureType.SWIPE_DOWN, + }[kind], + eventSource: 0, + systemExitReasonCode: 0, + frameId, + }; case "wakeword": // Same event the glasses report for the spoken wakeword (sid 0x07). return { @@ -1992,7 +2231,7 @@ class DashboardController { kind: "sys-event", containerName: "", eventType: OsEventTypeList.RING_LONG_PRESS_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2001,7 +2240,7 @@ class DashboardController { kind: "sys-event", containerName: "", eventType: OsEventTypeList.RING_LONG_PRESS_RELEASE_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2010,7 +2249,7 @@ class DashboardController { kind: "sys-event", containerName: "", eventType: OsEventTypeList.CLICK_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2019,7 +2258,7 @@ class DashboardController { kind: "sys-event", containerName: "", eventType: OsEventTypeList.DOUBLE_CLICK_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2028,7 +2267,7 @@ class DashboardController { kind: "text-click", containerName: "", eventType: OsEventTypeList.SCROLL_TOP_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2038,7 +2277,7 @@ class DashboardController { kind: "text-click", containerName: "", eventType: OsEventTypeList.SCROLL_BOTTOM_EVENT, - eventSource: EventSourceType.TOUCH_EVENT_FROM_RING, + eventSource, systemExitReasonCode: 0, frameId, }; @@ -2050,6 +2289,7 @@ class DashboardController { for (const listener of this.listeners) { listener(snapshot); } + this.wearRemote?.schedulePublish(); } } diff --git a/app/g2/events.ts b/app/g2/events.ts index 65581b01..139c261e 100644 --- a/app/g2/events.ts +++ b/app/g2/events.ts @@ -3,6 +3,12 @@ export const EventSourceType = { TOUCH_EVENT_FROM_GLASSES_R: 1, TOUCH_EVENT_FROM_RING: 2, TOUCH_EVENT_FROM_GLASSES_L: 3, + /** + * Synthetic: the Wear OS remote (app/g2/wear-remote.ts). Not a firmware + * value. Lets the UI give watch input its own, richer scheme (direct + * spatial selection) while ring input keeps the stock one. + */ + TOUCH_EVENT_FROM_WATCH: 4, } as const; export const EventSourceTypeName: Record = { @@ -10,6 +16,7 @@ export const EventSourceTypeName: Record = { 1: "TOUCH_EVENT_FROM_GLASSES_R", 2: "TOUCH_EVENT_FROM_RING", 3: "TOUCH_EVENT_FROM_GLASSES_L", + 4: "TOUCH_EVENT_FROM_WATCH", }; /** @@ -32,6 +39,24 @@ export const EvenAIStatusName: Record = { 3: "EVEN_AI_EXIT", }; +/** + * Synthetic gestures from the Wear OS remote (RawInputEvent kind + * "watch-gesture"). Not a firmware enum: the glasses never send these. + */ +export const WatchGestureType = { + SWIPE_LEFT: 0, + SWIPE_RIGHT: 1, + SWIPE_UP: 2, + SWIPE_DOWN: 3, +} as const; + +export const WatchGestureTypeName: Record = { + 0: "SWIPE_LEFT", + 1: "SWIPE_RIGHT", + 2: "SWIPE_UP", + 3: "SWIPE_DOWN", +}; + export const OsEventTypeList = { CLICK_EVENT: 0, SCROLL_TOP_EVENT: 1, diff --git a/app/g2/wear-remote.ts b/app/g2/wear-remote.ts new file mode 100644 index 00000000..cae1d982 --- /dev/null +++ b/app/g2/wear-remote.ts @@ -0,0 +1,490 @@ +/** + * The Wear OS watch as an input device and status mirror for the glasses. + * + * Inbound, the watch sends ring-style gestures (tap, double-tap, hold, + * swipe/crown scrolls, "Hey Even") which feed the exact synthetic-ring path + * the phone UI's test buttons use, plus higher-level commands the ring has no + * way to express: launch an app, switch or close a window, wake/sleep/lock + * the display, send a spoken or typed query straight to the assistant, or + * type text into the foreground app. + * + * Outbound, the dashboard state (connection, battery, screen, lock, open + * windows, launchable apps) is mirrored into a Data Layer item so the watch + * UI can show it and offer the right actions; assistant replies and alerts + * stream to the watch as events so a query made from the wrist can be read + * from the wrist. + * + * Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the + * message shapes are documented in wear/PROTOCOL.md. + */ +import { type AppDefinition } from "../apps/app-definition"; +import { getInstalledEvenHubApps, installedEvenHubAppId } from "../apps/evenhub/installed-apps"; +import { WEAR_PATHS, wearBridge, type WearMessage } from "../native/wear-bridge"; +import { + DISPLAY_MODE_VALUES, + displayModeSetting, + onAnySettingChanged, + watchCanUnlockSetting, + watchMirrorAssistantSetting, + watchRemoteEnabledSetting, +} from "../ui/dashboard-settings"; +import { shell, type AssistantActivityEvent } from "../ui/shell/shell"; +import { voiceActivity } from "../ui/shell/voice-activity"; +import { FACECLAW_VERSION } from "../version"; + +/** Protocol revision carried in the state item; bump on incompatible changes. */ +export const WEAR_PROTOCOL_VERSION = 1; + +export type WearRemoteInputKind = + | "click" + | "double-click" + | "scroll-up" + | "scroll-down" + | "long-press" + | "long-press-start" + | "long-press-release" + | "wakeword" + | "swipe-left" + | "swipe-right" + | "swipe-up" + | "swipe-down"; + +const INPUT_KINDS: readonly WearRemoteInputKind[] = [ + "click", + "double-click", + "scroll-up", + "scroll-down", + "long-press", + "long-press-start", + "long-press-release", + "wakeword", + "swipe-left", + "swipe-right", + "swipe-up", + "swipe-down", +]; + +const COMMANDS = [ + "launch-app", + "focus-window", + "close-window", + "sidebar", + "wake", + "sleep", + "lock", + "unlock", + "connect", + "disconnect", + "close-assistant", + "display-mode", +] as const; +type WearCommand = (typeof COMMANDS)[number]; + +const MAX_SCROLL_STEPS = 12; +const MAX_TEXT_LENGTH = 2000; +const STATE_PUBLISH_DEBOUNCE_MS = 150; +const STATE_REFRESH_INTERVAL_MS = 30_000; +const ASSISTANT_STREAM_MIN_INTERVAL_MS = 250; + +/** What the dashboard controller lends the watch remote. */ +export type WearRemoteHost = { + /** The launcher-grid apps; installed EvenHub packages are added dynamically. */ + apps: readonly AppDefinition[]; + injectInput: (kind: WearRemoteInputKind) => Promise; + launchApp: (appId: string) => Promise; + connect: () => Promise; + disconnect: () => Promise; + setGlassesLocked: (locked: boolean, reason: string) => void; + requestShellRender: () => void; + getState: () => { + phase: string; + status: string; + glassesLocked: boolean; + glassesWorn: boolean | null; + }; + appendLog: (line: string) => void; +}; + +export class WearRemote { + private readonly active: boolean; + private publishTimer: ReturnType | null = null; + private watchReachable = false; + private watchName = ""; + // Assistant text streams as deltas many times a second; the watch gets a + // snapshot every ASSISTANT_STREAM_MIN_INTERVAL_MS plus every phase change. + private assistantStreamTimer: ReturnType | null = null; + private assistantPendingText: string | null = null; + private lastAssistantSentAtMs = 0; + + constructor(private readonly host: WearRemoteHost) { + this.active = wearBridge.isAvailable(); + if (!this.active) { + host.appendLog("watch: Google Play services unavailable; watch remote disabled"); + return; + } + wearBridge.onMessage((message) => { + void this.handleMessage(message).catch((error) => { + host.appendLog(`watch: ${message.path} failed: ${formatError(error)}`); + }); + }); + wearBridge.onWatchConnection((connection) => { + const changed = connection.reachable !== this.watchReachable || connection.watchName !== this.watchName; + this.watchReachable = connection.reachable; + this.watchName = connection.watchName; + if (changed) { + host.appendLog( + connection.reachable ? `watch: ${connection.watchName || "watch"} reachable` : "watch: no watch reachable", + ); + } + // A watch that just came into range gets the current state at once. + if (connection.reachable) this.publishNow(true); + }); + shell.onAssistantActivity((event) => this.forwardAssistantActivity(event)); + shell.onAlertShown((text) => { + if (watchMirrorAssistantSetting.get()) { + wearBridge.sendEvent({ type: "alert", text }); + } + }); + voiceActivity.subscribe(() => this.schedulePublish()); + onAnySettingChanged(() => this.schedulePublish()); + // Catch-all refresh for state the controller does not announce (cheap: + // identical states are dropped on the Java side). + setInterval(() => this.schedulePublish(), STATE_REFRESH_INTERVAL_MS); + this.schedulePublish(); + } + + /** Whether a watch running the Faceclaw watch app is reachable right now. */ + isWatchReachable(): boolean { + return this.watchReachable; + } + + getWatchName(): string { + return this.watchName; + } + + /** Mirror the dashboard state to the watch soon; coalesces bursts. */ + schedulePublish(): void { + if (!this.active || this.publishTimer) return; + this.publishTimer = setTimeout(() => { + this.publishTimer = null; + this.publishNow(false); + }, STATE_PUBLISH_DEBOUNCE_MS); + } + + private publishNow(force: boolean): void { + if (!this.active) return; + try { + wearBridge.publishState(this.buildState(), force); + } catch (error) { + this.host.appendLog(`watch: state publish failed: ${formatError(error)}`); + } + } + + private buildState(): Record { + const state = this.host.getState(); + const battery = shell.getBatteryLevels(); + const foregroundId = shell.foregroundWindow()?.windowId ?? null; + const foreground = shell.getForegroundApp(); + return { + protocol: WEAR_PROTOCOL_VERSION, + version: FACECLAW_VERSION, + phase: state.phase, + status: state.status, + connected: state.phase === "connected" || state.phase === "charging", + screenOn: shell.isScreenOn(), + locked: state.glassesLocked, + worn: state.glassesWorn, + listening: voiceActivity.isActive(), + battery: battery.headset, + charging: battery.headsetCharging === true, + foreground: foreground ? { appId: foreground.appId, title: foreground.title } : null, + windows: shell.getWindows().map((window) => ({ + windowId: window.windowId, + title: window.title, + appId: window.appId, + focused: window.windowId === foregroundId, + closeable: window.closeable, + acceptsText: typeof window.receiveTextInput === "function", + })), + apps: this.listApps(), + displayMode: displayModeSetting.get(), + remoteEnabled: watchRemoteEnabledSetting.get(), + canUnlock: watchCanUnlockSetting.get(), + mirrorAssistant: watchMirrorAssistantSetting.get(), + assistantAvailable: shell.isAssistantAvailable(), + }; + } + + /** Launchable apps, matching the launcher grid and the apps.launch tool. */ + private listApps(): Array<{ appId: string; title: string }> { + const apps = this.host.apps.map((app) => ({ appId: app.appId, title: app.title })); + for (const installed of getInstalledEvenHubApps()) { + apps.push({ appId: installedEvenHubAppId(installed.packageId), title: installed.name }); + } + return apps; + } + + private async handleMessage(message: WearMessage): Promise { + const seq = readNumber(message.payload.seq, 0); + const ack = (ok: boolean, text = "") => wearBridge.sendAck(message.nodeId, seq, ok, text); + + if (message.path === WEAR_PATHS.stateRequest) { + this.publishNow(true); + ack(true); + return; + } + if (!watchRemoteEnabledSetting.get()) { + ack(false, "Watch control is turned off in Faceclaw's Settings > Watch."); + return; + } + + switch (message.path) { + case WEAR_PATHS.input: + await this.handleInput(message.payload, ack); + return; + case WEAR_PATHS.command: + await this.handleCommand(message.payload, ack); + return; + case WEAR_PATHS.assistant: { + const text = readText(message.payload.text); + if (!text) { + ack(false, "Nothing to send."); + return; + } + if (!shell.isAssistantAvailable()) { + ack(false, "Set up the assistant in Faceclaw's Settings first."); + return; + } + this.host.appendLog(`watch: assistant query (${text.length} chars)`); + shell.sendToAssistant(text); + ack(true); + return; + } + case WEAR_PATHS.text: { + const text = readText(message.payload.text); + if (!text) { + ack(false, "Nothing to type."); + return; + } + const window = shell.foregroundWindow(); + if (!window?.receiveTextInput) { + ack(false, `${window ? window.title : "The current app"} does not accept typed text.`); + return; + } + if (!shell.isScreenOn()) shell.wake("window"); + this.host.appendLog(`watch: typed ${text.length} chars into ${window.title}`); + shell.sendTextToForegroundWindow(text); + ack(true); + return; + } + default: + ack(false, `Unknown message: ${message.path}`); + } + } + + private async handleInput(payload: Record, ack: (ok: boolean, text?: string) => void): Promise { + const gesture = String(payload.gesture ?? ""); + if (!(INPUT_KINDS as readonly string[]).includes(gesture)) { + ack(false, `Unknown gesture: ${gesture}`); + return; + } + const kind = gesture as WearRemoteInputKind; + const state = this.host.getState(); + if (state.phase !== "connected" && state.phase !== "charging") { + ack(false, "The glasses are not connected."); + return; + } + // The lock only yields to a double-click, exactly as the ring's does + // (handled inside the controller); a dark display wakes on any watch- + // scheme gesture (injectSyntheticRingInput, shared with the phone's own + // pad and d-pad). + const repeatable = kind === "scroll-up" || kind === "scroll-down" || kind === "swipe-up" || kind === "swipe-down"; + const steps = repeatable ? Math.min(MAX_SCROLL_STEPS, Math.max(1, Math.round(readNumber(payload.steps, 1)))) : 1; + for (let i = 0; i < steps; i++) { + await this.host.injectInput(kind); + } + ack(true); + } + + private async handleCommand(payload: Record, ack: (ok: boolean, text?: string) => void): Promise { + const command = String(payload.command ?? ""); + if (!(COMMANDS as readonly string[]).includes(command)) { + ack(false, `Unknown command: ${command}`); + return; + } + const state = this.host.getState(); + const connected = state.phase === "connected" || state.phase === "charging"; + this.host.appendLog(`watch: ${command}`); + switch (command as WearCommand) { + case "connect": + if (connected) { + ack(true, "Already connected."); + return; + } + ack(true, "Connecting..."); + await this.host.connect(); + return; + case "disconnect": + if (!connected && state.phase === "disconnected") { + ack(true, "Already disconnected."); + return; + } + ack(true, "Disconnecting..."); + await this.host.disconnect(); + return; + default: + break; + } + if (!connected) { + ack(false, "The glasses are not connected."); + return; + } + if (state.glassesLocked && command !== "unlock" && command !== "lock") { + ack(false, "The glasses are locked."); + return; + } + switch (command as WearCommand) { + case "launch-app": { + const appId = String(payload.appId ?? "").trim(); + const app = this.listApps().find((entry) => entry.appId === appId); + if (!app) { + ack(false, `Unknown app: ${appId}`); + return; + } + shell.wake("window"); + await this.host.launchApp(appId); + this.schedulePublish(); + ack(true, `Opened ${app.title}.`); + return; + } + case "focus-window": { + const windowId = String(payload.windowId ?? "").trim(); + const window = shell.getWindows().find((entry) => entry.windowId === windowId); + if (!window) { + ack(false, "That window is no longer open."); + return; + } + shell.wake("window"); + shell.focusWindow(windowId); + this.host.requestShellRender(); + this.schedulePublish(); + ack(true); + return; + } + case "close-window": { + const windowId = String(payload.windowId ?? "").trim(); + const window = shell.getWindows().find((entry) => entry.windowId === windowId); + if (!window) { + ack(false, "That window is no longer open."); + return; + } + if (!window.closeable) { + ack(false, `${window.title} cannot be closed.`); + return; + } + shell.closeWindow(windowId); + this.host.requestShellRender(); + this.schedulePublish(); + ack(true, `Closed ${window.title}.`); + return; + } + case "sidebar": + shell.wake("sidebar"); + shell.yieldFocusToSidebar(); + this.host.requestShellRender(); + ack(true); + return; + case "wake": + if (!shell.wake("sidebar")) { + ack(true, "The display is already on."); + return; + } + this.host.requestShellRender(); + ack(true); + return; + case "sleep": + if (!shell.isScreenOn()) { + ack(true, "The display is already off."); + return; + } + shell.sleep(); + ack(true); + return; + case "lock": + this.host.setGlassesLocked(true, "locked from the watch"); + this.schedulePublish(); + ack(true); + return; + case "unlock": + if (!watchCanUnlockSetting.get()) { + ack(false, "Unlocking from the watch is turned off in Settings > Watch."); + return; + } + this.host.setGlassesLocked(false, "unlocked from the watch"); + this.schedulePublish(); + ack(true); + return; + case "close-assistant": + shell.closeAssistant(); + ack(true); + return; + case "display-mode": { + const value = String(payload.value ?? ""); + if (!(DISPLAY_MODE_VALUES as readonly string[]).includes(value)) { + ack(false, `Unknown display mode: ${value}`); + return; + } + displayModeSetting.set(value as (typeof DISPLAY_MODE_VALUES)[number]); + this.schedulePublish(); + ack(true, `Display mode: ${value}`); + return; + } + default: + ack(false, `Unhandled command: ${command}`); + } + } + + private forwardAssistantActivity(event: AssistantActivityEvent): void { + if (!watchMirrorAssistantSetting.get()) return; + if (event.phase === "streaming") { + // Coalesce: the newest text wins, sent no more often than the interval. + this.assistantPendingText = event.text; + if (this.assistantStreamTimer) return; + const wait = Math.max(0, ASSISTANT_STREAM_MIN_INTERVAL_MS - (Date.now() - this.lastAssistantSentAtMs)); + this.assistantStreamTimer = setTimeout(() => { + this.assistantStreamTimer = null; + if (this.assistantPendingText === null) return; + this.sendAssistantEvent("streaming", this.assistantPendingText); + this.assistantPendingText = null; + }, wait); + return; + } + // A phase change supersedes any queued stream snapshot. + if (this.assistantStreamTimer) { + clearTimeout(this.assistantStreamTimer); + this.assistantStreamTimer = null; + } + this.assistantPendingText = null; + this.sendAssistantEvent(event.phase, event.text); + this.schedulePublish(); + } + + private sendAssistantEvent(phase: AssistantActivityEvent["phase"], text: string): void { + this.lastAssistantSentAtMs = Date.now(); + wearBridge.sendEvent({ type: "assistant", phase, text }); + } +} + +function readNumber(value: unknown, fallback: number): number { + const numeric = typeof value === "number" ? value : Number(value); + return Number.isFinite(numeric) ? numeric : fallback; +} + +function readText(value: unknown): string { + const text = typeof value === "string" ? value : ""; + return text.replace(/[\x00-\x08\x0b\x0c\x0e-\x1f]/g, "").trim().slice(0, MAX_TEXT_LENGTH); +} + +function formatError(error: unknown): string { + return (error as Error)?.message ?? String(error); +} diff --git a/app/graphics/image.ts b/app/graphics/image.ts index 4a42271d..4a5d472e 100644 --- a/app/graphics/image.ts +++ b/app/graphics/image.ts @@ -533,6 +533,20 @@ export class GrayImage { return baked; } + /** + * Place this image into target at (dx, dy): raster via bitBlt, deferred + * draws carried over as deferred draws (translated), so firmware-text runs + * and cached glyphs stay on their cheap wire paths instead of being baked + * to raster. The composed-in draws render above target's earlier raster, + * matching the usual draw-above-own-raster rule. + */ + composeInto(target: GrayImage, dx: number, dy: number): void { + target.bitBlt(this, dx, dy); + for (const placed of this.drawList) { + target.drawList.push({ ...placed, x: placed.x + dx, y: placed.y + dy }); + } + } + /** Rasterize this image's deferred draws into target, translated by (dx, dy). */ bakeDrawsInto(target: GrayImage, dx: number, dy: number): void { for (const placed of this.drawList) { diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 4cfef948..c1338bd2 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -89,6 +89,18 @@ export type RawInputEvent = systemExitReasonCode: number; frameId: number; } + | { + /** + * Synthetic directional gesture from the Wear OS remote; `eventType` is + * a WatchGestureType (app/g2/events.ts). Never produced by Java. + */ + kind: "watch-gesture"; + containerName: string; + eventType: number; + eventSource: number; + systemExitReasonCode: number; + frameId: number; + } | { /** * Stock display-lifecycle wake observed after a ring or arm double tap diff --git a/app/native/wear-bridge.ts b/app/native/wear-bridge.ts new file mode 100644 index 00000000..7981809f --- /dev/null +++ b/app/native/wear-bridge.ts @@ -0,0 +1,178 @@ +/** + * TS wrapper for the Java FaceclawWearBridge: the phone side of the Wear OS + * watch remote. Inbound watch messages arrive as (path, JSON) pairs on the + * main isolate's thread; outbound calls publish dashboard state and send + * messages to the watch. The semantic mapping (what a watch "click" does) + * lives in app/g2/wear-remote.ts; this file only crosses the Java boundary. + * + * Every call is a no-op when Google Play services is missing (the Wearable + * Data Layer is part of it), so the rest of the app never has to care. + */ +import { Utils } from "@nativescript/core"; + +declare const com: any; + +/** Message paths shared with the watch app (wear/app/.../Protocol.kt). */ +export const WEAR_PATHS = { + /** Watch -> phone: a ring-style gesture. */ + input: "/faceclaw/input", + /** Watch -> phone: a shell command (launch app, wake, lock, ...). */ + command: "/faceclaw/command", + /** Watch -> phone: a text query for the assistant. */ + assistant: "/faceclaw/assistant", + /** Watch -> phone: text to type into the foreground app. */ + text: "/faceclaw/text", + /** Watch -> phone: please re-publish the state item. */ + stateRequest: "/faceclaw/state/request", + /** Phone -> watch: reply to any of the above. */ + ack: "/faceclaw/ack", + /** Phone -> watch: assistant activity, alerts, voice-dialog transcripts. */ + event: "/faceclaw/event", + /** Phone -> watch Data Layer item: the mirrored dashboard state. */ + state: "/faceclaw/state", +} as const; + +export type WearMessage = { + path: string; + /** Parsed JSON payload ({} when the watch sent none or it was malformed). */ + payload: Record; + nodeId: string; +}; + +export type WearWatchConnection = { + reachable: boolean; + watchName: string; +}; + +type MessageListener = (message: WearMessage) => void; +type ConnectionListener = (connection: WearWatchConnection) => void; + +class WearBridge { + private java: any | null | undefined = undefined; + // The Java-side listener proxy must stay referenced or it gets GC'd. + private retainedListenerProxy: any = null; + private readonly messageListeners = new Set(); + private readonly connectionListeners = new Set(); + private lastConnection: WearWatchConnection = { reachable: false, watchName: "" }; + + private getJava(): any | null { + if (this.java !== undefined) return this.java; + if (!global.isAndroid) { + this.java = null; + return null; + } + try { + const context = Utils.android.getApplicationContext(); + const bridge = context ? com.faceclaw.app.FaceclawWearBridge.getInstance(context) : null; + this.java = bridge && bridge.isAvailable() ? bridge : null; + } catch (error) { + console.warn("wear bridge unavailable", error); + this.java = null; + } + return this.java; + } + + /** Whether the Wearable Data Layer exists on this phone (Play services). */ + isAvailable(): boolean { + return this.getJava() !== null; + } + + private ensureListener(): void { + if (this.retainedListenerProxy) return; + const java = this.getJava(); + if (!java) return; + this.retainedListenerProxy = new com.faceclaw.app.FaceclawWearListener({ + onMessage: (path: string, json: string, nodeId: string) => { + const message: WearMessage = { + path: String(path), + payload: parsePayload(String(json)), + nodeId: String(nodeId), + }; + for (const listener of Array.from(this.messageListeners)) { + try { + listener(message); + } catch (error) { + console.warn("wear message listener failed", error); + } + } + }, + onWatchConnection: (reachable: boolean, watchName: string) => { + this.lastConnection = { reachable: Boolean(reachable), watchName: String(watchName ?? "") }; + for (const listener of Array.from(this.connectionListeners)) { + try { + listener(this.lastConnection); + } catch (error) { + console.warn("wear connection listener failed", error); + } + } + }, + }); + java.setListener(this.retainedListenerProxy); + } + + /** Subscribe to watch messages. Registers the Java listener on first use. */ + onMessage(listener: MessageListener): () => void { + this.messageListeners.add(listener); + this.ensureListener(); + return () => { + this.messageListeners.delete(listener); + }; + } + + /** Subscribe to watch reachability; the current value is delivered immediately. */ + onWatchConnection(listener: ConnectionListener): () => void { + this.connectionListeners.add(listener); + this.ensureListener(); + listener(this.getWatchConnection()); + return () => { + this.connectionListeners.delete(listener); + }; + } + + getWatchConnection(): WearWatchConnection { + const java = this.getJava(); + if (!java) return { reachable: false, watchName: "" }; + try { + return { + reachable: Boolean(java.isWatchReachable()), + watchName: String(java.getWatchName() ?? ""), + }; + } catch { + return this.lastConnection; + } + } + + /** Re-query reachable watches (e.g. when the phone UI wants a fresh answer). */ + refreshWatchNodes(): void { + this.getJava()?.refreshWatchNodes(); + } + + /** + * Mirror the dashboard state to the watch. Unchanged states are dropped on + * the Java side unless `force` (a watch explicitly asked for a refresh). + */ + publishState(state: Record, force = false): void { + this.getJava()?.publishState(JSON.stringify(state), force); + } + + /** Send an event to every reachable watch. */ + sendEvent(payload: Record): void { + this.getJava()?.sendToWatch(WEAR_PATHS.event, JSON.stringify(payload)); + } + + /** Acknowledge a watch message; `seq` echoes the watch's sequence number. */ + sendAck(nodeId: string, seq: number, ok: boolean, message = ""): void { + this.getJava()?.sendAck(nodeId, seq, ok, true, message); + } +} + +function parsePayload(json: string): Record { + try { + const parsed = JSON.parse(json); + return parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed : {}; + } catch { + return {}; + } +} + +export const wearBridge = new WearBridge(); diff --git a/app/phone-ui/main-page.xml b/app/phone-ui/main-page.xml index 98f51b4a..56d697fc 100644 --- a/app/phone-ui/main-page.xml +++ b/app/phone-ui/main-page.xml @@ -1,4 +1,4 @@ - + @@ -11,7 +11,8 @@ - + - -