diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index f910b748..da1b7ae2 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -26,7 +26,12 @@ export type DirectTurnOptions = { system: string; /** Owned by the session; this loop appends assistant + tool_result messages. */ messages: AnthropicMessage[]; - tools: AnthropicToolDefinition[]; + /** + * Build the tool list for the next request. Called once per loop iteration so + * the set reflects apps opening/closing and foreground changes mid-turn; it + * also refreshes the session's api-name map that resolveToolName reads. + */ + buildTools: () => AnthropicToolDefinition[]; registry: ToolRegistry; /** Map an API tool name (as the model calls it) back to its registry name. */ resolveToolName: (apiName: string) => string; @@ -62,7 +67,7 @@ export class DirectAnthropicBackend { system: options.system, effort: options.effort, messages: options.messages, - tools: options.tools, + tools: options.buildTools(), onTextDelta: (delta, textSoFar) => { if (cancelled) return; options.callbacks.onTextDelta(delta, textSoFar); diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 02d935b4..09e0b023 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -60,7 +60,6 @@ export class AssistantSession { this.messages.push({ role: "user", content: text }); this.trimHistory(); - const tools = this.buildToolDefinitions(); const system = `${SYSTEM_PROMPT_BASE}\n\n${describeContext(ctx)}`; const finish = () => { @@ -74,7 +73,7 @@ export class AssistantSession { effort: "low", system, messages: this.messages, - tools, + buildTools: () => this.buildToolDefinitions(), registry: this.registry, resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName, callbacks: { diff --git a/app/assistant/tool-registry.ts b/app/assistant/tool-registry.ts index e39fa923..b1d09b93 100644 --- a/app/assistant/tool-registry.ts +++ b/app/assistant/tool-registry.ts @@ -70,14 +70,28 @@ export type CallToolOptions = { const DEFAULT_TOOL_TIMEOUT_MS = 10_000; +/** How the shell reaches a window's tool handler and gates its foreground tools. */ +export type AppToolProvider = { + windowId: string; + appId: string; + /** The window's declared specs, with UNPREFIXED names (registry adds the prefix). */ + specs: ToolSpec[]; + /** Invoke a tool on the window by its unprefixed name. */ + invoke: (toolName: string, args: unknown) => Promise | ToolResult; + /** Whether the window currently owns the foreground (gates `foreground` tools). */ + isForeground: () => boolean; +}; + export class ToolRegistry { private readonly registrations = new Map(); private readonly changeListeners = new Set<() => void>(); + // windowId -> the canonical (prefixed) tool names it contributed, for bulk + // removal when the window closes or re-declares. + private readonly windowToolNames = new Map(); /** Register (or replace) a tool. Names are unique across all tiers. */ register(registration: ToolRegistration): void { - const { name } = registration.spec; - this.registrations.set(name, registration); + this.addRegistration(registration); this.fireToolsChanged(); } @@ -95,6 +109,52 @@ export class ToolRegistry { } } + /** + * Declare (replacing any prior set) the tools contributed by one app window. + * Names are prefixed `app..` so windows can't shadow system tools or + * each other. `open` tools stay live while the window exists; `foreground` + * tools are gated on `isForeground()` at both list and call time. + */ + setAppTools(provider: AppToolProvider): void { + this.clearWindowTools(provider.windowId, false); + const names: string[] = []; + for (const spec of provider.specs) { + if (spec.availability !== "open" && spec.availability !== "foreground") { + console.warn(`app tool ${spec.name} has non-app availability ${spec.availability}; skipping`); + continue; + } + const canonical = `app.${provider.appId}.${spec.name}`; + const toolName = spec.name; + this.addRegistration({ + spec: { ...spec, name: canonical }, + handler: (args) => provider.invoke(toolName, args), + isAvailable: spec.availability === "foreground" ? () => provider.isForeground() : undefined, + }); + names.push(canonical); + } + this.windowToolNames.set(provider.windowId, names); + this.fireToolsChanged(); + } + + /** Remove all tools contributed by a window (its worker closed or the window did). */ + removeAppTools(windowId: string): void { + this.clearWindowTools(windowId, true); + } + + private clearWindowTools(windowId: string, fire: boolean): void { + const names = this.windowToolNames.get(windowId); + if (!names) return; + for (const name of names) { + this.registrations.delete(name); + } + this.windowToolNames.delete(windowId); + if (fire) this.fireToolsChanged(); + } + + private addRegistration(registration: ToolRegistration): void { + this.registrations.set(registration.spec.name, registration); + } + /** Specs for the tools callable right now, given the availability tiers. */ listTools(options: ListToolsOptions = {}): ToolSpec[] { const specs: ToolSpec[] = []; diff --git a/app/ui/shell/worker-window.ts b/app/ui/shell/worker-window.ts index e3a618c4..d7fd26a2 100644 --- a/app/ui/shell/worker-window.ts +++ b/app/ui/shell/worker-window.ts @@ -1,6 +1,7 @@ import { GrayImage } from "../../graphics/image"; import { windowIcon } from "./chrome-layer"; import { type IconName } from "../../graphics/icons"; +import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/tool-registry"; import { shell, type ShellWindow } from "./shell"; /** @@ -16,7 +17,9 @@ export type WorkerAppMessage = | { type: "text-input"; windowId: string; text: string } | { type: "render"; windowId: string; focused: boolean } | { type: "foreground"; windowId: string; foreground: boolean; focused: boolean } - | { type: "screen"; on: boolean }; + | { type: "screen"; on: boolean } + /** Assistant tool invocation aimed at a window; reply with tool-result. */ + | { type: "tool-call"; callId: string; windowId: string; name: string; args: unknown }; export type WorkerAppReply = | { type: "yield-focus"; windowId: string } @@ -59,6 +62,21 @@ export type WorkerAppReply = */ type: "set-tray-icon"; icon: { width: number; height: number; pixels: number[] } | null; + } + | { + /** + * Declare (replacing the prior set) the assistant tools this window + * contributes. Names are unprefixed; the registry adds `app..`. + */ + type: "set-tools"; + windowId: string; + tools: ToolSpec[]; + } + | { + /** Result of a tool-call, matched to the request by callId. */ + type: "tool-result"; + callId: string; + result: ToolResult; }; export type WorkerWindowSpec = { @@ -92,8 +110,24 @@ export type WorkerAppHostOptions = { * and manages compositor surfaces for the app's windows. The worker submits * frames straight to the Java compositor, so no pixels cross this boundary. */ +/** A tool-call awaiting its worker reply; also its own leak-safety timeout. */ +type PendingToolCall = { + windowId: string; + resolve: (result: ToolResult) => void; + timer: ReturnType; +}; + +/** + * How long the host waits for a worker's tool-result before giving up. The + * registry applies its own (usually shorter) per-tool timeout; this is the + * backstop that also frees the pending-call entry if the worker never replies. + */ +const TOOL_CALL_HOST_TIMEOUT_MS = 15_000; + export class WorkerAppHost { private readonly openWindows = new Set(); + private readonly pendingToolCalls = new Map(); + private nextCallSerial = 1; constructor(private readonly options: WorkerAppHostOptions) { options.worker.onmessage = (event: MessageEvent) => { @@ -152,6 +186,27 @@ export class WorkerAppHost { case "set-title": // Titles are informational for now (sidebar shows icons only). break; + case "set-tools": + // Only a window we actually have open may contribute tools. + if (this.openWindows.has(message.windowId)) { + toolRegistry.setAppTools({ + windowId: message.windowId, + appId: this.options.appId, + specs: message.tools, + invoke: (toolName, args) => this.callWindowTool(message.windowId, toolName, args), + isForeground: () => shell.foregroundWindow()?.windowId === message.windowId, + }); + } + break; + case "tool-result": { + const pending = this.pendingToolCalls.get(message.callId); + if (pending) { + clearTimeout(pending.timer); + this.pendingToolCalls.delete(message.callId); + pending.resolve(message.result); + } + break; + } } }; options.worker.onerror = (error) => { @@ -181,6 +236,9 @@ export class WorkerAppHost { closeable: true, close: () => { this.openWindows.delete(spec.windowId); + // Withdraw this window's tools and fail any in-flight calls to it. + toolRegistry.removeAppTools(spec.windowId); + this.failPendingToolCallsFor(spec.windowId); this.post({ type: "close-window", windowId: spec.windowId }); this.options.removeSurface(surfaceId); }, @@ -235,6 +293,32 @@ export class WorkerAppHost { return window; } + /** + * Post a tool-call to the worker and resolve when its tool-result comes back. + * Resolves with a tool error on timeout so a hung worker yields a tool error + * rather than a stuck assistant turn. + */ + private callWindowTool(windowId: string, toolName: string, args: unknown): Promise { + return new Promise((resolve) => { + const callId = `${this.options.appId}:${this.nextCallSerial++}`; + const timer = setTimeout(() => { + this.pendingToolCalls.delete(callId); + resolve({ ok: false, error: `App ${this.options.appId} did not respond to ${toolName}` }); + }, TOOL_CALL_HOST_TIMEOUT_MS); + this.pendingToolCalls.set(callId, { windowId, resolve, timer }); + this.post({ type: "tool-call", callId, windowId, name: toolName, args }); + }); + } + + private failPendingToolCallsFor(windowId: string): void { + for (const [callId, pending] of this.pendingToolCalls) { + if (pending.windowId !== windowId) continue; + clearTimeout(pending.timer); + this.pendingToolCalls.delete(callId); + pending.resolve({ ok: false, error: "The target window was closed" }); + } + } + private post(message: WorkerAppMessage): void { this.options.worker.postMessage(message); } diff --git a/app/workers/terminal-app.worker.ts b/app/workers/terminal-app.worker.ts index ba8bbafb..d259a620 100644 --- a/app/workers/terminal-app.worker.ts +++ b/app/workers/terminal-app.worker.ts @@ -39,6 +39,7 @@ import { } from "../ui/menu"; import { defaultWindowMenuItems, WindowMenu } from "../ui/window-menu"; import type { WorkerAppMessage, WorkerAppReply } from "../ui/shell/worker-window"; +import type { ToolResult, ToolSpec } from "../assistant/tool-registry"; declare const global: any; declare const com: any; @@ -102,7 +103,43 @@ type TerminalWindow = HubWindow | ViewWindow; const windows = new Map(); const pendingViews = new Map(); +// The view an assistant tool acts on when the terminal isn't foregrounded: +// the last view to be foregrounded or receive input. +let activeViewId: string | null = null; let nextViewSerial = 1; + +/** + * Assistant tools this app contributes, declared on the hub window (which the + * launcher opens and which persists for the app's life). All `open`-tier so + * "rerun the build" works while the terminal is backgrounded. send_input and + * read_screen act on the active view session (see resolveActiveView). + */ +const TERMINAL_TOOLS: ToolSpec[] = [ + { + name: "list_sessions", + description: "List the live g2mirror terminal sessions the glasses can see.", + inputSchema: { type: "object", properties: {}, additionalProperties: false }, + availability: "open", + }, + { + name: "send_input", + description: + "Type a line into the active terminal session and submit it (as if typed and Enter pressed). Use to run a command in the terminal.", + inputSchema: { + type: "object", + properties: { text: { type: "string", description: "The line to type and run." } }, + required: ["text"], + additionalProperties: false, + }, + availability: "open", + }, + { + name: "read_screen", + description: "Return the current visible contents of the active terminal session's screen.", + inputSchema: { type: "object", properties: {}, additionalProperties: false }, + availability: "open", + }, +]; let viewportWidth = 0; let viewportHeight = 0; let gridCols = 0; @@ -161,6 +198,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { if (!window) break; window.foreground = message.foreground; window.focused = message.focused; + if (window.foreground && window.kind === "view") activeViewId = window.windowId; if (window.foreground) renderAndSubmit(window, 0); break; } @@ -172,6 +210,19 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { } } break; + case "tool-call": { + const callId = message.callId; + Promise.resolve(handleTerminalTool(message.name, message.args)) + .then((result) => post({ type: "tool-result", callId, result })) + .catch((error) => + post({ + type: "tool-result", + callId, + result: { ok: false, error: String((error as Error)?.message ?? error) }, + }), + ); + break; + } } }; @@ -209,6 +260,8 @@ function openWindow(windowId: string, surfaceId: string, viewport: { width: numb selectedIndex: 0, scrollRow: 0, }); + // The hub carries the app's assistant tools; declare them once it exists. + post({ type: "set-tools", windowId, tools: TERMINAL_TOOLS }); if (!controlClient) { startControlClient(); } @@ -224,6 +277,7 @@ function closeWindow(windowId: string): void { window.client.stop(); } windows.delete(windowId); + if (activeViewId === windowId) activeViewId = null; if (window.kind === "view") { renderHubWindows(); } @@ -414,6 +468,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] { } function handleInput(window: TerminalWindow, event: DashboardInputEvent, frameId: number): void { + if (window.kind === "view") activeViewId = window.windowId; // An open window menu owns all input (it closes itself via pop). if (window.menu?.isOpen()) { window.menu @@ -743,3 +798,71 @@ function sessionLabel(session: G2MirrorSession): string { const hint = session.cwdHint.replace(/^_+/, "").replace(/_+/g, "/"); return hint || "session"; } + +/** Dispatch an assistant tool-call (unprefixed name) to its handler. */ +function handleTerminalTool(name: string, args: any): ToolResult { + switch (name) { + case "list_sessions": + return toolListSessions(); + case "send_input": + return toolSendInput(args); + case "read_screen": + return toolReadScreen(); + default: + return { ok: false, error: `Unknown terminal tool: ${name}` }; + } +} + +function toolListSessions(): ToolResult { + const sessions = controlState?.sessions ?? []; + if (!sessions.length) { + return { ok: true, content: "No live terminal sessions (run g2mirror on the host)." }; + } + const lines = sessions.map( + (session) => `- ${sessionLabel(session)}${viewWindowIdForSocket(session.socket) ? " [open]" : ""}`, + ); + return { ok: true, content: lines.join("\n") }; +} + +function toolSendInput(args: any): ToolResult { + const text = String(args?.text ?? ""); + if (!text) return { ok: false, error: "send_input requires non-empty text." }; + const view = resolveActiveView(); + if (!view) return { ok: false, error: "No terminal session is open to send input to." }; + view.client.submitInput(text); + return { ok: true, content: `Sent to ${view.label}.` }; +} + +function toolReadScreen(): ToolResult { + const view = resolveActiveView(); + if (!view) return { ok: false, error: "No terminal session is open." }; + const length = view.emulator.bufferLength(); + const start = Math.max(0, length - gridRows); + const lines: string[] = []; + for (let index = start; index < length; index++) { + lines.push(view.emulator.lineAt(index)); + } + const text = lines.join("\n").replace(/\s+$/, ""); + return { ok: true, content: text || "(screen is empty)" }; +} + +/** + * The terminal view an `open`-tier tool acts on: a foregrounded view if any, + * else the last view to be active, else the sole open view. Null when no view + * is open (the model gets a tool error it can relay). + */ +function resolveActiveView(): ViewWindow | null { + const views: ViewWindow[] = []; + let foreground: ViewWindow | null = null; + for (const window of windows.values()) { + if (window.kind !== "view") continue; + views.push(window); + if (window.foreground) foreground = window; + } + if (foreground) return foreground; + if (activeViewId) { + const window = windows.get(activeViewId); + if (window && window.kind === "view") return window; + } + return views.length === 1 ? views[0]! : null; +} diff --git a/notes/apps.txt b/notes/apps.txt new file mode 100644 index 00000000..c6d66831 --- /dev/null +++ b/notes/apps.txt @@ -0,0 +1,87 @@ +# Even-Analogous Apps + +Ideally we'd like full feature parity with the stock Even app (or at least, +close enough that not-using the stock app doesn't feel like a sacrifice). +A rough list of things the stock app has that we currently don't: + +## Quicklist + +Quicklist is Even's version of a todo app, in the form of a dashboard card +that you add to and remove from with assistant voice commands. It's pretty +simple to build once a general voice assistant is in place, but I don't +actually like their design: a todo list really wants to be synchronized +with something external like Roam or Notion, so I think it'd be better to +build one of those integrations instead. + +## Stocks, News + +Even has a dashboard card which is a stock ticker. I think this is probably +misguided; while there do exist people who would want and benefit from a stock +tracker on their glasses, I do'm not sure there exist people who would want and +benefit from a stock tracker that isn't a professional-tier tool connected +to carefully chosen low-latency data sources, and neither I (nor Even) can +build that tool. + +The News dashboard card is more broadly useful, but has a similar issue: a +generic news ticker isn't really useful without a much higher degree of +customization. Probably the right thing to build instead is more like an RSS +reader. + +## Ring Connection Management + +The stock Even app is responsible for pairing the glasses with the ring (ie, +sending the glasses a configuration messages telling it to connect to the +ring's MAC address). If you pair them in the stock app and then switch to a +custom app, they remain paired indefinitely (this configuration is stored in +flash), and our custom app can make use of the fact that they're paired. + +We don't yet have support for connecting or disconnecting the glasses and ring +to/from each other. For latency reasons, it would be nice if we could unpair +the glasses from the ring and connect the phone to it directly; we don't yet +support that either. + +## Ring Health Metrics + +The R1 ring contains an oximeter-style LED sensor and an accelerometer, which +can be used for a step count, heart rate measurement, etc. I'm not sure +whether the sensor interpretation is done within the firmware (in which case +it would make sense to display them in the UI) or in the Even phone app (in +which case we'd be reimplementing all the interpretation algorithms, which +isn't worth it. + +## Navigate + +Even has a Navigate app, which is basically Google Maps for the specific use +case of "name a destination, get turn by turn directions to there". I believe +this is using Mapbox, which takes care of most of the heavy lifting (if you +have an API key). I don't think Even's version is very good (they ran into +trouble with image-upload performance preventing a good map display size), so +it's likely not too hard to do better. One caveat is the location permission; +Android normally only wants to prompt for location permission "while using the +app", but Android's idea of in use (phone screen unlocked and window focused) +doesn't match our idea of in use (G2 glasses on, phone could be locked). This +app also gets a lot better if we figure out how to use the device's +magnetometer (if it indeed has a working magnetometer). + +## Distance/Viewport Configuration + +During glasses onboarding, the stock firmware has you select a vertical offset +for the screen image. This accounts for glasses resting at different positions +on your face (the optics don't let you see the whole screen area), but cuts +the usable screen height from 480px to 288px. There is also a "distance" +config option in the stock Android app, which adjusts the horizontal position +of the image on the two lenses separately to create the impression of the +image being closer or farther; this cuts the usable screen width from 640px to +576px. Unlike the vertical size reduction, this 64px reduction is not +optically necessary. + +## Even AI +## Translate and Conversate + +## Other Configuration + +Brightness, head-up trigger + + +# Other Apps + diff --git a/notes/media-resume-via-media-key.md b/notes/media-resume-via-media-key.md new file mode 100644 index 00000000..0b052996 --- /dev/null +++ b/notes/media-resume-via-media-key.md @@ -0,0 +1,13 @@ +# Music: "Resume" action via media-key dispatch + +Follow-up to the MediaBrowserService browsing support. When no media session +is active, `AudioManager.dispatchMediaKeyEvent(KEYCODE_MEDIA_PLAY)` (down+up +pair) is routed by the system to the *last* media app's manifest-declared +`MEDIA_BUTTON` receiver, even if that app's process is dead — the same +mechanism a Bluetooth headset play button uses. Most players respond by +resuming the last queue. + +Idea: add a "Resume last" row to the Music app's no-active-session screen +(next to the browse entry). ~5 lines in `FaceclawMediaController` plus one UI +row; no per-app negotiation needed. Caveat: it's fire-and-forget — no feedback +if nothing handles it, so the UI should just wait for a session to appear. diff --git a/notes/mentraos-compat-assessment.md b/notes/mentraos-compat-assessment.md new file mode 100644 index 00000000..274d7d84 --- /dev/null +++ b/notes/mentraos-compat-assessment.md @@ -0,0 +1,133 @@ +# MentraOS app compatibility for Faceclaw — feasibility assessment (2026-07-23) + +MentraOS: https://github.com/Mentra-Community/MentraOS (MIT), docs at https://docs.mentraglass.com. + +## TL;DR + +Technically feasible with moderate effort, but the timing is awkward: MentraOS is +**deprecating its entire current app API**. MentraOS 3.0 ships **Aug 3, 2026** and +cloud-SDK apps stop working on it (a "MentraOS Legacy" mode preserves them through +**October 2026**). The replacement — a **Miniapp SDK** where apps are phone-local JS +bundles running in a WebView — gets a public spec in **September 2026**, and that model +is a *much* better fit for Faceclaw than the cloud model. Recommendation: wait for the +miniapp spec and target that; only build a minimal cloud-protocol emulator if access to +the existing app corpus is wanted in the interim. + +## MentraOS app model (current, "Cloud SDK" / TPA era) + +- Apps are developer-hosted server processes (Node/Bun, `@mentra/sdk`). Chain: + glasses → phone (BLE) → MentraOS Cloud (WS) → app server (WS). +- Start flow: cloud POSTs a `session_request` webhook `{sessionId, userId, websocketUrl}` + to the app's registered URL; the app dials back to `websocketUrl` and sends + `tpa_connection_init {packageName, sessionId, apiKey}`; cloud acks with settings + + device capabilities. **The webhook carries the WS URL, so an unmodified `@mentra/sdk` + app can be pointed at any server** — that is the compat hook. +- Protocol: JSON over WebSocket (binary frames for audio). 28 app→cloud + 28 cloud→app + message types, defined in `cloud/packages/sdk/src/types/` (`message-types.ts`, + `streams.ts`, `layouts.ts`, `webhooks.ts`, `capabilities.ts`). The TS types are the spec. +- Subscriptions: `subscription_update` with a full list of stream types, some + parameterized (`transcription:en-US`, `touch_event:forward_swipe`). Events arrive in a + `data_stream` envelope. +- Display: 7 layout types — `text_wall`, `double_text_wall`, `dashboard_card`, + `reference_card`, `bitmap_view` (base64), `bitmap_animation`, `clear_view` — sent as + `display_event {view: 'main'|'dashboard', layout, durationMs?, forceDisplay?}`. + Cloud throttles displays to 1 per 300 ms. Dashboard = shared look-up surface. +- Streams: `transcription` (`{text, isFinal, ...}`), `translation`, `VAD`, `audio_chunk` + (PCM), `button_press`, `touch_event`, `head_position` (up/down), `glasses_battery_update`, + `phone_battery_update`, `location_update`, `phone_notification`(+dismissed), + `calendar_event`, plus camera/RTMP/LED families irrelevant to G2. +- Other: ElevenLabs TTS (`audio.speak`), per-app settings (schema in dev console, 9 + setting types, live `settings_update`), webview UI with signed-token auth, cloud KV + storage, app-to-app messaging. +- Hardware abstraction: `Capabilities` flags per device. **Even Realities G2 is a + first-class device** (`capabilities/even-realities-g2.ts`): display 640×200 (their + rendering canvas, not the 576×288 EvenHub raster), maxTextLines 5, bitmap-capable, + capacitive swipe bar with TAP/DOUBLE_TAP/TRIPLE_TAP/PRESS_HOLD/SWIPE_UP/SWIPE_DOWN, + IMU, mic, no camera/speaker/LED. +- Licensing: MIT throughout; cloud is self-hostable in principle (Bun + MongoDB + S3 + + third-party STT/TTS keys) but heavy. + +## Coming model (Miniapp SDK, MentraOS 3.0) + +Miniapps are JS bundles (`index.html` + `miniapp.json`) running **on the phone** inside +the MentraOS host app's WebView ("Mentra Runtime"), packages `@mentra/miniapp` + +`@mentra/miniapp-cli`, sideloaded via QR. API is analogous but smaller: +`session.transcription`, `session.display.showTextWall`, `session.storage`. Example: +https://github.com/Mentra-Community/LiveCaptionsLocalSdkMiniapp. Public spec Sept 2026. + +## Fit with Faceclaw + +Faceclaw's worker-per-app architecture is unusually favorable: apps are already isolated +contexts speaking a small typed JSON vocabulary (`WorkerAppMessage`/`WorkerAppReply` in +`app/ui/shell/worker-window.ts`), with pixels out-of-band. The compat layer would be one +compiled-in "MentraOS host" app (a peer of `WorkerAppHost`) that presents each external +app as a `ShellWindow` and: + +- **Rasterizes layouts** → trivial. All 7 layout types map onto `GrayImage` + + `drawTextWrapped`/`bitBlt` into the 540×260 `APP_VIEWPORT`. Bitmaps need scaling from + their 640×200/576×136 assumptions; monochrome already matches. +- **Maps input** → good. `DashboardInputEvent` (`click`/`double-click`/`long-press`/ + `scroll-up`/`scroll-down`) covers their G2 button-event list almost 1:1 (TRIPLE_TAP + missing). Work item: input is currently routed only to the focused window; MentraOS + wants multi-subscriber fan-out (background apps), so a broadcast layer is needed. +- **Transcription** → the load-bearing stream, and Faceclaw already has it: + `voiceControlBridge.onTranscript` (`{text, isFinal}`, replace semantics — matches + MentraOS's `TranscriptionData` model well). Onboard Moonshine is English-only; + parameterized languages would fall back to ElevenLabs/Whisper cloud providers. + Needs multi-consumer fan-out and a policy for when the mic is open (MentraOS keeps it + effectively continuous; Faceclaw is PTT/wakeword-oriented — battery implications). +- **VAD** → sherpa endpointing exists. **audio_chunk** → PCM already surfaced on the + ElevenLabs path (`onPcm`). +- **Battery / phone notifications / calendar** → data already flows in Faceclaw + (top-bar battery, notification listener, calendar app); just needs re-emission as + streams. +- **head_position** → gap. Faceclaw exposes raw IMU only (no gesture classification); + the look-up dashboard gesture would need a small accelerometer classifier (cmd 19 + stream exists, see imu-accelerometer memory). +- **TTS / audio out** → G2 has no speaker; MentraOS plays through the phone. Faceclaw + would need phone-side audio playback (new but simple Android work). +- **Settings** → MentraOS per-app settings map onto the shared `FaceclawSettings` store + with a `mentra..*` prefix; the 9 setting types map onto + `ConfigSetting` subclasses, mostly. +- **Not applicable on G2**: camera/photo/RTMP/LiveKit, LED, WiFi setup — can be + rejected via capabilities, which the protocol explicitly supports. + +## The hard parts + +1. **Who plays "cloud"?** Cloud-SDK apps dial out to a WS URL given in a webhook, so + Faceclaw must (a) run a WS **server** — none exists anywhere in the codebase today, + all networking is outbound okhttp client (`FaceclawWebSocket`, g2mirror pattern) — + reachable from wherever the app server runs (fine on LAN; NAT pain otherwise), and + (b) send the session webhook itself (simple outbound HTTP POST). A phone-hosted + mini-cloud with Java-WebSocket/NanoWSD listening on LAN is the sane MVP; self-hosting + their actual cloud (Bun+Mongo+Docker) is overkill. +2. **Session lifecycle**: connection init/ack, `subscription_update` bookkeeping, + `reconnect`/`reconnect_ack/rejected/deferred`, `app_stopped`, per-app permission + gating. This is bookkeeping-heavy but not deep. +3. **Display arbitration**: foreground vs `background`+`forceDisplay` apps, `durationMs` + auto-revert, the shared dashboard view. Faceclaw's shell has focus/foreground but no + equivalent of background-app display preemption or a look-up dashboard. +4. **App discovery/registration UX**: no dev console; needs a simple "add app by URL + + API key" settings flow (API key check can be a no-op or shared secret). + +## Effort estimate + +MVP covering the bulk of display-glasses apps (live captions, teleprompters, dashboards, +notification relays): ~10–15 message types + ~8 stream types + webhook sender + LAN WS +server + layout rasterizer + settings mapping. Roughly a few weeks of focused work, with +transcription-stream policy and the WS server being the riskiest pieces. Full-surface +emulation (webview auth, app messaging rooms, TTS, location tiers) is substantially more +and mostly not worth it. + +## Recommendation + +- **Don't invest in the cloud-TPA protocol as the long-term target** — the vendor is + killing it within ~3 months of now, and the ecosystem's apps will migrate to miniapps. +- **Target the Miniapp SDK when its spec lands (Sept 2026).** It's strictly easier for + Faceclaw: JS bundle in an Android WebView with an injected `session` bridge — no + server, no webhooks, no reconnect protocol — and Faceclaw already has every underlying + facility the bridge needs. The rasterizer/input/transcription plumbing built for it + would be shared with any interim TPA emulator anyway. +- If interim access to existing cloud-SDK apps matters, build the minimal LAN emulator + described above, scoped to text layouts + transcription + button streams, and treat it + as disposable. diff --git a/notes/voice-assistant-design.md b/notes/voice-assistant-design.md index 317e17ed..eda40274 100644 --- a/notes/voice-assistant-design.md +++ b/notes/voice-assistant-design.md @@ -1,8 +1,8 @@ # Voice Assistant Architecture Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability -ontology, misc fixes). **Phase 1 is implemented** (see the Implementation plan -below); phases 2-4 are still design-only. +ontology, misc fixes). **Phases 1 and 2 are implemented** (see the +Implementation plan below); phases 3-4 are still design-only. ## Goal @@ -383,13 +383,26 @@ default highlight and wakeword skip-confirmation auto-send; *Deliverable: speak wakeword, ask "what's on my calendar", get an answer on-lens.* -**Phase 2 — app tools.** -Worker protocol extension (set-tools / tool-call / tool-result), -availability-tier filtering in the registry (`open` on window open/close, -`foreground` on foreground change), `installed`-tier registration on launcher -entries (shell-side handlers only; launch-on-call deferred), terminal app -tools as the reference. *Deliverable: "run the build again" typed into the -terminal even while it's backgrounded.* +**Phase 2 — app tools.** ✅ Landed 2026-07-25 (builds + typechecks; not yet +hardware-tested). +Worker protocol extended with `set-tools` / `tool-result` (WorkerAppReply) and +`tool-call` (WorkerAppMessage). `ToolRegistry.setAppTools`/`removeAppTools` +key tools by windowId, prefix names `app..`, and gate `foreground` +tools on an `isForeground()` predicate (checked at list and dispatch time). +`WorkerAppHost` proxies calls to its worker (callId round-trip, 15s host +timeout backstop on top of the registry's per-tool timeout), registers a +window's tools on `set-tools`, and withdraws them + fails in-flight calls on +window close. Direct mode now re-lists tools each loop iteration. +Terminal is the reference: the hub window declares `send_input` / +`read_screen` / `list_sessions` (all `open`-tier); send_input/read_screen act +on the active view (foregrounded → last-active → sole view). *Deliverable: +"run the build again" typed into the terminal even while it's backgrounded.* + +*Not yet done from the Phase 2 design:* `installed`-tier registration on +launcher entries (no `installed` app tool exists yet — timer stays deferred); +in-process-window tool registration (only worker apps wired); the per-window +same-name collision handling (the terminal sidesteps it by declaring all tools +on the single hub window). **Phase 3 — external bridge.** Bridge wire protocol; ExternalAgentBackend (phone side, reusing the