diff --git a/app/assistant/bridge-client.ts b/app/assistant/bridge-client.ts new file mode 100644 index 00000000..ebaa4c50 --- /dev/null +++ b/app/assistant/bridge-client.ts @@ -0,0 +1,334 @@ +import { AssistantMcpServer } from "./mcp-server"; +import { toolRegistry } from "./tool-registry"; +import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types"; + +declare const com: any; + +/** + * Dial-out websocket client for the external assistant bridge (an OpenClaw + * plugin or compatible server on the user's machine; see + * notes/voice-assistant-design.md "External mode"). One JSON object per text + * frame, three multiplexed channels: + * + * ctl: hello/hello-ack auth handshake, ping/pong, error + * chat: utterance in, streamed reply out (per-turn) + * mcp: raw MCP JSON-RPC; the phone is the MCP *server* (ToolRegistry) + * + * Unlike G2MirrorClient (per-terminal-window, no reconnect), this connection + * is a long-lived shell service: it stays up while configured so the remote + * agent can make proactive tool calls, and it re-dials with backoff. + */ + +const PROTOCOL_VERSION = 1; +const RECONNECT_MIN_MS = 1_000; +const RECONNECT_MAX_MS = 60_000; +/** Backstop on a turn the bridge never finishes (server-side cap is 2 min). */ +const TURN_TIMEOUT_MS = 3 * 60 * 1000; + +export type AssistantBridgePhase = "idle" | "connecting" | "connected" | "failed"; + +export type AssistantBridgeState = { + phase: AssistantBridgePhase; + status: string; +}; + +export type AssistantBridgeOptions = { + host: string; + port: number; + token: string; + deviceName: string; + /** Reads assistant.allowProactive; consulted per proactive tool call. */ + allowProactive: () => boolean; +}; + +type ActiveTurn = { + turnId: string; + callbacks: AssistantTurnCallbacks; + textSoFar: string; + timer: ReturnType; +}; + +export class AssistantBridgeClient { + private options: AssistantBridgeOptions | null = null; + private ws: any = null; + private listenerProxy: any = null; + private phase: AssistantBridgePhase = "idle"; + private status = "Not configured."; + private stopped = true; + private reconnectTimer: ReturnType | null = null; + private reconnectDelayMs = RECONNECT_MIN_MS; + private activeTurn: ActiveTurn | null = null; + private turnSeq = 0; + private mcpServer: AssistantMcpServer | null = null; + private unsubscribeToolsChanged: (() => void) | null = null; + private readonly stateListeners = new Set<(state: AssistantBridgeState) => void>(); + + state(): AssistantBridgeState { + return { phase: this.phase, status: this.status }; + } + + onStateChange(listener: (state: AssistantBridgeState) => void): () => void { + this.stateListeners.add(listener); + return () => this.stateListeners.delete(listener); + } + + isConnected(): boolean { + return this.phase === "connected"; + } + + /** + * Set (or replace) the bridge configuration and connect. Safe to call again + * with new settings; the old connection is torn down first. + */ + configure(options: AssistantBridgeOptions): void { + this.stop(); + this.options = options; + this.stopped = false; + this.mcpServer = new AssistantMcpServer({ + send: (msg) => this.send({ chan: "mcp", msg }), + isTurnActive: () => this.activeTurn !== null, + allowProactive: options.allowProactive, + }); + this.unsubscribeToolsChanged = toolRegistry.onToolsChanged(() => { + if (this.phase === "connected") this.mcpServer?.sendToolsChanged(); + }); + this.connect(); + } + + /** Disconnect and stay down until the next configure(). */ + stop(): void { + this.stopped = true; + this.clearReconnectTimer(); + this.failActiveTurn("Bridge connection closed"); + if (this.unsubscribeToolsChanged) { + this.unsubscribeToolsChanged(); + this.unsubscribeToolsChanged = null; + } + this.mcpServer = null; + if (this.ws) { + try { + this.ws.close(1000, "bye"); + } catch { + // ignore + } + this.ws = null; + } + this.listenerProxy = null; + this.setState("idle", "Not connected."); + } + + /** + * Send an utterance as a new turn. Streaming replies arrive via callbacks. + * A turn already in flight is superseded (the session serializes turns, so + * this is just a safety net). + */ + sendUtterance( + text: string, + ctx: AssistantContext, + callbacks: AssistantTurnCallbacks, + ): AssistantTurnHandle { + if (this.phase !== "connected") { + callbacks.onError(`Agent bridge is not connected (${this.status})`); + return { cancel: () => {} }; + } + this.failActiveTurn("Superseded by a new request"); + const turnId = `t${++this.turnSeq}`; + this.activeTurn = { + turnId, + callbacks, + textSoFar: "", + timer: setTimeout(() => { + if (this.activeTurn?.turnId !== turnId) return; + this.failActiveTurn("The agent took too long to reply"); + }, TURN_TIMEOUT_MS), + }; + this.send({ chan: "chat", type: "utterance", turnId, text, ctx }); + return { + cancel: () => { + if (this.activeTurn?.turnId !== turnId) return; + this.send({ chan: "chat", type: "cancel", turnId }); + this.clearActiveTurn(); + }, + }; + } + + private connect(): void { + if (this.stopped || this.ws || !this.options) return; + const { host, port } = this.options; + const url = `ws://${host}:${port}`; + this.setState("connecting", `Connecting to ${host}:${port}...`); + this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + onOpen: () => { + if (this.stopped) return; + this.setState("connecting", "Authenticating..."); + this.send({ + chan: "ctl", + type: "hello", + version: PROTOCOL_VERSION, + token: this.options!.token, + deviceName: this.options!.deviceName, + capabilities: ["chat", "mcp"], + }); + }, + onTextMessage: (message: string) => { + if (this.stopped) return; + this.handleMessage(String(message)); + }, + onClosed: (code: number, reason: string) => { + if (this.stopped) return; + this.handleConnectionLost( + `Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""})`, + ); + }, + onFailure: (message: string) => { + if (this.stopped) return; + this.handleConnectionLost(`Connection failed: ${String(message)}`); + }, + }); + try { + this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, null, null); + } catch (error) { + this.ws = null; + this.handleConnectionLost( + `Connection failed: ${String((error as Error)?.message ?? error)}`, + ); + } + } + + private handleMessage(raw: string): void { + let frame: any = null; + try { + frame = JSON.parse(raw); + } catch { + return; + } + if (!frame || typeof frame !== "object") return; + switch (frame.chan) { + case "ctl": + this.handleCtl(frame); + return; + case "chat": + this.handleChat(frame); + return; + case "mcp": + this.mcpServer?.handleMessage(frame.msg); + return; + default: + return; + } + } + + private handleCtl(frame: any): void { + if (frame.type === "hello-ack") { + this.reconnectDelayMs = RECONNECT_MIN_MS; + this.setState("connected", `Connected to ${String(frame.serverName ?? "bridge")}`); + return; + } + if (frame.type === "ping") { + this.send({ chan: "ctl", type: "pong", ts: frame.ts }); + return; + } + if (frame.type === "error") { + // The server closes after a ctl error; the close handler schedules the + // re-dial. Record the reason so the status is more useful than a bare + // close code (especially for a bad token). + this.status = `Bridge error: ${String(frame.message ?? "unknown")}`; + return; + } + } + + private handleChat(frame: any): void { + const turn = this.activeTurn; + if (!turn || frame.turnId !== turn.turnId) return; + switch (frame.type) { + case "text-delta": { + const text = typeof frame.text === "string" ? frame.text : ""; + turn.textSoFar = frame.replace === true ? text : turn.textSoFar + text; + turn.callbacks.onTextDelta(text, turn.textSoFar); + return; + } + case "tool-activity": + turn.callbacks.onToolActivity(String(frame.label ?? "tool")); + return; + case "turn-done": { + this.clearActiveTurn(); + turn.callbacks.onTurnDone({ + stopReason: typeof frame.stopReason === "string" ? frame.stopReason : null, + }); + return; + } + case "turn-error": + this.clearActiveTurn(); + turn.callbacks.onError(String(frame.message ?? "Agent error")); + return; + default: + return; + } + } + + private handleConnectionLost(status: string): void { + this.ws = null; + this.listenerProxy = null; + // Keep a ctl-error status (e.g. "invalid token") in preference to the + // generic close message that follows it. + const detail = this.status.startsWith("Bridge error:") ? this.status : status; + this.failActiveTurn("Bridge connection lost"); + this.setState("failed", detail); + this.scheduleReconnect(); + } + + private scheduleReconnect(): void { + if (this.stopped || this.reconnectTimer) return; + const delay = this.reconnectDelayMs; + this.reconnectDelayMs = Math.min(this.reconnectDelayMs * 2, RECONNECT_MAX_MS); + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + this.connect(); + }, delay); + } + + private clearReconnectTimer(): void { + if (this.reconnectTimer) { + clearTimeout(this.reconnectTimer); + this.reconnectTimer = null; + } + } + + private clearActiveTurn(): void { + if (!this.activeTurn) return; + clearTimeout(this.activeTurn.timer); + this.activeTurn = null; + } + + private failActiveTurn(message: string): void { + const turn = this.activeTurn; + if (!turn) return; + this.clearActiveTurn(); + turn.callbacks.onError(message); + } + + private send(frame: object): void { + if (!this.ws) return; + try { + this.ws.sendText(JSON.stringify({ v: PROTOCOL_VERSION, ...frame })); + } catch { + // A send on a dying socket; the close/failure callback handles recovery. + } + } + + private setState(phase: AssistantBridgePhase, status: string): void { + this.phase = phase; + this.status = status; + const state = this.state(); + for (const listener of this.stateListeners) { + try { + listener(state); + } catch { + // listeners must not break the client + } + } + } +} + +/** Process-wide bridge client (main isolate), configured by the dashboard controller. */ +export const assistantBridge = new AssistantBridgeClient(); diff --git a/app/assistant/mcp-server.ts b/app/assistant/mcp-server.ts new file mode 100644 index 00000000..4057cf27 --- /dev/null +++ b/app/assistant/mcp-server.ts @@ -0,0 +1,126 @@ +import { toolRegistry, type ToolRegistry } from "./tool-registry"; + +/** + * Minimal MCP (Model Context Protocol) server surface over the assistant + * bridge's `mcp` channel. The phone serves its ToolRegistry to the external + * agent: `tools/list`, `tools/call`, and `notifications/tools/list_changed` + * (emitted by the bridge client when the registry changes). + * + * Transport-agnostic: the bridge client hands decoded JSON-RPC messages to + * handleMessage and provides `send` to deliver replies/notifications. + */ + +const MCP_PROTOCOL_VERSION = "2025-06-18"; + +/** Proactive display actions are rate-limited to this many per minute. */ +const PROACTIVE_CALLS_PER_MINUTE = 6; + +export type McpServerOptions = { + send: (msg: object) => void; + /** Whether a voice turn is currently in flight (calls outside one are "proactive"). */ + isTurnActive: () => boolean; + /** Master setting gate for proactive calls (assistant.allowProactive). */ + allowProactive: () => boolean; + registry?: ToolRegistry; +}; + +export class AssistantMcpServer { + private readonly registry: ToolRegistry; + private readonly proactiveCallTimes: number[] = []; + + constructor(private readonly options: McpServerOptions) { + this.registry = options.registry ?? toolRegistry; + } + + /** Notify the agent that the live tool set changed. */ + sendToolsChanged(): void { + this.options.send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" }); + } + + handleMessage(msg: any): void { + if (!msg || msg.jsonrpc !== "2.0") return; + if (typeof msg.method !== "string") return; // responses to server-sent requests: none yet + const id = msg.id; + const isNotification = id === undefined || id === null; + + switch (msg.method) { + case "initialize": + this.reply(id, { + protocolVersion: + typeof msg.params?.protocolVersion === "string" + ? msg.params.protocolVersion + : MCP_PROTOCOL_VERSION, + capabilities: { tools: { listChanged: true } }, + serverInfo: { name: "faceclaw", version: "0.3.0" }, + }); + return; + case "notifications/initialized": + return; + case "ping": + this.reply(id, {}); + return; + case "tools/list": + this.reply(id, { + tools: this.registry.listTools().map((spec) => ({ + name: spec.name, + description: spec.description, + inputSchema: spec.inputSchema, + })), + }); + return; + case "tools/call": + void this.handleToolCall(id, msg.params); + return; + default: + if (!isNotification) { + this.options.send({ + jsonrpc: "2.0", + id, + error: { code: -32601, message: `Method not supported: ${msg.method}` }, + }); + } + return; + } + } + + private async handleToolCall(id: unknown, params: any): Promise { + const name = typeof params?.name === "string" ? params.name : ""; + const args = params?.arguments ?? {}; + // A call while no voice turn is in flight is a proactive action by the + // remote agent; the phone (never the bridge) enforces the gating. + const proactive = !this.options.isTurnActive(); + if (proactive && !this.options.allowProactive()) { + this.replyToolError(id, "Proactive assistant actions are disabled in Settings"); + return; + } + if (proactive && !this.admitProactiveCall()) { + this.replyToolError(id, "Proactive action rate limit exceeded; try again later"); + return; + } + const result = await this.registry.callTool(name, args, { proactive }); + this.reply(id, { + content: [{ type: "text", text: result.ok ? result.content ?? "" : result.error ?? "Tool error" }], + isError: !result.ok, + }); + } + + /** Sliding-window rate limiter for proactive calls. */ + private admitProactiveCall(): boolean { + const now = Date.now(); + while (this.proactiveCallTimes.length && now - this.proactiveCallTimes[0]! > 60_000) { + this.proactiveCallTimes.shift(); + } + if (this.proactiveCallTimes.length >= PROACTIVE_CALLS_PER_MINUTE) return false; + this.proactiveCallTimes.push(now); + return true; + } + + private reply(id: unknown, result: object): void { + if (id === undefined || id === null) return; + this.options.send({ jsonrpc: "2.0", id, result }); + } + + private replyToolError(id: unknown, text: string): void { + this.reply(id, { content: [{ type: "text", text }], isError: true }); + } +} diff --git a/app/assistant/session.ts b/app/assistant/session.ts index dd554879..103634a5 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -1,20 +1,33 @@ import { buildAssistantSystemPrompt } from "../prompts"; +import { assistantBridge } from "./bridge-client"; import { DirectAssistantBackend } from "./direct-backend"; import type { LlmMessage, LlmToolDefinition } from "./llm-protocol"; import type { ResolvedAssistantModel } from "./models"; import { toolRegistry, type ToolRegistry } from "./tool-registry"; -import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types"; +import type { + AssistantBridgeConfig, + AssistantContext, + AssistantTurnCallbacks, + AssistantTurnHandle, +} from "./types"; /** * One assistant conversation. Owns the message history and turn state and - * drives a direct provider backend. The UI talks only to - * this surface, so swapping in the external-agent backend later doesn't touch - * AssistantLayer. + * drives either the on-phone provider loop (direct) or the external agent + * bridge. The UI talks only to this surface, so AssistantLayer is + * backend-agnostic. * * Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh - * one for a new wakeword and reuses the current one for a follow-up. + * one for a new wakeword and reuses the current one for a follow-up. In + * external mode the conversation history lives on the agent's machine (one + * long-lived session there), so expiry here only affects the overlay UI. */ +/** Which engine answers utterances, plus what it needs to do so. */ +export type AssistantBackendConfig = + | { kind: "direct"; llm: ResolvedAssistantModel } + | { kind: "external"; bridge: AssistantBridgeConfig }; + /** A new PTT after this much idle starts a fresh conversation. */ const SESSION_IDLE_MS = 10 * 60 * 1000; @@ -22,7 +35,7 @@ const SESSION_IDLE_MS = 10 * 60 * 1000; const MAX_HISTORY_MESSAGES = 40; export class AssistantSession { - private readonly backend = new DirectAssistantBackend(); + private readonly directBackend = new DirectAssistantBackend(); private readonly messages: LlmMessage[] = []; private turnHandle: AssistantTurnHandle | null = null; private lastActivityMs = Date.now(); @@ -31,16 +44,27 @@ export class AssistantSession { private readonly toolNameMap = new Map(); constructor( - private readonly llm: ResolvedAssistantModel, + private readonly config: AssistantBackendConfig, private readonly registry: ToolRegistry = toolRegistry, ) {} - matchesConfiguration(llm: ResolvedAssistantModel): boolean { - return ( - this.llm.provider === llm.provider && - this.llm.model === llm.model && - this.llm.apiKey === llm.apiKey - ); + matchesConfiguration(config: AssistantBackendConfig): boolean { + if (this.config.kind !== config.kind) return false; + if (this.config.kind === "direct" && config.kind === "direct") { + return ( + this.config.llm.provider === config.llm.provider && + this.config.llm.model === config.llm.model && + this.config.llm.apiKey === config.llm.apiKey + ); + } + if (this.config.kind === "external" && config.kind === "external") { + return ( + this.config.bridge.host === config.bridge.host && + this.config.bridge.port === config.bridge.port && + this.config.bridge.token === config.bridge.token + ); + } + return false; } /** Whether this session has been idle long enough to retire. */ @@ -59,38 +83,45 @@ export class AssistantSession { return; } this.lastActivityMs = Date.now(); - this.messages.push({ role: "user", content: text }); - this.trimHistory(); - - const system = buildAssistantSystemPrompt(ctx); const finish = () => { this.turnHandle = null; this.lastActivityMs = Date.now(); }; + const wrappedCallbacks: AssistantTurnCallbacks = { + onTextDelta: callbacks.onTextDelta, + onToolActivity: callbacks.onToolActivity, + onTurnDone: (result) => { + finish(); + callbacks.onTurnDone(result); + }, + onError: (message) => { + finish(); + callbacks.onError(message); + }, + }; - this.turnHandle = this.backend.runTurn({ - provider: this.llm.provider, - apiKey: this.llm.apiKey, - model: this.llm.model, - effort: this.llm.effort, - system, + if (this.config.kind === "external") { + // History and the agent loop live on the agent's machine; the phone just + // streams this turn. The overlay keeps its own display state. + this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks); + return; + } + + this.messages.push({ role: "user", content: text }); + this.trimHistory(); + const llm = this.config.llm; + this.turnHandle = this.directBackend.runTurn({ + provider: llm.provider, + apiKey: llm.apiKey, + model: llm.model, + effort: llm.effort, + system: buildAssistantSystemPrompt(ctx), messages: this.messages, buildTools: () => this.buildToolDefinitions(), registry: this.registry, resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName, - callbacks: { - onTextDelta: callbacks.onTextDelta, - onToolActivity: callbacks.onToolActivity, - onTurnDone: (result) => { - finish(); - callbacks.onTurnDone(result); - }, - onError: (message) => { - finish(); - callbacks.onError(message); - }, - }, + callbacks: wrappedCallbacks, }); } diff --git a/app/assistant/types.ts b/app/assistant/types.ts index e310b2ff..297516f3 100644 --- a/app/assistant/types.ts +++ b/app/assistant/types.ts @@ -30,3 +30,10 @@ export type AssistantTurnCallbacks = { export type AssistantTurnHandle = { cancel(): void; }; + +/** Connection settings for the external agent bridge (assistant.bridge* settings). */ +export type AssistantBridgeConfig = { + host: string; + port: number; + token: string; +}; diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 425f1eee..e77493e6 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -27,6 +27,7 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; import { registerSystemTools } from "../assistant/system-tools"; import { registerNavigateTools } from "../assistant/navigate-tools"; +import { assistantBridge } from "../assistant/bridge-client"; import { registerWindowTools } from "../assistant/window-tools"; import { WorkerAppHost } from "../ui/shell/worker-window"; import { @@ -83,6 +84,11 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry"; import { type LayerActions } from "../ui/layers"; import { + assistantAllowProactiveSetting, + assistantBackendSetting, + assistantBridgeHostSetting, + assistantBridgePortSetting, + assistantBridgeTokenSetting, brightnessSetting, brightnessSettingToLevel, elevenLabsApiKeySetting, @@ -326,6 +332,46 @@ class DashboardController { this.pushBrightness(); this.syncEvenHubScreenOffSetting(); this.applyVerticalPositionIfChanged(); + this.syncAssistantBridgeIfChanged(); + }); + // Connect to the external agent bridge at boot if configured; the + // connection stays up (with re-dial) so proactive tool calls work + // outside voice turns. + this.syncAssistantBridge(); + } + + // Bridge settings changes re-dial the connection; unrelated setting changes + // must not, so the config it was last started with is tracked. + private lastBridgeConfigKey = ""; + + private bridgeConfigKey(): string { + return JSON.stringify([ + assistantBackendSetting.get(), + assistantBridgeHostSetting.get().trim(), + assistantBridgePortSetting.get().trim(), + assistantBridgeTokenSetting.get(), + ]); + } + + private syncAssistantBridgeIfChanged(): void { + if (this.bridgeConfigKey() === this.lastBridgeConfigKey) return; + this.syncAssistantBridge(); + } + + private syncAssistantBridge(): void { + this.lastBridgeConfigKey = this.bridgeConfigKey(); + const host = assistantBridgeHostSetting.get().trim(); + const token = assistantBridgeTokenSetting.get(); + if (assistantBackendSetting.get() !== "external" || !host || !token) { + assistantBridge.stop(); + return; + } + assistantBridge.configure({ + host, + port: parseInt(assistantBridgePortSetting.get(), 10) || 8790, + token, + deviceName: "faceclaw", + allowProactive: () => assistantAllowProactiveSetting.get(), }); } diff --git a/app/ui/dashboard-settings.ts b/app/ui/dashboard-settings.ts index 3a86faad..1f63114d 100644 --- a/app/ui/dashboard-settings.ts +++ b/app/ui/dashboard-settings.ts @@ -350,6 +350,65 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({ description: "After a wakeword utterance, send the transcript straight to the assistant instead of stopping at the Send/Type confirmation menu.", }); +export type AssistantBackendKind = "direct" | "external"; + +const assistantBackendLabels: Record = { + direct: "On-phone (API key)", + external: "My own agent (bridge)", +}; + +export const assistantBackendSetting = new ConfigSettingEnum({ + id: "assistant-backend", + label: "Assistant backend", + storageKey: "assistant.backend", + defaultValue: "direct", + values: ["direct", "external"], + formatValue: (value) => assistantBackendLabels[value] ?? value, + description: + "Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.", +}); + +export const assistantBridgeHostSetting = new ConfigSettingString({ + id: "assistant-bridge-host", + label: "Bridge host", + storageKey: "assistant.bridgeHost", + defaultValue: "", + editorTitle: "Agent bridge host (tailscale IP)", + glassesEditTitle: "Edit bridge host", + description: + "Hostname or IP address (e.g. a Tailscale address) of the machine running the agent bridge.", +}); + +export const assistantBridgePortSetting = new ConfigSettingString({ + id: "assistant-bridge-port", + label: "Bridge port", + storageKey: "assistant.bridgePort", + defaultValue: "8790", + editorTitle: "Agent bridge port", + glassesEditTitle: "Edit bridge port", + description: "TCP port the agent bridge listens on. The default is 8790.", +}); + +export const assistantBridgeTokenSetting = new ConfigSettingString({ + id: "assistant-bridge-token", + label: "Bridge token", + storageKey: "assistant.bridgeToken", + defaultValue: "", + editorTitle: "Agent bridge auth token", + glassesEditTitle: "Edit bridge token", + formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"), + description: "Shared secret that must match the bridge's configured token.", +}); + +export const assistantAllowProactiveSetting = new ConfigSettingBoolean({ + id: "assistant-allow-proactive", + label: "Allow proactive agent actions", + storageKey: "assistant.allowProactive", + defaultValue: true, + description: + "Let the external agent use glasses tools outside of a conversation, e.g. showing an alert when a long-running job finishes. Rate-limited; only tools marked proactive-safe are allowed.", +}); + export const elevenLabsApiKeySetting = new ConfigSettingString({ id: "elevenlabs-api-key", label: "ElevenLabs key", diff --git a/app/ui/dashboard/settings-menus.ts b/app/ui/dashboard/settings-menus.ts index db9a64d5..d86fabf5 100644 --- a/app/ui/dashboard/settings-menus.ts +++ b/app/ui/dashboard/settings-menus.ts @@ -8,6 +8,11 @@ import type { MenuItem } from "../menu"; import { shell } from "../shell/shell"; import { anthropicApiKeySetting, + assistantAllowProactiveSetting, + assistantBackendSetting, + assistantBridgeHostSetting, + assistantBridgePortSetting, + assistantBridgeTokenSetting, assistantModelSetting, assistantSkipConfirmationSetting, batteryDisplayModeSetting, @@ -69,10 +74,21 @@ function settingsSections(): SettingsSection[] { items: [ enumSettingMenuItem(wakeWordActionSetting), enumSettingMenuItem(voiceProviderSetting), + ], + }, + { + label: "Assistant", + items: [ + // On-phone LLM loop vs the user's own agent via the bridge plugin. + enumSettingMenuItem(assistantBackendSetting), enumSettingMenuItem(assistantModelSetting), // When on, a wakeword utterance goes straight to the assistant with no // Send/Type menu step. toggleSettingMenuItem(assistantSkipConfirmationSetting), + textSettingMenuItem(assistantBridgeHostSetting), + textSettingMenuItem(assistantBridgePortSetting), + textSettingMenuItem(assistantBridgeTokenSetting), + toggleSettingMenuItem(assistantAllowProactiveSetting), ], }, { diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 1d2cd08e..3981e43b 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -6,12 +6,16 @@ import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noo import { MenuLayer, type MenuItem } from "../menu"; import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input"; import { AssistantLayer } from "../apps/assistant"; -import { AssistantSession } from "../../assistant/session"; -import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models"; +import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session"; +import { resolveAssistantModel } from "../../assistant/models"; import type { AssistantContext } from "../../assistant/types"; import { SingleNotificationLayer } from "../notifications"; import { anthropicApiKeySetting, + assistantBackendSetting, + assistantBridgeHostSetting, + assistantBridgePortSetting, + assistantBridgeTokenSetting, assistantModelSetting, assistantSkipConfirmationSetting, batteryDisplayModeSetting, @@ -722,24 +726,32 @@ class Shell { } private ensureAssistantSession(): AssistantSession | null { - const llm = this.resolveAssistantConfiguration(); - if (!llm) return null; + const config = this.resolveAssistantConfiguration(); + if (!config) return null; if ( !this.assistantSession || this.assistantSession.isExpired() || - !this.assistantSession.matchesConfiguration(llm) + !this.assistantSession.matchesConfiguration(config) ) { this.assistantSession?.cancel(); - this.assistantSession = new AssistantSession(llm); + this.assistantSession = new AssistantSession(config); } return this.assistantSession; } - private resolveAssistantConfiguration(): ResolvedAssistantModel | null { - return resolveAssistantModel(assistantModelSetting.get(), { + private resolveAssistantConfiguration(): AssistantBackendConfig | null { + if (assistantBackendSetting.get() === "external") { + const host = assistantBridgeHostSetting.get().trim(); + const token = assistantBridgeTokenSetting.get(); + if (!host || !token) return null; + const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790; + return { kind: "external", bridge: { host, port, token } }; + } + const llm = resolveAssistantModel(assistantModelSetting.get(), { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(), }); + return llm ? { kind: "direct", llm } : null; } private buildAssistantContext(): AssistantContext { @@ -761,7 +773,11 @@ class Shell { sendToAssistant(text: string): void { const session = this.ensureAssistantSession(); if (!session) { - this.showAlert("Set an API key for the selected assistant model in Settings."); + this.showAlert( + assistantBackendSetting.get() === "external" + ? "Configure the agent bridge host and token in Settings." + : "Set an API key for the selected assistant model in Settings.", + ); return; } if (!this.screenOn) this.wake("sidebar"); diff --git a/notes/voice-assistant-design.md b/notes/voice-assistant-design.md index eda40274..675d1c8d 100644 --- a/notes/voice-assistant-design.md +++ b/notes/voice-assistant-design.md @@ -1,8 +1,20 @@ # Voice Assistant Architecture Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability -ontology, misc fixes). **Phases 1 and 2 are implemented** (see the -Implementation plan below); phases 3-4 are still design-only. +ontology, misc fixes); revised 2026-08-01 (external bridge landed). +**Phases 1 and 2 are implemented**, and **phase 3 + the OpenClaw part of +phase 4 landed 2026-08-01** with one architecture change: the bridge server +is not a standalone package — it *is* the OpenClaw plugin +(`~/repositories/faceclaw-agent-bridge`, installed on the OpenClaw host). +The plugin hosts the websocket endpoint the phone dials into, runs +utterances as embedded OpenClaw agent turns (`runEmbeddedAgent` with a +dedicated session, default key `faceclaw:glasses`), and exposes the +glasses' MCP-served tools to the agent via two fixed meta-tools +(`glasses_list_tools` / `glasses_call` — fixed names because OpenClaw's +manifest `contracts.tools` wants static declarations, while the glasses +toolset is dynamic). The generic "MCP face for any agent" from the original +design is unbuilt; the wire protocol is unchanged, so a standalone bridge +for other agents can be added later without phone changes. ## Goal @@ -404,17 +416,36 @@ in-process-window tool registration (only worker apps wired); the per-window same-name collision handling (the terminal sidesteps it by declaring all tools on the single hub window). -**Phase 3 — external bridge.** -Bridge wire protocol; ExternalAgentBackend (phone side, reusing the -FaceclawWebSocket plumbing); `faceclaw-agent-bridge` server with the MCP -face + fallback CLI chat adapter; settings; reconnect/offline behavior. -*Deliverable: utterances answered by the user's own agent, which can also -call glasses tools mid-task.* +**Phase 3 — external bridge.** ✅ Landed 2026-08-01 (see the status note at +the top for the architecture change: bridge server == OpenClaw plugin). +Phone side: `app/assistant/bridge-client.ts` (`assistantBridge` singleton — +dial-out FaceclawWebSocket, hello/token auth, auto-reconnect with 1s→60s +backoff, started at boot by the dashboard controller when configured), +`app/assistant/mcp-server.ts` (MCP over the `mcp` channel: initialize / +tools/list / tools/call from the ToolRegistry, list_changed on +onToolsChanged, proactive gating + 6/min rate limit enforced on-phone), +external mode in `AssistantSession` (`AssistantBackendConfig` union selects +direct vs external; external history lives on the agent's machine), settings +(`assistant.backend`, `assistant.bridgeHost/Port/Token`, +`assistant.allowProactive`) in a new Assistant settings section. +Server side: `~/repositories/faceclaw-agent-bridge` (own repo/package). +*Verified against the live OpenClaw on serac with a simulated phone +(`test/fake-phone.js`): auth, streamed turns, and both tool directions +work; a real model turn needs the OpenClaw instance to have provider auth.* -**Phase 4 — OpenClaw/Hermes adapters + proactive polish.** -Native chat adapters for both agents; proactive gating + rate limits; -`tools/list_changed` on foreground swap; session persistence across -reconnects. +**Phase 4 — OpenClaw/Hermes adapters + proactive polish.** Partially landed +2026-08-01: the OpenClaw adapter is the plugin itself (no separate chat +adapter needed); proactive gating + rate limits and `tools/list_changed` +are done. Remaining: Hermes (or generic-agent) support — likely a +standalone bridge speaking the same wire protocol with an MCP face + +CLI chat adapter; session persistence questions across reconnects +(currently: OpenClaw session is persistent, phone session is UI-only). + +**Hardware-test TODO (external mode):** end-to-end on the real phone + +glasses against serac — connect status, a streamed turn, cancel, follow-up, +proactive `glasses.show_alert` from another OpenClaw channel, and behavior +across phone network transitions (the reconnect backoff is untested on +device). ## Open questions (deferred, not blockers):