mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
External assistant backend: agent bridge client + MCP server
Phase 3 of the voice-assistant design: assistant.backend=external routes utterances over a dial-out websocket to faceclaw-agent-bridge (an OpenClaw plugin on the user's machine). The phone serves its ToolRegistry as MCP on the same socket, with proactive gating and rate limiting enforced locally. New Assistant settings section with backend/bridge settings. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01XCdkQyB3RUVPTuEmhxVEdp
This commit is contained in:
1 parent
474844f961
commit
48bac85cbe
9 files changed
+722
-56
No files matched your search
@@ -0,0 +1,334 @@
|
||||
import { AssistantMcpServer } from "./mcp-server";
|
||||
import { toolRegistry } from "./tool-registry";
|
||||
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Dial-out websocket client for the external assistant bridge (an OpenClaw
|
||||
* plugin or compatible server on the user's machine; see
|
||||
* notes/voice-assistant-design.md "External mode"). One JSON object per text
|
||||
* frame, three multiplexed channels:
|
||||
*
|
||||
* ctl: hello/hello-ack auth handshake, ping/pong, error
|
||||
* chat: utterance in, streamed reply out (per-turn)
|
||||
* mcp: raw MCP JSON-RPC; the phone is the MCP *server* (ToolRegistry)
|
||||
*
|
||||
* Unlike G2MirrorClient (per-terminal-window, no reconnect), this connection
|
||||
* is a long-lived shell service: it stays up while configured so the remote
|
||||
* agent can make proactive tool calls, and it re-dials with backoff.
|
||||
*/
|
||||
|
||||
const PROTOCOL_VERSION = 1;
|
||||
const RECONNECT_MIN_MS = 1_000;
|
||||
const RECONNECT_MAX_MS = 60_000;
|
||||
/** Backstop on a turn the bridge never finishes (server-side cap is 2 min). */
|
||||
const TURN_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
|
||||
export type AssistantBridgePhase = "idle" | "connecting" | "connected" | "failed";
|
||||
|
||||
export type AssistantBridgeState = {
|
||||
phase: AssistantBridgePhase;
|
||||
status: string;
|
||||
};
|
||||
|
||||
export type AssistantBridgeOptions = {
|
||||
host: string;
|
||||
port: number;
|
||||
token: string;
|
||||
deviceName: string;
|
||||
/** Reads assistant.allowProactive; consulted per proactive tool call. */
|
||||
allowProactive: () => boolean;
|
||||
};
|
||||
|
||||
type ActiveTurn = {
|
||||
turnId: string;
|
||||
callbacks: AssistantTurnCallbacks;
|
||||
textSoFar: string;
|
||||
timer: ReturnType<typeof setTimeout>;
|
||||
};
|
||||
|
||||
export class AssistantBridgeClient {
|
||||
private options: AssistantBridgeOptions | null = null;
|
||||
private ws: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private phase: AssistantBridgePhase = "idle";
|
||||
private status = "Not configured.";
|
||||
private stopped = true;
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private reconnectDelayMs = RECONNECT_MIN_MS;
|
||||
private activeTurn: ActiveTurn | null = null;
|
||||
private turnSeq = 0;
|
||||
private mcpServer: AssistantMcpServer | null = null;
|
||||
private unsubscribeToolsChanged: (() => void) | null = null;
|
||||
private readonly stateListeners = new Set<(state: AssistantBridgeState) => void>();
|
||||
|
||||
state(): AssistantBridgeState {
|
||||
return { phase: this.phase, status: this.status };
|
||||
}
|
||||
|
||||
onStateChange(listener: (state: AssistantBridgeState) => void): () => void {
|
||||
this.stateListeners.add(listener);
|
||||
return () => this.stateListeners.delete(listener);
|
||||
}
|
||||
|
||||
isConnected(): boolean {
|
||||
return this.phase === "connected";
|
||||
}
|
||||
|
||||
/**
|
||||
* Set (or replace) the bridge configuration and connect. Safe to call again
|
||||
* with new settings; the old connection is torn down first.
|
||||
*/
|
||||
configure(options: AssistantBridgeOptions): void {
|
||||
this.stop();
|
||||
this.options = options;
|
||||
this.stopped = false;
|
||||
this.mcpServer = new AssistantMcpServer({
|
||||
send: (msg) => this.send({ chan: "mcp", msg }),
|
||||
isTurnActive: () => this.activeTurn !== null,
|
||||
allowProactive: options.allowProactive,
|
||||
});
|
||||
this.unsubscribeToolsChanged = toolRegistry.onToolsChanged(() => {
|
||||
if (this.phase === "connected") this.mcpServer?.sendToolsChanged();
|
||||
});
|
||||
this.connect();
|
||||
}
|
||||
|
||||
/** Disconnect and stay down until the next configure(). */
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
this.clearReconnectTimer();
|
||||
this.failActiveTurn("Bridge connection closed");
|
||||
if (this.unsubscribeToolsChanged) {
|
||||
this.unsubscribeToolsChanged();
|
||||
this.unsubscribeToolsChanged = null;
|
||||
}
|
||||
this.mcpServer = null;
|
||||
if (this.ws) {
|
||||
try {
|
||||
this.ws.close(1000, "bye");
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
this.ws = null;
|
||||
}
|
||||
this.listenerProxy = null;
|
||||
this.setState("idle", "Not connected.");
|
||||
}
|
||||
|
||||
/**
|
||||
* Send an utterance as a new turn. Streaming replies arrive via callbacks.
|
||||
* A turn already in flight is superseded (the session serializes turns, so
|
||||
* this is just a safety net).
|
||||
*/
|
||||
sendUtterance(
|
||||
text: string,
|
||||
ctx: AssistantContext,
|
||||
callbacks: AssistantTurnCallbacks,
|
||||
): AssistantTurnHandle {
|
||||
if (this.phase !== "connected") {
|
||||
callbacks.onError(`Agent bridge is not connected (${this.status})`);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
this.failActiveTurn("Superseded by a new request");
|
||||
const turnId = `t${++this.turnSeq}`;
|
||||
this.activeTurn = {
|
||||
turnId,
|
||||
callbacks,
|
||||
textSoFar: "",
|
||||
timer: setTimeout(() => {
|
||||
if (this.activeTurn?.turnId !== turnId) return;
|
||||
this.failActiveTurn("The agent took too long to reply");
|
||||
}, TURN_TIMEOUT_MS),
|
||||
};
|
||||
this.send({ chan: "chat", type: "utterance", turnId, text, ctx });
|
||||
return {
|
||||
cancel: () => {
|
||||
if (this.activeTurn?.turnId !== turnId) return;
|
||||
this.send({ chan: "chat", type: "cancel", turnId });
|
||||
this.clearActiveTurn();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
private connect(): void {
|
||||
if (this.stopped || this.ws || !this.options) return;
|
||||
const { host, port } = this.options;
|
||||
const url = `ws://${host}:${port}`;
|
||||
this.setState("connecting", `Connecting to ${host}:${port}...`);
|
||||
this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({
|
||||
onOpen: () => {
|
||||
if (this.stopped) return;
|
||||
this.setState("connecting", "Authenticating...");
|
||||
this.send({
|
||||
chan: "ctl",
|
||||
type: "hello",
|
||||
version: PROTOCOL_VERSION,
|
||||
token: this.options!.token,
|
||||
deviceName: this.options!.deviceName,
|
||||
capabilities: ["chat", "mcp"],
|
||||
});
|
||||
},
|
||||
onTextMessage: (message: string) => {
|
||||
if (this.stopped) return;
|
||||
this.handleMessage(String(message));
|
||||
},
|
||||
onClosed: (code: number, reason: string) => {
|
||||
if (this.stopped) return;
|
||||
this.handleConnectionLost(
|
||||
`Connection closed (${Number(code)}${reason ? `: ${String(reason)}` : ""})`,
|
||||
);
|
||||
},
|
||||
onFailure: (message: string) => {
|
||||
if (this.stopped) return;
|
||||
this.handleConnectionLost(`Connection failed: ${String(message)}`);
|
||||
},
|
||||
});
|
||||
try {
|
||||
this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, null, null);
|
||||
} catch (error) {
|
||||
this.ws = null;
|
||||
this.handleConnectionLost(
|
||||
`Connection failed: ${String((error as Error)?.message ?? error)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
private handleMessage(raw: string): void {
|
||||
let frame: any = null;
|
||||
try {
|
||||
frame = JSON.parse(raw);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (!frame || typeof frame !== "object") return;
|
||||
switch (frame.chan) {
|
||||
case "ctl":
|
||||
this.handleCtl(frame);
|
||||
return;
|
||||
case "chat":
|
||||
this.handleChat(frame);
|
||||
return;
|
||||
case "mcp":
|
||||
this.mcpServer?.handleMessage(frame.msg);
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private handleCtl(frame: any): void {
|
||||
if (frame.type === "hello-ack") {
|
||||
this.reconnectDelayMs = RECONNECT_MIN_MS;
|
||||
this.setState("connected", `Connected to ${String(frame.serverName ?? "bridge")}`);
|
||||
return;
|
||||
}
|
||||
if (frame.type === "ping") {
|
||||
this.send({ chan: "ctl", type: "pong", ts: frame.ts });
|
||||
return;
|
||||
}
|
||||
if (frame.type === "error") {
|
||||
// The server closes after a ctl error; the close handler schedules the
|
||||
// re-dial. Record the reason so the status is more useful than a bare
|
||||
// close code (especially for a bad token).
|
||||
this.status = `Bridge error: ${String(frame.message ?? "unknown")}`;
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private handleChat(frame: any): void {
|
||||
const turn = this.activeTurn;
|
||||
if (!turn || frame.turnId !== turn.turnId) return;
|
||||
switch (frame.type) {
|
||||
case "text-delta": {
|
||||
const text = typeof frame.text === "string" ? frame.text : "";
|
||||
turn.textSoFar = frame.replace === true ? text : turn.textSoFar + text;
|
||||
turn.callbacks.onTextDelta(text, turn.textSoFar);
|
||||
return;
|
||||
}
|
||||
case "tool-activity":
|
||||
turn.callbacks.onToolActivity(String(frame.label ?? "tool"));
|
||||
return;
|
||||
case "turn-done": {
|
||||
this.clearActiveTurn();
|
||||
turn.callbacks.onTurnDone({
|
||||
stopReason: typeof frame.stopReason === "string" ? frame.stopReason : null,
|
||||
});
|
||||
return;
|
||||
}
|
||||
case "turn-error":
|
||||
this.clearActiveTurn();
|
||||
turn.callbacks.onError(String(frame.message ?? "Agent error"));
|
||||
return;
|
||||
default:
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private handleConnectionLost(status: string): void {
|
||||
this.ws = null;
|
||||
this.listenerProxy = null;
|
||||
// Keep a ctl-error status (e.g. "invalid token") in preference to the
|
||||
// generic close message that follows it.
|
||||
const detail = this.status.startsWith("Bridge error:") ? this.status : status;
|
||||
this.failActiveTurn("Bridge connection lost");
|
||||
this.setState("failed", detail);
|
||||
this.scheduleReconnect();
|
||||
}
|
||||
|
||||
private scheduleReconnect(): void {
|
||||
if (this.stopped || this.reconnectTimer) return;
|
||||
const delay = this.reconnectDelayMs;
|
||||
this.reconnectDelayMs = Math.min(this.reconnectDelayMs * 2, RECONNECT_MAX_MS);
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
this.connect();
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private clearReconnectTimer(): void {
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private clearActiveTurn(): void {
|
||||
if (!this.activeTurn) return;
|
||||
clearTimeout(this.activeTurn.timer);
|
||||
this.activeTurn = null;
|
||||
}
|
||||
|
||||
private failActiveTurn(message: string): void {
|
||||
const turn = this.activeTurn;
|
||||
if (!turn) return;
|
||||
this.clearActiveTurn();
|
||||
turn.callbacks.onError(message);
|
||||
}
|
||||
|
||||
private send(frame: object): void {
|
||||
if (!this.ws) return;
|
||||
try {
|
||||
this.ws.sendText(JSON.stringify({ v: PROTOCOL_VERSION, ...frame }));
|
||||
} catch {
|
||||
// A send on a dying socket; the close/failure callback handles recovery.
|
||||
}
|
||||
}
|
||||
|
||||
private setState(phase: AssistantBridgePhase, status: string): void {
|
||||
this.phase = phase;
|
||||
this.status = status;
|
||||
const state = this.state();
|
||||
for (const listener of this.stateListeners) {
|
||||
try {
|
||||
listener(state);
|
||||
} catch {
|
||||
// listeners must not break the client
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Process-wide bridge client (main isolate), configured by the dashboard controller. */
|
||||
export const assistantBridge = new AssistantBridgeClient();
|
||||
@@ -0,0 +1,126 @@
|
||||
import { toolRegistry, type ToolRegistry } from "./tool-registry";
|
||||
|
||||
/**
|
||||
* Minimal MCP (Model Context Protocol) server surface over the assistant
|
||||
* bridge's `mcp` channel. The phone serves its ToolRegistry to the external
|
||||
* agent: `tools/list`, `tools/call`, and `notifications/tools/list_changed`
|
||||
* (emitted by the bridge client when the registry changes).
|
||||
*
|
||||
* Transport-agnostic: the bridge client hands decoded JSON-RPC messages to
|
||||
* handleMessage and provides `send` to deliver replies/notifications.
|
||||
*/
|
||||
|
||||
const MCP_PROTOCOL_VERSION = "2025-06-18";
|
||||
|
||||
/** Proactive display actions are rate-limited to this many per minute. */
|
||||
const PROACTIVE_CALLS_PER_MINUTE = 6;
|
||||
|
||||
export type McpServerOptions = {
|
||||
send: (msg: object) => void;
|
||||
/** Whether a voice turn is currently in flight (calls outside one are "proactive"). */
|
||||
isTurnActive: () => boolean;
|
||||
/** Master setting gate for proactive calls (assistant.allowProactive). */
|
||||
allowProactive: () => boolean;
|
||||
registry?: ToolRegistry;
|
||||
};
|
||||
|
||||
export class AssistantMcpServer {
|
||||
private readonly registry: ToolRegistry;
|
||||
private readonly proactiveCallTimes: number[] = [];
|
||||
|
||||
constructor(private readonly options: McpServerOptions) {
|
||||
this.registry = options.registry ?? toolRegistry;
|
||||
}
|
||||
|
||||
/** Notify the agent that the live tool set changed. */
|
||||
sendToolsChanged(): void {
|
||||
this.options.send({ jsonrpc: "2.0", method: "notifications/tools/list_changed" });
|
||||
}
|
||||
|
||||
handleMessage(msg: any): void {
|
||||
if (!msg || msg.jsonrpc !== "2.0") return;
|
||||
if (typeof msg.method !== "string") return; // responses to server-sent requests: none yet
|
||||
const id = msg.id;
|
||||
const isNotification = id === undefined || id === null;
|
||||
|
||||
switch (msg.method) {
|
||||
case "initialize":
|
||||
this.reply(id, {
|
||||
protocolVersion:
|
||||
typeof msg.params?.protocolVersion === "string"
|
||||
? msg.params.protocolVersion
|
||||
: MCP_PROTOCOL_VERSION,
|
||||
capabilities: { tools: { listChanged: true } },
|
||||
serverInfo: { name: "faceclaw", version: "0.3.0" },
|
||||
});
|
||||
return;
|
||||
case "notifications/initialized":
|
||||
return;
|
||||
case "ping":
|
||||
this.reply(id, {});
|
||||
return;
|
||||
case "tools/list":
|
||||
this.reply(id, {
|
||||
tools: this.registry.listTools().map((spec) => ({
|
||||
name: spec.name,
|
||||
description: spec.description,
|
||||
inputSchema: spec.inputSchema,
|
||||
})),
|
||||
});
|
||||
return;
|
||||
case "tools/call":
|
||||
void this.handleToolCall(id, msg.params);
|
||||
return;
|
||||
default:
|
||||
if (!isNotification) {
|
||||
this.options.send({
|
||||
jsonrpc: "2.0",
|
||||
id,
|
||||
error: { code: -32601, message: `Method not supported: ${msg.method}` },
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
private async handleToolCall(id: unknown, params: any): Promise<void> {
|
||||
const name = typeof params?.name === "string" ? params.name : "";
|
||||
const args = params?.arguments ?? {};
|
||||
// A call while no voice turn is in flight is a proactive action by the
|
||||
// remote agent; the phone (never the bridge) enforces the gating.
|
||||
const proactive = !this.options.isTurnActive();
|
||||
if (proactive && !this.options.allowProactive()) {
|
||||
this.replyToolError(id, "Proactive assistant actions are disabled in Settings");
|
||||
return;
|
||||
}
|
||||
if (proactive && !this.admitProactiveCall()) {
|
||||
this.replyToolError(id, "Proactive action rate limit exceeded; try again later");
|
||||
return;
|
||||
}
|
||||
const result = await this.registry.callTool(name, args, { proactive });
|
||||
this.reply(id, {
|
||||
content: [{ type: "text", text: result.ok ? result.content ?? "" : result.error ?? "Tool error" }],
|
||||
isError: !result.ok,
|
||||
});
|
||||
}
|
||||
|
||||
/** Sliding-window rate limiter for proactive calls. */
|
||||
private admitProactiveCall(): boolean {
|
||||
const now = Date.now();
|
||||
while (this.proactiveCallTimes.length && now - this.proactiveCallTimes[0]! > 60_000) {
|
||||
this.proactiveCallTimes.shift();
|
||||
}
|
||||
if (this.proactiveCallTimes.length >= PROACTIVE_CALLS_PER_MINUTE) return false;
|
||||
this.proactiveCallTimes.push(now);
|
||||
return true;
|
||||
}
|
||||
|
||||
private reply(id: unknown, result: object): void {
|
||||
if (id === undefined || id === null) return;
|
||||
this.options.send({ jsonrpc: "2.0", id, result });
|
||||
}
|
||||
|
||||
private replyToolError(id: unknown, text: string): void {
|
||||
this.reply(id, { content: [{ type: "text", text }], isError: true });
|
||||
}
|
||||
}
|
||||
+66
-35
@@ -1,20 +1,33 @@
|
||||
import { buildAssistantSystemPrompt } from "../prompts";
|
||||
import { assistantBridge } from "./bridge-client";
|
||||
import { DirectAssistantBackend } from "./direct-backend";
|
||||
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
|
||||
import type { ResolvedAssistantModel } from "./models";
|
||||
import { toolRegistry, type ToolRegistry } from "./tool-registry";
|
||||
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
|
||||
import type {
|
||||
AssistantBridgeConfig,
|
||||
AssistantContext,
|
||||
AssistantTurnCallbacks,
|
||||
AssistantTurnHandle,
|
||||
} from "./types";
|
||||
|
||||
/**
|
||||
* One assistant conversation. Owns the message history and turn state and
|
||||
* drives a direct provider backend. The UI talks only to
|
||||
* this surface, so swapping in the external-agent backend later doesn't touch
|
||||
* AssistantLayer.
|
||||
* drives either the on-phone provider loop (direct) or the external agent
|
||||
* bridge. The UI talks only to this surface, so AssistantLayer is
|
||||
* backend-agnostic.
|
||||
*
|
||||
* Sessions are cheap and idle-expire (see isExpired): the shell starts a fresh
|
||||
* one for a new wakeword and reuses the current one for a follow-up.
|
||||
* one for a new wakeword and reuses the current one for a follow-up. In
|
||||
* external mode the conversation history lives on the agent's machine (one
|
||||
* long-lived session there), so expiry here only affects the overlay UI.
|
||||
*/
|
||||
|
||||
/** Which engine answers utterances, plus what it needs to do so. */
|
||||
export type AssistantBackendConfig =
|
||||
| { kind: "direct"; llm: ResolvedAssistantModel }
|
||||
| { kind: "external"; bridge: AssistantBridgeConfig };
|
||||
|
||||
/** A new PTT after this much idle starts a fresh conversation. */
|
||||
const SESSION_IDLE_MS = 10 * 60 * 1000;
|
||||
|
||||
@@ -22,7 +35,7 @@ const SESSION_IDLE_MS = 10 * 60 * 1000;
|
||||
const MAX_HISTORY_MESSAGES = 40;
|
||||
|
||||
export class AssistantSession {
|
||||
private readonly backend = new DirectAssistantBackend();
|
||||
private readonly directBackend = new DirectAssistantBackend();
|
||||
private readonly messages: LlmMessage[] = [];
|
||||
private turnHandle: AssistantTurnHandle | null = null;
|
||||
private lastActivityMs = Date.now();
|
||||
@@ -31,16 +44,27 @@ export class AssistantSession {
|
||||
private readonly toolNameMap = new Map<string, string>();
|
||||
|
||||
constructor(
|
||||
private readonly llm: ResolvedAssistantModel,
|
||||
private readonly config: AssistantBackendConfig,
|
||||
private readonly registry: ToolRegistry = toolRegistry,
|
||||
) {}
|
||||
|
||||
matchesConfiguration(llm: ResolvedAssistantModel): boolean {
|
||||
return (
|
||||
this.llm.provider === llm.provider &&
|
||||
this.llm.model === llm.model &&
|
||||
this.llm.apiKey === llm.apiKey
|
||||
);
|
||||
matchesConfiguration(config: AssistantBackendConfig): boolean {
|
||||
if (this.config.kind !== config.kind) return false;
|
||||
if (this.config.kind === "direct" && config.kind === "direct") {
|
||||
return (
|
||||
this.config.llm.provider === config.llm.provider &&
|
||||
this.config.llm.model === config.llm.model &&
|
||||
this.config.llm.apiKey === config.llm.apiKey
|
||||
);
|
||||
}
|
||||
if (this.config.kind === "external" && config.kind === "external") {
|
||||
return (
|
||||
this.config.bridge.host === config.bridge.host &&
|
||||
this.config.bridge.port === config.bridge.port &&
|
||||
this.config.bridge.token === config.bridge.token
|
||||
);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Whether this session has been idle long enough to retire. */
|
||||
@@ -59,38 +83,45 @@ export class AssistantSession {
|
||||
return;
|
||||
}
|
||||
this.lastActivityMs = Date.now();
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
|
||||
const system = buildAssistantSystemPrompt(ctx);
|
||||
|
||||
const finish = () => {
|
||||
this.turnHandle = null;
|
||||
this.lastActivityMs = Date.now();
|
||||
};
|
||||
const wrappedCallbacks: AssistantTurnCallbacks = {
|
||||
onTextDelta: callbacks.onTextDelta,
|
||||
onToolActivity: callbacks.onToolActivity,
|
||||
onTurnDone: (result) => {
|
||||
finish();
|
||||
callbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
finish();
|
||||
callbacks.onError(message);
|
||||
},
|
||||
};
|
||||
|
||||
this.turnHandle = this.backend.runTurn({
|
||||
provider: this.llm.provider,
|
||||
apiKey: this.llm.apiKey,
|
||||
model: this.llm.model,
|
||||
effort: this.llm.effort,
|
||||
system,
|
||||
if (this.config.kind === "external") {
|
||||
// History and the agent loop live on the agent's machine; the phone just
|
||||
// streams this turn. The overlay keeps its own display state.
|
||||
this.turnHandle = assistantBridge.sendUtterance(text, ctx, wrappedCallbacks);
|
||||
return;
|
||||
}
|
||||
|
||||
this.messages.push({ role: "user", content: text });
|
||||
this.trimHistory();
|
||||
const llm = this.config.llm;
|
||||
this.turnHandle = this.directBackend.runTurn({
|
||||
provider: llm.provider,
|
||||
apiKey: llm.apiKey,
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
system: buildAssistantSystemPrompt(ctx),
|
||||
messages: this.messages,
|
||||
buildTools: () => this.buildToolDefinitions(),
|
||||
registry: this.registry,
|
||||
resolveToolName: (apiName) => this.toolNameMap.get(apiName) ?? apiName,
|
||||
callbacks: {
|
||||
onTextDelta: callbacks.onTextDelta,
|
||||
onToolActivity: callbacks.onToolActivity,
|
||||
onTurnDone: (result) => {
|
||||
finish();
|
||||
callbacks.onTurnDone(result);
|
||||
},
|
||||
onError: (message) => {
|
||||
finish();
|
||||
callbacks.onError(message);
|
||||
},
|
||||
},
|
||||
callbacks: wrappedCallbacks,
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -30,3 +30,10 @@ export type AssistantTurnCallbacks = {
|
||||
export type AssistantTurnHandle = {
|
||||
cancel(): void;
|
||||
};
|
||||
|
||||
/** Connection settings for the external agent bridge (assistant.bridge* settings). */
|
||||
export type AssistantBridgeConfig = {
|
||||
host: string;
|
||||
port: number;
|
||||
token: string;
|
||||
};
|
||||
@@ -27,6 +27,7 @@ import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image";
|
||||
import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell";
|
||||
import { registerSystemTools } from "../assistant/system-tools";
|
||||
import { registerNavigateTools } from "../assistant/navigate-tools";
|
||||
import { assistantBridge } from "../assistant/bridge-client";
|
||||
import { registerWindowTools } from "../assistant/window-tools";
|
||||
import { WorkerAppHost } from "../ui/shell/worker-window";
|
||||
import {
|
||||
@@ -83,6 +84,11 @@ import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-a
|
||||
import { appViewportRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type LayerActions } from "../ui/layers";
|
||||
import {
|
||||
assistantAllowProactiveSetting,
|
||||
assistantBackendSetting,
|
||||
assistantBridgeHostSetting,
|
||||
assistantBridgePortSetting,
|
||||
assistantBridgeTokenSetting,
|
||||
brightnessSetting,
|
||||
brightnessSettingToLevel,
|
||||
elevenLabsApiKeySetting,
|
||||
@@ -326,6 +332,46 @@ class DashboardController {
|
||||
this.pushBrightness();
|
||||
this.syncEvenHubScreenOffSetting();
|
||||
this.applyVerticalPositionIfChanged();
|
||||
this.syncAssistantBridgeIfChanged();
|
||||
});
|
||||
// Connect to the external agent bridge at boot if configured; the
|
||||
// connection stays up (with re-dial) so proactive tool calls work
|
||||
// outside voice turns.
|
||||
this.syncAssistantBridge();
|
||||
}
|
||||
|
||||
// Bridge settings changes re-dial the connection; unrelated setting changes
|
||||
// must not, so the config it was last started with is tracked.
|
||||
private lastBridgeConfigKey = "";
|
||||
|
||||
private bridgeConfigKey(): string {
|
||||
return JSON.stringify([
|
||||
assistantBackendSetting.get(),
|
||||
assistantBridgeHostSetting.get().trim(),
|
||||
assistantBridgePortSetting.get().trim(),
|
||||
assistantBridgeTokenSetting.get(),
|
||||
]);
|
||||
}
|
||||
|
||||
private syncAssistantBridgeIfChanged(): void {
|
||||
if (this.bridgeConfigKey() === this.lastBridgeConfigKey) return;
|
||||
this.syncAssistantBridge();
|
||||
}
|
||||
|
||||
private syncAssistantBridge(): void {
|
||||
this.lastBridgeConfigKey = this.bridgeConfigKey();
|
||||
const host = assistantBridgeHostSetting.get().trim();
|
||||
const token = assistantBridgeTokenSetting.get();
|
||||
if (assistantBackendSetting.get() !== "external" || !host || !token) {
|
||||
assistantBridge.stop();
|
||||
return;
|
||||
}
|
||||
assistantBridge.configure({
|
||||
host,
|
||||
port: parseInt(assistantBridgePortSetting.get(), 10) || 8790,
|
||||
token,
|
||||
deviceName: "faceclaw",
|
||||
allowProactive: () => assistantAllowProactiveSetting.get(),
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -350,6 +350,65 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
|
||||
description: "After a wakeword utterance, send the transcript straight to the assistant instead of stopping at the Send/Type confirmation menu.",
|
||||
});
|
||||
|
||||
export type AssistantBackendKind = "direct" | "external";
|
||||
|
||||
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
|
||||
direct: "On-phone (API key)",
|
||||
external: "My own agent (bridge)",
|
||||
};
|
||||
|
||||
export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKind>({
|
||||
id: "assistant-backend",
|
||||
label: "Assistant backend",
|
||||
storageKey: "assistant.backend",
|
||||
defaultValue: "direct",
|
||||
values: ["direct", "external"],
|
||||
formatValue: (value) => assistantBackendLabels[value] ?? value,
|
||||
description:
|
||||
"Who answers assistant queries: an LLM called directly from the phone (needs an API key), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
|
||||
});
|
||||
|
||||
export const assistantBridgeHostSetting = new ConfigSettingString({
|
||||
id: "assistant-bridge-host",
|
||||
label: "Bridge host",
|
||||
storageKey: "assistant.bridgeHost",
|
||||
defaultValue: "",
|
||||
editorTitle: "Agent bridge host (tailscale IP)",
|
||||
glassesEditTitle: "Edit bridge host",
|
||||
description:
|
||||
"Hostname or IP address (e.g. a Tailscale address) of the machine running the agent bridge.",
|
||||
});
|
||||
|
||||
export const assistantBridgePortSetting = new ConfigSettingString({
|
||||
id: "assistant-bridge-port",
|
||||
label: "Bridge port",
|
||||
storageKey: "assistant.bridgePort",
|
||||
defaultValue: "8790",
|
||||
editorTitle: "Agent bridge port",
|
||||
glassesEditTitle: "Edit bridge port",
|
||||
description: "TCP port the agent bridge listens on. The default is 8790.",
|
||||
});
|
||||
|
||||
export const assistantBridgeTokenSetting = new ConfigSettingString({
|
||||
id: "assistant-bridge-token",
|
||||
label: "Bridge token",
|
||||
storageKey: "assistant.bridgeToken",
|
||||
defaultValue: "",
|
||||
editorTitle: "Agent bridge auth token",
|
||||
glassesEditTitle: "Edit bridge token",
|
||||
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
|
||||
description: "Shared secret that must match the bridge's configured token.",
|
||||
});
|
||||
|
||||
export const assistantAllowProactiveSetting = new ConfigSettingBoolean({
|
||||
id: "assistant-allow-proactive",
|
||||
label: "Allow proactive agent actions",
|
||||
storageKey: "assistant.allowProactive",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Let the external agent use glasses tools outside of a conversation, e.g. showing an alert when a long-running job finishes. Rate-limited; only tools marked proactive-safe are allowed.",
|
||||
});
|
||||
|
||||
export const elevenLabsApiKeySetting = new ConfigSettingString({
|
||||
id: "elevenlabs-api-key",
|
||||
label: "ElevenLabs key",
|
||||
|
||||
@@ -8,6 +8,11 @@ import type { MenuItem } from "../menu";
|
||||
import { shell } from "../shell/shell";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantAllowProactiveSetting,
|
||||
assistantBackendSetting,
|
||||
assistantBridgeHostSetting,
|
||||
assistantBridgePortSetting,
|
||||
assistantBridgeTokenSetting,
|
||||
assistantModelSetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
@@ -69,10 +74,21 @@ function settingsSections(): SettingsSection[] {
|
||||
items: [
|
||||
enumSettingMenuItem(wakeWordActionSetting),
|
||||
enumSettingMenuItem(voiceProviderSetting),
|
||||
],
|
||||
},
|
||||
{
|
||||
label: "Assistant",
|
||||
items: [
|
||||
// On-phone LLM loop vs the user's own agent via the bridge plugin.
|
||||
enumSettingMenuItem(assistantBackendSetting),
|
||||
enumSettingMenuItem(assistantModelSetting),
|
||||
// When on, a wakeword utterance goes straight to the assistant with no
|
||||
// Send/Type menu step.
|
||||
toggleSettingMenuItem(assistantSkipConfirmationSetting),
|
||||
textSettingMenuItem(assistantBridgeHostSetting),
|
||||
textSettingMenuItem(assistantBridgePortSetting),
|
||||
textSettingMenuItem(assistantBridgeTokenSetting),
|
||||
toggleSettingMenuItem(assistantAllowProactiveSetting),
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
+25
-9
@@ -6,12 +6,16 @@ import { DashboardInputEvent, Layer, LayerActions, LayerContext, LayerStack, noo
|
||||
import { MenuLayer, type MenuItem } from "../menu";
|
||||
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
|
||||
import { AssistantLayer } from "../apps/assistant";
|
||||
import { AssistantSession } from "../../assistant/session";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { AssistantSession, type AssistantBackendConfig } from "../../assistant/session";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantContext } from "../../assistant/types";
|
||||
import { SingleNotificationLayer } from "../notifications";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantBackendSetting,
|
||||
assistantBridgeHostSetting,
|
||||
assistantBridgePortSetting,
|
||||
assistantBridgeTokenSetting,
|
||||
assistantModelSetting,
|
||||
assistantSkipConfirmationSetting,
|
||||
batteryDisplayModeSetting,
|
||||
@@ -722,24 +726,32 @@ class Shell {
|
||||
}
|
||||
|
||||
private ensureAssistantSession(): AssistantSession | null {
|
||||
const llm = this.resolveAssistantConfiguration();
|
||||
if (!llm) return null;
|
||||
const config = this.resolveAssistantConfiguration();
|
||||
if (!config) return null;
|
||||
if (
|
||||
!this.assistantSession ||
|
||||
this.assistantSession.isExpired() ||
|
||||
!this.assistantSession.matchesConfiguration(llm)
|
||||
!this.assistantSession.matchesConfiguration(config)
|
||||
) {
|
||||
this.assistantSession?.cancel();
|
||||
this.assistantSession = new AssistantSession(llm);
|
||||
this.assistantSession = new AssistantSession(config);
|
||||
}
|
||||
return this.assistantSession;
|
||||
}
|
||||
|
||||
private resolveAssistantConfiguration(): ResolvedAssistantModel | null {
|
||||
return resolveAssistantModel(assistantModelSetting.get(), {
|
||||
private resolveAssistantConfiguration(): AssistantBackendConfig | null {
|
||||
if (assistantBackendSetting.get() === "external") {
|
||||
const host = assistantBridgeHostSetting.get().trim();
|
||||
const token = assistantBridgeTokenSetting.get();
|
||||
if (!host || !token) return null;
|
||||
const port = parseInt(assistantBridgePortSetting.get(), 10) || 8790;
|
||||
return { kind: "external", bridge: { host, port, token } };
|
||||
}
|
||||
const llm = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
});
|
||||
return llm ? { kind: "direct", llm } : null;
|
||||
}
|
||||
|
||||
private buildAssistantContext(): AssistantContext {
|
||||
@@ -761,7 +773,11 @@ class Shell {
|
||||
sendToAssistant(text: string): void {
|
||||
const session = this.ensureAssistantSession();
|
||||
if (!session) {
|
||||
this.showAlert("Set an API key for the selected assistant model in Settings.");
|
||||
this.showAlert(
|
||||
assistantBackendSetting.get() === "external"
|
||||
? "Configure the agent bridge host and token in Settings."
|
||||
: "Set an API key for the selected assistant model in Settings.",
|
||||
);
|
||||
return;
|
||||
}
|
||||
if (!this.screenOn) this.wake("sidebar");
|
||||
|
||||
@@ -1,8 +1,20 @@
|
||||
# Voice Assistant Architecture
|
||||
|
||||
Status: design draft, 2026-07-19; revised 2026-07-24 (tool availability
|
||||
ontology, misc fixes). **Phases 1 and 2 are implemented** (see the
|
||||
Implementation plan below); phases 3-4 are still design-only.
|
||||
ontology, misc fixes); revised 2026-08-01 (external bridge landed).
|
||||
**Phases 1 and 2 are implemented**, and **phase 3 + the OpenClaw part of
|
||||
phase 4 landed 2026-08-01** with one architecture change: the bridge server
|
||||
is not a standalone package — it *is* the OpenClaw plugin
|
||||
(`~/repositories/faceclaw-agent-bridge`, installed on the OpenClaw host).
|
||||
The plugin hosts the websocket endpoint the phone dials into, runs
|
||||
utterances as embedded OpenClaw agent turns (`runEmbeddedAgent` with a
|
||||
dedicated session, default key `faceclaw:glasses`), and exposes the
|
||||
glasses' MCP-served tools to the agent via two fixed meta-tools
|
||||
(`glasses_list_tools` / `glasses_call` — fixed names because OpenClaw's
|
||||
manifest `contracts.tools` wants static declarations, while the glasses
|
||||
toolset is dynamic). The generic "MCP face for any agent" from the original
|
||||
design is unbuilt; the wire protocol is unchanged, so a standalone bridge
|
||||
for other agents can be added later without phone changes.
|
||||
|
||||
## Goal
|
||||
|
||||
@@ -404,17 +416,36 @@ in-process-window tool registration (only worker apps wired); the per-window
|
||||
same-name collision handling (the terminal sidesteps it by declaring all tools
|
||||
on the single hub window).
|
||||
|
||||
**Phase 3 — external bridge.**
|
||||
Bridge wire protocol; ExternalAgentBackend (phone side, reusing the
|
||||
FaceclawWebSocket plumbing); `faceclaw-agent-bridge` server with the MCP
|
||||
face + fallback CLI chat adapter; settings; reconnect/offline behavior.
|
||||
*Deliverable: utterances answered by the user's own agent, which can also
|
||||
call glasses tools mid-task.*
|
||||
**Phase 3 — external bridge.** ✅ Landed 2026-08-01 (see the status note at
|
||||
the top for the architecture change: bridge server == OpenClaw plugin).
|
||||
Phone side: `app/assistant/bridge-client.ts` (`assistantBridge` singleton —
|
||||
dial-out FaceclawWebSocket, hello/token auth, auto-reconnect with 1s→60s
|
||||
backoff, started at boot by the dashboard controller when configured),
|
||||
`app/assistant/mcp-server.ts` (MCP over the `mcp` channel: initialize /
|
||||
tools/list / tools/call from the ToolRegistry, list_changed on
|
||||
onToolsChanged, proactive gating + 6/min rate limit enforced on-phone),
|
||||
external mode in `AssistantSession` (`AssistantBackendConfig` union selects
|
||||
direct vs external; external history lives on the agent's machine), settings
|
||||
(`assistant.backend`, `assistant.bridgeHost/Port/Token`,
|
||||
`assistant.allowProactive`) in a new Assistant settings section.
|
||||
Server side: `~/repositories/faceclaw-agent-bridge` (own repo/package).
|
||||
*Verified against the live OpenClaw on serac with a simulated phone
|
||||
(`test/fake-phone.js`): auth, streamed turns, and both tool directions
|
||||
work; a real model turn needs the OpenClaw instance to have provider auth.*
|
||||
|
||||
**Phase 4 — OpenClaw/Hermes adapters + proactive polish.**
|
||||
Native chat adapters for both agents; proactive gating + rate limits;
|
||||
`tools/list_changed` on foreground swap; session persistence across
|
||||
reconnects.
|
||||
**Phase 4 — OpenClaw/Hermes adapters + proactive polish.** Partially landed
|
||||
2026-08-01: the OpenClaw adapter is the plugin itself (no separate chat
|
||||
adapter needed); proactive gating + rate limits and `tools/list_changed`
|
||||
are done. Remaining: Hermes (or generic-agent) support — likely a
|
||||
standalone bridge speaking the same wire protocol with an MCP face +
|
||||
CLI chat adapter; session persistence questions across reconnects
|
||||
(currently: OpenClaw session is persistent, phone session is UI-only).
|
||||
|
||||
**Hardware-test TODO (external mode):** end-to-end on the real phone +
|
||||
glasses against serac — connect status, a streamed turn, cancel, follow-up,
|
||||
proactive `glasses.show_alert` from another OpenClaw channel, and behavior
|
||||
across phone network transitions (the reconnect backoff is untested on
|
||||
device).
|
||||
|
||||
## Open questions (deferred, not blockers):
|
||||
|
||||
|
||||
Reference in new issue
Block a user