Merge commit 'd9924ee' into hermes-integration

This commit is contained in:
James Babcock committed 2026-10-17 13:45:43 -07:00
commit 825559aa82
9 files changed
+297 -4

No files matched your search

+9 -2
View File
@@ -1,4 +1,5 @@
import { streamAnthropicMessage } from "../native/anthropic";
import { streamHermesChat } from "../native/hermes";
import { streamLocalQwen } from "../native/llama";
import { streamOpenAiResponse } from "../native/openai";
import type {
@@ -30,6 +31,9 @@ export type DirectTurnOptions = {
apiKey: string;
model: string;
effort?: "low" | "medium" | "high";
/** Hermes: the API root, and the server-side session this conversation maps to. */
baseUrl?: string;
sessionId?: string;
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: LlmMessage[];
@@ -79,7 +83,8 @@ export class DirectAssistantBackend {
system: options.system,
effort: options.effort,
messages: options.messages,
tools: options.buildTools(),
// Hermes runs its own tools; Faceclaw's are not offered to it.
tools: options.provider === "hermes" ? [] : options.buildTools(),
onTextDelta: (delta, textSoFar) => {
if (cancelled) return;
options.callbacks.onTextDelta(delta, priorText + textSoFar);
@@ -128,7 +133,9 @@ export class DirectAssistantBackend {
},
};
streamHandle =
options.provider === "openai"
options.provider === "hermes"
? streamHermesChat({ ...streamOptions, baseUrl: options.baseUrl ?? "", sessionId: options.sessionId ?? "" })
: options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: options.provider === "local"
? streamLocalQwen(streamOptions)
+14 -1
View File
@@ -10,6 +10,7 @@ export const ASSISTANT_MODEL_VALUES = [
"terra",
"sol",
"qwen",
"hermes",
] as const;
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
@@ -20,11 +21,14 @@ export function supportedAssistantModel(value: AssistantModel): AssistantModel {
return global.isIOS && value === "qwen" ? "auto" : value;
}
export type AssistantProvider = "anthropic" | "openai" | "local";
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
export type AssistantApiKeys = {
anthropic: string;
openai: string;
/** A Hermes Agent API server; only the voice assistant passes these. */
hermesUrl?: string;
hermesKey?: string;
};
export type ResolvedAssistantModel = {
@@ -33,6 +37,8 @@ export type ResolvedAssistantModel = {
model: string;
apiKey: string;
effort?: "low" | "medium" | "high";
/** API root for providers that are not fixed services (Hermes). */
baseUrl?: string;
};
type ModelDefinition = {
@@ -55,6 +61,8 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
// not an API key. The default for users who haven't set any key.
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
// Your own Hermes Agent (its API server), with its own tools and memory.
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
};
export function assistantModelLabel(value: AssistantModel): string {
@@ -93,6 +101,11 @@ export function resolveAssistantModel(
}
const definition = MODEL_DEFINITIONS[resolvedSelection as Exclude<AssistantModel, "auto">];
if (definition.provider === "hermes") {
const baseUrl = keys.hermesUrl?.trim() ?? "";
if (!baseUrl) return null;
return { selection, provider: "hermes", model: definition.model, apiKey: keys.hermesKey?.trim() ?? "", baseUrl };
}
if (definition.provider === "local") {
if (!isLocalModelReady()) return null;
return {
+11 -1
View File
@@ -32,6 +32,11 @@ export type AssistantBackendConfig =
/** Trim history from the head once it grows past this many messages. */
const MAX_HISTORY_MESSAGES = 40;
/** Hermes brings its own persona and tools; it only needs to know where the reply is shown. */
const HERMES_SYSTEM_PROMPT =
"The user is talking to you through smart glasses with a small text display. Reply briefly in plain text, without Markdown.";
export class AssistantSession {
private readonly directBackend = new DirectAssistantBackend();
private readonly messages: LlmMessage[] = [];
@@ -43,6 +48,8 @@ export class AssistantSession {
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
// names, so dotted registry names are sanitized and mapped back on calls.
private readonly toolNameMap = new Map<string, string>();
// The Hermes server-side session for this conversation (Hermes keeps the history).
private readonly hermesSessionId = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`;
constructor(
private config: AssistantBackendConfig,
@@ -173,7 +180,10 @@ export class AssistantSession {
apiKey: llm.apiKey,
model: llm.model,
effort: llm.effort,
system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
baseUrl: llm.baseUrl,
sessionId: this.hermesSessionId,
system: llm.provider === "hermes" ? HERMES_SYSTEM_PROMPT
: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
messages: turnMessages,
buildTools: () => this.buildToolDefinitions(),
registry: this.registry,
+121
View File
@@ -0,0 +1,121 @@
import type { LlmStreamHandle, LlmStreamOptions } from "../assistant/llm-protocol";
import { openSseRequest, type SseListener } from "./sse";
/**
* Streaming client for a Hermes Agent gateway's OpenAI-compatible API server
* (POST {baseUrl}/chat/completions). Hermes is an agent with its own tools and
* memory, so only the latest user message is sent: the X-Hermes-Session-Id header
* keeps the conversation on the server. Faceclaw's own tools are not offered.
*/
export type HermesStreamOptions = LlmStreamOptions & {
baseUrl: string;
sessionId: string;
};
/** The API root from what a user typed: adds /v1 to a bare host. Empty when unusable. */
export function normalizeHermesUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
if (!/^https?:\/\/[^/\s]+/i.test(url)) return "";
url = url.replace(/\/chat\/completions$/i, "");
if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1";
return url;
}
export function streamHermesChat(options: HermesStreamOptions): LlmStreamHandle {
let request: { cancel(): void } | null = null;
let settled = false;
let text = "";
let finishReason: string | null = null;
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
settled = true;
request?.cancel();
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: finishReason === "length" ? "max_tokens" : "end_turn" });
};
const baseUrl = normalizeHermesUrl(options.baseUrl);
const last = [...options.messages].reverse().find((message) => message.role === "user");
const prompt = typeof last?.content === "string" ? last.content : "";
if (!baseUrl || !prompt) {
setTimeout(() => fail(baseUrl ? "Nothing to send to Hermes" : "No Hermes URL set"), 0);
return { cancel: () => { settled = true; } };
}
const messages = [
...(options.system ? [{ role: "system", content: options.system }] : []),
{ role: "user", content: prompt },
];
const body = JSON.stringify({ model: options.model, stream: true, messages });
const headers: Record<string, string> = { "Content-Type": "application/json", "X-Hermes-Session-Id": options.sessionId };
if (options.apiKey.trim()) headers.Authorization = `Bearer ${options.apiKey.trim()}`;
const listener: SseListener = {
onLine: (line: string) => {
if (settled || !line.startsWith("data:")) return;
const payload = line.slice(5).trim();
if (payload === "[DONE]") {
done();
return;
}
let event: any;
try {
event = JSON.parse(payload);
} catch {
return;
}
if (event?.error) {
fail(`Hermes: ${String(event.error.message ?? event.error)}`);
return;
}
const choice = event?.choices?.[0];
// reasoning_content is the agent's thinking; only the answer goes to the glasses.
// Hermes opens its answer with blank lines after the reasoning; drop them.
const delta = text ? choice?.delta?.content : String(choice?.delta?.content ?? "").trimStart();
if (typeof delta === "string" && delta) {
text += delta;
options.onTextDelta?.(delta, text);
}
if (choice?.finish_reason) finishReason = String(choice.finish_reason);
},
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
// Some servers close without [DONE] after the final chunk.
onComplete: () => (finishReason || text ? done() : fail("Hermes stream ended without a reply")),
onFailure: (message: string) => fail(`Hermes connection failed: ${String(message)}`),
};
try {
request = openSseRequest(`${baseUrl}/chat/completions`, body, headers, listener);
} catch (error) {
setTimeout(() => fail(`Hermes request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => { settled = true; } };
}
return {
cancel: () => {
settled = true;
try {
request?.cancel();
} catch {
// ignore
}
},
};
}
function describeHttpError(code: number, body: string): string {
if (code === 401 || code === 403) return "Hermes: the API server rejected the key";
let detail = "";
try {
detail = String(JSON.parse(body)?.error?.message ?? "");
} catch {
// non-JSON error body
}
return `Hermes: HTTP ${code}${detail ? ` (${detail})` : ""}`;
}
+23
View File
@@ -916,6 +916,28 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantHermesUrlSetting = new ConfigSettingString({
id: "assistant-hermes-url",
label: "Hermes URL",
storageKey: "assistant.hermesUrl",
defaultValue: "",
editorTitle: "Hermes API server (e.g. http://host:8642/v1)",
glassesEditTitle: "Edit Hermes URL",
formatValue: (value) => value || "(not set)",
description: "Address of a Hermes Agent gateway's API server, used when Hermes agent is the assistant model.",
});
export const assistantHermesKeySetting = new ConfigSettingString({
id: "assistant-hermes-key",
label: "Hermes key",
storageKey: "assistant.hermesKey",
defaultValue: "",
editorTitle: "Hermes API server key",
glassesEditTitle: "Edit Hermes key",
formatValue: (value) => (value ? `${value.slice(0, 6)}...` : "(not set)"),
description: "The API server's key (API_SERVER_KEY in Hermes's .env).",
});
export const assistantBridgeHostSetting = new ConfigSettingString({
id: "assistant-bridge-host",
label: "Bridge host",
@@ -1061,6 +1083,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
if (provider === "local") return !isLocalModelReady();
if (provider === "hermes") return assistantHermesUrlSetting.get().trim().length === 0;
return false;
},
description:
+4
View File
@@ -36,6 +36,8 @@ import {
uiDepthSetting,
assistantAllowProactiveSetting,
assistantBackendSetting,
assistantHermesKeySetting,
assistantHermesUrlSetting,
assistantBridgeHostSetting,
assistantBridgePortSetting,
assistantBridgeTokenSetting,
@@ -142,6 +144,8 @@ function settingsSections(): SettingsSection[] {
// iOS offers cloud APIs; Android also supports local models and the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
textSettingMenuItem(assistantHermesUrlSetting),
textSettingMenuItem(assistantHermesKeySetting),
...(!global.isIOS ? [localModelMenuItem()] : []),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
+4
View File
@@ -31,6 +31,8 @@ import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
anthropicApiKeySetting,
assistantHermesKeySetting,
assistantHermesUrlSetting,
assistantBackendSetting,
assistantBridgeHostSetting,
assistantBridgePortSetting,
@@ -1686,6 +1688,8 @@ class Shell {
const llm = resolveAssistantModel(model, {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
hermesUrl: assistantHermesUrlSetting.get(),
hermesKey: assistantHermesKeySetting.get(),
});
if (llm && reasoning !== "default" && llm.effort) llm.effort = reasoning;
return llm ? { kind: "direct", llm } : null;
+34
View File
@@ -0,0 +1,34 @@
# Hermes agent as the assistant
The "Hermes agent" assistant model sends what you say to a
[Hermes Agent](https://github.com/NousResearch/hermes-agent) gateway through its
OpenAI-compatible API server and streams the answer back to the glasses. Hermes
runs its own tools and keeps the conversation: each Faceclaw assistant
conversation maps to one Hermes session (`X-Hermes-Session-Id`), and only the
latest message is sent. Faceclaw's own glasses tools are not offered to Hermes.
## Enable the API server in Hermes
In the Hermes config, enable the `api_server` platform and bind it where the
phone can reach it, then set a key in Hermes's `.env` and restart the gateway:
hermes config set platforms.api_server.enabled true
hermes config set platforms.api_server.extra.host <address the phone can reach>
hermes config set platforms.api_server.extra.port 8642
echo "API_SERVER_KEY=$(openssl rand -hex 24)" >> ~/.hermes/.env
Check it with `curl -H "Authorization: Bearer <key>" http://<host>:8642/v1/models`.
## Settings (Settings > Assistant)
- **Assistant model**: Hermes agent. Disabled until the URL is set.
- **Hermes URL**: e.g. `http://host:8642/v1` (a bare host gets `/v1` added).
- **Hermes key**: the `API_SERVER_KEY` value.
On iOS, App Transport Security blocks plain `http://` to IP addresses unless
`App_Resources/iOS/Info.plist` has an `NSExceptionDomains` entry for that host
(`NSExceptionAllowsInsecureHTTPLoads`); otherwise serve the API over https, for
example with `tailscale serve`.
Actions that Hermes holds for approval (`approvals.mode manual`) cannot be
approved from the glasses; approve them from Hermes itself.
+77
View File
@@ -0,0 +1,77 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function hermes() {
const requests = [];
const load = loader({ setTimeout, clearTimeout, JSON, Object, String, Number, Error }, {
'./sse': { openSseRequest: (url, body, headers, listener) => {
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false, cancel() { this.cancelled = true; } };
requests.push(request);
return request;
} },
});
return { ...load('app/native/hermes.ts'), requests };
}
function stream(api, extra = {}) {
const deltas = [], results = [], errors = [];
api.streamHermesChat({
apiKey: 'k', model: 'hermes-agent', system: 'glasses', baseUrl: 'http://hermes-host:8642', sessionId: 'faceclaw-1',
messages: [{ role: 'user', content: 'Remember kiwi.' }, { role: 'assistant', content: 'OK' }, { role: 'user', content: 'Which word?' }],
onTextDelta: (delta) => deltas.push(delta), onDone: (r) => results.push(r), onError: (e) => errors.push(e), ...extra,
});
return { deltas, results, errors, request: api.requests.at(-1) };
}
const chunk = (delta, finish = null) => `data: ${JSON.stringify({ choices: [{ index: 0, delta, finish_reason: finish }] })}`;
test('Hermes gets only the latest user message, the session header and the key', () => {
const api = hermes();
const { request } = stream(api);
assert.equal(request.url, 'http://hermes-host:8642/v1/chat/completions');
assert.equal(request.headers['X-Hermes-Session-Id'], 'faceclaw-1');
assert.equal(request.headers.Authorization, 'Bearer k');
assert.equal(request.body.stream, true);
assert.deepEqual(request.body.messages.map((m) => [m.role, m.content]), [['system', 'glasses'], ['user', 'Which word?']]);
});
test('answer deltas stream, reasoning is not shown, [DONE] finishes the turn', () => {
const api = hermes();
const s = stream(api);
s.request.listener.onLine(chunk({ role: 'assistant' }));
s.request.listener.onLine(chunk({ reasoning_content: 'The word was kiwi.' }));
s.request.listener.onLine(chunk({ content: '\n\n' }));
s.request.listener.onLine(chunk({ content: 'Ki' }));
s.request.listener.onLine(chunk({ content: 'wi' }, 'stop'));
s.request.listener.onLine('data: [DONE]');
assert.deepEqual(s.deltas, ['Ki', 'wi']);
assert.deepEqual(s.results.map((r) => [r.text, r.stopReason]), [['Kiwi', 'end_turn']]);
assert.deepEqual(s.errors, []);
});
test('a rejected key and an empty stream are errors', () => {
const api = hermes();
const rejected = stream(api);
rejected.request.listener.onHttpError(401, '{}');
assert.deepEqual(rejected.errors, ['Hermes: the API server rejected the key']);
const empty = stream(api);
empty.request.listener.onComplete();
assert.deepEqual(empty.errors, ['Hermes stream ended without a reply']);
});
test('URL normalization adds /v1 to a bare host and drops the endpoint path', () => {
const { normalizeHermesUrl } = hermes();
assert.equal(normalizeHermesUrl('http://192.0.2.10:8642'), 'http://192.0.2.10:8642/v1');
assert.equal(normalizeHermesUrl('http://h:8642/v1/chat/completions'), 'http://h:8642/v1');
assert.equal(normalizeHermesUrl('h:8642'), '');
});
test('the Hermes model resolves only with a URL, carrying the key and base URL', () => {
const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
const { resolveAssistantModel } = load('app/assistant/models.ts');
assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null);
const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' });
assert.deepEqual({ ...resolved }, { selection: 'hermes', provider: 'hermes', model: 'hermes-agent', apiKey: 'k', baseUrl: 'http://hermes-host:8642' });
assert.equal(resolveAssistantModel('auto', { anthropic: '', openai: '', hermesUrl: 'http://hermes-host:8642' }), null, 'Auto never picks Hermes');
});