Add a model picker and support for the OpenAI API

This commit is contained in:
James Babcock committed 2026-07-26 13:09:28 -07:00
1 parent 1c2e445519
commit ccdeedd687
13 files changed
+688 -131

No files matched your search

+25 -18
View File
@@ -1,10 +1,13 @@
import {
streamAnthropicMessage,
type AnthropicContentBlock,
type AnthropicMessage,
type AnthropicStreamHandle,
type AnthropicToolDefinition,
} from "../native/anthropic";
import { streamAnthropicMessage } from "../native/anthropic";
import { streamOpenAiResponse } from "../native/openai";
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
LlmToolDefinition,
} from "./llm-protocol";
import type { AssistantProvider } from "./models";
import type { ToolRegistry } from "./tool-registry";
import type { AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
@@ -20,29 +23,30 @@ const MAX_ITERATIONS = 8;
const MAX_TURN_MS = 2 * 60 * 1000;
export type DirectTurnOptions = {
provider: AssistantProvider;
apiKey: string;
model: string;
effort?: "low" | "medium" | "high";
system: string;
/** Owned by the session; this loop appends assistant + tool_result messages. */
messages: AnthropicMessage[];
messages: LlmMessage[];
/**
* Build the tool list for the next request. Called once per loop iteration so
* the set reflects apps opening/closing and foreground changes mid-turn; it
* also refreshes the session's api-name map that resolveToolName reads.
*/
buildTools: () => AnthropicToolDefinition[];
buildTools: () => LlmToolDefinition[];
registry: ToolRegistry;
/** Map an API tool name (as the model calls it) back to its registry name. */
resolveToolName: (apiName: string) => string;
callbacks: AssistantTurnCallbacks;
};
export class DirectAnthropicBackend {
export class DirectAssistantBackend {
runTurn(options: DirectTurnOptions): AssistantTurnHandle {
const startMs = Date.now();
let cancelled = false;
let streamHandle: AnthropicStreamHandle | null = null;
let streamHandle: LlmStreamHandle | null = null;
const finishError = (message: string) => {
if (cancelled) return;
@@ -61,7 +65,7 @@ export class DirectAnthropicBackend {
return;
}
streamHandle = streamAnthropicMessage({
const streamOptions: LlmStreamOptions = {
apiKey: options.apiKey,
model: options.model,
system: options.system,
@@ -80,13 +84,13 @@ export class DirectAnthropicBackend {
return;
}
// Record what the model produced this iteration.
const assistantContent: AnthropicContentBlock[] = result.content.length
const assistantContent: LlmContentBlock[] = result.content.length
? result.content
: [{ type: "text", text: result.text }];
options.messages.push({ role: "assistant", content: assistantContent });
const toolUses = assistantContent.filter(
(block): block is Extract<AnthropicContentBlock, { type: "tool_use" }> =>
(block): block is Extract<LlmContentBlock, { type: "tool_use" }> =>
block.type === "tool_use",
);
if (result.stopReason !== "tool_use" || toolUses.length === 0) {
@@ -110,7 +114,10 @@ export class DirectAnthropicBackend {
streamHandle = null;
finishError(message);
},
});
};
streamHandle = options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: streamAnthropicMessage(streamOptions);
};
runIteration(0);
@@ -125,10 +132,10 @@ export class DirectAnthropicBackend {
}
private async runToolCalls(
toolUses: Array<Extract<AnthropicContentBlock, { type: "tool_use" }>>,
toolUses: Array<Extract<LlmContentBlock, { type: "tool_use" }>>,
options: DirectTurnOptions,
): Promise<AnthropicContentBlock[]> {
const results: AnthropicContentBlock[] = [];
): Promise<LlmContentBlock[]> {
const results: LlmContentBlock[] = [];
for (const toolUse of toolUses) {
const canonicalName = options.resolveToolName(toolUse.name);
options.callbacks.onToolActivity(canonicalName);
+56
View File
@@ -0,0 +1,56 @@
/**
* Provider-neutral shapes used by the direct assistant loop. Each native LLM
* client translates these messages and tools to its provider's wire format.
*/
export type LlmContentBlock =
| { type: "text"; text: string }
| {
type: "tool_use";
id: string;
name: string;
input: any;
/** Exact provider output item to replay during stateless continuation. */
provider_item?: { provider: "openai"; item: any };
}
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }
/** Opaque state required to continue a provider's stateless conversation. */
| { type: "provider_item"; provider: "openai"; item: any };
export type LlmMessage = {
role: "user" | "assistant";
content: string | LlmContentBlock[];
};
export type LlmToolDefinition = {
name: string;
description: string;
input_schema: object;
};
export type LlmStreamResult = {
/** Concatenated text of all text blocks. */
text: string;
/** The assembled assistant content blocks (text + any tool calls). */
content: LlmContentBlock[];
/** Provider-normalized stop reason: end_turn, tool_use, refusal, etc. */
stopReason: string | null;
};
export type LlmStreamOptions = {
apiKey: string;
model: string;
system?: string;
messages: LlmMessage[];
tools?: LlmToolDefinition[];
maxTokens?: number;
/** Thinking/latency tradeoff; voice flows want "low". */
effort?: "low" | "medium" | "high";
onTextDelta?: (delta: string, textSoFar: string) => void;
onDone: (result: LlmStreamResult) => void;
onError: (message: string) => void;
};
export type LlmStreamHandle = {
cancel(): void;
};
+88
View File
@@ -0,0 +1,88 @@
export const ASSISTANT_MODEL_VALUES = [
"auto",
"hauku",
"sonnet",
"opus",
"fable",
"luna",
"terra",
"sol",
] as const;
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
export type AssistantProvider = "anthropic" | "openai";
export type AssistantApiKeys = {
anthropic: string;
openai: string;
};
export type ResolvedAssistantModel = {
selection: AssistantModel;
provider: AssistantProvider;
model: string;
apiKey: string;
effort?: "low" | "medium" | "high";
};
type ModelDefinition = {
label: string;
provider: AssistantProvider;
model: string;
effort?: "low" | "medium" | "high";
};
const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition> = {
// Keep the user-facing spelling requested for the picker while using
// Anthropic's canonical Haiku alias on the wire.
hauku: { label: "Hauku", provider: "anthropic", model: "claude-haiku-4-5" },
sonnet: { label: "Sonnet", provider: "anthropic", model: "claude-sonnet-5", effort: "low" },
opus: { label: "Opus", provider: "anthropic", model: "claude-opus-5", effort: "low" },
fable: { label: "Fable", provider: "anthropic", model: "claude-fable-5", effort: "low" },
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
};
export function assistantModelLabel(value: AssistantModel): string {
return value === "auto" ? "Auto" : MODEL_DEFINITIONS[value].label;
}
export function assistantModelProvider(value: AssistantModel): AssistantProvider | null {
return value === "auto" ? null : MODEL_DEFINITIONS[value].provider;
}
/**
* Resolve Auto and attach the credential needed by the chosen provider.
* Auto intentionally prefers Terra over Sonnet when both keys are available.
*/
export function resolveAssistantModel(
selection: AssistantModel,
keys: AssistantApiKeys,
): ResolvedAssistantModel | null {
const normalizedKeys: AssistantApiKeys = {
anthropic: keys.anthropic.trim(),
openai: keys.openai.trim(),
};
let resolvedSelection = selection;
if (selection === "auto") {
if (normalizedKeys.openai) {
resolvedSelection = "terra";
} else if (normalizedKeys.anthropic) {
resolvedSelection = "sonnet";
} else {
return null;
}
}
const definition = MODEL_DEFINITIONS[resolvedSelection as Exclude<AssistantModel, "auto">];
const apiKey = normalizedKeys[definition.provider];
if (!apiKey) return null;
return {
selection,
provider: definition.provider,
model: definition.model,
apiKey,
effort: definition.effort,
};
}
+24 -15
View File
@@ -1,12 +1,13 @@
import { DEFAULT_LLM_MODEL, type AnthropicMessage, type AnthropicToolDefinition } from "../native/anthropic";
import { buildAssistantSystemPrompt } from "../prompts";
import { DirectAnthropicBackend } from "./direct-backend";
import { DirectAssistantBackend } from "./direct-backend";
import type { LlmMessage, LlmToolDefinition } from "./llm-protocol";
import type { ResolvedAssistantModel } from "./models";
import { toolRegistry, type ToolRegistry } from "./tool-registry";
import type { AssistantContext, AssistantTurnCallbacks, AssistantTurnHandle } from "./types";
/**
* One assistant conversation. Owns the message history and turn state and
* drives a backend (direct Anthropic loop for phase 1). The UI talks only to
* drives a direct provider backend. The UI talks only to
* this surface, so swapping in the external-agent backend later doesn't touch
* AssistantLayer.
*
@@ -21,20 +22,27 @@ const SESSION_IDLE_MS = 10 * 60 * 1000;
const MAX_HISTORY_MESSAGES = 40;
export class AssistantSession {
private readonly backend = new DirectAnthropicBackend();
private readonly messages: AnthropicMessage[] = [];
private readonly backend = new DirectAssistantBackend();
private readonly messages: LlmMessage[] = [];
private turnHandle: AssistantTurnHandle | null = null;
private lastActivityMs = Date.now();
// API-safe tool name -> canonical registry name. The Anthropic API only
// allows [a-zA-Z0-9_-] in tool names, so dotted registry names ("glasses.
// get_state") are sanitized for the request and mapped back on tool calls.
// API-safe tool name -> canonical registry name. Provider APIs restrict tool
// names, so dotted registry names are sanitized and mapped back on calls.
private readonly toolNameMap = new Map<string, string>();
constructor(
private readonly apiKey: string,
private readonly llm: ResolvedAssistantModel,
private readonly registry: ToolRegistry = toolRegistry,
) {}
matchesConfiguration(llm: ResolvedAssistantModel): boolean {
return (
this.llm.provider === llm.provider &&
this.llm.model === llm.model &&
this.llm.apiKey === llm.apiKey
);
}
/** Whether this session has been idle long enough to retire. */
isExpired(nowMs = Date.now()): boolean {
return nowMs - this.lastActivityMs > SESSION_IDLE_MS;
@@ -62,9 +70,10 @@ export class AssistantSession {
};
this.turnHandle = this.backend.runTurn({
apiKey: this.apiKey,
model: DEFAULT_LLM_MODEL,
effort: "low",
provider: this.llm.provider,
apiKey: this.llm.apiKey,
model: this.llm.model,
effort: this.llm.effort,
system,
messages: this.messages,
buildTools: () => this.buildToolDefinitions(),
@@ -91,7 +100,7 @@ export class AssistantSession {
this.lastActivityMs = Date.now();
}
private buildToolDefinitions(): AnthropicToolDefinition[] {
private buildToolDefinitions(): LlmToolDefinition[] {
this.toolNameMap.clear();
return this.registry.listTools().map((spec) => {
const apiName = this.toApiToolName(spec.name);
@@ -100,7 +109,7 @@ export class AssistantSession {
});
}
/** Convert a canonical registry name to an Anthropic-legal tool name. */
/** Convert a canonical registry name to a provider-legal tool name. */
private toApiToolName(name: string): string {
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
// Disambiguate the rare case where two canonical names sanitize alike.
@@ -125,7 +134,7 @@ export class AssistantSession {
}
}
function startsWithToolResult(message: AnthropicMessage): boolean {
function startsWithToolResult(message: LlmMessage): boolean {
return (
Array.isArray(message.content) &&
message.content.length > 0 &&
+1 -1
View File
@@ -1,7 +1,7 @@
/**
* The assistant's tool registry: the single place that knows which tools exist,
* whether each is currently callable, and how to invoke it. Both assistant
* backends (direct Anthropic loop, external agent bridge) list and call tools
* backends (direct provider loop, external agent bridge) list and call tools
* through here, so gating and timeouts are enforced in one spot rather than
* trusted to a model or a remote agent.
*
+1 -1
View File
@@ -1,7 +1,7 @@
/**
* Shared assistant types. Kept backend-agnostic so the UI (AssistantLayer)
* talks to the same session surface whether the turn runs on the phone
* (direct Anthropic loop) or against the user's own agent (external bridge,
* (direct provider loop) or against the user's own agent (external bridge,
* later phases).
*/
+14 -42
View File
@@ -1,4 +1,12 @@
import { buildRefineUserMessage, REFINE_SYSTEM_PROMPT } from "../prompts";
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
LlmStreamResult,
LlmToolDefinition,
} from "../assistant/llm-protocol";
declare const com: any;
@@ -20,56 +28,20 @@ export const DEFAULT_LLM_MODEL = "claude-sonnet-5";
* still pass a plain string for content; the assistant tool loop uses the
* block array to carry tool_use / tool_result alongside text.
*/
export type AnthropicContentBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; input: any }
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean };
export type AnthropicMessage = {
role: "user" | "assistant";
content: string | AnthropicContentBlock[];
};
export type AnthropicContentBlock = LlmContentBlock;
export type AnthropicMessage = LlmMessage;
/** An Anthropic tool definition (input_schema is snake_case as the API wants). */
export type AnthropicToolDefinition = {
name: string;
description: string;
input_schema: object;
};
export type AnthropicStreamResult = {
/** Concatenated text of all text blocks. */
text: string;
/** The assembled assistant content blocks (text + any tool_use). */
content: AnthropicContentBlock[];
/** e.g. "end_turn", "tool_use", "max_tokens", "refusal"; null if never said. */
stopReason: string | null;
};
export type AnthropicStreamOptions = {
apiKey: string;
model?: string;
system?: string;
messages: AnthropicMessage[];
tools?: AnthropicToolDefinition[];
maxTokens?: number;
/** Thinking/latency tradeoff; voice flows want "low". Default: model default. */
effort?: "low" | "medium" | "high";
/** Called with each incremental text delta as it streams in. */
onTextDelta?: (delta: string, textSoFar: string) => void;
onDone: (result: AnthropicStreamResult) => void;
/** Called at most once, instead of onDone, with a user-displayable message. */
onError: (message: string) => void;
};
export type AnthropicToolDefinition = LlmToolDefinition;
export type AnthropicStreamResult = LlmStreamResult;
export type AnthropicStreamOptions = Omit<LlmStreamOptions, "model"> & { model?: string };
/** Accumulator for one streamed content block before it is finalized. */
type StreamingBlock =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; inputJson: string };
export type AnthropicStreamHandle = {
cancel(): void;
};
export type AnthropicStreamHandle = LlmStreamHandle;
export function streamAnthropicMessage(options: AnthropicStreamOptions): AnthropicStreamHandle {
const apiKey = options.apiKey.trim();
+333
View File
@@ -0,0 +1,333 @@
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
declare const com: any;
/**
* Minimal streaming client for the OpenAI Responses API. NativeScript cannot
* use the Node SDK, so this shares the okhttp/SSE bridge used by Anthropic.
*/
const OPENAI_RESPONSES_URL = "https://api.openai.com/v1/responses";
type StreamingOutput =
| { type: "text"; text: string }
| { type: "tool_use"; id: string; name: string; inputJson: string; itemId: string; item: any }
| { type: "provider_item"; item: any };
export function streamOpenAiResponse(options: LlmStreamOptions): LlmStreamHandle {
const apiKey = options.apiKey.trim();
let request: any = null;
let settled = false;
let text = "";
let stopReason: string | null = null;
let refusal = false;
const outputs = new Map<number, StreamingOutput>();
const fail = (message: string) => {
if (settled) return;
settled = true;
options.onError(message);
};
const done = () => {
if (settled) return;
settled = true;
const content = finalizeOutputs(outputs);
const hasToolUse = content.some((block) => block.type === "tool_use");
options.onDone({
text,
content,
stopReason: refusal ? "refusal" : hasToolUse ? "tool_use" : stopReason ?? "end_turn",
});
};
if (!global.isAndroid) {
setTimeout(() => fail("OpenAI API is only wired up on Android"), 0);
return { cancel: () => {} };
}
if (!apiKey) {
setTimeout(() => fail("No OpenAI API key set"), 0);
return { cancel: () => {} };
}
const body: any = {
model: options.model,
input: buildOpenAiInput(options.messages),
stream: true,
store: false,
include: ["reasoning.encrypted_content"],
max_output_tokens: options.maxTokens ?? 8192,
};
if (options.system) body.instructions = options.system;
if (options.tools?.length) {
body.tools = options.tools.map((tool) => ({
type: "function",
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
}));
}
if (options.effort) body.reasoning = { effort: options.effort };
const listener = new com.faceclaw.app.FaceclawSseListener({
onLine: (line: string) => {
if (settled) return;
const event = parseSseDataLine(String(line));
if (!event) return;
const outputIndex = Number(event.output_index);
switch (event.type) {
case "response.output_item.added": {
const item = event.item;
if (item?.type === "function_call") {
outputs.set(outputIndex, {
type: "tool_use",
id: String(item.call_id ?? item.id ?? ""),
name: String(item.name ?? ""),
inputJson: String(item.arguments ?? ""),
itemId: String(item.id ?? ""),
item,
});
} else if (item?.type === "message") {
outputs.set(outputIndex, { type: "text", text: "" });
} else if (item?.type === "reasoning") {
outputs.set(outputIndex, { type: "provider_item", item });
}
return;
}
case "response.output_text.delta": {
const delta = String(event.delta ?? "");
text += delta;
const output = outputs.get(outputIndex);
if (output?.type === "text") {
output.text += delta;
} else {
outputs.set(outputIndex, { type: "text", text: delta });
}
options.onTextDelta?.(delta, text);
return;
}
case "response.function_call_arguments.delta": {
const output = findToolOutput(outputs, outputIndex, String(event.item_id ?? ""));
if (output) output.inputJson += String(event.delta ?? "");
return;
}
case "response.output_item.done": {
const item = event.item;
if (item?.type === "function_call") {
outputs.set(outputIndex, {
type: "tool_use",
id: String(item.call_id ?? item.id ?? ""),
name: String(item.name ?? ""),
inputJson: String(item.arguments ?? ""),
itemId: String(item.id ?? ""),
item,
});
} else if (item?.type === "message") {
const output = outputs.get(outputIndex);
const completedText = extractMessageText(item);
outputs.set(outputIndex, {
type: "text",
text: completedText || (output?.type === "text" ? output.text : ""),
});
if (completedText && !text) text = completedText;
} else if (item?.type === "reasoning") {
outputs.set(outputIndex, { type: "provider_item", item });
}
return;
}
case "response.refusal.delta":
case "response.refusal.done":
refusal = true;
return;
case "response.completed":
done();
return;
case "response.incomplete":
stopReason = String(event.response?.incomplete_details?.reason ?? "incomplete");
done();
return;
case "response.failed":
fail(`OpenAI: ${String(event.response?.error?.message ?? "response failed")}`);
return;
case "error":
fail(`OpenAI: ${String(event.message ?? event.error?.message ?? "stream error")}`);
return;
default:
return;
}
},
onHttpError: (code: number, errorBody: string) => {
fail(describeHttpError(Number(code), String(errorBody)));
},
onComplete: () => {
done();
},
onFailure: (message: string) => {
fail(`OpenAI connection failed: ${String(message)}`);
},
});
const headers = Array.create("java.lang.String", 2) as string[];
headers[0] = "Authorization";
headers[1] = `Bearer ${apiKey}`;
try {
request = new com.faceclaw.app.FaceclawSseRequest(
OPENAI_RESPONSES_URL,
JSON.stringify(body),
headers,
listener,
);
} catch (error) {
setTimeout(() => fail(`OpenAI request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
return {
cancel: () => {
settled = true;
try {
request?.cancel();
} catch {
// ignore
}
},
};
}
/** Convert neutral conversation history to Responses API input items. */
function buildOpenAiInput(messages: LlmMessage[]): any[] {
const input: any[] = [];
for (const message of messages) {
if (typeof message.content === "string") {
input.push({ role: message.role, content: message.content });
continue;
}
let pendingText = "";
const flushText = () => {
if (!pendingText) return;
input.push({ role: message.role, content: pendingText });
pendingText = "";
};
for (const block of message.content) {
if (block.type === "text") {
pendingText += block.text;
} else if (block.type === "tool_use") {
flushText();
if (block.provider_item?.provider === "openai") {
input.push(block.provider_item.item);
} else {
input.push({
type: "function_call",
call_id: block.id,
name: block.name,
arguments: JSON.stringify(block.input ?? {}),
});
}
} else if (block.type === "tool_result") {
flushText();
input.push({
type: "function_call_output",
call_id: block.tool_use_id,
output: block.is_error ? `Error: ${block.content}` : block.content,
});
} else if (block.provider === "openai") {
flushText();
input.push(block.item);
}
}
flushText();
}
return input;
}
function finalizeOutputs(outputs: Map<number, StreamingOutput>): LlmContentBlock[] {
const content: LlmContentBlock[] = [];
for (const index of Array.from(outputs.keys()).sort((a, b) => a - b)) {
const output = outputs.get(index)!;
if (output.type === "text") {
if (output.text) content.push({ type: "text", text: output.text });
continue;
}
if (output.type === "provider_item") {
content.push({ type: "provider_item", provider: "openai", item: output.item });
continue;
}
let input: any = {};
if (output.inputJson.trim()) {
try {
input = JSON.parse(output.inputJson);
} catch {
// Let the tool handler report any required arguments that are missing.
}
}
content.push({
type: "tool_use",
id: output.id,
name: output.name,
input,
provider_item: { provider: "openai", item: output.item },
});
}
return content;
}
function findToolOutput(
outputs: Map<number, StreamingOutput>,
outputIndex: number,
itemId: string,
): Extract<StreamingOutput, { type: "tool_use" }> | null {
const indexed = outputs.get(outputIndex);
if (indexed?.type === "tool_use") return indexed;
for (const output of outputs.values()) {
if (output.type === "tool_use" && output.itemId === itemId) return output;
}
return null;
}
function extractMessageText(item: any): string {
if (!Array.isArray(item?.content)) return "";
return item.content
.filter((part: any) => part?.type === "output_text")
.map((part: any) => String(part.text ?? ""))
.join("");
}
function parseSseDataLine(line: string): any | null {
if (!line.startsWith("data:")) return null;
const payload = line.slice(5).trim();
if (!payload || payload === "[DONE]") return null;
try {
return JSON.parse(payload);
} catch {
return null;
}
}
function describeHttpError(code: number, body: string): string {
let detail = "";
try {
detail = String(JSON.parse(body)?.error?.message ?? "");
} catch {
// non-JSON error body
}
switch (code) {
case 401:
return "OpenAI: invalid API key";
case 403:
return "OpenAI: API key lacks permission";
case 429:
return "OpenAI: rate limited, try again shortly";
case 500:
case 502:
case 503:
case 504:
return "OpenAI: service unavailable, try again shortly";
default:
return `OpenAI: HTTP ${code}${detail ? ` (${detail})` : ""}`;
}
}
+63 -30
View File
@@ -13,7 +13,13 @@ import {
type UiFontChoice,
} from "~/graphics/bdffont";
import { wrapText } from "~/graphics/textwrap";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, MenuLayer } from "./menu";
import {
ASSISTANT_MODEL_VALUES,
assistantModelLabel,
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
import { DashboardInputEvent, Layer, type LayerContext } from "./layers";
import { GrayImage } from "~/graphics/image";
@@ -109,15 +115,19 @@ export class ConfigSettingBoolean<TId extends string = string> extends ConfigSet
type ConfigSettingEnumOptions<TValue extends string, TId extends string> = ConfigSettingOptions<TValue, TId> & {
values: readonly TValue[];
normalize?: (value: string | null | undefined) => TValue;
/** Dynamic availability check used by enum picker rows. */
isDisabled?: (value: TValue) => boolean;
};
export class ConfigSettingEnum<TValue extends string, TId extends string = string> extends ConfigSetting<TValue, TId> {
readonly values: readonly TValue[];
private readonly normalizer: (value: string | null | undefined) => TValue;
private readonly disabledPredicate: (value: TValue) => boolean;
constructor(options: ConfigSettingEnumOptions<TValue, TId>) {
super(options);
this.values = options.values;
this.disabledPredicate = options.isDisabled ?? (() => false);
if (options.normalize) {
this.normalizer = (value: string | null | undefined) => {
const normalized = options.normalize(value) as TValue|undefined;
@@ -139,9 +149,17 @@ export class ConfigSettingEnum<TValue extends string, TId extends string = strin
return normalized;
}
isDisabled(value: TValue): boolean {
return this.disabledPredicate(value);
}
next(value = this.get()): TValue {
const index = this.values.indexOf(value);
return this.values[(index + 1) % this.values.length] ?? this.defaultValue;
for (let offset = 1; offset <= this.values.length; offset++) {
const candidate = this.values[(index + offset) % this.values.length];
if (candidate !== undefined && !this.isDisabled(candidate)) return candidate;
}
return value;
}
}
@@ -352,6 +370,22 @@ export const anthropicApiKeySetting = new ConfigSettingString({
description: "Anthropic API key, used when an Anthropic model is selected for the voice assistant.",
});
export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
id: "assistant-model",
label: "Assistant model",
storageKey: "assistant.model",
defaultValue: "auto",
values: ASSISTANT_MODEL_VALUES,
formatValue: assistantModelLabel,
isDisabled: (value) => {
const provider = assistantModelProvider(value);
if (provider === "anthropic") return anthropicApiKeySetting.get().trim().length === 0;
if (provider === "openai") return openAiApiKeySetting.get().trim().length === 0;
return false;
},
description: "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
id: "mapbox-api-key",
label: "Mapbox token",
@@ -494,7 +528,6 @@ function maskToken(token: string): string {
type SettingsMenuOptions<T> = {
style?: "cycle"|"submenu"
onChange?: (ctx: LayerContext, newValue: T, oldValue: T) => void
}
@@ -502,36 +535,36 @@ export function enumSettingMenuItem<TValue extends string, TId extends string =
setting: ConfigSettingEnum<TValue, TId>,
opts?: SettingsMenuOptions<TValue>
): MenuItem {
const style = opts?.style ?? "cycle";
return {
label: setting.label,
description: setting.description,
onSelect: (ctx) => {
if (style === "cycle") {
const oldValue = setting.get();
const newValue = setting.next(oldValue);
setting.set(newValue);
opts?.onChange?.(ctx, newValue, oldValue);
} else {
const submenu = new MenuLayer(setting.label, setting.values.map((value) => ({
label: setting.displayValue(value),
onSelect: () => {
const oldValue = setting.get();
setting.set(value);
opts?.onChange?.(ctx, value, oldValue);
ctx.stack.pop();
},
render: ({ image, x, y }) => {
const selected = setting.get() === value ? " *" : "";
image.drawText(getDefaultSmallFont(), x, y + 3, `${setting.displayValue(value)}${selected}`, 200);
}
})));
ctx.stack.push(submenu);
}
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, setting.label, setting.displayValue(setting.get()));
},
onSelect: (ctx) => {
const current = setting.get();
const items = setting.values.map((value): MenuItem => ({
label: setting.displayValue(value),
disabled: () => setting.isDisabled(value),
onSelect: () => {
const oldValue = setting.get();
setting.set(value);
opts?.onChange?.(ctx, value, oldValue);
ctx.stack.pop();
},
render: ({ image, x, y, disabled }) => {
const selected = setting.get() === value ? " *" : "";
image.drawText(
getDefaultSmallFont(),
x,
y + 3,
`${setting.displayValue(value)}${selected}`,
disabled ? 70 : 200,
);
},
}));
openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current)));
},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, setting.label, setting.displayValue(setting.get()));
},
};
}
+3 -1
View File
@@ -4,6 +4,7 @@ import { getDashboardLogo } from "../../graphics/logo";
import { shell } from "../shell/shell";
import {
anthropicApiKeySetting,
assistantModelSetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
brightnessSetting,
@@ -40,7 +41,7 @@ function settingsSections(): SettingsSection[] {
items: [
// Auto (ambient sensor) or an exact level; pushed to the glasses by
// the dashboard controller when changed and on each connect.
enumSettingMenuItem(brightnessSetting, { style: "submenu" }),
enumSettingMenuItem(brightnessSetting),
enumSettingMenuItem(screenTimeoutSetting, {
onChange: () => {
shell.noteUserActivity();
@@ -59,6 +60,7 @@ function settingsSections(): SettingsSection[] {
items: [
enumSettingMenuItem(wakeWordActionSetting),
enumSettingMenuItem(voiceProviderSetting),
enumSettingMenuItem(assistantModelSetting),
// When on, a wakeword utterance goes straight to the assistant with no
// Send/Type menu step.
toggleSettingMenuItem(assistantSkipConfirmationSetting),
+19 -15
View File
@@ -4,7 +4,7 @@ import { wrapText } from "../../graphics/textwrap";
import { clamp } from "../../util/numeric-util";
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL } from "../gestures";
import { DashboardInputEvent, Layer, LayerContext } from "../layers";
import { drawSelectionHighlight, MenuItem, MenuLayer, type MenuLayout } from "../menu";
import { drawSelectionHighlight, isMenuItemDisabled, MenuItem, MenuLayer, openModalMenu } from "../menu";
import { shell } from "../shell/shell";
/**
@@ -114,13 +114,24 @@ export class SettingsPanelLayer implements Layer {
// A selection only appears once focus is in the right column; before
// that the pane is a preview of what tapping would open.
const selected = this.focus === "right" && i === this.rightIndex;
const disabled = isMenuItemDisabled(item);
if (selected) {
drawSelectionHighlight(image, rightX - 2, rowY, rightW + 2, ROW_H - 2, appFocused, 6);
}
if (item.render) {
item.render({ image, x: rightX + 8, y: rowY, width: rightW - 16, height: ROW_H - 3, selected, text: item.label, ctx });
item.render({
image,
x: rightX + 8,
y: rowY,
width: rightW - 16,
height: ROW_H - 3,
selected,
disabled,
text: item.label,
ctx,
});
} else {
image.drawText(font, rightX + 8, rowY + 4, item.label, selected ? 255 : 200);
image.drawText(font, rightX + 8, rowY + 4, item.label, disabled ? 70 : selected ? 255 : 200);
}
}
if (descriptionLines.length) {
@@ -174,7 +185,10 @@ export class SettingsPanelLayer implements Layer {
if (items.length) this.rightIndex = (this.rightIndex + 1) % items.length;
return;
case "click":
if (items.length) await items[clamp(this.rightIndex, 0, items.length - 1)]!.onSelect(ctx, NO_MENU);
if (items.length) {
const item = items[clamp(this.rightIndex, 0, items.length - 1)]!;
if (!isMenuItemDisabled(item)) await item.onSelect(ctx, NO_MENU);
}
return;
case "double-click":
this.focus = "left";
@@ -195,17 +209,7 @@ export class SettingsPanelLayer implements Layer {
* MenuLayer (which draws the border and self-closes on double-click).
*/
export function openSettingsSubMenu(ctx: LayerContext, title: string, items: MenuItem[]): void {
const { width, height } = ctx.stack.getBaseSize();
const menuW = Math.min(320, width - 40);
const menuH = 16 + PAD + items.length * ROW_H + PAD;
const layout: MenuLayout = {
x: ((width - menuW) / 2) | 0,
y: ((height - menuH) / 2) | 0,
width: menuW,
minHeight: menuH,
maxHeight: menuH,
};
ctx.stack.push(new MenuLayer(title, items, layout));
openModalMenu(ctx, title, items);
}
function clampScroll(selected: number, scroll: number, visible: number, count: number): number {
+40 -2
View File
@@ -37,6 +37,7 @@ export type MenuItemRenderArgs = {
width: number;
height: number;
selected: boolean;
disabled: boolean;
text: string;
ctx: LayerContext;
};
@@ -45,6 +46,8 @@ export type MenuItem = {
label: string;
/** Extended description; the Settings panel shows it below the list while the item is selected. */
description?: string;
/** Disabled rows stay visible but render dimly and ignore clicks. */
disabled?: boolean | (() => boolean);
onSelect: (ctx: LayerContext, menu: MenuLayer) => Promise<void> | void;
render?: (args: MenuItemRenderArgs) => void;
};
@@ -181,6 +184,12 @@ export class MenuLayer implements Layer {
public readonly paintOverBase = false,
) {}
/** Start a newly opened picker on its current value. */
selectItem(index: number): this {
this.selectedIndex = clamp(index, 0, Math.max(0, this.items.length - 1));
return this;
}
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
const font = getDefaultSmallFont();
const { x, y, width } = this.layout;
@@ -216,6 +225,7 @@ export class MenuLayer implements Layer {
const item = this.items[index]!;
const rowY = bodyY + (index - this.scrollRow) * MENU_ROW_HEIGHT;
const selected = index === this.selectedIndex;
const disabled = isMenuItemDisabled(item);
if (selected) {
drawSelectionHighlight(
image,
@@ -235,11 +245,12 @@ export class MenuLayer implements Layer {
width: width - 44,
height: MENU_ROW_HEIGHT - 3,
selected,
disabled,
text: item.label,
ctx,
});
} else {
image.drawText(font, x + 22, rowY + 3, item.label, selected ? 255 : 200);
image.drawText(font, x + 22, rowY + 3, item.label, disabled ? 70 : selected ? 255 : 200);
}
}
@@ -276,7 +287,9 @@ export class MenuLayer implements Layer {
ctx.stack.pop();
return;
case "click":
await this.items[this.selectedIndex]!.onSelect(ctx, this);
if (!isMenuItemDisabled(this.items[this.selectedIndex]!)) {
await this.items[this.selectedIndex]!.onSelect(ctx, this);
}
return;
default:
return;
@@ -284,6 +297,31 @@ export class MenuLayer implements Layer {
}
}
/** Open a centered, bordered menu over the current layer. */
export function openModalMenu(
ctx: LayerContext,
title: string,
items: MenuItem[],
initialSelectedIndex = 0,
): void {
const { width, height } = ctx.stack.getBaseSize();
const menuWidth = Math.min(320, width - 40);
const naturalHeight = MENU_TITLE_HEIGHT + 2 * MENU_BODY_PADDING + items.length * MENU_ROW_HEIGHT;
const menuHeight = Math.min(naturalHeight, height - 40);
const layout: MenuLayout = {
x: ((width - menuWidth) / 2) | 0,
y: ((height - menuHeight) / 2) | 0,
width: menuWidth,
minHeight: menuHeight,
maxHeight: menuHeight,
};
ctx.stack.push(new MenuLayer(title, items, layout).selectItem(initialSelectedIndex));
}
export function isMenuItemDisabled(item: MenuItem): boolean {
return typeof item.disabled === "function" ? item.disabled() : item.disabled === true;
}
export class TextPageLayer implements Layer {
constructor(
private readonly title: string,
+21 -6
View File
@@ -7,13 +7,16 @@ import { MenuLayer, type MenuItem } from "../menu";
import { VoiceInputLayer, type VoiceSendTarget } from "../apps/voice-input";
import { AssistantLayer } from "../apps/assistant";
import { AssistantSession } from "../../assistant/session";
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
import type { AssistantContext } from "../../assistant/types";
import { SingleNotificationLayer } from "../notifications";
import {
anthropicApiKeySetting,
assistantModelSetting,
assistantSkipConfirmationSetting,
batteryDisplayModeSetting,
onAnySettingChanged,
openAiApiKeySetting,
timeFormatSetting,
wakeWordActionSetting,
} from "../dashboard-settings";
@@ -673,18 +676,30 @@ class Shell {
}
private isAssistantAvailable(): boolean {
return anthropicApiKeySetting.get().trim().length > 0;
return this.resolveAssistantConfiguration() !== null;
}
private ensureAssistantSession(): AssistantSession | null {
const apiKey = anthropicApiKeySetting.get().trim();
if (!apiKey) return null;
if (!this.assistantSession || this.assistantSession.isExpired()) {
this.assistantSession = new AssistantSession(apiKey);
const llm = this.resolveAssistantConfiguration();
if (!llm) return null;
if (
!this.assistantSession ||
this.assistantSession.isExpired() ||
!this.assistantSession.matchesConfiguration(llm)
) {
this.assistantSession?.cancel();
this.assistantSession = new AssistantSession(llm);
}
return this.assistantSession;
}
private resolveAssistantConfiguration(): ResolvedAssistantModel | null {
return resolveAssistantModel(assistantModelSetting.get(), {
anthropic: anthropicApiKeySetting.get(),
openai: openAiApiKeySetting.get(),
});
}
private buildAssistantContext(): AssistantContext {
const foreground = this.getForegroundApp();
return {
@@ -704,7 +719,7 @@ class Shell {
sendToAssistant(text: string): void {
const session = this.ensureAssistantSession();
if (!session) {
this.showAlert("Set an Anthropic API key in Settings to use the assistant.");
this.showAlert("Set an API key for the selected assistant model in Settings.");
return;
}
if (!this.screenOn) this.wake("sidebar");