Files
faceclaw/app/native/openai-chat.ts
T

331 lines
12 KiB
TypeScript

import type {
LlmContentBlock,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { openSseRequest, type SseListener } from "./sse";
/** True only for Venice's official API root, never for lookalike hosts. */
export function isVeniceApiBaseUrl(input: string): boolean {
return normalizeOpenAiCompatibleUrl(input) === "https://api.venice.ai/api/v1";
}
/** Streaming client for OpenAI-compatible Chat Completions APIs. */
export type OpenAiCompatibleStreamOptions = LlmStreamOptions & {
/** Explicit API root, for example https://api.venice.ai/api/v1 or https://host/api. */
baseUrl: string;
/** Venice-only web search; never sent to other compatible providers. */
webSearch?: boolean;
};
type PendingToolCall = {
id: string;
name: string;
inputJson: string;
};
/** Remove an optional endpoint suffix, but never guess a provider's /v1 or /api prefix. */
export function normalizeOpenAiCompatibleUrl(input: string): string {
let url = input.trim().replace(/\/+$/, "");
url = url.replace(/\/chat\/completions$/i, "").replace(/\/+$/, "");
return /^https?:\/\/[^/\s?#@\\]+(?:\/[^\s?#\\]*)?$/i.test(url) ? url : "";
}
export function streamOpenAiCompatibleChat(options: OpenAiCompatibleStreamOptions): LlmStreamHandle {
const apiKey = options.apiKey.trim();
const baseUrl = normalizeOpenAiCompatibleUrl(options.baseUrl);
let request: { cancel(): void } | null = null;
let settled = false;
let text = "";
let finishReason: string | null = null;
const toolCalls = new Map<number, PendingToolCall>();
const providerMetadata: Record<string, any> = {};
const reasoningDetails = new ReasoningDetailsAccumulator();
const fail = (message: string) => {
if (settled) return;
settled = true;
request?.cancel();
options.onError(message);
};
const done = () => {
if (settled) return;
const hasToolCalls = toolCalls.size > 0;
// Never execute calls from a truncated stream or a length-limited response.
if (hasToolCalls && finishReason !== "tool_calls" && finishReason !== "stop") {
fail("OpenAI-compatible stream ended before its tool calls completed");
return;
}
if (!text && !hasToolCalls) {
fail("OpenAI-compatible provider ended the turn without a reply");
return;
}
const content: LlmContentBlock[] = [];
if (text) content.push({ type: "text", text });
const details = reasoningDetails.entries();
if (details.length) providerMetadata.reasoning_details = details;
if (Object.keys(providerMetadata).length) {
content.push({ type: "provider_item", provider: "openai-compatible", item: providerMetadata });
}
for (const index of Array.from(toolCalls.keys()).sort((a, b) => a - b)) {
const call = toolCalls.get(index)!;
if (!call.name) {
fail("OpenAI-compatible provider returned a tool call without a name");
return;
}
let input: any = {};
if (call.inputJson.trim()) {
try {
input = JSON.parse(call.inputJson);
} catch {
fail("OpenAI-compatible provider returned malformed tool arguments");
return;
}
}
content.push({
type: "tool_use",
id: call.id || `call_${index + 1}`,
name: call.name,
input,
});
}
settled = true;
request?.cancel();
options.onDone({
text,
content,
stopReason: finishReason === "content_filter" ? "refusal"
: hasToolCalls ? "tool_use"
: finishReason === "length" ? "max_tokens" : "end_turn",
});
};
if (!baseUrl || !options.model.trim()) {
setTimeout(() => fail(!baseUrl
? "Set a valid OpenAI-compatible API URL"
: "Set an OpenAI-compatible model ID"), 0);
return { cancel: () => { settled = true; } };
}
const body: any = {
model: options.model.trim(),
messages: buildChatMessages(options),
stream: true,
};
// Let the provider choose its default unless a one-shot feature supplies a cap.
if (options.maxTokens !== undefined) body.max_tokens = options.maxTokens;
if (options.tools?.length) {
body.tools = options.tools.map((tool) => ({
type: "function",
function: {
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
},
}));
}
// Provider-specific parameters must never leak to another endpoint, even
// when a saved Venice preference is still enabled.
if (options.webSearch === true && isVeniceApiBaseUrl(baseUrl)) {
body.venice_parameters = { enable_web_search: "on" };
}
// Do not send provider-specific reasoning fields by default. Compatible
// servers vary widely; advanced reasoning controls can be added separately.
const headers: Record<string, string> = { "Content-Type": "application/json" };
if (apiKey) headers.Authorization = `Bearer ${apiKey}`;
const listener: SseListener = {
onLine: (line: string) => {
if (settled || !line.startsWith("data:")) return;
const payload = line.slice(5).trim();
if (!payload) return;
if (payload === "[DONE]") {
if (finishReason || (!toolCalls.size && text)) done();
else fail("OpenAI-compatible stream ended before the response completed");
return;
}
let event: any;
try {
event = JSON.parse(payload);
} catch {
return;
}
if (event?.error) {
fail(`OpenAI-compatible: ${String(event.error.message ?? event.error)}`);
return;
}
const choice = event?.choices?.[0];
if (!choice) return; // Usage and provider-metadata chunks may have no choices.
const delta = choice.delta ?? {};
if (typeof delta.content === "string" && delta.content) {
text += delta.content;
options.onTextDelta?.(delta.content, text);
}
// Reasoning is not displayed. Preserve provider metadata for replay in
// multi-turn tool loops when the service supplies it.
if (Array.isArray(delta.reasoning_details)) reasoningDetails.add(delta.reasoning_details);
if (delta.thought_signature !== undefined) providerMetadata.thought_signature = delta.thought_signature;
if (Array.isArray(delta.tool_calls)) {
delta.tool_calls.forEach((call: any, position: number) => {
const index = Number.isInteger(call?.index) ? Number(call.index) : position;
const pending = toolCalls.get(index) ?? { id: "", name: "", inputJson: "" };
if (call?.id) pending.id = String(call.id);
if (call?.function?.name) pending.name = String(call.function.name);
if (typeof call?.function?.arguments === "string") pending.inputJson += call.function.arguments;
toolCalls.set(index, pending);
});
}
if (choice.finish_reason) finishReason = String(choice.finish_reason);
},
onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))),
onComplete: () => {
// Some compatible servers close after the finish chunk without [DONE].
if (finishReason) done();
else fail("OpenAI-compatible stream ended before the response completed");
},
onFailure: (message: string) => fail(`OpenAI-compatible connection failed: ${String(message)}`),
};
try {
request = openSseRequest(`${baseUrl}/chat/completions`, JSON.stringify(body), headers, listener);
} catch (error) {
setTimeout(() => fail(`OpenAI-compatible request failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => { settled = true; } };
}
return {
cancel: () => {
settled = true;
try { request?.cancel(); } catch { /* ignore */ }
},
};
}
/** String fields of a reasoning_details entry that stream in pieces. */
const REASONING_DETAIL_TEXT_FIELDS = ["text", "summary", "data"];
/**
* Rebuilds the complete reasoning_details array (OpenRouter's format) from
* streamed deltas. Each delta carries fragments; fragments with the same
* `index` belong to one entry, so their text, summary, and data concatenate
* and their other fields (e.g. a signature that arrives after the text) fill
* in. Entries without an index are kept as sent. The replayed array must
* match what the model produced, so it is reassembled whole rather than
* replaying only the latest chunk.
*/
class ReasoningDetailsAccumulator {
private readonly ordered: Record<string, any>[] = [];
private readonly byIndex = new Map<number, Record<string, any>>();
add(fragments: any[]): void {
for (const fragment of fragments) {
if (!fragment || typeof fragment !== "object" || Array.isArray(fragment)) continue;
const index = Number.isInteger(fragment.index) ? Number(fragment.index) : null;
const existing = index === null ? undefined : this.byIndex.get(index);
if (!existing) {
const entry = { ...fragment };
this.ordered.push(entry);
if (index !== null) this.byIndex.set(index, entry);
continue;
}
for (const [key, value] of Object.entries(fragment)) {
if (value === undefined || value === null) continue;
if (REASONING_DETAIL_TEXT_FIELDS.includes(key) && typeof value === "string" && typeof existing[key] === "string") {
existing[key] += value;
} else if (existing[key] === undefined || existing[key] === null) {
existing[key] = value;
}
}
}
}
entries(): Record<string, any>[] {
return this.ordered;
}
}
/** Convert neutral conversation history to Chat Completions messages. */
function buildChatMessages(options: OpenAiCompatibleStreamOptions): any[] {
const messages: any[] = [];
if (options.system) messages.push({ role: "system", content: options.system });
for (const message of options.messages) {
if (typeof message.content === "string") {
messages.push({ role: message.role, content: message.content });
continue;
}
if (message.role === "assistant") {
let text = "";
const toolCalls: any[] = [];
let providerMetadata: Record<string, any> = {};
for (const block of message.content) {
if (block.type === "text") text += block.text;
else if (block.type === "tool_use") {
toolCalls.push({
id: block.id,
type: "function",
function: { name: block.name, arguments: JSON.stringify(block.input ?? {}) },
});
} else if (block.type === "provider_item" && block.provider === "openai-compatible") {
providerMetadata = { ...providerMetadata, ...block.item };
}
}
if (text || toolCalls.length) {
messages.push({
role: "assistant",
content: text || (toolCalls.length ? null : ""),
...(toolCalls.length ? { tool_calls: toolCalls } : {}),
...providerMetadata,
});
}
continue;
}
let pendingText = "";
const flushText = () => {
if (!pendingText) return;
messages.push({ role: "user", content: pendingText });
pendingText = "";
};
for (const block of message.content) {
if (block.type === "text") pendingText += block.text;
else if (block.type === "tool_result") {
flushText();
messages.push({
role: "tool",
tool_call_id: block.tool_use_id,
content: block.is_error ? `Error: ${block.content}` : block.content,
});
}
}
flushText();
}
return messages;
}
function describeHttpError(code: number, body: string): string {
let detail = "";
try {
const parsed = JSON.parse(body);
detail = String(parsed?.error?.message ?? parsed?.error ?? parsed?.message ?? "");
} catch {
// Non-JSON error body.
}
switch (code) {
case 401: return "OpenAI-compatible: invalid API key";
case 403: return "OpenAI-compatible: API key lacks permission";
case 429: return "OpenAI-compatible: rate limited, try again shortly";
case 500:
case 502:
case 503:
case 504:
return "OpenAI-compatible: service unavailable, try again shortly";
default: return `OpenAI-compatible: HTTP ${code}${detail ? ` (${detail})` : ""}`;
}
}