Support Apple Intelligence as an on-device AI model for the voice assistant/etc

This commit is contained in:
James Babcock committed 2026-10-31 17:48:45 -07:00
1 parent 3c3aa31542
commit 4973fd2aea
22 files changed
+1551 -40

No files matched your search

+3 -1
View File
@@ -3,7 +3,9 @@ TARGETED_DEVICE_FAMILY = 1;
CODE_SIGN_STYLE = Automatic;
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
@@ -0,0 +1,694 @@
import Foundation
#if canImport(FoundationModels)
import FoundationModels
#endif
/// The on-phone LLM on iOS: Apple's Foundation Models framework (the Apple
/// Intelligence system model, iOS 26+, eligible hardware). It runs out of
/// process on the Neural Engine, so it costs the app no model download or
/// memory, and unlike Metal work it is allowed (within a system budget) while
/// Faceclaw is in the background driving the glasses.
///
/// JavaScript (app/native/local-llm.ios.ts) drives the same provider-neutral
/// agent loop as the cloud models: each start() is one loop iteration. The
/// request is JSON:
///
/// {"system":String,
/// "tools":[{"name","description","parameters":<JSON Schema>}],
/// "history":[{"type":"prompt"|"response","text"} |
/// {"type":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]} |
/// {"type":"toolOutput","id","name","text"}],
/// "prompt":String, "maxTokens":Int, "permissive":Bool}
///
/// and events arrive on the main queue through eventHandler:
///
/// {"kind":"delta","text":String} streamed reply text
/// {"kind":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]}
/// {"kind":"done"}
/// {"kind":"error","code":String,"message":String}
/// {"kind":"log","message":String}
///
/// The framework runs tools itself, mid-response. When the model calls one,
/// the call is reported as "toolCalls" and the generation stays suspended
/// inside the tool until resolveToolCalls() hands back the results that
/// JavaScript's tool loop produced; the reply then continues on the same
/// instance.
///
/// The model's context window is small (4096 tokens on iOS 26, shared by input
/// and reply) and the framework spends ~50 tokens on each tool before its
/// parameters, so the assistant's full tool list (30+ tools, ~5000 tokens) can't
/// be offered at once. When the tools don't fit their share of the window, a
/// quick schema-constrained routing request first asks the model which few
/// tools the request needs. History is then trimmed to fit before each start().
/// Called from JavaScript, so on the main thread.
@MainActor
@objc(FaceclawSystemLanguageModel)
@objcMembers
public class FaceclawSystemLanguageModel: NSObject {
public var eventHandler: ((String) -> Void)?
// A LanguageModelEngine, typed loosely so the class itself loads before iOS 26.
private var engine: AnyObject?
/// "available", or why not: "unsupportedOS", "deviceNotEligible",
/// "appleIntelligenceNotEnabled", "modelNotReady" (still downloading), "unavailable".
/// Nonisolated: worker isolates load the model-choice list too, off the main thread.
nonisolated public static func availability() -> String {
#if canImport(FoundationModels)
if #available(iOS 26.0, *) {
switch SystemLanguageModel.default.availability {
case .available:
return "available"
case .unavailable(let reason):
switch reason {
case .deviceNotEligible: return "deviceNotEligible"
case .appleIntelligenceNotEnabled: return "appleIntelligenceNotEnabled"
case .modelNotReady: return "modelNotReady"
@unknown default: return "unavailable"
}
}
}
#endif
return "unsupportedOS"
}
/// The model's context window in tokens (input and output share it), or 0 when unavailable.
nonisolated public static func contextSize() -> Int {
#if canImport(FoundationModels)
if #available(iOS 26.0, *) { return SystemLanguageModel.default.contextSize }
#endif
return 0
}
public func start(_ requestJson: String) {
cancel()
#if canImport(FoundationModels)
if #available(iOS 26.0, *) {
guard let data = requestJson.data(using: .utf8),
let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else {
emit(["kind": "error", "code": "other", "message": "Invalid on-device model request"])
return
}
let engine = LanguageModelEngine { [weak self] event in self?.emit(event) }
self.engine = engine
engine.start(request)
return
}
#endif
emit(["kind": "error", "code": "unavailable", "message": "The on-device model needs iOS 26 or later."])
}
/// `[{"id","content","isError"}]` for the calls of the last "toolCalls" event.
public func resolveToolCalls(_ resultsJson: String) {
#if canImport(FoundationModels)
if #available(iOS 26.0, *) {
let data = resultsJson.data(using: .utf8) ?? Data()
let results = ((try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]]) ?? []
(engine as? LanguageModelEngine)?.resolve(results)
}
#endif
}
/// Abandon the generation (and any suspended tool calls) without further events.
public func cancel() {
#if canImport(FoundationModels)
if #available(iOS 26.0, *) { (engine as? LanguageModelEngine)?.cancel() }
#endif
engine = nil
}
private func emit(_ event: [String: Any]) {
// JSONSerialization raises (not throws) on NaN and other invalid values.
guard JSONSerialization.isValidJSONObject(event),
let data = try? JSONSerialization.data(withJSONObject: event),
let json = String(data: data, encoding: .utf8) else { return }
eventHandler?(json)
}
}
#if canImport(FoundationModels)
/// A JavaScript-side tool: the framework calls it mid-response, and the
/// engine parks the call until JavaScript resolves it.
@available(iOS 26.0, *)
private struct BridgedTool: Tool {
typealias Arguments = GeneratedContent
typealias Output = String
let name: String
let description: String
let parameters: GenerationSchema
let handler: @Sendable (String, String) async throws -> String
func call(arguments: GeneratedContent) async throws -> String {
try await handler(name, arguments.jsonString)
}
}
/// One tool from the request, convertible at two levels of verbosity.
@available(iOS 26.0, *)
private struct ToolSpec {
let name: String
let description: String
let parameters: [String: Any]
}
@available(iOS 26.0, *)
@MainActor
final class LanguageModelEngine {
private struct PendingCall {
let id: String
let name: String
let arguments: String
let continuation: CheckedContinuation<String, Error>
}
/// How long a call waits for any parallel calls before they're reported together.
private static let toolBatchDelayNs: UInt64 = 30_000_000
/// Tokens kept free beyond the reply budget: the framework's own framing and rounding.
private static let contextMargin = 64
/// Share of the context window tool definitions may take before tools are routed.
private static let toolBudgetFraction = 0.4
/// Most tools a routing request picks.
private static let maxRoutedTools = 5
/// Tools used in the latest exchanges stay offered, for follow-ups ("make it ten minutes").
private static let keptRecentTools = 2
private let emit: ([String: Any]) -> Void
private var task: Task<Void, Never>?
// Calls not yet reported, then reported calls awaiting results.
private var unreported: [PendingCall] = []
private var awaiting: [String: PendingCall] = [:]
private var flushScheduled = false
private var closed = false
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
func start(_ request: [String: Any]) {
task = Task { [weak self] in await self?.run(request) }
}
func resolve(_ results: [[String: Any]]) {
for result in results {
guard let id = result["id"] as? String, let call = awaiting.removeValue(forKey: id) else { continue }
let content = (result["content"] as? String) ?? ""
let isError = (result["isError"] as? Bool) ?? false
// An error goes back to the model as text: a thrown error would end the whole response.
call.continuation.resume(returning: isError ? "Error: \(content)" : (content.isEmpty ? "(no output)" : content))
}
// A call JavaScript didn't answer can't be left hanging.
for (_, call) in awaiting {
call.continuation.resume(returning: "Error: the tool returned no result")
}
awaiting.removeAll()
}
func cancel() {
closed = true
task?.cancel()
task = nil
for call in unreported { call.continuation.resume(throwing: CancellationError()) }
for (_, call) in awaiting { call.continuation.resume(throwing: CancellationError()) }
unreported.removeAll()
awaiting.removeAll()
}
/// Events stop once the session is cancelled, even from work still winding down.
private func send(_ event: [String: Any]) {
if !closed { emit(event) }
}
private func run(_ request: [String: Any]) async {
let system = (request["system"] as? String) ?? ""
let prompt = (request["prompt"] as? String) ?? ""
let maxTokens = (request["maxTokens"] as? NSNumber)?.intValue ?? 512
let permissive = (request["permissive"] as? Bool) ?? false
let toolSpecs: [ToolSpec] = ((request["tools"] as? [[String: Any]]) ?? []).compactMap { tool in
guard let name = tool["name"] as? String else { return nil }
return ToolSpec(name: name, description: (tool["description"] as? String) ?? "",
parameters: (tool["parameters"] as? [String: Any]) ?? [:])
}
let history = (request["history"] as? [[String: Any]]) ?? []
// Transformations of user text (translate, summarize) get the permissive
// guardrails; they apply to plain string replies, which tool-free requests are.
let model = SystemLanguageModel(useCase: .general,
guardrails: permissive ? .permissiveContentTransformations : .default)
var offered = toolSpecs
if !toolSpecs.isEmpty {
switch await selectTools(model: model, specs: toolSpecs, history: history, prompt: prompt) {
case .tools(let specs): offered = specs
case .failed(let event):
send(event)
return
}
if closed { return }
}
let fitted = await fit(model: model, system: system, tools: offered, history: history,
prompt: prompt, maxTokens: maxTokens)
if closed { return }
let handler: @Sendable (String, String) async throws -> String = { [weak self] name, arguments in
guard let self = self else { throw CancellationError() }
return try await self.toolInvoked(name: name, arguments: arguments)
}
let tools: [any Tool] = fitted.tools.map {
BridgedTool(name: $0.name, description: $0.description, parameters: $0.schema, handler: handler)
}
let session = LanguageModelSession(model: model, tools: tools, transcript: Transcript(entries: fitted.entries))
let options = GenerationOptions(maximumResponseTokens: maxTokens)
var last = ""
do {
for try await snapshot in session.streamResponse(to: prompt, options: options) {
if closed { return }
let text = snapshot.content
// Snapshots are cumulative; one that doesn't extend the last is a new segment.
let delta = text.hasPrefix(last) ? String(text.dropFirst(last.count)) : text
last = text
if !delta.isEmpty { send(["kind": "delta", "text": delta]) }
}
send(["kind": "done"])
} catch {
if closed || error is CancellationError { return }
send(describe(error))
}
}
// MARK: - Tool calls
private func toolInvoked(name: String, arguments: String) async throws -> String {
if closed { throw CancellationError() }
return try await withCheckedThrowingContinuation { continuation in
if closed {
continuation.resume(throwing: CancellationError())
return
}
let id = "fm_" + UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased()
unreported.append(PendingCall(id: id, name: name, arguments: arguments, continuation: continuation))
scheduleFlush()
}
}
/// Report calls in one event, after a moment for any parallel calls to arrive.
private func scheduleFlush() {
if flushScheduled { return }
flushScheduled = true
Task { [weak self] in
try? await Task.sleep(nanoseconds: LanguageModelEngine.toolBatchDelayNs)
self?.flush()
}
}
private func flush() {
flushScheduled = false
if closed || unreported.isEmpty { return }
let calls = unreported
unreported.removeAll()
for call in calls { awaiting[call.id] = call }
send(["kind": "toolCalls",
"calls": calls.map { ["id": $0.id, "name": $0.name, "arguments": $0.arguments] }])
}
// MARK: - Tool routing
private enum Selection {
case tools([ToolSpec])
case failed([String: Any])
}
/// All tools when their definitions fit their share of the context window;
/// otherwise the few the model picks for this request, plus recently used ones.
private func selectTools(model: SystemLanguageModel, specs: [ToolSpec], history: [[String: Any]],
prompt: String) async -> Selection {
let budget = Int(Double(model.contextSize) * LanguageModelEngine.toolBudgetFraction)
let cost = await toolTokens(model, buildTools(specs, compact: false))
if cost <= budget { return .tools(specs) }
let names = specs.map { $0.name }
let catalog = specs.map { "- \($0.name): \(SchemaBuilder.shorten($0.description, sentences: 1, limit: 100))" }
.joined(separator: "\n")
let instructions = "You pick the tools a voice assistant needs to handle the user's latest request. " +
"Choose at most \(LanguageModelEngine.maxRoutedTools) tools from this list that the assistant should be able to use, " +
"most important first. Choose none for small talk or questions it can answer from general knowledge.\n\n" + catalog
var query = prompt
if let previous = lastPromptText(history) {
query = "Previous request: \(SchemaBuilder.shorten(previous, sentences: 3, limit: 300))\n\nLatest request: \(prompt)"
}
do {
let schema = try GenerationSchema(root: DynamicGenerationSchema(name: "ToolSelection", properties: [
DynamicGenerationSchema.Property(
name: "tools", description: "Names of the tools needed, most important first.",
schema: DynamicGenerationSchema(arrayOf: DynamicGenerationSchema(type: String.self, guides: [.anyOf(names)]),
minimumElements: 0,
maximumElements: LanguageModelEngine.maxRoutedTools)),
]), dependencies: [])
let session = LanguageModelSession(model: model, instructions: instructions)
let response = try await session.respond(to: query, schema: schema,
options: GenerationOptions(sampling: .greedy, maximumResponseTokens: 100))
var chosen = Set(try response.content.value([String].self, forProperty: "tools"))
let recent = recentToolNames(history).prefix(LanguageModelEngine.keptRecentTools)
chosen.formUnion(recent)
let selected = specs.filter { chosen.contains($0.name) }
send(["kind": "log", "message": "routed \(specs.count) tools (\(cost) tokens) to: " +
(selected.isEmpty ? "none" : selected.map { $0.name }.joined(separator: ", "))])
return .tools(selected)
} catch {
if let generationError = error as? LanguageModelSession.GenerationError {
switch generationError {
case .rateLimited, .concurrentRequests, .assetsUnavailable, .guardrailViolation, .refusal,
.unsupportedLanguageOrLocale:
// The real request would fail the same way.
return .failed(describe(error))
default:
break
}
}
send(["kind": "log", "message": "tool routing failed (\(error.localizedDescription)); offering tools in order"])
return .tools(specs)
}
}
private func toolTokens(_ model: SystemLanguageModel, _ tools: [BuiltTool]) async -> Int {
let definitions = tools.map {
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
}
let instructions = Transcript.Entry.instructions(Transcript.Instructions(segments: [], toolDefinitions: definitions))
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
return await tokenCount(model, [instructions], extraCharacters: schemaCharacters + 150 * tools.count)
}
private func lastPromptText(_ history: [[String: Any]]) -> String? {
for item in history.reversed() where item["type"] as? String == "prompt" {
if let text = item["text"] as? String, !text.isEmpty { return text }
}
return nil
}
/// Tool names from the newest tool calls back, without repeats.
private func recentToolNames(_ history: [[String: Any]]) -> [String] {
var names: [String] = []
for item in history.reversed() where item["type"] as? String == "toolCalls" {
for call in (item["calls"] as? [[String: Any]]) ?? [] {
if let name = call["name"] as? String, !names.contains(name) { names.append(name) }
}
}
return names
}
// MARK: - Context fitting
private struct BuiltTool {
let name: String
let description: String
let schema: GenerationSchema
}
private struct Fitted {
let tools: [BuiltTool]
let entries: [Transcript.Entry]
}
/// Instructions + tool definitions + history + prompt must leave the reply
/// budget free. In order, until it fits: shorten tool descriptions, drop the
/// oldest exchanges, then drop the last-registered tools.
private func fit(model: SystemLanguageModel, system: String, tools specs: [ToolSpec],
history: [[String: Any]], prompt: String, maxTokens: Int) async -> Fitted {
let contextSize = model.contextSize
let reserve = min(maxTokens, contextSize / 3) + LanguageModelEngine.contextMargin
var tools = buildTools(specs, compact: false)
var past = transcriptEntries(history)
let promptEntry = Transcript.Entry.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: prompt))]))
func entries() -> [Transcript.Entry] {
let definitions = tools.map {
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
}
let instructions = Transcript.Instructions(segments: [.text(Transcript.TextSegment(content: system))],
toolDefinitions: definitions)
return [.instructions(instructions)] + past
}
func overBudget() async -> Bool {
// The tokenizer can't see tool schemas' size from a ToolDefinition; the fallback estimate adds them.
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
let used = await tokenCount(model, entries() + [promptEntry], extraCharacters: schemaCharacters)
return used + reserve > contextSize
}
if !tools.isEmpty, await overBudget() {
tools = buildTools(specs, compact: true)
send(["kind": "log", "message": "shortened tool descriptions to fit the context window"])
}
var droppedEntries = 0
while !past.isEmpty, await overBudget() {
// Drop the oldest exchange: through to the next user prompt.
past.removeFirst()
droppedEntries += 1
while let first = past.first, !isPrompt(first) {
past.removeFirst()
droppedEntries += 1
}
}
if droppedEntries > 0 {
send(["kind": "log", "message": "dropped \(droppedEntries) history entries to fit the context window"])
}
var droppedTools: [String] = []
while !tools.isEmpty, await overBudget() {
droppedTools.append(tools.removeLast().name)
}
if !droppedTools.isEmpty {
send(["kind": "log", "message": "dropped tools to fit the context window: \(droppedTools.joined(separator: ", "))"])
}
return Fitted(tools: tools, entries: entries())
}
private func tokenCount(_ model: SystemLanguageModel, _ entries: [Transcript.Entry], extraCharacters: Int) async -> Int {
if #available(iOS 26.4, macOS 26.4, *) {
if let count = try? await model.tokenCount(for: entries) { return count }
}
// Before iOS 26.4 there's no tokenizer API: estimate conservatively.
var characters = extraCharacters
for entry in entries { characters += estimateCharacters(entry) + 16 }
return characters / 3
}
private func estimateCharacters(_ entry: Transcript.Entry) -> Int {
func segments(_ list: [Transcript.Segment]) -> Int {
list.reduce(0) { total, segment in
switch segment {
case .text(let text): return total + text.content.count
case .structure(let structure): return total + structure.content.jsonString.count
@unknown default: return total
}
}
}
switch entry {
case .instructions(let instructions):
var total = segments(instructions.segments)
for definition in instructions.toolDefinitions {
total += definition.name.count + definition.description.count
}
return total
case .prompt(let prompt): return segments(prompt.segments)
case .response(let response): return segments(response.segments)
case .toolCalls(let calls): return calls.reduce(0) { $0 + $1.toolName.count + $1.arguments.jsonString.count }
case .toolOutput(let output): return segments(output.segments)
@unknown default: return 0
}
}
private func isPrompt(_ entry: Transcript.Entry) -> Bool {
if case .prompt = entry { return true }
return false
}
// MARK: - Request conversion
private func transcriptEntries(_ history: [[String: Any]]) -> [Transcript.Entry] {
var entries: [Transcript.Entry] = []
for item in history {
let text = (item["text"] as? String) ?? ""
switch item["type"] as? String {
case "prompt":
entries.append(.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: text))])))
case "response":
entries.append(.response(Transcript.Response(assetIDs: [], segments: [.text(Transcript.TextSegment(content: text))])))
case "toolCalls":
let calls: [Transcript.ToolCall] = ((item["calls"] as? [[String: Any]]) ?? []).compactMap { call in
guard let id = call["id"] as? String, let name = call["name"] as? String else { return nil }
let arguments = (try? GeneratedContent(json: (call["arguments"] as? String) ?? "{}"))
?? GeneratedContent(properties: [:])
return Transcript.ToolCall(id: id, toolName: name, arguments: arguments)
}
if !calls.isEmpty { entries.append(.toolCalls(Transcript.ToolCalls(calls))) }
case "toolOutput":
guard let id = item["id"] as? String else { continue }
entries.append(.toolOutput(Transcript.ToolOutput(
id: id, toolName: (item["name"] as? String) ?? "tool",
segments: [.text(Transcript.TextSegment(content: text))])))
default:
continue
}
}
return entries
}
private func buildTools(_ specs: [ToolSpec], compact: Bool) -> [BuiltTool] {
specs.map { spec in
var builder = SchemaBuilder(compact: compact)
let schema: GenerationSchema
do {
schema = try GenerationSchema(root: builder.object(spec.parameters, name: "\(spec.name)_arguments"),
dependencies: [])
} catch {
send(["kind": "log", "message": "tool \(spec.name): unusable parameter schema (\(error)); offering it without parameters"])
schema = (try? GenerationSchema(root: DynamicGenerationSchema(name: "\(spec.name)_arguments", properties: []),
dependencies: []))
?? GenerationSchema(type: GeneratedContent.self, properties: [])
}
let description = compact ? SchemaBuilder.shorten(spec.description, sentences: 1, limit: 160) : spec.description
return BuiltTool(name: spec.name, description: description, schema: schema)
}
}
// MARK: - Errors
private func describe(_ error: Error) -> [String: Any] {
if let error = error as? LanguageModelSession.GenerationError {
switch error {
case .exceededContextWindowSize:
return failure("context", "That conversation is too long for the on-device model. Start a new conversation and try again.")
case .assetsUnavailable:
return failure("unavailable", "The Apple Intelligence model isn't available right now. Check that Apple Intelligence is on and finished downloading.")
case .guardrailViolation, .refusal:
return failure("refusal", "The on-device model declined this request.")
case .unsupportedLanguageOrLocale:
return failure("language", "The on-device model doesn't support this language.")
case .rateLimited:
return failure("rateLimited", "iOS is limiting the on-device model while Faceclaw is in the background. Try again in a moment, or with the phone unlocked.")
case .concurrentRequests:
return failure("busy", "The on-device model is busy with another request. Try again in a moment.")
case .decodingFailure, .unsupportedGuide:
return failure("other", "The on-device model produced an unreadable reply: \(error.localizedDescription)")
@unknown default:
return failure("other", "On-device model failed: \(error.localizedDescription)")
}
}
if let error = error as? LanguageModelSession.ToolCallError {
return failure("other", "On-device model tool call failed: \(error.underlyingError.localizedDescription)")
}
return failure("other", "On-device model failed: \(error.localizedDescription)")
}
private func failure(_ code: String, _ message: String) -> [String: Any] {
["kind": "error", "code": code, "message": message]
}
}
/// JSON Schema (the subset our tool definitions use: objects of string, number,
/// integer, boolean, enum and array properties) to a DynamicGenerationSchema, so
/// the framework constrains tool arguments to valid values while decoding.
/// Anything it doesn't understand becomes a string rather than failing.
@available(iOS 26.0, *)
private struct SchemaBuilder {
let compact: Bool
private var names = Set<String>()
init(compact: Bool) { self.compact = compact }
mutating func object(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
let properties = (json["properties"] as? [String: Any]) ?? [:]
let required = Set((json["required"] as? [String]) ?? [])
// JSON objects lose declaration order; keep required properties first, then by name.
let keys = properties.keys.sorted { a, b in
let ra = required.contains(a), rb = required.contains(b)
return ra != rb ? ra : a < b
}
let members: [DynamicGenerationSchema.Property] = keys.compactMap { key in
guard let property = properties[key] as? [String: Any] else { return nil }
return DynamicGenerationSchema.Property(
name: key,
description: describe(property),
schema: value(property, name: "\(name)_\(key)"),
isOptional: !required.contains(key))
}
return DynamicGenerationSchema(name: unique(name), description: nil, properties: members)
}
private mutating func value(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
if let choices = json["enum"] as? [Any], !choices.isEmpty {
let strings = choices.compactMap { $0 as? String }
if strings.count == choices.count {
// A guide renders as one compact JSON Schema enum; an anyOf schema lists
// every choice as its own subschema, at over twice the tokens.
return DynamicGenerationSchema(type: String.self, guides: [.anyOf(strings)])
}
}
switch schemaType(json) {
case "integer":
var guides: [GenerationGuide<Int>] = []
if let minimum = (json["minimum"] as? NSNumber)?.intValue { guides.append(.minimum(minimum)) }
if let maximum = (json["maximum"] as? NSNumber)?.intValue { guides.append(.maximum(maximum)) }
return DynamicGenerationSchema(type: Int.self, guides: guides)
case "number":
var guides: [GenerationGuide<Double>] = []
if let minimum = (json["minimum"] as? NSNumber)?.doubleValue { guides.append(.minimum(minimum)) }
if let maximum = (json["maximum"] as? NSNumber)?.doubleValue { guides.append(.maximum(maximum)) }
return DynamicGenerationSchema(type: Double.self, guides: guides)
case "boolean":
return DynamicGenerationSchema(type: Bool.self)
case "array":
let items = (json["items"] as? [String: Any]) ?? ["type": "string"]
return DynamicGenerationSchema(arrayOf: value(items, name: "\(name)_item"),
minimumElements: (json["minItems"] as? NSNumber)?.intValue,
maximumElements: (json["maxItems"] as? NSNumber)?.intValue)
case "object":
return object(json, name: name)
default:
return DynamicGenerationSchema(type: String.self)
}
}
private func schemaType(_ json: [String: Any]) -> String {
if let type = json["type"] as? String { return type }
if let types = json["type"] as? [String], let type = types.first(where: { $0 != "null" }) { return type }
return json["properties"] != nil ? "object" : "string"
}
private func describe(_ json: [String: Any]) -> String? {
guard let text = json["description"] as? String, !text.isEmpty else { return nil }
return compact ? SchemaBuilder.shorten(text, sentences: 1, limit: 80) : text
}
/// Schema names must be unique within one GenerationSchema.
private mutating func unique(_ base: String) -> String {
var name = base
var suffix = 2
while names.contains(name) {
name = "\(base)_\(suffix)"
suffix += 1
}
names.insert(name)
return name
}
/// The first `sentences` sentences, cut to `limit` characters.
static func shorten(_ text: String, sentences: Int, limit: Int) -> String {
var result = ""
var count = 0
var remainder = Substring(text)
while count < sentences, !remainder.isEmpty {
if let end = remainder.range(of: ". ") {
result += remainder[..<end.lowerBound] + "."
remainder = remainder[end.upperBound...]
if count + 1 < sentences { result += " " }
} else {
result += remainder
remainder = ""
}
count += 1
}
if result.count > limit { result = String(result.prefix(limit - 1)) + "…" }
return result
}
}
#endif
+1
View File
@@ -7,6 +7,7 @@
* Ensure connection is re-set to 2M on reconnect
* Add support for embedded icons and custom layouts in Android notifications
* Port the Developer app's resource-usage monitor and various test pages to iOS
* iOS: Support Apple Intelligence as an on-device model for voice assistant/etc
* iOS: Notifications are selectable via the task switcher
* Android: Optionally sign in to Spotify's Web API for library browsing and other features
+2 -2
View File
@@ -7,7 +7,7 @@
* surfacing a transport error.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamLocalModel } from "../../native/local-llm";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel } from "../../assistant/models";
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
@@ -41,7 +41,7 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
streamOpenAiResponse(options);
break;
case "local":
streamLocalQwen(options);
streamLocalModel(options);
break;
default:
streamAnthropicMessage(options);
+2 -2
View File
@@ -1,6 +1,6 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
import { parseConversationQuery } from "../../phone-ui/conversation-search";
@@ -245,7 +245,7 @@ export function askConversations(
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
return { cancel: () => {} };
}
return streamLocalQwen({
return streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -1,4 +1,4 @@
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import type { NameInference } from "./name-guess";
/**
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
};
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
try {
streamLocalQwen({
streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -1,6 +1,6 @@
import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
import { speakerRegistry } from "./speakers";
declare const com: any;
@@ -124,7 +124,7 @@ function generateForSpeaker(
};
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
try {
streamLocalQwen({
streamLocalModel({
apiKey: "",
model: "local",
system:
+2 -2
View File
@@ -4,7 +4,7 @@
* TranslationModel for how stretches are batched and given context.
*/
import { streamAnthropicMessage } from "../../native/anthropic";
import { streamLocalQwen } from "../../native/llama";
import { streamLocalModel } from "../../native/local-llm";
import { streamOpenAiResponse } from "../../native/openai";
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
@@ -55,7 +55,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
case "openai":
return streamOpenAiResponse(options);
case "local":
return streamLocalQwen(options);
return streamLocalModel(options);
default:
return streamAnthropicMessage(options);
}
+2 -2
View File
@@ -1,6 +1,6 @@
import { streamAnthropicMessage } from "../native/anthropic";
import { streamHermesChat } from "../native/hermes";
import { streamLocalQwen } from "../native/llama";
import { streamLocalModel } from "../native/local-llm";
import { streamOpenAiResponse } from "../native/openai";
import type {
LlmContentBlock,
@@ -146,7 +146,7 @@ export class DirectAssistantBackend {
: options.provider === "openai"
? streamOpenAiResponse(streamOptions)
: options.provider === "local"
? streamLocalQwen(streamOptions)
? streamLocalModel(streamOptions)
: streamAnthropicMessage(streamOptions);
};
+13 -7
View File
@@ -1,4 +1,4 @@
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
export const ASSISTANT_MODEL_VALUES = [
"auto",
@@ -13,12 +13,17 @@ export const ASSISTANT_MODEL_VALUES = [
"hermes",
] as const;
/**
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
* support and stays for saved settings and conversations).
*/
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
/** Keep saved model IDs portable, while offering only supported choices. */
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
return global.isIOS && value === "qwen" ? "auto" : value;
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
}
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
@@ -58,8 +63,9 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
// not an API key. The default for users who haven't set any key.
// Runs on the phone itself (Android: llama.cpp with the downloaded model
// file; iOS: Apple Intelligence), not an API key. The default for users who
// haven't set any key.
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
// Your own Hermes Agent (its API server), with its own tools and memory.
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
@@ -76,8 +82,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
/**
* Resolve Auto and attach the credential needed by the chosen provider.
* Auto intentionally prefers Terra over Sonnet when both keys are available,
* and falls back to the on-phone model when no key is set but the model file
* has been downloaded.
* and falls back to the on-phone model when no key is set but that model is
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
*/
export function resolveAssistantModel(
selection: AssistantModel,
+4 -6
View File
@@ -6,12 +6,14 @@ import type {
LlmStreamOptions,
} from "../assistant/llm-protocol";
import { buildQwenPrompt } from "../assistant/qwen-prompt";
import type { LocalModelState } from "./local-llm-types";
declare const com: any;
declare const java: any;
/**
* On-phone LLM backend: model download management plus a streaming client
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
* local-llm.ios.ts instead): model download management plus a streaming client
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
* the direct assistant loop can treat "local" as just another provider.
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
@@ -38,11 +40,7 @@ const TOP_K = 20;
/** Free the ~3GB of model + KV this long after the last generation. */
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
export type LocalModelState = {
status: "absent" | "downloading" | "ready";
bytesDownloaded: number;
totalBytes: number;
};
export type { LocalModelState } from "./local-llm-types";
let runner: any = null;
let downloader: any = null;
+13
View File
@@ -0,0 +1,13 @@
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
export type LocalModelState = {
/**
* Android downloads its model: absent → downloading → ready. iOS uses the
* system's Apple Intelligence model: ready, or unavailable (see detail).
*/
status: "absent" | "downloading" | "ready" | "unavailable";
bytesDownloaded: number;
totalBytes: number;
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
detail?: string;
};
+383
View File
@@ -0,0 +1,383 @@
import type {
LlmContentBlock,
LlmMessage,
LlmStreamHandle,
LlmStreamOptions,
} from "../assistant/llm-protocol";
import type { LocalModelState } from "./local-llm-types";
export type { LocalModelState } from "./local-llm-types";
declare const FaceclawSystemLanguageModel: any;
/**
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
* Calculator and the Microphones helpers treat it as the "local" provider.
*
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
* background (where Faceclaw usually is, driving the glasses with the phone
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
* ~3 GB in-process model makes the app the first thing jetsam kills. The
* system model runs out of process on the Neural Engine, needs no download,
* and decodes tool arguments under a schema constraint (like our GBNF grammar
* on Android). Its costs: a 4096-token context (the Swift side fits history
* and tool definitions into it), Apple's guardrails, a background request
* budget, and Apple Intelligence-capable hardware.
*
* Tool calls: the framework runs tools itself, mid-response. The Swift side
* parks each call and reports it; here that ends the loop iteration with
* stopReason "tool_use", the direct backend executes the tools as for any
* provider, and its next request (ending in the matching tool_result blocks)
* resumes the parked generation instead of starting a new one.
*/
export const LOCAL_MODEL = {
id: "apple-foundation-model",
label: "Apple Intelligence",
sizeBytes: 0,
};
const DEFAULT_MAX_TOKENS = 512;
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
/**
* Only used when tool results arrive with no parked generation to resume (it
* timed out, or the app restarted mid-turn): the framework has no way to
* continue after a tool output without a new prompt.
*/
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
type Availability =
| "available"
| "unsupportedOS"
| "deviceNotEligible"
| "appleIntelligenceNotEnabled"
| "modelNotReady"
| "unavailable";
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "needs iOS 26",
deviceNotEligible: "not on this iPhone",
appleIntelligenceNotEnabled: "Apple Intelligence off",
modelNotReady: "downloading",
unavailable: "unavailable",
};
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
appleIntelligenceNotEnabled:
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
};
export function localModelAvailability(): Availability {
try {
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
const value = String(FaceclawSystemLanguageModel.availability());
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
} catch {
return "unsupportedOS";
}
}
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
export function isLocalModelSupported(): boolean {
const availability = localModelAvailability();
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
}
export function isLocalModelReady(): boolean {
return localModelAvailability() === "available";
}
export function localModelState(): LocalModelState {
const availability = localModelAvailability();
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
}
/** The user-facing reason the model can't run, or "" when it can. */
export function localModelUnavailableMessage(): string {
const availability = localModelAvailability();
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
}
// iOS manages the system model itself: there is nothing to download, cancel,
// delete or unload, and no progress to watch.
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
return () => {};
}
export function startLocalModelDownload(): void {}
export function cancelLocalModelDownload(): void {}
export function deleteLocalModel(): void {}
export function unloadLocalModel(): void {}
type HistoryEntry =
| { type: "prompt" | "response"; text: string }
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
| { type: "toolOutput"; id: string; name: string; text: string };
export type SystemModelRequest = {
system: string;
tools: Array<{ name: string; description: string; parameters: object }>;
history: HistoryEntry[];
prompt: string;
maxTokens: number;
permissive: boolean;
};
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
/**
* Translate provider-neutral history into the Swift request: everything but
* the newest user prompt becomes Transcript entries, and that prompt is what
* the model answers.
*/
export function buildSystemModelRequest(
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
): SystemModelRequest {
const messages = options.messages;
const last = messages[messages.length - 1];
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
const history: HistoryEntry[] = [];
const toolNames = new Map<string, string>();
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
appendHistory(history, message, toolNames);
}
const tools = (options.tools ?? []).map((tool) => ({
name: tool.name,
description: tool.description,
parameters: tool.input_schema,
}));
return {
system: options.system?.trim() || "You are a helpful assistant.",
tools,
history,
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
permissive: tools.length === 0,
};
}
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
if (message.role === "user") {
for (const result of toolResultsOf(message)) {
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
}
const text = textOf(message.content);
if (text) history.push({ type: "prompt", text });
return;
}
const text = textOf(message.content);
if (text) history.push({ type: "response", text });
if (typeof message.content === "string") return;
const calls: Array<{ id: string; name: string; arguments: string }> = [];
for (const block of message.content) {
if (block.type !== "tool_use") continue;
toolNames.set(block.id, block.name);
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
}
if (calls.length) history.push({ type: "toolCalls", calls });
}
function textOf(content: string | LlmContentBlock[]): string {
if (typeof content === "string") return content.trim();
return content
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
.map((block) => block.text.trim())
.filter(Boolean)
.join("\n");
}
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
if (!message || message.role !== "user" || typeof message.content === "string") return [];
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
}
function parseArguments(json: string): any {
try {
const value = JSON.parse(json);
return value && typeof value === "object" ? value : {};
} catch {
return {};
}
}
type NativeEvent = {
kind: string;
text?: string;
calls?: Array<{ id: string; name: string; arguments: string }>;
code?: string;
message?: string;
};
/** Generations parked in tool calls, by call id. */
const parked = new Map<string, Generation>();
/** Every unfinished generation, so its native object outlives the caller's handle. */
const live = new Set<Generation>();
/**
* One native generation, possibly spanning several loop iterations: each
* iteration ends at a batch of tool calls (or the end of the reply), and the
* next one resumes it with the results.
*/
class Generation {
private readonly native: any;
private options: LlmStreamOptions | null;
private iteration = 0;
private text = "";
private callIds: string[] = [];
private expiry: ReturnType<typeof setTimeout> | null = null;
private closed = false;
constructor(options: LlmStreamOptions) {
this.options = options;
this.native = FaceclawSystemLanguageModel.new();
this.native.eventHandler = (json: string) => this.onEvent(String(json));
live.add(this);
}
start(request: SystemModelRequest): LlmStreamHandle {
this.native.start(JSON.stringify(request));
return this.handle();
}
/** Whether these results answer this generation's parked calls. */
awaits(results: ToolResultBlock[]): boolean {
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
}
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
this.unpark();
this.options = options;
this.iteration++;
this.text = "";
this.native.resolveToolCalls(
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
);
return this.handle();
}
cancel(): void {
if (this.closed) return;
this.close();
try {
this.native.cancel();
} catch {
// ignore
}
}
private handle(): LlmStreamHandle {
const iteration = this.iteration;
// A stale handle (an earlier iteration's) still cancels a generation parked
// since then: its owner gave up on the turn.
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
}
private close(): void {
this.closed = true;
this.unpark();
this.options = null;
this.native.eventHandler = null;
live.delete(this);
}
private park(ids: string[]): void {
this.callIds = ids;
for (const id of ids) parked.set(id, this);
this.expiry = setTimeout(() => {
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
this.cancel();
}, PARKED_TIMEOUT_MS);
}
private unpark(): void {
for (const id of this.callIds) {
if (parked.get(id) === this) parked.delete(id);
}
this.callIds = [];
if (this.expiry !== null) clearTimeout(this.expiry);
this.expiry = null;
}
private onEvent(json: string): void {
if (this.closed) return;
let event: NativeEvent;
try {
event = JSON.parse(json);
} catch {
return;
}
if (event.kind === "log") {
console.log(`[local-llm] ${event.message ?? ""}`);
return;
}
const options = this.options;
if (!options) return;
switch (event.kind) {
case "delta": {
const delta = event.text ?? "";
this.text += delta;
options.onTextDelta?.(delta, this.text);
return;
}
case "toolCalls": {
const calls = event.calls ?? [];
const text = this.text.trim();
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
for (const call of calls) {
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
}
this.options = null;
this.park(calls.map((call) => call.id));
options.onDone({ text, content, stopReason: "tool_use" });
return;
}
case "done": {
this.close();
const text = this.text.trim();
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
return;
}
case "error": {
this.close();
if (event.code === "refusal") {
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
} else {
options.onError(event.message || "The on-device model failed");
}
return;
}
}
}
}
/**
* Stream a reply from the system model. A request whose history ends in tool
* results for a parked generation resumes it; anything else starts afresh.
*/
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
const availability = localModelAvailability();
if (availability !== "available") {
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
return { cancel: () => {} };
}
const results = toolResultsOf(options.messages[options.messages.length - 1]);
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
if (waiting?.awaits(results)) return waiting.resume(options, results);
let generation: Generation | null = null;
try {
generation = new Generation(options);
return generation.start(buildSystemModelRequest(options));
} catch (error) {
generation?.cancel();
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
return { cancel: () => {} };
}
}
+28
View File
@@ -0,0 +1,28 @@
/**
* The on-phone LLM: one entry point for every feature that can run on the
* phone without an API key (the direct assistant's "local" provider,
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
* contract, plus model status for Settings.
*
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
* to download, available on Apple Intelligence devices running iOS 26+.
*/
export {
LOCAL_MODEL,
isLocalModelReady,
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
cancelLocalModelDownload,
deleteLocalModel,
unloadLocalModel,
streamLocalQwen as streamLocalModel,
} from "./llama";
export type { LocalModelState } from "./local-llm-types";
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
export function isLocalModelSupported(): boolean {
return true;
}
+4 -4
View File
@@ -16,7 +16,7 @@ import {
assistantModelProvider,
type AssistantModel,
} from "~/assistant/models";
import { isLocalModelReady } from "../native/llama";
import { isLocalModelReady } from "../native/local-llm";
import { normalizeWhisperServerUrl } from "../native/whisper-server-stt";
import { isValidSpotifyClientId, SPOTIFY_CLIENT_ID_KEY } from "../native/spotify-auth";
import { drawRightValueMenuItem, drawToggleMenuItem, MenuItem, openModalMenu } from "./menu";
@@ -930,7 +930,7 @@ export const assistantSkipConfirmationSetting = new ConfigSettingBoolean({
export type AssistantBackendKind = "direct" | "external";
const assistantBackendLabels: Record<AssistantBackendKind, string> = {
direct: global.isIOS ? "Cloud API" : "On-phone",
direct: "On-phone",
external: "My own agent (bridge)",
};
@@ -942,7 +942,7 @@ export const assistantBackendSetting = new ConfigSettingEnum<AssistantBackendKin
values: global.isIOS ? ["direct"] : ["direct", "external"],
formatValue: (value) => assistantBackendLabels[value] ?? value,
description:
global.isIOS ? "Cloud models called with your OpenAI or Anthropic API key. Add your key in Settings > API Keys." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
global.isIOS ? "An LLM called from the phone: a cloud model with your OpenAI or Anthropic API key (Settings > API Keys), or Apple Intelligence's on-device model." : "Who answers assistant queries: an LLM called from the phone (a cloud API with your key, or the downloaded on-phone model), or your own long-running agent (e.g. OpenClaw) reached through the faceclaw-agent-bridge plugin.",
});
export const assistantHermesUrlSetting = new ConfigSettingString({
@@ -1116,7 +1116,7 @@ export const assistantModelSetting = new ConfigSettingEnum<AssistantModel>({
return false;
},
description:
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
global.isIOS ? "Model used by the voice assistant. Auto prefers Terra with an OpenAI key, then Sonnet with an Anthropic key, then Apple Intelligence's on-device model." : "Model used by the voice assistant. Auto prefers Terra when an OpenAI key is set, then Sonnet when an Anthropic key is set, then the downloaded on-phone model.",
});
export const mapboxApiKeySetting = new ConfigSettingString({
+27 -4
View File
@@ -12,7 +12,7 @@ import {
localModelState,
onLocalModelStateChanged,
startLocalModelDownload,
} from "../../native/llama";
} from "../../native/local-llm";
import {
ASR_MODELS,
asrModelState,
@@ -143,14 +143,14 @@ function settingsSections(): SettingsSection[] {
{
label: "Assistant",
items: [
// iOS offers cloud APIs; Android also supports local models and the bridge.
// iOS's on-phone model is Apple Intelligence; Android downloads one. Only Android has the bridge.
enumSettingMenuItem(assistantBackendSetting),
enumSettingMenuItem(assistantModelSetting),
textSettingMenuItem(assistantHermesUrlSetting),
textSettingMenuItem(assistantHermesKeySetting),
...(!global.isIOS ? [localModelMenuItem()] : []),
global.isIOS ? systemModelMenuItem() : localModelMenuItem(),
toggleSettingMenuItem(assistantSkipConfirmationSetting),
// iOS has no local LLM or agent bridge yet.
// iOS has no agent bridge yet.
...(!global.isIOS ? [
textSettingMenuItem(assistantBridgeHostSetting),
textSettingMenuItem(assistantBridgePortSetting),
@@ -399,6 +399,29 @@ function localModelStatusText(): string {
return "not downloaded";
}
/**
* iOS: the on-phone model is the Apple Intelligence system model, which iOS
* installs and manages; the row only reports whether it can be used.
*/
function systemModelMenuItem(): MenuItem {
const statusText = (): string => {
const state = localModelState();
return state.status === "ready" ? "ready" : state.detail ?? "unavailable";
};
return {
label: "On-phone model",
description:
"Apple Intelligence's on-device model answers assistant queries on the phone itself, with no API key or cloud service. " +
"Needs iOS 26 and an iPhone that supports Apple Intelligence, with Apple Intelligence turned on in iOS Settings. " +
"Faster and more private than the cloud models, but simpler, with a short memory. " +
"Used automatically when no API key is set.",
onSelect: () => {},
render: ({ image, x, y, width }) => {
drawRightValueMenuItem(image, getDefaultSmallFont(), x, y, width, "On-phone model", statusText());
},
};
}
/** Download/cancel/delete management for the on-phone assistant model. */
function localModelMenuItem(): MenuItem {
return {
@@ -0,0 +1,52 @@
// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]),
// pulled from the tool files' object literals via the TypeScript AST, with the dynamic
// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of
// what ToolRegistry.listTools() offers with no app windows open.
// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json
const path = require('path');
const fs = require('fs');
const root = path.resolve(__dirname, '../..');
const ts = require(path.join(root, 'node_modules/typescript'));
const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools']
.map(name => path.join(root, 'app/assistant', `${name}.ts`));
const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true })
.filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name))
.map(entry => entry.name).sort();
const NUMBER_ARG = (noun, listTool) => ({
type: 'number',
description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`,
});
const out = [];
for (const file of FILES) {
const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true);
const consts = {};
src.forEachChild(function collect(node) {
if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) {
consts[node.name.getText()] = node.initializer.text;
}
node.forEachChild(collect);
});
const evaluate = node => {
const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText;
const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG'];
return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG);
};
src.forEachChild(function visit(node) {
if (ts.isObjectLiteralExpression(node)) {
const keys = node.properties.map(p => p.name && p.name.getText());
if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) {
try {
const spec = evaluate(node);
out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema });
} catch (error) {
console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`);
}
return;
}
}
node.forEachChild(visit);
});
}
process.stdout.write(JSON.stringify(out, null, 1));
+50
View File
@@ -0,0 +1,50 @@
// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one
// assistant request against the Apple Intelligence model the way local-llm.ios.ts does,
// answering every tool call with a canned result. See run.sh.
import Foundation
@main struct Harness {
@MainActor static func main() async {
let args = CommandLine.arguments
guard args.count >= 3 else {
print("usage: harness <tools.json> <prompt> [previous prompt]")
return
}
print("availability:", FaceclawSystemLanguageModel.availability(),
"context:", FaceclawSystemLanguageModel.contextSize())
let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? []
var history: [[String: Any]] = []
if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] }
let request: [String: Any] = [
"system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
"tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false,
]
let model = FaceclawSystemLanguageModel()
let started = Date()
var finished = false
var firstText: TimeInterval?
model.eventHandler = { json in
let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started))
guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return }
switch event["kind"] as? String {
case "delta":
if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") }
print(event["text"] as? String ?? "", terminator: "")
case "toolCalls":
let calls = event["calls"] as? [[String: Any]] ?? []
for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") }
let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] }
let data = try! JSONSerialization.data(withJSONObject: results)
model.resolveToolCalls(String(data: data, encoding: .utf8)!)
case "done":
print("\n\(elapsed) done"); finished = true
default:
print("\(elapsed) \(json)")
if event["kind"] as? String == "error" { finished = true }
}
}
let data = try! JSONSerialization.data(withJSONObject: request)
model.start(String(data: data, encoding: .utf8)!)
while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) }
}
}
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+,
# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the
# assistant's real tool list. Prints tool routing, tool calls and the streamed reply.
# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"]
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness"
mkdir -p "$out"
node "$here/extract-tools.cjs" > "$out/tools.json"
swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \
"$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness"
"$out/harness" "$out/tools.json" "$@"
+2 -2
View File
@@ -91,7 +91,7 @@ test('a restored conversation keeps its Hermes session, and Hermes turns get a l
requests.push(request);
return request;
} },
'../native/llama': { streamLocalQwen: () => assert.fail('no local model') },
'../native/local-llm': { streamLocalModel: () => assert.fail('no local model') },
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: {} },
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' },
});
@@ -122,7 +122,7 @@ test('URL normalization adds /v1 to a bare host and drops the endpoint path', ()
});
test('the Hermes model resolves only with a URL, carrying the key and base URL', () => {
const load = loader({ global: { isIOS: true } }, { '../native/llama': { isLocalModelReady: () => false, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
const load = loader({ global: { isIOS: true } }, { '../native/local-llm': { isLocalModelReady: () => false, isLocalModelSupported: () => true, LOCAL_MODEL: { id: 'q', label: 'Q' } } });
const { resolveAssistantModel } = load('app/assistant/models.ts');
assert.equal(resolveAssistantModel('hermes', { anthropic: '', openai: '' }), null);
const resolved = resolveAssistantModel('hermes', { anthropic: '', openai: '', hermesUrl: ' http://hermes-host:8642 ', hermesKey: ' k ' });
+4 -4
View File
@@ -12,7 +12,7 @@ function assistant(provider) {
const request = { url, body: JSON.parse(body), headers, listener, cancelled: false };
requests.push(request); return { cancel: () => { request.cancelled = true; } };
} },
'../native/llama': { streamLocalQwen: () => assert.fail('iOS must not invoke a local LLM') },
'../native/local-llm': { streamLocalModel: () => assert.fail('cloud providers must not invoke the local LLM') },
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'test', buildAssistantSystemPrompt: () => 'test', describeAssistantContext: () => '' },
});
@@ -100,8 +100,8 @@ test('iOS SSE adapter forwards the Kotlin stream events once and stops after can
assert.deepEqual(seen.at(-1), 'http:429:slow down');
});
test('iOS only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
const api = loader({ global: { isIOS: true } }, { '../native/llama': { LOCAL_MODEL: { label: 'Qwen', id: 'qwen' }, isLocalModelReady: () => false } })('app/assistant/models.ts');
test('iOS without Apple Intelligence support only offers cloud assistant models and preserves old local-model conversations as Auto', () => {
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => false, isLocalModelSupported: () => false } })('app/assistant/models.ts');
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), false);
assert.equal(api.supportedAssistantModel('qwen'), 'auto');
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai');
@@ -119,7 +119,7 @@ test('cancelling during a tool prevents subsequent tool side effects and a stall
const calls = [], failures = [];
const { DirectAssistantBackend } = loader({ setTimeout: fn => { deadline = fn; return 1; }, clearTimeout() {} }, {
'../native/openai': { streamOpenAiResponse: options => { stream = options; return { cancel() {} }; } },
'../native/anthropic': {}, '../native/llama': {},
'../native/anthropic': {}, '../native/local-llm': {},
})('app/assistant/direct-backend.ts');
const backend = new DirectAssistantBackend();
const options = { provider: 'openai', messages: [], buildTools: () => [], resolveToolName: name => name,
+248
View File
@@ -0,0 +1,248 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const flush = () => new Promise(resolve => setImmediate(resolve));
// Values built inside the loader's VM context have that realm's prototypes.
const plain = value => JSON.parse(JSON.stringify(value));
/** A stand-in for FaceclawSystemLanguageModel.swift: records requests and lets tests emit its events. */
function fakeNative(availability = 'available') {
const instances = [];
const cls = {
availability: () => availability,
new() {
const native = {
request: null, resolved: [], cancelled: false, eventHandler: null,
start(json) { this.request = JSON.parse(json); },
resolveToolCalls(json) { this.resolved.push(JSON.parse(json)); },
cancel() { this.cancelled = true; },
emit(event) { if (this.eventHandler) this.eventHandler(JSON.stringify(event)); },
};
instances.push(native);
return native;
},
};
return { cls, instances, setAvailability(value) { availability = value; } };
}
function loadLocalLlm(native, extraGlobals = {}) {
const load = loader({ FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout, ...extraGlobals }, {});
return load('app/native/local-llm.ios.ts');
}
function collector() {
const seen = { deltas: [], done: [], errors: [] };
return {
seen,
callbacks: {
onTextDelta: (_delta, full) => seen.deltas.push(full),
onDone: result => seen.done.push(result),
onError: message => seen.errors.push(message),
},
};
}
test('iOS local LLM converts provider-neutral history into transcript entries and a final prompt', () => {
const api = loadLocalLlm(fakeNative());
const request = api.buildSystemModelRequest({
system: 'Be brief.',
maxTokens: 200,
tools: [{ name: 'timer_set', description: 'Set a timer', input_schema: { type: 'object', properties: { seconds: { type: 'integer' } } } }],
messages: [
{ role: 'user', content: 'Set a timer' },
{ role: 'assistant', content: [{ type: 'text', text: 'On it.' }, { type: 'tool_use', id: 'fm_1', name: 'timer_set', input: { seconds: 60 } }] },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'Timer set' }] },
{ role: 'assistant', content: 'Done.' },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'gone', content: 'boom', is_error: true }, { type: 'text', text: 'And now?' }] },
{ role: 'assistant', content: [{ type: 'provider_item', provider: 'openai', item: {} }] },
{ role: 'user', content: 'What time is it?' },
],
});
assert.equal(request.system, 'Be brief.');
assert.equal(request.prompt, 'What time is it?');
assert.equal(request.maxTokens, 200);
assert.equal(request.permissive, false);
assert.deepEqual(plain(request.tools), [{ name: 'timer_set', description: 'Set a timer', parameters: { type: 'object', properties: { seconds: { type: 'integer' } } } }]);
assert.deepEqual(plain(request.history), [
{ type: 'prompt', text: 'Set a timer' },
{ type: 'response', text: 'On it.' },
{ type: 'toolCalls', calls: [{ id: 'fm_1', name: 'timer_set', arguments: '{"seconds":60}' }] },
{ type: 'toolOutput', id: 'fm_1', name: 'timer_set', text: 'Timer set' },
{ type: 'response', text: 'Done.' },
{ type: 'toolOutput', id: 'gone', name: 'tool', text: 'Error: boom' },
{ type: 'prompt', text: 'And now?' },
]);
// Tool-free requests (translate, Q&A) get the permissive guardrails; a
// trailing tool result with nothing to resume falls back to a continuation prompt.
const orphan = api.buildSystemModelRequest({
messages: [{ role: 'user', content: 'x' }, { role: 'assistant', content: [{ type: 'tool_use', id: 'a', name: 't', input: {} }] },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'a', content: 'ok' }] }],
});
assert.equal(orphan.permissive, true);
assert.equal(orphan.maxTokens, 512);
assert.equal(orphan.prompt, api.CONTINUE_PROMPT);
assert.deepEqual(plain(orphan.history.at(-1)), { type: 'toolOutput', id: 'a', name: 't', text: 'ok' });
});
test('iOS local LLM streams text deltas, then ends the turn', () => {
const native = fakeNative();
const api = loadLocalLlm(native);
const { seen, callbacks } = collector();
api.streamLocalModel({ apiKey: '', model: 'm', system: 's', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
const [instance] = native.instances;
assert.equal(instance.request.prompt, 'Hi');
instance.emit({ kind: 'log', message: 'shortened tool descriptions' });
instance.emit({ kind: 'delta', text: 'Hello' });
instance.emit({ kind: 'delta', text: ' there' });
instance.emit({ kind: 'done' });
instance.emit({ kind: 'delta', text: 'late' });
assert.deepEqual(plain(seen.deltas), ['Hello', 'Hello there']);
assert.equal(seen.done.length, 1);
assert.deepEqual(plain(seen.done[0]), { text: 'Hello there', content: [{ type: 'text', text: 'Hello there' }], stopReason: 'end_turn' });
assert.equal(instance.eventHandler, null, 'finished generations drop their handler');
});
test('iOS local LLM reports why the model is unavailable without touching native code', async () => {
const native = fakeNative('appleIntelligenceNotEnabled');
const api = loadLocalLlm(native);
assert.equal(api.isLocalModelReady(), false);
assert.equal(api.isLocalModelSupported(), true);
assert.deepEqual(plain(api.localModelState()), { status: 'unavailable', bytesDownloaded: 0, totalBytes: 0, detail: 'Apple Intelligence off' });
const { seen, callbacks } = collector();
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'Hi' }], ...callbacks });
await new Promise(resolve => setTimeout(resolve, 0));
assert.match(seen.errors[0], /Turn on Apple Intelligence/);
assert.equal(native.instances.length, 0);
native.setAvailability('deviceNotEligible');
assert.equal(api.isLocalModelSupported(), false);
native.setAvailability('somethingNew');
assert.equal(api.localModelState().detail, 'unavailable');
const missing = loader({ setTimeout, clearTimeout }, {})('app/native/local-llm.ios.ts');
assert.equal(missing.localModelAvailability(), 'unsupportedOS', 'pre-iOS-26 builds have no Swift class to call');
native.setAvailability('available');
assert.deepEqual(plain(api.localModelState()), { status: 'ready', bytesDownloaded: 0, totalBytes: 0 });
});
test('iOS local LLM maps refusals to the refusal stop reason and other failures to errors', () => {
const native = fakeNative();
const api = loadLocalLlm(native);
const first = collector();
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...first.callbacks });
native.instances[0].emit({ kind: 'error', code: 'refusal', message: 'declined' });
assert.equal(first.seen.done[0].stopReason, 'refusal');
const second = collector();
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...second.callbacks });
native.instances[1].emit({ kind: 'error', code: 'rateLimited', message: 'iOS is limiting the on-device model' });
assert.deepEqual(plain(second.seen.errors), ['iOS is limiting the on-device model']);
});
test('iOS local LLM cancellation stops the native generation and silences later events', () => {
const native = fakeNative();
const api = loadLocalLlm(native);
const { seen, callbacks } = collector();
const handle = api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], ...callbacks });
handle.cancel();
native.instances[0].emit({ kind: 'delta', text: 'late' });
native.instances[0].emit({ kind: 'done' });
assert.equal(native.instances[0].cancelled, true);
assert.deepEqual(seen, { deltas: [], done: [], errors: [] });
});
test('iOS assistant runs the on-device model through the shared tool loop, resuming the parked generation', async () => {
const native = fakeNative();
const calls = [], deltas = [], errors = [], done = [];
const registry = {
listTools: () => [{ name: 'timer.set', description: 'Set a timer', inputSchema: { type: 'object', properties: { seconds: { type: 'integer' } }, required: ['seconds'] } }],
callTool: async (name, args) => { calls.push({ name, args }); return { ok: true, content: 'Timer set' }; },
};
const mocks = {
'./bridge-client': { assistantBridge: {} }, './tool-registry': { toolRegistry: registry },
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: 'base prompt', buildAssistantSystemPrompt: () => 'cloud prompt', describeAssistantContext: () => 'Context: now' },
};
const load = loader({ global: { isIOS: true, isAndroid: false }, FaceclawSystemLanguageModel: native.cls, setTimeout, clearTimeout }, mocks);
mocks['../native/local-llm'] = load('app/native/local-llm.ios.ts');
const { AssistantSession } = load('app/assistant/session.ts');
const session = new AssistantSession({ kind: 'direct', llm: { selection: 'qwen', provider: 'local', model: 'apple-foundation-model', apiKey: '' } }, registry);
const send = text => session.sendUtterance(text, {}, { onTextDelta: (_d, full) => deltas.push(full),
onToolActivity() {}, onError: error => errors.push(error), onTurnDone: result => done.push(result) });
send('Set a one-minute timer');
assert.equal(native.instances.length, 1);
const generation = native.instances[0];
assert.equal(generation.request.system, 'base prompt', 'the local provider keeps the system prompt stable');
assert.equal(generation.request.prompt, 'Set a one-minute timer\n\n(Context: now)');
assert.deepEqual(plain(generation.request.tools.map(tool => tool.name)), ['timer_set']);
generation.emit({ kind: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] });
await flush();
assert.deepEqual(plain(calls), [{ name: 'timer.set', args: { seconds: 60 } }]);
assert.equal(native.instances.length, 1, 'tool results resume the parked generation instead of starting another');
assert.deepEqual(plain(generation.resolved), [[{ id: 'fm_abc', content: 'Timer set', isError: false }]]);
generation.emit({ kind: 'delta', text: 'Your timer is running.' });
generation.emit({ kind: 'done' });
assert.equal(done.length, 1);
assert.equal(errors.length, 0);
assert.equal(deltas.at(-1), 'Your timer is running.');
assert.equal(session.isTurnActive(), false);
// The next turn starts a fresh generation that carries the whole exchange as transcript history.
send('Thanks');
const next = native.instances[1];
assert.deepEqual(plain(next.request.history), [
{ type: 'prompt', text: 'Set a one-minute timer\n\n(Context: now)' },
{ type: 'toolCalls', calls: [{ id: 'fm_abc', name: 'timer_set', arguments: '{"seconds":60}' }] },
{ type: 'toolOutput', id: 'fm_abc', name: 'timer_set', text: 'Timer set' },
{ type: 'response', text: 'Your timer is running.' },
]);
session.cancel();
assert.equal(next.cancelled, true);
});
test('a parked iOS generation survives unrelated requests and is abandoned if its results never arrive', () => {
const native = fakeNative();
let expire = null;
const timers = {
setTimeout: (fn, ms) => { if (ms > 0) expire = fn; else fn(); return 1; },
clearTimeout: () => { expire = null; },
};
const api = loadLocalLlm(native, timers);
const turn = collector();
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'x' }], tools: [], ...turn.callbacks });
native.instances[0].emit({ kind: 'toolCalls', calls: [{ id: 'fm_1', name: 't', arguments: 'not json' }] });
assert.deepEqual(plain(turn.seen.done[0].content), [{ type: 'tool_use', id: 'fm_1', name: 't', input: {} }]);
// e.g. Translate asking the model something while the assistant's tool runs.
const other = collector();
api.streamLocalModel({ apiKey: '', model: 'm', messages: [{ role: 'user', content: 'translate this' }], ...other.callbacks });
assert.equal(native.instances.length, 2);
assert.equal(native.instances[0].cancelled, false);
assert.equal(typeof expire, 'function');
expire();
assert.equal(native.instances[0].cancelled, true);
// Results for the abandoned call now start a fresh generation with the fallback prompt.
api.streamLocalModel({ apiKey: '', model: 'm', ...turn.callbacks, messages: [
{ role: 'user', content: 'x' },
{ role: 'assistant', content: turn.seen.done[0].content },
{ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'fm_1', content: 'ok' }] },
] });
assert.equal(native.instances.length, 3);
assert.equal(native.instances[2].request.prompt, api.CONTINUE_PROMPT);
});
test('iOS offers the on-phone model where Apple Intelligence is supported', () => {
const local = { LOCAL_MODEL: { label: 'Apple Intelligence', id: 'apple-foundation-model' }, isLocalModelReady: () => true, isLocalModelSupported: () => true };
const api = loader({ global: { isIOS: true } }, { '../native/local-llm': local })('app/assistant/models.ts');
assert.equal(api.ASSISTANT_MODEL_CHOICES.includes('qwen'), true);
assert.equal(api.supportedAssistantModel('qwen'), 'qwen');
assert.equal(api.assistantModelLabel('qwen'), 'Apple Intelligence (on-phone)');
const resolved = api.resolveAssistantModel('auto', { openai: '', anthropic: '' });
assert.equal(resolved.provider, 'local');
assert.equal(resolved.model, 'apple-foundation-model');
assert.equal(api.resolveAssistantModel('auto', { openai: 'key', anthropic: '' }).provider, 'openai', 'API keys still win under Auto');
local.isLocalModelReady = () => false;
assert.equal(api.resolveAssistantModel('qwen', { openai: '', anthropic: '' }), null);
});