Support Apple Intelligence as an on-device AI model for the voice assistant/etc

This commit is contained in:
James Babcock committed 2026-10-31 17:48:45 -07:00
1 parent 3c3aa31542
commit 4973fd2aea
22 files changed
+1551 -40

No files matched your search

@@ -0,0 +1,52 @@
// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]),
// pulled from the tool files' object literals via the TypeScript AST, with the dynamic
// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of
// what ToolRegistry.listTools() offers with no app windows open.
// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json
const path = require('path');
const fs = require('fs');
const root = path.resolve(__dirname, '../..');
const ts = require(path.join(root, 'node_modules/typescript'));
const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools']
.map(name => path.join(root, 'app/assistant', `${name}.ts`));
const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true })
.filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name))
.map(entry => entry.name).sort();
const NUMBER_ARG = (noun, listTool) => ({
type: 'number',
description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`,
});
const out = [];
for (const file of FILES) {
const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true);
const consts = {};
src.forEachChild(function collect(node) {
if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) {
consts[node.name.getText()] = node.initializer.text;
}
node.forEachChild(collect);
});
const evaluate = node => {
const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText;
const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG'];
return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG);
};
src.forEachChild(function visit(node) {
if (ts.isObjectLiteralExpression(node)) {
const keys = node.properties.map(p => p.name && p.name.getText());
if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) {
try {
const spec = evaluate(node);
out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema });
} catch (error) {
console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`);
}
return;
}
}
node.forEachChild(visit);
});
}
process.stdout.write(JSON.stringify(out, null, 1));
+50
View File
@@ -0,0 +1,50 @@
// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one
// assistant request against the Apple Intelligence model the way local-llm.ios.ts does,
// answering every tool call with a canned result. See run.sh.
import Foundation
@main struct Harness {
@MainActor static func main() async {
let args = CommandLine.arguments
guard args.count >= 3 else {
print("usage: harness <tools.json> <prompt> [previous prompt]")
return
}
print("availability:", FaceclawSystemLanguageModel.availability(),
"context:", FaceclawSystemLanguageModel.contextSize())
let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? []
var history: [[String: Any]] = []
if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] }
let request: [String: Any] = [
"system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
"tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false,
]
let model = FaceclawSystemLanguageModel()
let started = Date()
var finished = false
var firstText: TimeInterval?
model.eventHandler = { json in
let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started))
guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return }
switch event["kind"] as? String {
case "delta":
if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") }
print(event["text"] as? String ?? "", terminator: "")
case "toolCalls":
let calls = event["calls"] as? [[String: Any]] ?? []
for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") }
let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] }
let data = try! JSONSerialization.data(withJSONObject: results)
model.resolveToolCalls(String(data: data, encoding: .utf8)!)
case "done":
print("\n\(elapsed) done"); finished = true
default:
print("\(elapsed) \(json)")
if event["kind"] as? String == "error" { finished = true }
}
}
let data = try! JSONSerialization.data(withJSONObject: request)
model.start(String(data: data, encoding: .utf8)!)
while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) }
}
}
+13
View File
@@ -0,0 +1,13 @@
#!/bin/bash
# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+,
# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the
# assistant's real tool list. Prints tool routing, tool calls and the streamed reply.
# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"]
set -euo pipefail
here="$(cd "$(dirname "$0")" && pwd)"
out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness"
mkdir -p "$out"
node "$here/extract-tools.cjs" > "$out/tools.json"
swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \
"$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness"
"$out/harness" "$out/tools.json" "$@"