mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Support Apple Intelligence as an on-device AI model for the voice assistant/etc
This commit is contained in:
1 parent
3c3aa31542
commit
4973fd2aea
22 files changed
+1551
-40
No files matched your search
@@ -0,0 +1,52 @@
|
||||
// Print the assistant's built-in tool definitions as JSON ([{name, description, parameters}]),
|
||||
// pulled from the tool files' object literals via the TypeScript AST, with the dynamic
|
||||
// parts (the app-id enum, NUMBER_ARG) filled in from a fixed list. An approximation of
|
||||
// what ToolRegistry.listTools() offers with no app windows open.
|
||||
// Usage: node scripts/apple-llm-harness/extract-tools.cjs > tools.json
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const root = path.resolve(__dirname, '../..');
|
||||
const ts = require(path.join(root, 'node_modules/typescript'));
|
||||
|
||||
const FILES = ['system-tools', 'timer-tools', 'window-tools', 'navigate-tools', 'roam-tools']
|
||||
.map(name => path.join(root, 'app/assistant', `${name}.ts`));
|
||||
const APP_IDS = fs.readdirSync(path.join(root, 'app/apps'), { withFileTypes: true })
|
||||
.filter(entry => entry.isDirectory() && !['launcher', 'onboarding', 'evenhub'].includes(entry.name))
|
||||
.map(entry => entry.name).sort();
|
||||
const NUMBER_ARG = (noun, listTool) => ({
|
||||
type: 'number',
|
||||
description: `1-based ${noun} number, as shown by ${listTool}. May be omitted when only one ${noun} exists.`,
|
||||
});
|
||||
|
||||
const out = [];
|
||||
for (const file of FILES) {
|
||||
const src = ts.createSourceFile(file, fs.readFileSync(file, 'utf8'), ts.ScriptTarget.ES2020, true);
|
||||
const consts = {};
|
||||
src.forEachChild(function collect(node) {
|
||||
if (ts.isVariableDeclaration(node) && node.initializer && ts.isStringLiteral(node.initializer)) {
|
||||
consts[node.name.getText()] = node.initializer.text;
|
||||
}
|
||||
node.forEachChild(collect);
|
||||
});
|
||||
const evaluate = node => {
|
||||
const js = ts.transpileModule(`(${node.getText()})`, { compilerOptions: { target: ts.ScriptTarget.ES2020 } }).outputText;
|
||||
const names = [...Object.keys(consts), 'MAX_TIMER_DURATION_MS', 'appIds', 'installedNote', 'NUMBER_ARG'];
|
||||
return Function(...names, `return ${js}`)(...Object.values(consts), 86400000, APP_IDS, '', NUMBER_ARG);
|
||||
};
|
||||
src.forEachChild(function visit(node) {
|
||||
if (ts.isObjectLiteralExpression(node)) {
|
||||
const keys = node.properties.map(p => p.name && p.name.getText());
|
||||
if (keys.includes('name') && keys.includes('description') && keys.includes('inputSchema')) {
|
||||
try {
|
||||
const spec = evaluate(node);
|
||||
out.push({ name: String(spec.name).replace(/[^a-zA-Z0-9_-]/g, '_'), description: spec.description, parameters: spec.inputSchema });
|
||||
} catch (error) {
|
||||
console.error(`skipped a tool in ${path.basename(file)}: ${error.message}`);
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
node.forEachChild(visit);
|
||||
});
|
||||
}
|
||||
process.stdout.write(JSON.stringify(out, null, 1));
|
||||
@@ -0,0 +1,50 @@
|
||||
// macOS harness for App_Resources/iOS/src/FaceclawSystemLanguageModel.swift: runs one
|
||||
// assistant request against the Apple Intelligence model the way local-llm.ios.ts does,
|
||||
// answering every tool call with a canned result. See run.sh.
|
||||
import Foundation
|
||||
|
||||
@main struct Harness {
|
||||
@MainActor static func main() async {
|
||||
let args = CommandLine.arguments
|
||||
guard args.count >= 3 else {
|
||||
print("usage: harness <tools.json> <prompt> [previous prompt]")
|
||||
return
|
||||
}
|
||||
print("availability:", FaceclawSystemLanguageModel.availability(),
|
||||
"context:", FaceclawSystemLanguageModel.contextSize())
|
||||
let tools = (try? JSONSerialization.jsonObject(with: Data(contentsOf: URL(fileURLWithPath: args[1])))) ?? []
|
||||
var history: [[String: Any]] = []
|
||||
if args.count > 3 { history = [["type": "prompt", "text": args[3]], ["type": "response", "text": "OK."]] }
|
||||
let request: [String: Any] = [
|
||||
"system": "You are the voice assistant built into a pair of Even Realities G2 smart glasses. Your reply is shown as text on a 640x480 monochrome heads-up display and may also be read aloud, so keep it short: 1-3 plain sentences, no markdown, no bullet lists unless the user explicitly asks for a list. Prefer doing things with the tools you have over describing how the user could do them; when a tool can answer or act, use it. If a tool fails or a capability is missing, say so briefly rather than inventing a result.",
|
||||
"tools": tools, "history": history, "prompt": args[2], "maxTokens": 512, "permissive": false,
|
||||
]
|
||||
let model = FaceclawSystemLanguageModel()
|
||||
let started = Date()
|
||||
var finished = false
|
||||
var firstText: TimeInterval?
|
||||
model.eventHandler = { json in
|
||||
let elapsed = String(format: "%6.2fs", Date().timeIntervalSince(started))
|
||||
guard let event = (try? JSONSerialization.jsonObject(with: Data(json.utf8))) as? [String: Any] else { return }
|
||||
switch event["kind"] as? String {
|
||||
case "delta":
|
||||
if firstText == nil { firstText = Date().timeIntervalSince(started); print("\(elapsed) first text") }
|
||||
print(event["text"] as? String ?? "", terminator: "")
|
||||
case "toolCalls":
|
||||
let calls = event["calls"] as? [[String: Any]] ?? []
|
||||
for call in calls { print("\(elapsed) tool call: \(call["name"] ?? "") \(call["arguments"] ?? "")") }
|
||||
let results = calls.map { ["id": $0["id"] ?? "", "content": "Done (simulated \($0["name"] ?? "tool") result).", "isError": false] }
|
||||
let data = try! JSONSerialization.data(withJSONObject: results)
|
||||
model.resolveToolCalls(String(data: data, encoding: .utf8)!)
|
||||
case "done":
|
||||
print("\n\(elapsed) done"); finished = true
|
||||
default:
|
||||
print("\(elapsed) \(json)")
|
||||
if event["kind"] as? String == "error" { finished = true }
|
||||
}
|
||||
}
|
||||
let data = try! JSONSerialization.data(withJSONObject: request)
|
||||
model.start(String(data: data, encoding: .utf8)!)
|
||||
while !finished && Date().timeIntervalSince(started) < 120 { try? await Task.sleep(nanoseconds: 50_000_000) }
|
||||
}
|
||||
}
|
||||
Executable
+13
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
# Run one assistant request against Apple's on-device model on this Mac (macOS 26.4+,
|
||||
# Apple Intelligence on), through the same Swift bridge the iOS app uses, with the
|
||||
# assistant's real tool list. Prints tool routing, tool calls and the streamed reply.
|
||||
# scripts/apple-llm-harness/run.sh "set a timer for five minutes" ["previous prompt"]
|
||||
set -euo pipefail
|
||||
here="$(cd "$(dirname "$0")" && pwd)"
|
||||
out="${TMPDIR:-/tmp}/faceclaw-apple-llm-harness"
|
||||
mkdir -p "$out"
|
||||
node "$here/extract-tools.cjs" > "$out/tools.json"
|
||||
swiftc -swift-version 5 -parse-as-library -target "$(uname -m)-apple-macos26.4" \
|
||||
"$here/../../App_Resources/iOS/src/FaceclawSystemLanguageModel.swift" "$here/main.swift" -o "$out/harness"
|
||||
"$out/harness" "$out/tools.json" "$@"
|
||||
Reference in new issue
Block a user