Files
faceclaw/tests/ai-chat.test.cjs
T
James Babcock a02b849200 Borderless modes have different padding inside the builtin apps
Most of the builtin apps had some padding between the window edge and
their content, because they expect to be displayed with a line border on
their edge. We added a customization option to remove the border, so,
that padding is not always necessary. Make the layouts dynamically
adjust, to omit the padding when it isn't needed.
2026-11-11 15:19:29 -08:00

480 lines
24 KiB
JavaScript

const test = require('node:test');
const assert = require('node:assert/strict');
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
const { bordered, unbordered, edged } = require('./helpers/viewport.cjs');
function load(file, deps, globals = {}) {
const exports = {};
const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8');
vm.runInNewContext(ts.transpileModule(source, {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText, { exports, console, setTimeout, clearTimeout, ...globals, require(name) {
assert.ok(name in deps, `Unexpected import ${name} in ${file}`);
return deps[name];
} });
return exports;
}
const callbacks = () => ({ onTextDelta() {}, onToolActivity() {}, onTurnDone() {}, onError() {} });
const config = (model = 'terra', effort = 'low') => ({ kind: 'direct', llm: {
provider: model === 'sonnet' ? 'anthropic' : 'openai', selection: model, model, effort, apiKey: 'secret-key',
} });
function sessions() {
const turns = [];
let synchronousError = false;
const { AssistantSession } = load('app/assistant/session.ts', {
'../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' },
'./bridge-client': { assistantBridge: {} },
'./direct-backend': { DirectAssistantBackend: class { runTurn(options) {
const turn = { ...options, cancelled: false };
turns.push(turn);
if (synchronousError) options.callbacks.onError('failed');
return { cancel() { turn.cancelled = true; } };
} } },
'./tool-registry': { toolRegistry: { listTools: () => [] } },
});
function done(text) {
const turn = turns.at(-1);
turn.messages.push({ role: 'assistant', content: text });
turn.callbacks.onTextDelta(text, text);
turn.callbacks.onTurnDone({ stopReason: 'end_turn' });
}
return { AssistantSession, turns, done, setError() { synchronousError = true; } };
}
function conversations(saved = '') {
const env = sessions();
const { AssistantConversations } = load('app/assistant/conversations.ts', {
'./session': { AssistantSession: env.AssistantSession },
'./models': { supportedAssistantModel: value => value, ASSISTANT_MODEL_VALUES: ['auto', 'terra', 'sonnet'] },
});
let stored = '';
const store = new AssistantConversations((model, effort) => config(model, effort), () => 'terra', (value) => { stored = value; }, saved);
return { ...env, store, saved: () => stored };
}
test('voice and chat reuse selected history; sessions isolate context and persist without credentials', () => {
const env = conversations();
const firstId = env.store.current().id;
const voice = env.store.ensureSession();
voice.sendUtterance('Remember the blue door', {}, callbacks());
assert.equal(env.store.create(), false);
env.done('I will remember.');
const chat = env.store.ensureSession();
assert.equal(chat, voice);
chat.sendUtterance('What color?', {}, callbacks());
assert.equal(env.turns.at(-1).messages[0].content, 'Remember the blue door');
assert.equal(chat.transcript[1].text, 'I will remember.');
env.done('Blue.');
assert.ok(env.store.create());
env.store.ensureSession().sendUtterance('A separate topic', {}, callbacks());
assert.equal(env.turns.at(-1).messages.length, 1);
env.done('New topic.');
assert.ok(env.store.select(firstId));
assert.equal(env.store.ensureSession(), voice);
const restored = conversations(env.saved());
assert.equal(restored.store.current().id, firstId);
assert.equal(restored.store.ensureSession().transcript.at(-1).text, 'Blue.');
assert.equal(env.saved().includes('secret-key'), false);
restored.store.ensureSession().sendUtterance('Continue after restart', {}, callbacks());
assert.equal(restored.turns.at(-1).messages[0].content, 'Remember the blue door');
});
test('model and effort changes retain transcript and apply only to the chosen session', () => {
const env = conversations();
const original = env.store.current().id;
env.store.ensureSession().sendUtterance('hello', {}, callbacks());
assert.equal(env.store.configure('sonnet', 'high'), false);
env.done('hi');
env.store.configure('sonnet', 'high');
env.store.ensureSession().sendUtterance('follow up', {}, callbacks());
assert.equal(env.turns.at(-1).provider, 'anthropic');
assert.equal(env.turns.at(-1).effort, 'high');
assert.equal(env.turns.at(-1).messages[0].content, 'hello');
env.done('still here');
env.store.create();
assert.equal(env.store.current().model, 'terra');
assert.equal(env.store.current().reasoning, 'default');
env.store.select(original);
assert.equal(env.store.current().reasoning, 'high');
});
test('cancelled tools and late callbacks cannot mutate the next turn or its history', () => {
const env = sessions();
const session = new env.AssistantSession(config());
session.sendUtterance('first', {}, callbacks());
const old = env.turns[0];
old.callbacks.onTextDelta('partial', 'partial');
session.cancel();
session.sendUtterance('second', {}, callbacks());
old.messages.push({ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'old', content: 'late' }] });
old.callbacks.onTextDelta('late', 'late');
old.callbacks.onTurnDone({ stopReason: 'end_turn' });
assert.equal(session.isTurnActive(), true);
assert.equal(session.transcript[1].text, 'partial');
assert.equal(JSON.stringify(env.turns[1].messages).includes('tool_result'), false);
env.done('new answer');
assert.equal(session.isTurnActive(), false);
});
test('tool names are sanitized, shortened to 64 characters, kept distinct, and mapped back', () => {
const env = sessions();
const pkg = 'com.example.some-long-developer-name.evenhub';
const names = ['glasses.get_state', `app.${pkg}.set_metronome_tempo_and_signature`,
`app.${pkg}.set_metronome_tempo_and_signature_v2`, 'a.b', 'a_b'];
const registry = { listTools: () => names.map(name => ({ name, description: '', inputSchema: { type: 'object' } })) };
const session = new env.AssistantSession(config(), registry);
session.sendUtterance('hi', {}, callbacks());
const turn = env.turns.at(-1);
const apiNames = turn.buildTools().map(tool => tool.name);
assert.equal(apiNames[0], 'glasses_get_state');
assert.ok(apiNames.every(name => /^[a-zA-Z0-9_-]{1,64}$/.test(name)), apiNames.join(', '));
assert.equal(new Set(apiNames).size, names.length);
assert.match(apiNames[1], /^app_com_example_/);
assert.match(apiNames[1], /tempo_and_signature$/);
assert.deepEqual(apiNames.slice(3), ['a_b', 'a_b_2']);
assert.deepEqual(apiNames.map(turn.resolveToolName), names);
// The same canonical name shortens the same way on the next turn.
assert.deepEqual(turn.buildTools().map(tool => tool.name), apiNames);
});
test('synchronous provider errors do not leave a permanently active turn', () => {
const env = sessions(); env.setError();
const session = new env.AssistantSession(config());
session.sendUtterance('hello', {}, callbacks());
assert.equal(session.isTurnActive(), false);
assert.equal(session.status, 'failed');
});
function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) {
const transcripts = new Set(), statuses = new Set(), timers = new Map();
const sent = [];
let starts = 0, stops = 0, mic = false, timerId = 0;
const { VoiceDraft } = load('app/apps/ai-chat/voice-draft.ts', {
'../../native/voice-control': { voiceControlBridge: {
onTranscript(cb) { transcripts.add(cb); return () => transcripts.delete(cb); },
onStatus(cb) { statuses.add(cb); cb({ status: initialStatus }); return () => statuses.delete(cb); },
} },
'../../ui/shell/voice-activity': { voiceActivity: { setActive(value) { mic = value; } } },
}, {
setTimeout(cb) { timers.set(++timerId, cb); return timerId; },
clearTimeout(id) { timers.delete(id); },
});
const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text));
return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic,
transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); },
status(status) { for (const cb of [...statuses]) cb({ status }); },
timeout() { for (const cb of [...timers.values()]) cb(); },
subscriptions: () => transcripts.size + statuses.size,
};
}
test('hold displays partials, release waits for final and sends exactly once', async () => {
const env = draftEnv();
await env.draft.start();
env.transcript('hello');
assert.equal(env.draft.text, 'hello');
assert.deepEqual(env.sent, []);
env.draft.release();
assert.equal(env.stops(), 1);
env.transcript('hello world', true);
env.timeout();
env.transcript('duplicate', true);
assert.deepEqual(env.sent, ['hello world']);
assert.equal(env.subscriptions(), 0);
assert.equal(env.mic(), false);
});
test('release before permission resolves never starts a microphone', async () => {
let ready;
const env = draftEnv({ prepare: () => new Promise((resolve) => { ready = resolve; }) });
const start = env.draft.start();
env.draft.release(); ready(true); await start;
assert.equal(env.starts(), 0);
assert.equal(env.draft.active, false);
assert.deepEqual(env.sent, []);
});
test('chat waits past its fallback deadline for native recognition', async () => {
let complete;
const stopped = new Promise(resolve => { complete = resolve; });
const env = draftEnv({ stop: () => stopped });
await env.draft.start();
env.transcript('first segment');
env.draft.release(); env.timeout();
assert.deepEqual(env.sent, []);
assert.equal(env.draft.phase, 'finishing');
env.transcript('first segment and the rest', true);
complete(); await stopped; env.timeout();
assert.deepEqual(env.sent, ['first segment and the rest']);
});
test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => {
let complete;
const stopped = new Promise(resolve => { complete = resolve; });
const env = draftEnv({ stop: () => stopped });
await env.draft.start();
env.draft.release(); env.draft.cancel();
await env.draft.start();
env.transcript('new speech');
complete(); await stopped; env.timeout();
assert.deepEqual(env.sent, []);
assert.equal(env.draft.phase, 'listening');
env.draft.cancel();
});
test('release during async mic startup stops capture after it starts', async () => {
let started;
const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) });
const pending = env.draft.start();
await new Promise(setImmediate);
env.transcript('quick message');
env.draft.release(); started(); await pending;
assert.equal(env.stops(), 1);
env.timeout();
assert.deepEqual(env.sent, ['quick message']);
});
test('empty capture, rejection and closing never submit stale speech', async () => {
for (const cancel of ['empty', 'rejection', 'close']) {
const env = draftEnv({ initialStatus: 'old error' });
await env.draft.start();
assert.equal(env.starts(), 1);
if (cancel !== 'empty') env.transcript('should not send');
if (cancel === 'rejection') env.status('Ignored — not your enrolled voice');
if (cancel === 'close') env.draft.cancel();
env.draft.release(); env.timeout(); env.transcript('late', true);
assert.deepEqual(env.sent, []);
assert.equal(env.subscriptions(), 0);
}
});
function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) {
const gestures = load('app/ui/gestures.ts', {});
class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } }
const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 };
const timings = { spanCurrent: (_, fn) => fn(), span: (_, __, fn) => fn(), spanAsync: (_, __, fn) => fn(),
startFrame: () => 0, annotateFrame() {}, finishFrame() {}, runWithFrame: (_, fn) => fn() };
const layers = load('app/ui/layers.ts', {
'./viewport': require('../.test-build/app/ui/viewport.js'),
'../graphics/image': images, '../graphics/plane': { dimPlanes: (planes) => planes },
'../native/frame-timings': timings, './gestures': gestures,
'../graphics/screen-size': { screenWidth: () => 640, screenHeight: () => 480 },
});
class Menu {
constructor(title, items) { this.title = title; this.items = items; this.selectedIndex = 0; }
selectItem(index) { this.selectedIndex = index; return this; }
paint(_ctx, below) { return below(); }
handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); }
}
const voiceDialogs = [];
const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' },
assistantSkipConfirmationSetting: { get: () => skipConfirmation } };
const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', {
'../../graphics/shell-scene': { encodeShellScene: () => new Uint8Array([0, 0]) },
'../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}),
'../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu },
'../input-monitor': load('app/ui/input-monitor.ts', {}),
'./voice-input': { VoiceInputLayer: class {
constructor(options) { this.options = options; voiceDialogs.push(options); }
startCapture() {}
onRemoved() { this.options.onClosed(); }
} }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {},
'../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {},
'../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {},
'./chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {},
'./brightness-picker-layer': {}, './notification-modal-queue': load('app/ui/shell/notification-modal-queue.ts', {}),
'./notification-popup-timeout': {},
'../../native/notification-icons': {}, '../../native/notification-sources': {},
'../../assistant/tool-registry': {}, './geometry': {
sidebarWidth: () => 64, minWindowTop: () => 96, switcherPosition: () => 'left', uiDepth: () => 0,
screenWidth: () => 640, screenHeight: () => 480,
appViewportRect: () => ({ x: 64, y: 124, width: 576, height: 260 }),
},
});
const { createInProcessWindow } = load('app/ui/shell/in-process-window.ts', {
'../../graphics/image': images, '../../native/frame-timings': timings,
'../../util/render-freshness': { beginRenderPass() {}, endRenderPass: () => false },
'../layers': layers, '../window-menu': { WindowMenuLayer: Menu },
'./chrome-layer': { windowIcon: () => () => null }, './geometry': { appViewport: () => ({ width: 576, height: 260 }), setAppImmersive: () => false },
'./shell': { shell },
});
const input = [], menus = [];
let microphone = false;
const app = createInProcessWindow({
appId: 'ai-chat', windowId: 'ai-chat', title: 'AI Chat', iconLetter: 'AI', closeable: true, holdToTalk: true,
baseLayer: { paint: () => new Image(), handleInput(event) {
input.push(event.type);
if (event.type === 'long-press') microphone = true;
if (event.type === 'long-press-release') microphone = false;
if (event.type === 'double-click') shell.yieldFocusToSidebar();
} },
menuItems: () => [{ label: 'New session', onSelect() {} }],
onAppMenuOpened() { menus.push('app'); microphone = false; },
onSystemMenuOpened() { menus.push('system'); microphone = false; },
isVoiceCapturing: () => microphone,
actions: layers.noopLayerActions, submitFrame: async () => {}, setSurfaceVisible() {},
});
shell.registerWindow(app.window);
shell.focusWindow('ai-chat');
return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) };
}
test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => {
for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) {
const env = shellEnv({ wakeAction, skipConfirmation: true });
env.shell.isAssistantAvailable = () => true;
env.shell.sleep();
const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 });
assert.equal(event.type, 'wakeword');
await env.shell.receiveInput(event);
await new Promise(resolve => setImmediate(resolve));
assert.equal(env.shell.isScreenOn(), wakeAction !== 'off');
assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0);
if (wakeAction === 'voice-input') {
const options = env.voiceDialogs[0];
assert.equal(options.handsFree, true);
assert.equal(options.autoSend, true);
assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant');
await env.shell.receiveInput(event);
assert.equal(env.voiceDialogs.length, 1);
}
env.shell.stack.clearToBase();
}
});
test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => {
const env = shellEnv();
await env.shell.receiveInput(env.event('long-press'));
assert.equal(env.microphone(), true);
assert.equal(env.shell.escapeMenuTimer, null);
await env.shell.receiveInput(env.event('long-press-release'));
assert.equal(env.microphone(), false);
assert.deepEqual(env.input, ['long-press', 'long-press-release']);
await env.shell.receiveInput(env.event('click'));
assert.deepEqual(env.menus, ['app']);
assert.equal(env.app.stack.isAtBase(), false);
await env.shell.receiveInput(env.event('short-then-long-press'));
assert.deepEqual(env.menus, ['app', 'system']);
assert.equal(env.app.stack.isAtBase(), true);
assert.equal(env.shell.stack.isAtBase(), false);
await env.shell.receiveInput(env.event('long-press-release'));
assert.deepEqual(env.input, ['long-press', 'long-press-release']);
env.shell.stack.clearToBase();
env.shell.focusWindow('ai-chat');
await env.shell.receiveInput(env.event('double-click'));
assert.equal(env.shell.isWindowFocused('ai-chat'), false);
});
test('ordinary apps retain long-press system menu and games retain the escape timer', async () => {
const env = shellEnv();
env.app.window.holdToTalk = false;
await env.shell.receiveInput(env.event('long-press'));
assert.deepEqual(env.menus, ['system']);
assert.deepEqual(env.input, []);
env.shell.stack.clearToBase();
env.app.window.claimsLongPress = () => true;
await env.shell.receiveInput(env.event('long-press'));
assert.notEqual(env.shell.escapeMenuTimer, null);
await env.shell.receiveInput(env.event('long-press-release'));
assert.equal(env.shell.escapeMenuTimer, null);
});
function chatLayerEnv() {
const env = conversations();
const textwrap = load('app/graphics/textwrap.ts', {});
const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} });
const graphics = require('../.test-build/app/graphics/image.js');
const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8'));
let windowOptions;
const { createAiChatWindow } = load('app/apps/ai-chat/ai-chat-app.ts', {
'../../graphics/image': graphics, '../../graphics/ui-fonts': { getDefaultSmallFont: () => font },
'../../graphics/textwrap': textwrap, '../../assistant/models': { assistantModelLabel: (s) => s },
'../../ui/dashboard-settings': { assistantBackendSetting: { get: () => 'direct' } },
'../../ui/gestures': load('app/ui/gestures.ts', {}), '../../ui/window-menu': {},
'../../ui/shell/in-process-window': { createInProcessWindow(options) {
windowOptions = options; return { requestRender() {} };
} },
'../../ui/shell/shell': { shell: { getAssistantConversations: () => env.store } },
'./voice-draft': { VoiceDraft: class { active = false; text = ''; status = ''; cancel() {} } },
});
createAiChatWindow({ actions: {}, setSurfaceVisible() {}, onClosed() {} });
const layer = windowOptions.baseLayer;
// A framed window (8px of padding all round) unless a viewport says otherwise.
return { ...env, layer, paint: (width = 576, height = 260, viewport = bordered(width, height)) =>
layer.paint({ stack: { getBaseSize: () => viewport } }) };
}
test('scrollback stays anchored during streaming, follows at the bottom and resets on session switch', () => {
const env = chatLayerEnv();
const session = env.store.ensureSession();
for (let i = 0; i < 20; i++) session.transcript.push({ role: i % 2 ? 'assistant' : 'user', text: `Message ${i}: sample conversation text.` });
env.paint();
assert.ok(env.layer.maxFirstLine > 0);
env.layer.handleInput({ type: 'swipe-up' });
const first = env.layer.firstLine;
session.transcript.push({ role: 'assistant', text: 'New streamed text '.repeat(30) });
env.paint();
assert.equal(env.layer.firstLine, first);
for (let i = 0; i < 50; i++) env.layer.handleInput({ type: 'swipe-down' });
assert.equal(env.layer.firstLine, null);
env.layer.handleInput({ type: 'scroll-up' });
env.store.create(); env.paint();
assert.equal(env.layer.firstLine, null);
});
test('chat paints at both window heights with pending speech pinned beneath history', () => {
const env = chatLayerEnv();
env.store.ensureSession().transcript.push({ role: 'user', text: 'Plan my afternoon.' }, {
role: 'assistant', text: 'You have a meeting at two. There is time for a walk beforehand. '.repeat(8),
});
env.layer.draft.active = true;
env.layer.draft.status = 'Listening... release to send';
env.layer.draft.text = 'Actually, leave some time to pick up lunch on the way.';
for (const [width, height] of [[576, 260], [576, 452], [640, 452]]) {
const image = env.paint(width, height);
assert.equal(image.width, width); assert.equal(image.height, height);
assert.equal(image.to8bppBuffer().length, width * height);
if (process.env.FACECLAW_CHAT_RENDER_DIR) {
const pixels = image.to8bppBuffer();
const rgba = new Uint8Array(pixels.length * 4);
for (let i = 0; i < pixels.length; i++) rgba.set([pixels[i], pixels[i], pixels[i], 255], i * 4);
const png = require('upng-js').encode([rgba.buffer], width, height, 0);
fs.writeFileSync(path.join(process.env.FACECLAW_CHAT_RENDER_DIR, `chat-${width}x${height}.png`), Buffer.from(png));
}
}
});
test('chat keeps inside the padding of the edges a border runs along, and uses the room where there is none', () => {
const env = chatLayerEnv();
const session = env.store.ensureSession();
for (let i = 0; i < 30; i++) session.transcript.push({ role: i % 2 ? 'assistant' : 'user', text: `Message ${i}: sample conversation text.` });
// The rows and columns holding any ink, text included.
const inkBounds = (image) => {
const baked = image.withDrawsBaked();
const bounds = { left: Infinity, top: Infinity, right: -1, bottom: -1 };
for (let y = 0; y < baked.height; y++) for (let x = 0; x < baked.width; x++) {
if (!baked.getPixel(x, y)) continue;
bounds.left = Math.min(bounds.left, x); bounds.right = Math.max(bounds.right, x);
bounds.top = Math.min(bounds.top, y); bounds.bottom = Math.max(bounds.bottom, y);
}
return bounds;
};
for (const viewport of [
bordered(576, 260), unbordered(576, 260), edged(576, 260, 'top', 'left'), edged(576, 452, 'bottom', 'right'),
bordered(476, 155), unbordered(476, 155), edged(640, 480, 'bottom'),
]) {
const { width, height, padding } = viewport;
const where = JSON.stringify(viewport);
const ink = inkBounds(env.paint(width, height, viewport));
assert.ok(env.layer.maxFirstLine > 0, where);
// The footer's rule starts right at the left padding and the scrollbar hugs the right.
assert.equal(ink.left, padding.left, where);
assert.equal(ink.right, width - padding.right - 1, where);
// The title's line starts at the top padding and the footer's last line ends at the bottom.
assert.ok(ink.top >= padding.top && ink.top < padding.top + 12, where);
assert.ok(ink.bottom < height - padding.bottom && ink.bottom >= height - padding.bottom - 12, where);
}
});