const test = require('node:test'); const assert = require('node:assert/strict'); const fs = require('node:fs'); const path = require('node:path'); const vm = require('node:vm'); const ts = require('typescript'); const { bordered, unbordered, edged } = require('./helpers/viewport.cjs'); function load(file, deps, globals = {}) { const exports = {}; const source = fs.readFileSync(path.join(__dirname, '..', file), 'utf8'); vm.runInNewContext(ts.transpileModule(source, { compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, }).outputText, { exports, console, setTimeout, clearTimeout, ...globals, require(name) { assert.ok(name in deps, `Unexpected import ${name} in ${file}`); return deps[name]; } }); return exports; } const callbacks = () => ({ onTextDelta() {}, onToolActivity() {}, onTurnDone() {}, onError() {} }); const config = (model = 'terra', effort = 'low') => ({ kind: 'direct', llm: { provider: model === 'sonnet' ? 'anthropic' : 'openai', selection: model, model, effort, apiKey: 'secret-key', } }); function sessions() { const turns = []; let synchronousError = false; const { AssistantSession } = load('app/assistant/session.ts', { '../prompts': { ASSISTANT_SYSTEM_PROMPT_BASE: '', buildAssistantSystemPrompt: () => '', describeAssistantContext: () => '' }, './bridge-client': { assistantBridge: {} }, './direct-backend': { DirectAssistantBackend: class { runTurn(options) { const turn = { ...options, cancelled: false }; turns.push(turn); if (synchronousError) options.callbacks.onError('failed'); return { cancel() { turn.cancelled = true; } }; } } }, './tool-registry': { toolRegistry: { listTools: () => [] } }, }); function done(text) { const turn = turns.at(-1); turn.messages.push({ role: 'assistant', content: text }); turn.callbacks.onTextDelta(text, text); turn.callbacks.onTurnDone({ stopReason: 'end_turn' }); } return { AssistantSession, turns, done, setError() { synchronousError = true; } }; } function conversations(saved = '') { const env = sessions(); const { AssistantConversations } = load('app/assistant/conversations.ts', { './session': { AssistantSession: env.AssistantSession }, './models': { supportedAssistantModel: value => value, ASSISTANT_MODEL_VALUES: ['auto', 'terra', 'sonnet'] }, }); let stored = ''; const store = new AssistantConversations((model, effort) => config(model, effort), () => 'terra', (value) => { stored = value; }, saved); return { ...env, store, saved: () => stored }; } test('voice and chat reuse selected history; sessions isolate context and persist without credentials', () => { const env = conversations(); const firstId = env.store.current().id; const voice = env.store.ensureSession(); voice.sendUtterance('Remember the blue door', {}, callbacks()); assert.equal(env.store.create(), false); env.done('I will remember.'); const chat = env.store.ensureSession(); assert.equal(chat, voice); chat.sendUtterance('What color?', {}, callbacks()); assert.equal(env.turns.at(-1).messages[0].content, 'Remember the blue door'); assert.equal(chat.transcript[1].text, 'I will remember.'); env.done('Blue.'); assert.ok(env.store.create()); env.store.ensureSession().sendUtterance('A separate topic', {}, callbacks()); assert.equal(env.turns.at(-1).messages.length, 1); env.done('New topic.'); assert.ok(env.store.select(firstId)); assert.equal(env.store.ensureSession(), voice); const restored = conversations(env.saved()); assert.equal(restored.store.current().id, firstId); assert.equal(restored.store.ensureSession().transcript.at(-1).text, 'Blue.'); assert.equal(env.saved().includes('secret-key'), false); restored.store.ensureSession().sendUtterance('Continue after restart', {}, callbacks()); assert.equal(restored.turns.at(-1).messages[0].content, 'Remember the blue door'); }); test('model and effort changes retain transcript and apply only to the chosen session', () => { const env = conversations(); const original = env.store.current().id; env.store.ensureSession().sendUtterance('hello', {}, callbacks()); assert.equal(env.store.configure('sonnet', 'high'), false); env.done('hi'); env.store.configure('sonnet', 'high'); env.store.ensureSession().sendUtterance('follow up', {}, callbacks()); assert.equal(env.turns.at(-1).provider, 'anthropic'); assert.equal(env.turns.at(-1).effort, 'high'); assert.equal(env.turns.at(-1).messages[0].content, 'hello'); env.done('still here'); env.store.create(); assert.equal(env.store.current().model, 'terra'); assert.equal(env.store.current().reasoning, 'default'); env.store.select(original); assert.equal(env.store.current().reasoning, 'high'); }); test('cancelled tools and late callbacks cannot mutate the next turn or its history', () => { const env = sessions(); const session = new env.AssistantSession(config()); session.sendUtterance('first', {}, callbacks()); const old = env.turns[0]; old.callbacks.onTextDelta('partial', 'partial'); session.cancel(); session.sendUtterance('second', {}, callbacks()); old.messages.push({ role: 'user', content: [{ type: 'tool_result', tool_use_id: 'old', content: 'late' }] }); old.callbacks.onTextDelta('late', 'late'); old.callbacks.onTurnDone({ stopReason: 'end_turn' }); assert.equal(session.isTurnActive(), true); assert.equal(session.transcript[1].text, 'partial'); assert.equal(JSON.stringify(env.turns[1].messages).includes('tool_result'), false); env.done('new answer'); assert.equal(session.isTurnActive(), false); }); test('tool names are sanitized, shortened to 64 characters, kept distinct, and mapped back', () => { const env = sessions(); const pkg = 'com.example.some-long-developer-name.evenhub'; const names = ['glasses.get_state', `app.${pkg}.set_metronome_tempo_and_signature`, `app.${pkg}.set_metronome_tempo_and_signature_v2`, 'a.b', 'a_b']; const registry = { listTools: () => names.map(name => ({ name, description: '', inputSchema: { type: 'object' } })) }; const session = new env.AssistantSession(config(), registry); session.sendUtterance('hi', {}, callbacks()); const turn = env.turns.at(-1); const apiNames = turn.buildTools().map(tool => tool.name); assert.equal(apiNames[0], 'glasses_get_state'); assert.ok(apiNames.every(name => /^[a-zA-Z0-9_-]{1,64}$/.test(name)), apiNames.join(', ')); assert.equal(new Set(apiNames).size, names.length); assert.match(apiNames[1], /^app_com_example_/); assert.match(apiNames[1], /tempo_and_signature$/); assert.deepEqual(apiNames.slice(3), ['a_b', 'a_b_2']); assert.deepEqual(apiNames.map(turn.resolveToolName), names); // The same canonical name shortens the same way on the next turn. assert.deepEqual(turn.buildTools().map(tool => tool.name), apiNames); }); test('synchronous provider errors do not leave a permanently active turn', () => { const env = sessions(); env.setError(); const session = new env.AssistantSession(config()); session.sendUtterance('hello', {}, callbacks()); assert.equal(session.isTurnActive(), false); assert.equal(session.status, 'failed'); }); function draftEnv({ prepare = async () => true, start = async () => {}, stop = () => {}, initialStatus = 'Ready' } = {}) { const transcripts = new Set(), statuses = new Set(), timers = new Map(); const sent = []; let starts = 0, stops = 0, mic = false, timerId = 0; const { VoiceDraft } = load('app/apps/ai-chat/voice-draft.ts', { '../../native/voice-control': { voiceControlBridge: { onTranscript(cb) { transcripts.add(cb); return () => transcripts.delete(cb); }, onStatus(cb) { statuses.add(cb); cb({ status: initialStatus }); return () => statuses.delete(cb); }, } }, '../../ui/shell/voice-activity': { voiceActivity: { setActive(value) { mic = value; } } }, }, { setTimeout(cb) { timers.set(++timerId, cb); return timerId; }, clearTimeout(id) { timers.delete(id); }, }); const draft = new VoiceDraft({ startVoiceCapture: async () => { starts++; await start(); }, stopVoiceCapture: () => { stops++; return stop(); } }, prepare, () => {}, (text) => sent.push(text)); return { draft, sent, starts: () => starts, stops: () => stops, mic: () => mic, transcript(text, isFinal = false) { for (const cb of [...transcripts]) cb({ text, isFinal }); }, status(status) { for (const cb of [...statuses]) cb({ status }); }, timeout() { for (const cb of [...timers.values()]) cb(); }, subscriptions: () => transcripts.size + statuses.size, }; } test('hold displays partials, release waits for final and sends exactly once', async () => { const env = draftEnv(); await env.draft.start(); env.transcript('hello'); assert.equal(env.draft.text, 'hello'); assert.deepEqual(env.sent, []); env.draft.release(); assert.equal(env.stops(), 1); env.transcript('hello world', true); env.timeout(); env.transcript('duplicate', true); assert.deepEqual(env.sent, ['hello world']); assert.equal(env.subscriptions(), 0); assert.equal(env.mic(), false); }); test('release before permission resolves never starts a microphone', async () => { let ready; const env = draftEnv({ prepare: () => new Promise((resolve) => { ready = resolve; }) }); const start = env.draft.start(); env.draft.release(); ready(true); await start; assert.equal(env.starts(), 0); assert.equal(env.draft.active, false); assert.deepEqual(env.sent, []); }); test('chat waits past its fallback deadline for native recognition', async () => { let complete; const stopped = new Promise(resolve => { complete = resolve; }); const env = draftEnv({ stop: () => stopped }); await env.draft.start(); env.transcript('first segment'); env.draft.release(); env.timeout(); assert.deepEqual(env.sent, []); assert.equal(env.draft.phase, 'finishing'); env.transcript('first segment and the rest', true); complete(); await stopped; env.timeout(); assert.deepEqual(env.sent, ['first segment and the rest']); }); test('a cancelled chat capture cannot finish a newer capture when native stop resolves', async () => { let complete; const stopped = new Promise(resolve => { complete = resolve; }); const env = draftEnv({ stop: () => stopped }); await env.draft.start(); env.draft.release(); env.draft.cancel(); await env.draft.start(); env.transcript('new speech'); complete(); await stopped; env.timeout(); assert.deepEqual(env.sent, []); assert.equal(env.draft.phase, 'listening'); env.draft.cancel(); }); test('release during async mic startup stops capture after it starts', async () => { let started; const env = draftEnv({ start: () => new Promise((resolve) => { started = resolve; }) }); const pending = env.draft.start(); await new Promise(setImmediate); env.transcript('quick message'); env.draft.release(); started(); await pending; assert.equal(env.stops(), 1); env.timeout(); assert.deepEqual(env.sent, ['quick message']); }); test('empty capture, rejection and closing never submit stale speech', async () => { for (const cancel of ['empty', 'rejection', 'close']) { const env = draftEnv({ initialStatus: 'old error' }); await env.draft.start(); assert.equal(env.starts(), 1); if (cancel !== 'empty') env.transcript('should not send'); if (cancel === 'rejection') env.status('Ignored — not your enrolled voice'); if (cancel === 'close') env.draft.cancel(); env.draft.release(); env.timeout(); env.transcript('late', true); assert.deepEqual(env.sent, []); assert.equal(env.subscriptions(), 0); } }); function shellEnv({ wakeAction = 'voice-input', skipConfirmation = false } = {}) { const gestures = load('app/ui/gestures.ts', {}); class Image { constructor(width = 576, height = 260) { this.width = width; this.height = height; } } const images = { GrayImage: Image, G2_LENS_WIDTH: 640, G2_LENS_HEIGHT: 480 }; const timings = { spanCurrent: (_, fn) => fn(), span: (_, __, fn) => fn(), spanAsync: (_, __, fn) => fn(), startFrame: () => 0, annotateFrame() {}, finishFrame() {}, runWithFrame: (_, fn) => fn() }; const layers = load('app/ui/layers.ts', { './viewport': require('../.test-build/app/ui/viewport.js'), '../graphics/image': images, '../graphics/plane': { dimPlanes: (planes) => planes }, '../native/frame-timings': timings, './gestures': gestures, '../graphics/screen-size': { screenWidth: () => 640, screenHeight: () => 480 }, }); class Menu { constructor(title, items) { this.title = title; this.items = items; this.selectedIndex = 0; } selectItem(index) { this.selectedIndex = index; return this; } paint(_ctx, below) { return below(); } handleInput(event, ctx) { if (event.type === 'double-click') ctx.stack.pop(); } } const voiceDialogs = []; const settings = { wakeWordActionSetting: { get: () => wakeAction }, brightnessSetting: { get: () => 'auto' }, assistantSkipConfirmationSetting: { get: () => skipConfirmation } }; const { shell, rawInputEventToInputEvent } = load('app/ui/shell/shell.ts', { '../../graphics/shell-scene': { encodeShellScene: () => new Uint8Array([0, 0]) }, '../../graphics/image': images, '../../graphics/plane': {}, '../../graphics/ui-fonts': {}, '../../g2/events': load('app/g2/events.ts', {}), '../gestures': gestures, '../layers': layers, '../menu': { MenuLayer: Menu }, '../input-monitor': load('app/ui/input-monitor.ts', {}), './voice-input': { VoiceInputLayer: class { constructor(options) { this.options = options; voiceDialogs.push(options); } startCapture() {} onRemoved() { this.options.onClosed(); } } }, './keyboard-input': {}, './voice-activity': { voiceActivity: { setActive() {} } }, './assistant': {}, '../../assistant/conversations': {}, '../../assistant/models': {}, '../../native/settings-store': {}, '../notifications': {}, '../dashboard-settings': settings, './ambient-cards': {}, './chrome-layer': { ShellChromeLayer: class {} }, './modal-layer': {}, './tool-debug-layer': {}, './brightness-picker-layer': {}, './notification-modal-queue': load('app/ui/shell/notification-modal-queue.ts', {}), './notification-popup-timeout': {}, '../../native/notification-icons': {}, '../../native/notification-sources': {}, '../../assistant/tool-registry': {}, './geometry': { sidebarWidth: () => 64, minWindowTop: () => 96, switcherPosition: () => 'left', uiDepth: () => 0, screenWidth: () => 640, screenHeight: () => 480, appViewportRect: () => ({ x: 64, y: 124, width: 576, height: 260 }), }, }); const { createInProcessWindow } = load('app/ui/shell/in-process-window.ts', { '../../graphics/image': images, '../../native/frame-timings': timings, '../../util/render-freshness': { beginRenderPass() {}, endRenderPass: () => false }, '../layers': layers, '../window-menu': { WindowMenuLayer: Menu }, './chrome-layer': { windowIcon: () => () => null }, './geometry': { appViewport: () => ({ width: 576, height: 260 }), setAppImmersive: () => false }, './shell': { shell }, }); const input = [], menus = []; let microphone = false; const app = createInProcessWindow({ appId: 'ai-chat', windowId: 'ai-chat', title: 'AI Chat', iconLetter: 'AI', closeable: true, holdToTalk: true, baseLayer: { paint: () => new Image(), handleInput(event) { input.push(event.type); if (event.type === 'long-press') microphone = true; if (event.type === 'long-press-release') microphone = false; if (event.type === 'double-click') shell.yieldFocusToSidebar(); } }, menuItems: () => [{ label: 'New session', onSelect() {} }], onAppMenuOpened() { menus.push('app'); microphone = false; }, onSystemMenuOpened() { menus.push('system'); microphone = false; }, isVoiceCapturing: () => microphone, actions: layers.noopLayerActions, submitFrame: async () => {}, setSurfaceVisible() {}, }); shell.registerWindow(app.window); shell.focusWindow('ai-chat'); return { shell, app, input, menus, voiceDialogs, rawInputEventToInputEvent, microphone: () => microphone, event: (type) => gestures.makeInputEvent({ type }) }; } test('wakeword wakes the shared shell, respects its action, and opens only one hands-free assistant dialog', async () => { for (const wakeAction of ['off', 'turn-screen-on', 'voice-input']) { const env = shellEnv({ wakeAction, skipConfirmation: true }); env.shell.isAssistantAvailable = () => true; env.shell.sleep(); const event = env.rawInputEventToInputEvent({ kind: 'even-ai', eventType: 1, eventSource: 0 }); assert.equal(event.type, 'wakeword'); await env.shell.receiveInput(event); await new Promise(resolve => setImmediate(resolve)); assert.equal(env.shell.isScreenOn(), wakeAction !== 'off'); assert.equal(env.voiceDialogs.length, wakeAction === 'voice-input' ? 1 : 0); if (wakeAction === 'voice-input') { const options = env.voiceDialogs[0]; assert.equal(options.handsFree, true); assert.equal(options.autoSend, true); assert.equal(options.sendTargets[options.defaultTargetIndex].id, 'assistant'); await env.shell.receiveInput(event); assert.equal(env.voiceDialogs.length, 1); } env.shell.stack.clearToBase(); } }); test('chat holds bypass the escape timer; release, menus and double tap follow the requested routing', async () => { const env = shellEnv(); await env.shell.receiveInput(env.event('long-press')); assert.equal(env.microphone(), true); assert.equal(env.shell.escapeMenuTimer, null); await env.shell.receiveInput(env.event('long-press-release')); assert.equal(env.microphone(), false); assert.deepEqual(env.input, ['long-press', 'long-press-release']); await env.shell.receiveInput(env.event('click')); assert.deepEqual(env.menus, ['app']); assert.equal(env.app.stack.isAtBase(), false); await env.shell.receiveInput(env.event('short-then-long-press')); assert.deepEqual(env.menus, ['app', 'system']); assert.equal(env.app.stack.isAtBase(), true); assert.equal(env.shell.stack.isAtBase(), false); await env.shell.receiveInput(env.event('long-press-release')); assert.deepEqual(env.input, ['long-press', 'long-press-release']); env.shell.stack.clearToBase(); env.shell.focusWindow('ai-chat'); await env.shell.receiveInput(env.event('double-click')); assert.equal(env.shell.isWindowFocused('ai-chat'), false); }); test('ordinary apps retain long-press system menu and games retain the escape timer', async () => { const env = shellEnv(); env.app.window.holdToTalk = false; await env.shell.receiveInput(env.event('long-press')); assert.deepEqual(env.menus, ['system']); assert.deepEqual(env.input, []); env.shell.stack.clearToBase(); env.app.window.claimsLongPress = () => true; await env.shell.receiveInput(env.event('long-press')); assert.notEqual(env.shell.escapeMenuTimer, null); await env.shell.receiveInput(env.event('long-press-release')); assert.equal(env.shell.escapeMenuTimer, null); }); function chatLayerEnv() { const env = conversations(); const textwrap = load('app/graphics/textwrap.ts', {}); const { BdfFont } = load('app/graphics/bdffont.ts', { '@nativescript/core': {} }); const graphics = require('../.test-build/app/graphics/image.js'); const font = BdfFont.parse(fs.readFileSync(path.join(__dirname, '../app/fonts/terminus/ter-u12n.bdf'), 'utf8')); let windowOptions; const { createAiChatWindow } = load('app/apps/ai-chat/ai-chat-app.ts', { '../../graphics/image': graphics, '../../graphics/ui-fonts': { getDefaultSmallFont: () => font }, '../../graphics/textwrap': textwrap, '../../assistant/models': { assistantModelLabel: (s) => s }, '../../ui/dashboard-settings': { assistantBackendSetting: { get: () => 'direct' } }, '../../ui/gestures': load('app/ui/gestures.ts', {}), '../../ui/window-menu': {}, '../../ui/shell/in-process-window': { createInProcessWindow(options) { windowOptions = options; return { requestRender() {} }; } }, '../../ui/shell/shell': { shell: { getAssistantConversations: () => env.store } }, './voice-draft': { VoiceDraft: class { active = false; text = ''; status = ''; cancel() {} } }, }); createAiChatWindow({ actions: {}, setSurfaceVisible() {}, onClosed() {} }); const layer = windowOptions.baseLayer; // A framed window (8px of padding all round) unless a viewport says otherwise. return { ...env, layer, paint: (width = 576, height = 260, viewport = bordered(width, height)) => layer.paint({ stack: { getBaseSize: () => viewport } }) }; } test('scrollback stays anchored during streaming, follows at the bottom and resets on session switch', () => { const env = chatLayerEnv(); const session = env.store.ensureSession(); for (let i = 0; i < 20; i++) session.transcript.push({ role: i % 2 ? 'assistant' : 'user', text: `Message ${i}: sample conversation text.` }); env.paint(); assert.ok(env.layer.maxFirstLine > 0); env.layer.handleInput({ type: 'swipe-up' }); const first = env.layer.firstLine; session.transcript.push({ role: 'assistant', text: 'New streamed text '.repeat(30) }); env.paint(); assert.equal(env.layer.firstLine, first); for (let i = 0; i < 50; i++) env.layer.handleInput({ type: 'swipe-down' }); assert.equal(env.layer.firstLine, null); env.layer.handleInput({ type: 'scroll-up' }); env.store.create(); env.paint(); assert.equal(env.layer.firstLine, null); }); test('chat paints at both window heights with pending speech pinned beneath history', () => { const env = chatLayerEnv(); env.store.ensureSession().transcript.push({ role: 'user', text: 'Plan my afternoon.' }, { role: 'assistant', text: 'You have a meeting at two. There is time for a walk beforehand. '.repeat(8), }); env.layer.draft.active = true; env.layer.draft.status = 'Listening... release to send'; env.layer.draft.text = 'Actually, leave some time to pick up lunch on the way.'; for (const [width, height] of [[576, 260], [576, 452], [640, 452]]) { const image = env.paint(width, height); assert.equal(image.width, width); assert.equal(image.height, height); assert.equal(image.to8bppBuffer().length, width * height); if (process.env.FACECLAW_CHAT_RENDER_DIR) { const pixels = image.to8bppBuffer(); const rgba = new Uint8Array(pixels.length * 4); for (let i = 0; i < pixels.length; i++) rgba.set([pixels[i], pixels[i], pixels[i], 255], i * 4); const png = require('upng-js').encode([rgba.buffer], width, height, 0); fs.writeFileSync(path.join(process.env.FACECLAW_CHAT_RENDER_DIR, `chat-${width}x${height}.png`), Buffer.from(png)); } } }); test('chat keeps inside the padding of the edges a border runs along, and uses the room where there is none', () => { const env = chatLayerEnv(); const session = env.store.ensureSession(); for (let i = 0; i < 30; i++) session.transcript.push({ role: i % 2 ? 'assistant' : 'user', text: `Message ${i}: sample conversation text.` }); // The rows and columns holding any ink, text included. const inkBounds = (image) => { const baked = image.withDrawsBaked(); const bounds = { left: Infinity, top: Infinity, right: -1, bottom: -1 }; for (let y = 0; y < baked.height; y++) for (let x = 0; x < baked.width; x++) { if (!baked.getPixel(x, y)) continue; bounds.left = Math.min(bounds.left, x); bounds.right = Math.max(bounds.right, x); bounds.top = Math.min(bounds.top, y); bounds.bottom = Math.max(bounds.bottom, y); } return bounds; }; for (const viewport of [ bordered(576, 260), unbordered(576, 260), edged(576, 260, 'top', 'left'), edged(576, 452, 'bottom', 'right'), bordered(476, 155), unbordered(476, 155), edged(640, 480, 'bottom'), ]) { const { width, height, padding } = viewport; const where = JSON.stringify(viewport); const ink = inkBounds(env.paint(width, height, viewport)); assert.ok(env.layer.maxFirstLine > 0, where); // The footer's rule starts right at the left padding and the scrollbar hugs the right. assert.equal(ink.left, padding.left, where); assert.equal(ink.right, width - padding.right - 1, where); // The title's line starts at the top padding and the footer's last line ends at the bottom. assert.ok(ink.top >= padding.top && ink.top < padding.top + 12, where); assert.ok(ink.bottom < height - padding.bottom && ink.bottom >= height - padding.bottom - 12, where); } });