iOS porting: Main screen and compositor

This commit is contained in:
James Babcock committed 2026-09-05 22:59:37 -07:00
1 parent 61b3bf8b7c
commit 99793a37a7
24 files changed
+1124 -70

No files matched your search

+2
View File
@@ -23,6 +23,8 @@
<key>UISupportedInterfaceOrientations</key>
<array>
<string>UIInterfaceOrientationPortrait</string>
<string>UIInterfaceOrientationLandscapeLeft</string>
<string>UIInterfaceOrientationLandscapeRight</string>
</array>
<key>UIRequiresFullScreen</key>
<true/>
+13
View File
@@ -0,0 +1,13 @@
platform :ios, '15.1'
pod 'SVGKit', '3.0.0'
# SVGKit 3's physical-DPI lookup asserts on arm64 simulators. Our icons use
# only viewBox/pixel units, so that lookup is irrelevant to their rendering.
post_install do |installer|
installer.pods_project.targets.each do |target|
next unless target.name == 'SVGKit'
target.build_configurations.each do |config|
config.build_settings['ENABLE_NS_ASSERTIONS'] = 'NO'
end
end
end
+6
View File
@@ -0,0 +1,6 @@
#import <UIKit/UIKit.h>
@interface FaceclawGraphics : NSObject
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
@end
+56
View File
@@ -0,0 +1,56 @@
#import "FaceclawGraphics.h"
#import <SVGKit/SVGKit.h>
@implementation FaceclawGraphics
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green {
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
const uint8_t *src = gray.bytes;
uint8_t *dst = rgba.mutableBytes;
uint8_t lut[256];
for (int i = 0; i < 256; i++) lut[i] = (uint8_t)round(pow(i / 255.0, 0.7) * 255.0);
for (NSInteger i = 0; i < width * height; i++) {
uint8_t value = lut[src[i]];
dst[i * 4] = green ? 0 : value;
dst[i * 4 + 1] = value;
dst[i * 4 + 2] = green ? 0 : value;
dst[i * 4 + 3] = 255;
}
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
if (image) CGImageRelease(image);
CGDataProviderRelease(provider);
CGColorSpaceRelease(space);
return result;
}
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
if (size <= 0 || size > 1024) return nil;
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
if (!image) return nil;
image.size = CGSizeMake(size, size);
UIGraphicsImageRendererFormat *format = [UIGraphicsImageRendererFormat defaultFormat];
format.scale = 1;
format.opaque = NO;
UIGraphicsImageRenderer *renderer = [[UIGraphicsImageRenderer alloc] initWithSize:image.size format:format];
UIImage *raster = [renderer imageWithActions:^(UIGraphicsImageRendererContext *context) {
[image.CALayerTree renderInContext:context.CGContext];
}];
NSMutableData *rgba = [NSMutableData dataWithLength:size * size * 4];
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
CGContextRef context = CGBitmapContextCreate(rgba.mutableBytes, size, size, 8, size * 4, space,
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
CGContextDrawImage(context, CGRectMake(0, 0, size, size), raster.CGImage);
NSMutableData *gray = [NSMutableData dataWithLength:size * size];
const uint8_t *src = rgba.bytes;
uint8_t *dst = gray.mutableBytes;
for (NSInteger i = 0; i < size * size; i++) {
dst[i] = (uint8_t)round(0.2126 * src[i * 4] + 0.7152 * src[i * 4 + 1] + 0.0722 * src[i * 4 + 2]);
}
CGContextRelease(context);
CGColorSpaceRelease(space);
return gray;
}
@end
+4
View File
@@ -0,0 +1,4 @@
module FaceclawGraphics {
header "FaceclawGraphics.h"
export *
}
+8 -4
View File
@@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer {
constructor(
private readonly coordinator: MathCoordinator,
private readonly actions: LayerActions,
private readonly textInput?: () => Promise<void>,
) {
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
}
private get mode(): CalculatorMode {
@@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer {
}
private refreshListening(): void {
if (this.textInput) return;
if (!this.foreground) return;
if (this.listening === "continuous") {
if (!this.continuousActive) {
@@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer {
* that did not.
*/
beginDictation(): void {
if (this.textInput) { void this.textInput(); return; }
if (this.dictating) return;
this.dictating = true;
void this.actions.startVoiceCapture(true);
@@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer {
bodyBottom: number,
): void {
const prompt =
this.listening === "continuous"
this.textInput ? "Tap to type a problem on your phone"
: this.listening === "continuous"
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
: "Tap to say a problem — or use Voice input to type one";
const lines = wrapText(font, prompt, width - 96);
@@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer {
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
if (!this.lastInput) return;
const label = "Heard";
const label = this.textInput ? "Entered" : "Heard";
image.drawText(font, 24, y, label, 110);
const x = 24 + font.measureText(label) + 10;
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
@@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer {
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
return gestureHints([
[GESTURE_SCROLL, scrollTarget],
[GESTURE_CLICK, "mic"],
[GESTURE_CLICK, this.textInput ? "type" : "mic"],
[GESTURE_DOUBLE_CLICK, "back"],
]);
}
+3 -46
View File
@@ -1,47 +1,4 @@
import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core'
import { bytesToHex, hexToBytes } from './util/hex-util'
import { Application } from '@nativescript/core'
import { createMainPage } from './phone-ui/main-page.ios'
// Milestone 1: exercise native UI, TypeScript, and an existing shared utility
// without loading the Android services used by the full glasses application.
Application.run({
create: () => {
const page = new Page()
page.actionBarHidden = true
const scroll = new ScrollView()
const content = new StackLayout()
content.padding = 28
content.verticalAlignment = 'middle'
const label = (text: string, fontSize = 17) => {
const view = new Label()
view.text = text
view.fontSize = fontSize
view.textWrap = true
view.marginBottom = 20
content.addChild(view)
return view
}
label('Faceclaw', 36)
label('iOS port · First launch', 24)
label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`)
label('Glasses pairing and the Android features are still to be ported.')
const status = label('Tap below to test native input and shared TypeScript code.')
const button = new Button()
button.text = 'Run shared-code check'
button.accessibilityIdentifier = 'shared-code-check'
let checks = 0
button.on('tap', () => {
const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77'))
const passed = result === '46616365636C6177'
checks += 1
status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.`
console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`)
})
content.addChild(button)
scroll.content = content
page.content = scroll
page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded'))
return page
},
})
Application.run({ create: createMainPage })
+220
View File
@@ -0,0 +1,220 @@
import { Dialogs, type ImageSource } from '@nativescript/core'
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
import { CalculatorLayer } from '../apps/calculator/calculator'
import { MathCoordinator } from '../apps/calculator/math/coordinator'
import { SurfaceCompositor } from '../graphics/surface-compositor'
import { flattenPlanes, type Plane } from '../graphics/plane'
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
import { previewPixels } from '../native/ios-graphics'
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
import { noopLayerActions, type LayerActions } from '../ui/layers'
import { MenuLayer } from '../ui/menu'
import { createInProcessWindow } from '../ui/shell/in-process-window'
import { shell, type ShellWindow } from '../ui/shell/shell'
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings'
import type { PhoneGesture } from '../phone-ui/phone-gestures'
/** Local display host: shared shell/windows, no device-service initialization. */
export class IosPreviewController {
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
private renderTimer: ReturnType<typeof setTimeout> | null = null
private clockTimer: ReturnType<typeof setInterval> | null = null
private offSettings: (() => void) | null = null
private active = false
private shellDirty = true
private inputQueue: Promise<void> = Promise.resolve()
private lastLayout = ''
private prompting = false
private readonly actions: LayerActions = {
...noopLayerActions,
requestRender: () => this.requestShellRender(),
startTextSettingEdit: setting => this.editSetting(setting),
startTextSettingsEdit: async (settings, _title, finished) => {
for (const setting of settings) await this.editSetting(setting)
finished?.()
},
}
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
private readonly onError: (message: string) => void) {
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
shell.configure({
actions: this.actions,
voiceInputEnabled: false,
getScreenTimeoutMs: () => null,
requestShellRender: () => this.requestShellRender(),
onWindowsChanged: () => this.requestShellRender(),
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
})
const launcher = createLauncherWindow({
actions: this.actions,
apps: () => [
{ appId: 'calculator', label: 'Calculator', icon: 'calculator' },
{ appId: 'display', label: 'Display', icon: 'settings' },
],
launchApp: id => this.launchApp(id),
uninstallApp: () => {},
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() },
})
this.configureWindow(launcher)
shell.registerWindow(launcher)
shell.wake('window')
shell.focusWindow(launcher.windowId)
}
resume(): void {
if (this.active) return
this.active = true
this.offSettings = onAnySettingChanged(() => {
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
})
this.clockTimer = setInterval(() => this.requestShellRender(), 60_000)
this.relayout()
shell.foregroundWindow()?.requestRender()
this.requestShellRender()
console.log('[ios-preview] Main screen active')
}
pause(): void {
this.active = false
this.offSettings?.(); this.offSettings = null
if (this.clockTimer) clearInterval(this.clockTimer)
if (this.renderTimer) clearTimeout(this.renderTimer)
this.clockTimer = this.renderTimer = null
}
private configureWindow(window: ShellWindow): void {
this.compositor.configureSurface(window.surfaceId, {
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
})
this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
}
private relayout(): void {
const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}`
if (layout === this.lastLayout) return
this.lastLayout = layout
for (const window of shell.getWindows()) {
this.configureWindow(window)
window.relayout?.()
}
}
private async submit(id: string, planes: Plane[]): Promise<void> {
const image = flattenPlanes(planes)
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
this.scheduleFrame()
}
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
private scheduleFrame(): void {
if (!this.active || this.renderTimer !== null) return
// Coalesce window/chrome updates into one preview, capped at 30 fps.
this.renderTimer = setTimeout(() => {
this.renderTimer = null
try {
if (this.shellDirty) {
this.shellDirty = false
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
this.compositor.setUnderlayDim(1, shell.underlayDim())
}
const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green')
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
} catch (error) { this.fail(error) }
}, 33)
}
private fail(error: unknown): void {
const message = error instanceof Error ? error.stack ?? error.message : String(error)
console.error(`[ios-preview] ${message}`)
this.onError(error instanceof Error ? error.message : String(error))
}
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
this.inputQueue = this.inputQueue.then(async () => {
if (!this.active) return
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
await this.mirrorTap(nx, ny)
} else {
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
if (origin === 'ring' && type.startsWith('swipe-')) {
if (type === 'swipe-left' || type === 'swipe-right') return
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
}
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
await shell.receiveInput(event)
this.requestShellRender()
}
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
}).catch(error => this.fail(error))
}
private async mirrorTap(nx: number, ny: number): Promise<void> {
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
const window = shell.foregroundWindow()
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) {
const target = shell.windowAtSidebarPoint(x, y)
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
return
}
if (window && !shell.hasOverlay()) {
const rect = appViewportRect(window.heightMode, window.appId)
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
shell.focusWindow(window.windowId)
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
}
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
this.requestShellRender()
}
launchApp(id: string): void {
const existing = shell.getWindows().find(window => window.appId === id)
if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return }
const surfaceId = `window:${id}`
let render = () => {}
const actions = { ...this.actions, requestRender: () => render() }
const layer = id === 'calculator'
? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp())
: new MenuLayer('Display', [
{ label: 'Change screen size', onSelect: () => this.cycleDisplayMode() },
{ label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } },
{ label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } },
], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true })
const app = createInProcessWindow({
appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display',
iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings',
closeable: true, actions, baseLayer: layer,
receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined,
submitFrame: planes => this.submit(surfaceId, planes),
setSurfaceVisible: visible => {
if (layer instanceof CalculatorLayer) layer.setForeground(visible)
this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame()
},
removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() },
onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() },
})
render = app.requestRender
if (layer instanceof CalculatorLayer) layer.requestRender = render
this.configureWindow(app.window)
shell.registerWindow(app.window)
shell.focusWindow(app.window.windowId)
app.requestRender(); this.requestShellRender()
}
cycleDisplayMode(): void {
const values = DISPLAY_MODE_VALUES
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
}
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
async typeIntoApp(): Promise<void> {
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
this.prompting = true
try {
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
if (result.result) shell.sendTextToForegroundWindow(result.text)
} catch (error) { this.fail(error) }
finally { this.prompting = false }
}
private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise<void> {
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' })
if (result.result) setting.set(result.text)
}
}
+7 -17
View File
@@ -1,12 +1,10 @@
import { GrayImage } from "./image";
declare const com: any;
declare const global: any;
import { rasterizeSvg } from "../native/svg-rasterizer";
/**
* Vector icons for window indicators (and anywhere else). SVGs are rendered
* once to a correctly sized grayscale bitmap by the Java IconRenderer (a
* small SVG subset: path/circle/rect/line/polyline) and cached here. To add
* once to a correctly sized grayscale bitmap by the platform rasterizer
* (Android IconRenderer or iOS SVGKit) and cached here. To add
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
* Noun Project glyphs also work.
@@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag
if (cached !== undefined) return cached;
let icon: GrayImage | null = null;
if (global.isAndroid) {
try {
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH);
if (bytes && bytes.length >= size * size) {
icon = new GrayImage(size, size, 0);
for (let i = 0; i < size * size; i++) {
icon.pixels[i] = bytes[i] & 0xff;
}
}
} catch (error) {
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
}
try {
icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH);
} catch (error) {
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
}
cache.set(key, icon);
return icon;
+75
View File
@@ -0,0 +1,75 @@
/** Platform-independent 8bpp surface composition for the local display. */
export type SurfaceRect = { x: number; y: number; width: number; height: number }
export type SurfaceConfiguration = SurfaceRect & {
zOrder: number
transparency: 'opaque' | 'color-key'
}
type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean }
export class SurfaceCompositor {
private readonly surfaces = new Map<string, Surface>()
private dimBelow = 0
private dimFactor = 1
private blanked = false
constructor(readonly width: number, readonly height: number) {
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
throw new Error('Invalid compositor size')
}
}
configureSurface(id: string, options: SurfaceConfiguration): void {
if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger)
|| options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry')
const previous = this.surfaces.get(id)
const pixels = previous?.width === options.width && previous.height === options.height
? previous.pixels : new Uint8Array(options.width * options.height)
this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true })
}
removeSurface(id: string): void { this.surfaces.delete(id) }
setSurfaceVisible(id: string, visible: boolean): void {
const surface = this.surfaces.get(id)
if (surface) surface.visible = visible
}
setUnderlayDim(belowZOrder: number, factor: number): void {
if (!Number.isFinite(factor)) throw new Error('Invalid dim factor')
this.dimBelow = belowZOrder
this.dimFactor = Math.max(0, Math.min(1, factor))
}
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
const surface = this.surfaces.get(id)
if (!surface) throw new Error(`Unknown surface: ${id}`)
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) {
throw new Error('Invalid frame buffer or rectangle')
}
const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width)
const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height)
for (let y = top; y < bottom; y++) {
if (right <= left) break
const source = (y - rect.y) * rect.width + left - rect.x
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
}
}
composite(): Uint8Array {
const output = new Uint8Array(this.width * this.height)
if (this.blanked) return output
const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder)
for (const surface of surfaces) {
const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width)
const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height)
const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1
for (let y = top; y < bottom; y++) {
let source = (y - surface.y) * surface.width + left - surface.x
let target = y * this.width + left
for (let x = left; x < right; x++, source++, target++) {
const value = surface.pixels[source]
// Test the original value: opaque near-black must still cover the
// window beneath, even when dimming rounds it down to black.
if (surface.transparency === 'color-key' && value === 0) continue
output[target] = Math.round(value * dim)
}
}
}
return output
}
}
+21
View File
@@ -0,0 +1,21 @@
import { ImageSource } from '@nativescript/core'
import { GrayImage } from '../graphics/image'
declare const FaceclawGraphics: any
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
// NSData copies the buffer; the UIImage owns its pixels after this call.
const copy = new Uint8Array(pixels)
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
if (!native) throw new Error('Could not create preview image')
return new ImageSource(native)
}
export function rasterizeSvg(svg: string, size: number): GrayImage | null {
const data = FaceclawGraphics.renderSVGSize(svg, size)
if (!data) return null
const image = new GrayImage(size, size, 0)
image.pixels.set(new Uint8Array(interop.bufferFromData(data)))
return image
}
+28
View File
@@ -0,0 +1,28 @@
import { ApplicationSettings } from '@nativescript/core'
// Main-isolate settings for the initial iOS port. Worker synchronization will
// be added when worker-hosted apps are enabled on iOS.
const listeners = new Set<(key: string) => void>()
function changed(key: string): void {
setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0)
}
export function getStringSetting(key: string, fallback: string): string {
return ApplicationSettings.getString(key, fallback)
}
export function setStringSetting(key: string, value: string): void {
if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return
ApplicationSettings.setString(key, value)
changed(key)
}
export function getBooleanSetting(key: string, fallback: boolean): boolean {
return ApplicationSettings.getBoolean(key, fallback)
}
export function setBooleanSetting(key: string, value: boolean): void {
if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return
ApplicationSettings.setBoolean(key, value)
changed(key)
}
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
listeners.add(listener)
return () => { listeners.delete(listener) }
}
+14
View File
@@ -0,0 +1,14 @@
import { rasterizeSvg as render } from './ios-graphics'
export function rasterizeSvg(svg: string, size: number, strokeWidth: number) {
// Shared icon sources rely on the Android renderer's white, rounded
// stroke defaults. Supply those explicitly for a standards SVG renderer.
const source = svg.replace(/<svg\b([^>]*)>/, (_tag, attributes: string) => {
const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24',
stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth),
'stroke-linecap': 'round', 'stroke-linejoin': 'round' }
const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes))
.map(([key, value]) => ` ${key}="${value}"`).join('')
return `<svg${attributes}${missing}>`
})
return render(source, size)
}
+9
View File
@@ -0,0 +1,9 @@
import { GrayImage } from '../graphics/image'
declare const com: any
export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null {
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth)
if (!bytes || bytes.length < size * size) return null
const image = new GrayImage(size, size, 0)
for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff
return image
}
+141
View File
@@ -0,0 +1,141 @@
import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core'
import { IosPreviewController } from '../g2/ios-preview-controller'
import { PhoneGestureRecognizer } from './phone-gestures'
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
export function createMainPage(): Page {
const page = new Page()
page.actionBarHidden = true
const root = new GridLayout()
root.rows = 'auto,*'
page.content = root
const header = new GridLayout()
header.columns = '*,auto'; header.padding = '12 20'
const title = new Label()
title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold'
header.addChild(title)
const state = new Label()
state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle'
GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header)
const body = new GridLayout()
GridLayout.setRow(body, 1); root.addChild(body)
const mirror = new Image()
mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black')
mirror.accessibilityIdentifier = 'glasses-preview'
mirror.accessibilityLabel = 'Interactive glasses screen preview'
mirror.verticalAlignment = 'middle'; body.addChild(mirror)
const controls = new GridLayout()
controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls)
const tabs = new GridLayout()
tabs.columns = '*,*,*'; controls.addChild(tabs)
const pad = new GridLayout()
pad.borderRadius = 18; pad.backgroundColor = new Color('#202522')
pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center'
pad.accessibilityIdentifier = 'gesture-pad'
GridLayout.setRow(pad, 1); controls.addChild(pad)
const focus = new Label()
focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true
focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle'
focus.isUserInteractionEnabled = false; pad.addChild(focus)
const legend = new Label()
legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center'
legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom'
legend.isUserInteractionEnabled = false; pad.addChild(legend)
const settings = new StackLayout()
settings.verticalAlignment = 'middle'
GridLayout.setRow(settings, 1); controls.addChild(settings)
const errorLabel = new Label()
errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse'
settings.addChild(errorLabel)
let selectedTab: 'watch' | 'ring' | 'settings' = 'watch'
const controller = new IosPreviewController((image, label) => {
mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label
}, message => {
errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings')
})
const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch'))
const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => {
const size = mirror.getActualSize()
if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height)
})
const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => {
if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return
recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() })
}
pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput))
function button(text: string, action: () => void): Button {
const view = new Button()
view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action)
return view
}
const tabButtons: Button[] = []
function selectTab(tab: typeof selectedTab): void {
padInput.cancel(); selectedTab = tab
pad.visibility = tab === 'settings' ? 'collapse' : 'visible'
settings.visibility = tab === 'settings' ? 'visible' : 'collapse'
for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5
focus.text = tab === 'ring' ? 'Ring' : 'Watch controls'
legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu'
layout()
}
for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) {
const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab))
GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view)
}
const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode())
settings.addChild(sizeButton)
settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green')))
const help = new Label()
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
const footer = new GridLayout()
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
const menu = button('Menu', () => {
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
})
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
for (const [index, view] of [back, menu, keyboard].entries()) {
view.padding = '16 8'; view.fontSize = 14
GridLayout.setColumn(view, index); footer.addChild(view)
}
let lastLayout = ''
function layout(): void {
const size = root.getActualSize()
if (size.width <= 0 || size.height <= 0) return
const key = `${size.width}:${size.height}:${selectedTab}`
if (key === lastLayout) return
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
const landscape = size.width > size.height
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
mirror.width = width; mirror.height = width * 3 / 4
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
}
// iOS emits layoutChanged inside its layout pass. Defer mutations so
// NativeScript does not clear our new layout request at the end of that pass.
let layoutTimer: ReturnType<typeof setTimeout> | null = null
root.on('layoutChanged', () => {
if (layoutTimer !== null) return
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
})
const pause = () => {
if (layoutTimer !== null) clearTimeout(layoutTimer)
layoutTimer = null
padInput.cancel(); mirrorInput.cancel(); controller.pause()
}
const resume = () => controller.resume()
let offSettings: (() => void) | null = null
page.on('loaded', () => {
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
selectTab(selectedTab); controller.resume()
})
page.on('unloaded', () => {
pause(); offSettings?.(); offSettings = null
Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume)
})
return page
}
+70
View File
@@ -0,0 +1,70 @@
/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */
declare function setTimeout(callback: () => void, ms: number): number
declare function clearTimeout(id: number): void
export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release'
| 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right'
export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number }
export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void }
const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType<typeof setTimeout>) }
export class PhoneGestureRecognizer {
private start: PhoneTouch | null = null
private hold: unknown = null
private tap: unknown = null
private lastTap: { x: number; y: number } | null = null
private holding = false
private moved = false
private multi = false
private secondTap = false
constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void,
private readonly timer: GestureClock = clock) {}
touch(event: PhoneTouch): void {
if (event.action === 'cancel') { this.cancel(); return }
if (event.action === 'down' && !this.start) {
this.start = event
this.holding = this.moved = this.multi = false
this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36
if (this.secondTap) { this.clearTap() }
else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) }
this.hold = this.timer.set(() => {
this.hold = null
if (!this.start || this.moved || this.multi) return
this.holding = true
this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y)
}, 500)
}
if (!this.start) return
if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() }
const dx = event.x - this.start.x, dy = event.y - this.start.y
if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() }
if (event.action !== 'up') return
// Some platforms emit an up for each finger. Complete the multi-touch
// gesture only once, and suppress the trailing one-finger up events.
this.clearHold()
const start = this.start
this.start = null
if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return }
if (this.multi) { this.emit('double-tap', start.x, start.y); return }
if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) {
this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y)
} else if (!this.moved) {
if (this.secondTap) this.emit('double-tap', start.x, start.y)
else {
this.lastTap = { x: start.x, y: start.y }
this.tap = this.timer.set(() => {
const tap = this.lastTap
this.tap = null; this.lastTap = null
if (tap) this.emit('tap', tap.x, tap.y)
}, 280)
}
}
}
cancel(): void {
this.clearHold(); this.clearTap()
if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y)
this.start = null; this.holding = false
}
private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null }
private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null }
}
+6 -3
View File
@@ -144,6 +144,8 @@ export type ShellWindow = {
};
export type ShellConfig = {
/** Hosts without microphone support hide and reject voice entry points. */
voiceInputEnabled?: boolean;
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
actions: LayerActions;
getScreenTimeoutMs: () => number | null;
@@ -901,7 +903,7 @@ class Shell {
handsFree?: boolean;
defaultTarget: "assistant" | "app";
}): void {
if (this.voiceDialogPending) return;
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
this.voiceDialogPending = true;
void (async () => {
let ready = true;
@@ -1005,6 +1007,7 @@ class Shell {
* the dialog finishes on click instead of long-press-release.
*/
startVoiceInput(): void {
if (this.config.voiceInputEnabled === false) return;
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
// The transcript is aimed at the window whose menu requested it; the menu
// entry point defaults the highlight to Type Into App.
@@ -1334,7 +1337,7 @@ class Shell {
ctx.stack.pop();
},
},
{
...(this.config.voiceInputEnabled === false ? [] : [{
label: "Voice input",
onSelect: (ctx) => {
// The transcript is aimed at the foreground window, so keep focus
@@ -1343,7 +1346,7 @@ class Shell {
ctx.stack.pop();
this.startVoiceInput();
},
},
}]),
);
items.push({
label: "Debug",
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env bash
set -euo pipefail
cd "$(dirname "$0")/.."
# Prefer Homebrew Ruby over the older macOS system Ruby when available.
for ruby_bin in /opt/homebrew/opt/ruby/bin /usr/local/opt/ruby/bin; do
if [[ -x "$ruby_bin/ruby" ]]; then
export PATH="$ruby_bin:$PATH"
break
fi
done
if command -v ruby >/dev/null 2>&1; then
gem_bin="$(ruby -e 'puts Gem.bindir')"
export PATH="$gem_bin:$PATH"
fi
exec ns "$@"
+195
View File
@@ -0,0 +1,195 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
function harness(provider = 'elevenlabs', continuous = true) {
const timers = new Map();
let timerId = 0;
const sockets = [];
const events = [], statuses = [], errors = [];
const load = loader({
setTimeout(fn, delay) { const id = ++timerId; timers.set(id, { fn, delay }); return id; },
clearTimeout(id) { timers.delete(id); },
console: { log() {}, warn() {} },
com: { faceclaw: { app: {
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
FaceclawWebSocket: class {
constructor(url, listener) { Object.assign(this, { url, listener, sent: [], closed: false, reject: false }); sockets.push(this); }
sendText(text) { if (this.reject) return false; this.sent.push(text); return true; }
sendBinary(bytes) { if (this.reject) return false; this.sent.push(bytes); return true; }
close() { this.closed = true; }
}, } }, },
}, {
'./cloud-stt': {
CLOUD_STT_SAMPLE_RATE: 16000,
encodeBase64: bytes => Buffer.from(bytes).toString('base64'),
toJavaBytes: bytes => bytes,
},
});
const names = { elevenlabs: 'ElevenLabsSttClient', openai: 'OpenAiRealtimeSttClient', soniox: 'SonioxSttClient' };
const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]];
const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts');
const client = new ReconnectingSttClient(options => new Provider(options), {
apiKey: 'test', onTranscript: event => events.push(event),
onStatus: status => statuses.push(status), onError: error => errors.push(error),
}, () => continuous);
client.start();
return { client, sockets, events, statuses, errors, timers,
open() { sockets.at(-1).listener.onOpen(); },
message(message, socket = sockets.at(-1)) { socket.listener.onTextMessage(JSON.stringify(message)); },
tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); return timer.delay; },
};
}
function pcm(amplitude, ms = 50) {
const bytes = new Uint8Array(ms * 32);
for (let i = 0; i < bytes.length; i += 2) { bytes[i] = amplitude & 255; bytes[i + 1] = amplitude >> 8; }
return bytes;
}
for (const provider of ['elevenlabs', 'openai', 'soniox']) {
test(`${provider}: failure and close reconnect, stale callbacks are ignored, stop cancels retry`, () => {
const h = harness(provider);
h.open();
const old = h.sockets[0];
old.listener.onFailure('network unavailable');
old.listener.onClosed(); // Both callbacks must schedule only one retry.
assert.equal(h.timers.size, 1);
assert.equal(old.closed, true);
h.client.acceptPcm(pcm(2000));
assert.equal(h.tick(), 1000);
assert.equal(h.sockets.length, 2);
h.open();
assert.ok(h.sockets[1].sent.length > 0);
const eventCount = h.events.length;
old.listener.onTextMessage(JSON.stringify({ message_type: 'partial_transcript', text: 'stale' }));
old.listener.onOpen();
assert.equal(h.events.length, eventCount);
h.sockets[1].listener.onClosed();
assert.equal(h.timers.size, 1);
h.client.stop();
assert.equal(h.timers.size, 0);
});
test(`${provider}: rejected audio send triggers recovery and requeues the packet`, () => {
const h = harness(provider);
h.open();
h.sockets[0].reject = true;
h.client.acceptPcm(pcm(2000));
assert.equal(h.timers.size, 1);
h.tick(); h.open();
assert.ok(h.sockets[1].sent.length > (provider === 'elevenlabs' ? 0 : 1));
h.client.stop();
});
test(`${provider}: auth failure stops instead of retrying indefinitely`, () => {
const h = harness(provider);
h.sockets[0].listener.onFailure("Expected HTTP 101 response but was '401 Unauthorized'");
assert.equal(h.timers.size, 0);
assert.equal(h.errors.length, 1);
});
}
test('reconnect preserves visible partials once and does not replay already-sent audio', () => {
const h = harness(); h.open();
h.client.acceptPcm(pcm(2000));
h.message({ message_type: 'partial_transcript', text: 'Before outage' });
h.sockets[0].listener.onFailure('offline');
assert.equal(h.events.at(-1).text, 'Before outage');
assert.equal(h.events.at(-1).isFinal, true);
h.tick(); h.open();
assert.equal(h.sockets[1].sent.length, 0);
h.message({ message_type: 'partial_transcript', text: 'After outage' });
assert.equal(h.events.at(-1).text, 'After outage');
h.client.stop();
});
test('pending audio is bounded to ten seconds and backoff is capped', () => {
const h = harness();
for (let i = 0; i < 20; i++) h.client.acceptPcm(pcm(i, 1000));
h.open();
const sentBytes = h.sockets[0].sent.reduce((sum, text) => sum + Buffer.from(JSON.parse(text).audio_base_64, 'base64').length, 0);
assert.equal(sentBytes, 320000);
for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {
h.sockets.at(-1).listener.onFailure('offline');
assert.equal(h.tick(), delay);
}
h.client.stop();
});
test('finishing while disconnected cancels retries; finishing on an open socket times out cleanly', () => {
const h = harness(); h.open();
h.sockets[0].listener.onClosed();
h.client.finish();
assert.equal(h.timers.size, 0);
const live = harness(); live.open();
live.client.finish();
assert.equal(live.tick(), 5000);
assert.equal(live.sockets[0].closed, true);
live.sockets[0].listener.onFailure('late');
assert.equal(live.timers.size, 0);
});
test('only continuous capture commits at pauses, keeping ElevenLabs connected', () => {
for (const continuous of [false, true]) {
const h = harness('elevenlabs', continuous); h.open();
h.client.acceptPcm(pcm(3000));
h.client.acceptPcm(pcm(0, 1500));
const commits = h.sockets[0].sent.map(JSON.parse).filter(m => m.commit);
assert.equal(commits.length, continuous ? 1 : 0);
if (continuous) {
h.message({ message_type: 'committed_transcript', text: 'First.' });
h.message({ message_type: 'committed_transcript_with_timestamps', text: 'First.' });
assert.equal(h.events.length, 1);
assert.equal(h.events[0].paragraphBreakAfter, true);
assert.equal(h.sockets[0].closed, false);
}
h.client.stop();
}
});
test('OpenAI keeps successive items separate and drains out-of-order completions in audio order', () => {
const h = harness('openai'); h.open();
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
h.message({ type: 'input_audio_buffer.committed', item_id: 'a' });
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'a', delta: 'Fir' });
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
h.message({ type: 'input_audio_buffer.committed', item_id: 'b' });
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'Second.' });
assert.equal(h.events.some(e => e.isFinal), false);
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'First.' });
assert.deepEqual(h.events.filter(e => e.isFinal).map(e => e.text), ['First.', 'Second.']);
assert.equal(h.sockets[0].closed, false);
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'c', delta: 'Third.' });
assert.equal(h.events.at(-1).text, 'Third.');
h.client.stop();
});
test('Soniox speaker and pause formatting remains separate from dictation text and resets on finalize', () => {
const h = harness('soniox'); h.open();
h.message({ tokens: [
{ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1', is_final: true },
{ text: ' Hi.', start_ms: 550, end_ms: 800, speaker: '2', is_final: false },
] });
assert.equal(h.events.at(-1).text, 'Hello. Hi.');
assert.equal(h.events.at(-1).transcribeText, 'Hello.\nHi.');
h.message({ tokens: [
{ text: ' Yes.', start_ms: 550, end_ms: 800, speaker: '2', is_final: true },
{ text: '<fin>', is_final: true },
] });
assert.equal(h.events.find(e => e.isFinal).transcribeText, 'Hello.\nYes.');
h.message({ tokens: [{ text: 'New.', start_ms: 3000, end_ms: 3400, speaker: '2', is_final: false }] });
assert.equal(h.events.at(-1).text, 'New.');
h.client.stop();
});
test('an unsolicited Soniox session end finalizes visible text and reconnects', () => {
const h = harness('soniox'); h.open();
h.message({ tokens: [{ text: 'Before expiry.', is_final: true }], finished: true });
assert.equal(h.events.at(-1).isFinal, true);
assert.equal(h.events.at(-1).text, 'Before expiry.');
assert.equal(h.timers.size, 1);
h.tick(); h.open();
assert.equal(h.sockets.length, 2);
h.client.stop();
});
+28
View File
@@ -0,0 +1,28 @@
const fs = require('node:fs');
const path = require('node:path');
const vm = require('node:vm');
const ts = require('typescript');
// Execute production TypeScript with only the native boundary and clock stubbed.
exports.loader = (globals = {}, mocks = {}) => {
const cache = new Map();
function load(file) {
file = path.resolve(__dirname, '../..', file);
if (cache.has(file)) return cache.get(file);
const output = {};
cache.set(file, output);
const source = ts.transpileModule(fs.readFileSync(file, 'utf8'), {
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
}).outputText;
vm.runInNewContext(source, {
exports: output, console, Uint8Array, ...globals,
require(name) {
if (name in mocks) return mocks[name];
if (!name.startsWith('.')) throw new Error(`Unmocked module: ${name}`);
return load(path.resolve(path.dirname(file), name + '.ts'));
},
}, { filename: file });
return output;
}
return load;
};
+56
View File
@@ -0,0 +1,56 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { PhoneGestureRecognizer } = require('../.test-build/app/phone-ui/phone-gestures.js');
function setup() {
let now = 0, id = 0;
const timers = new Map(), events = [];
const r = new PhoneGestureRecognizer((type, x, y) => events.push({ type, x, y }), {
set(fn, ms) { timers.set(++id, { fn, at: now + ms }); return id; },
clear(id) { timers.delete(id); },
});
return {
r, events,
touch: (action, x = 20, y = 30, pointers = 1) => r.touch({ action, x, y, pointers }),
advance(ms) {
now += ms;
for (const [id, timer] of [...timers]) if (timer.at <= now && timers.delete(id)) timer.fn();
},
types: () => events.map(e => e.type),
};
}
test('single tap retains mirror coordinates and waits to disambiguate double tap', () => {
const s = setup(); s.touch('down'); s.touch('up');
assert.deepEqual(s.types(), []); s.advance(280);
assert.deepEqual(s.events, [{ type: 'tap', x: 20, y: 30 }]);
});
test('double tap emits back without accidental selection', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.advance(150);
s.touch('down'); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['double-tap']);
});
test('a hold emits a press and release, never a tap', () => {
const s = setup(); s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
});
test('tap then hold emits the app-menu gesture without an earlier selection', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.advance(100);
s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['short-then-long-press', 'long-press-release']);
});
test('four directional swipes cancel pending holds and taps', () => {
for (const [x, y, expected] of [[80, 30, 'swipe-right'], [-40, 30, 'swipe-left'], [20, 90, 'swipe-down'], [20, -30, 'swipe-up']]) {
const s = setup(); s.touch('down'); s.touch('move', x, y); s.advance(500); s.touch('up', x, y); s.advance(500);
assert.deepEqual(s.types(), [expected]);
}
});
test('two fingers send back exactly once and cancel the long-press timer', () => {
const s = setup(); s.touch('down'); s.touch('down', 21, 30, 2); s.advance(500);
s.touch('up', 21, 30, 2); s.touch('up'); s.advance(1000);
assert.deepEqual(s.types(), ['double-tap']);
});
test('cancellation prevents delayed selections and releases active holds', () => {
const s = setup(); s.touch('down'); s.touch('up'); s.r.cancel(); s.advance(1000);
assert.deepEqual(s.types(), []);
s.touch('down'); s.advance(500); s.touch('cancel'); s.advance(1000);
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
});
+49
View File
@@ -0,0 +1,49 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
const rect = (x, y, width, height) => ({ x, y, width, height });
function surface(c, id, box, values, zOrder = 0, transparency = 'opaque') {
c.configureSurface(id, { ...box, zOrder, transparency });
c.submitSurfaceFrame(id, Uint8Array.from(values), rect(0, 0, box.width, box.height));
}
test('shell color key preserves underlying text while opaque black covers it', () => {
const c = new SurfaceCompositor(3, 2);
surface(c, 'app', rect(0, 0, 3, 2), [200, 200, 200, 200, 200, 200]);
surface(c, 'shell', rect(0, 0, 3, 2), [0, 1, 255, 0, 0, 0], 1, 'color-key');
c.setUnderlayDim(1, 0.5);
assert.deepEqual([...c.composite()], [100, 1, 255, 100, 100, 100]);
c.setSurfaceVisible('shell', false);
assert.deepEqual([...c.composite()], [100, 100, 100, 100, 100, 100]);
surface(c, 'opaque', rect(1, 0, 1, 2), [0, 0], 2);
assert.deepEqual([...c.composite()], [100, 0, 100, 100, 0, 100]);
});
test('partial frames clip in surface coordinates, then surfaces clip to the screen', () => {
const c = new SurfaceCompositor(3, 2);
surface(c, 'app', rect(-1, 0, 3, 2), [1, 2, 3, 4, 5, 6]);
const source = Uint8Array.from([99, 10, 20, 99, 30, 40, 99]);
c.submitSurfaceFrame('app', source.subarray(1, 5), rect(1, 1, 2, 2));
source.fill(0);
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
c.submitSurfaceFrame('app', Uint8Array.from([50, 60, 70, 80]), rect(-1, -1, 2, 2));
// The clipped update changed the offscreen column, leaving visible pixels intact.
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
});
test('moving a surface retains its frame; resizing clears it; blanking is reversible', () => {
const c = new SurfaceCompositor(3, 1);
surface(c, 'app', rect(0, 0, 2, 1), [7, 9]);
c.configureSurface('app', { ...rect(1, 0, 2, 1), zOrder: 0, transparency: 'opaque' });
assert.deepEqual([...c.composite()], [0, 7, 9]);
c.setScreenBlanked(true); assert.deepEqual([...c.composite()], [0, 0, 0]);
c.setScreenBlanked(false); assert.deepEqual([...c.composite()], [0, 7, 9]);
c.configureSurface('app', { ...rect(0, 0, 3, 1), zOrder: 0, transparency: 'opaque' });
assert.deepEqual([...c.composite()], [0, 0, 0]);
});
test('layer order is independent of creation order; removal reveals lower surfaces', () => {
const c = new SurfaceCompositor(1, 1);
surface(c, 'top', rect(0, 0, 1, 1), [20], 10);
surface(c, 'bottom', rect(0, 0, 1, 1), [80], -1);
assert.equal(c.composite()[0], 20);
c.removeSurface('top'); assert.equal(c.composite()[0], 80);
assert.throws(() => c.submitSurfaceFrame('missing', new Uint8Array(1), rect(0, 0, 1, 1)));
assert.throws(() => c.submitSurfaceFrame('bottom', new Uint8Array(2), rect(0, 0, 1, 1)));
});
+94
View File
@@ -0,0 +1,94 @@
const test = require('node:test');
const assert = require('node:assert/strict');
const { loader } = require('./helpers/load-typescript.cjs');
const load = loader();
const { TranscriptModel, TranscriptScroll, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts');
const { SpeechPauseDetector } = load('app/native/speech-pause.ts');
const { TimedTranscript } = load('app/native/transcript-format.ts');
function pcm(amplitude, ms = 50) {
const bytes = new Uint8Array(ms * 16 * 2);
for (let i = 0; i < bytes.length; i += 2) {
bytes[i] = amplitude & 255;
bytes[i + 1] = (amplitude >> 8) & 255;
}
return bytes;
}
test('partials replace, finals correct, and pause breaks survive display and saving', () => {
const m = new TranscriptModel();
m.accept({ text: 'Hello wor', isFinal: false });
m.accept({ text: 'Hello world.', isFinal: true, paragraphBreakAfter: true });
m.accept({ text: 'Next', isFinal: false });
m.accept({ text: 'Next paragraph.', isFinal: false });
assert.equal(m.text, 'Hello world.\nNext paragraph.');
m.accept({ text: 'Next paragraph!', isFinal: true });
assert.equal(m.text, 'Hello world.\nNext paragraph!');
m.accept({ text: '', isFinal: true });
assert.equal(m.text, 'Hello world.\nNext paragraph!');
});
test('onboard pauses can precede or follow finalization', () => {
for (const before of [true, false]) {
const m = new TranscriptModel();
m.accept({ text: 'first', isFinal: false });
if (before) m.pause();
m.accept({ text: 'First.', isFinal: true });
if (!before) m.pause();
m.accept({ text: 'Second.', isFinal: false });
assert.equal(m.text, 'First.\nSecond.');
}
});
test('formatting metadata is used only by Transcribe, with revisable live speaker tokens', () => {
const final = new TimedTranscript();
final.append({ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1' });
const preview = final.copy();
preview.append({ text: ' Wrong', start_ms: 600, end_ms: 900, speaker: '2' });
assert.equal(preview.text, 'Hello.\nWrong');
assert.equal(final.text, 'Hello.');
final.append({ text: ' Right.', start_ms: 600, end_ms: 900, speaker: '1' });
final.append({ text: ' Later.', start_ms: 2400, end_ms: 2800, speaker: '1' });
assert.equal(final.text, 'Hello. Right.\nLater.');
const event = { text: 'Hello. Right. Later.', transcribeText: final.text, isFinal: false };
const m = new TranscriptModel();
m.accept(event);
assert.equal(m.text, final.text);
assert.equal(event.text.includes('\n'), false);
});
test('wrapping preserves explicit newlines, blank paragraphs, and long unspaced words', () => {
assert.deepEqual([...wrapTranscribeText(s => s.length, 'one two\nthree\n\nabcdefghij', 5)],
['one', 'two', 'three', '', 'abcde', 'fghij']);
});
test('scroll follows the tail until scrolled back, then resumes only at the bottom', () => {
const s = new TranscriptScroll();
assert.equal(s.layout(30, 10), 20);
assert.equal(s.layout(33, 10), 23);
s.scroll(-3);
assert.equal(s.layout(50, 10), 20);
assert.equal(s.layout(51, 10), 20);
s.scroll(100);
assert.equal(s.layout(55, 10), 45);
s.scroll(-100);
assert.equal(s.layout(60, 10), 0);
});
test('short transcripts become scrollable without disabling automatic following', () => {
const s = new TranscriptScroll();
assert.equal(s.layout(3, 10), 0);
s.scroll(-3);
assert.equal(s.layout(12, 10), 2);
});
test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores leading silence', () => {
const d = new SpeechPauseDetector();
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
d.accept(pcm(3000));
for (let i = 0; i < 29; i++) assert.equal(d.accept(pcm(0)), false);
assert.equal(d.accept(pcm(0)), true);
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
d.accept(pcm(3000));
assert.equal(d.accept(pcm(0, 1500)), true);
});
+2
View File
@@ -11,6 +11,8 @@
"outDir": "../.test-build"
},
"include": [
"../app/graphics/surface-compositor.ts",
"../app/phone-ui/phone-gestures.ts",
"../app/g2/glasses-hardware-identity.ts",
"../app/g2/ble-proximity.ts",
"../app/g2/even-advertisement.ts",