diff --git a/App_Resources/iOS/Info.plist b/App_Resources/iOS/Info.plist index 98cc8471..4be2ce2c 100644 --- a/App_Resources/iOS/Info.plist +++ b/App_Resources/iOS/Info.plist @@ -23,6 +23,8 @@ UISupportedInterfaceOrientations UIInterfaceOrientationPortrait + UIInterfaceOrientationLandscapeLeft + UIInterfaceOrientationLandscapeRight UIRequiresFullScreen diff --git a/App_Resources/iOS/Podfile b/App_Resources/iOS/Podfile new file mode 100644 index 00000000..c643ad84 --- /dev/null +++ b/App_Resources/iOS/Podfile @@ -0,0 +1,13 @@ +platform :ios, '15.1' +pod 'SVGKit', '3.0.0' + +# SVGKit 3's physical-DPI lookup asserts on arm64 simulators. Our icons use +# only viewBox/pixel units, so that lookup is irrelevant to their rendering. +post_install do |installer| + installer.pods_project.targets.each do |target| + next unless target.name == 'SVGKit' + target.build_configurations.each do |config| + config.build_settings['ENABLE_NS_ASSERTIONS'] = 'NO' + end + end +end diff --git a/App_Resources/iOS/src/FaceclawGraphics.h b/App_Resources/iOS/src/FaceclawGraphics.h new file mode 100644 index 00000000..3dc2e7d5 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawGraphics.h @@ -0,0 +1,6 @@ +#import + +@interface FaceclawGraphics : NSObject ++ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green; ++ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size; +@end diff --git a/App_Resources/iOS/src/FaceclawGraphics.m b/App_Resources/iOS/src/FaceclawGraphics.m new file mode 100644 index 00000000..96a5e25e --- /dev/null +++ b/App_Resources/iOS/src/FaceclawGraphics.m @@ -0,0 +1,56 @@ +#import "FaceclawGraphics.h" +#import + +@implementation FaceclawGraphics ++ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green { + if (width <= 0 || height <= 0 || gray.length != width * height) return nil; + NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4]; + const uint8_t *src = gray.bytes; + uint8_t *dst = rgba.mutableBytes; + uint8_t lut[256]; + for (int i = 0; i < 256; i++) lut[i] = (uint8_t)round(pow(i / 255.0, 0.7) * 255.0); + for (NSInteger i = 0; i < width * height; i++) { + uint8_t value = lut[src[i]]; + dst[i * 4] = green ? 0 : value; + dst[i * 4 + 1] = value; + dst[i * 4 + 2] = green ? 0 : value; + dst[i * 4 + 3] = 255; + } + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba); + CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space, + kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault); + UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil; + if (image) CGImageRelease(image); + CGDataProviderRelease(provider); + CGColorSpaceRelease(space); + return result; +} ++ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size { + if (size <= 0 || size > 1024) return nil; + SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]]; + if (!image) return nil; + image.size = CGSizeMake(size, size); + UIGraphicsImageRendererFormat *format = [UIGraphicsImageRendererFormat defaultFormat]; + format.scale = 1; + format.opaque = NO; + UIGraphicsImageRenderer *renderer = [[UIGraphicsImageRenderer alloc] initWithSize:image.size format:format]; + UIImage *raster = [renderer imageWithActions:^(UIGraphicsImageRendererContext *context) { + [image.CALayerTree renderInContext:context.CGContext]; + }]; + NSMutableData *rgba = [NSMutableData dataWithLength:size * size * 4]; + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + CGContextRef context = CGBitmapContextCreate(rgba.mutableBytes, size, size, 8, size * 4, space, + kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast); + CGContextDrawImage(context, CGRectMake(0, 0, size, size), raster.CGImage); + NSMutableData *gray = [NSMutableData dataWithLength:size * size]; + const uint8_t *src = rgba.bytes; + uint8_t *dst = gray.mutableBytes; + for (NSInteger i = 0; i < size * size; i++) { + dst[i] = (uint8_t)round(0.2126 * src[i * 4] + 0.7152 * src[i * 4 + 1] + 0.0722 * src[i * 4 + 2]); + } + CGContextRelease(context); + CGColorSpaceRelease(space); + return gray; +} +@end diff --git a/App_Resources/iOS/src/module.modulemap b/App_Resources/iOS/src/module.modulemap new file mode 100644 index 00000000..560d5493 --- /dev/null +++ b/App_Resources/iOS/src/module.modulemap @@ -0,0 +1,4 @@ +module FaceclawGraphics { + header "FaceclawGraphics.h" + export * +} diff --git a/app/apps/calculator/calculator.ts b/app/apps/calculator/calculator.ts index 5f47d785..1a02dfed 100644 --- a/app/apps/calculator/calculator.ts +++ b/app/apps/calculator/calculator.ts @@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer { constructor( private readonly coordinator: MathCoordinator, private readonly actions: LayerActions, + private readonly textInput?: () => Promise, ) { this.coordinator.setUsesDegrees(calculatorDegreesSetting.get()); this.coordinator.onChanged = () => this.handleCoordinatorChanged(); - this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); + if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event)); } private get mode(): CalculatorMode { @@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer { } private refreshListening(): void { + if (this.textInput) return; if (!this.foreground) return; if (this.listening === "continuous") { if (!this.continuousActive) { @@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer { * that did not. */ beginDictation(): void { + if (this.textInput) { void this.textInput(); return; } if (this.dictating) return; this.dictating = true; void this.actions.startVoiceCapture(true); @@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer { bodyBottom: number, ): void { const prompt = - this.listening === "continuous" + this.textInput ? "Tap to type a problem on your phone" + : this.listening === "continuous" ? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"` : "Tap to say a problem — or use Voice input to type one"; const lines = wrapText(font, prompt, width - 96); @@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer { private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void { if (!this.lastInput) return; - const label = "Heard"; + const label = this.textInput ? "Entered" : "Heard"; image.drawText(font, 24, y, label, 110); const x = 24 + font.measureText(label) + 10; image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160); @@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer { this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode"; return gestureHints([ [GESTURE_SCROLL, scrollTarget], - [GESTURE_CLICK, "mic"], + [GESTURE_CLICK, this.textInput ? "type" : "mic"], [GESTURE_DOUBLE_CLICK, "back"], ]); } diff --git a/app/bootstrap.ios.ts b/app/bootstrap.ios.ts index c2a43d2c..977ab199 100644 --- a/app/bootstrap.ios.ts +++ b/app/bootstrap.ios.ts @@ -1,47 +1,4 @@ -import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core' -import { bytesToHex, hexToBytes } from './util/hex-util' +import { Application } from '@nativescript/core' +import { createMainPage } from './phone-ui/main-page.ios' -// Milestone 1: exercise native UI, TypeScript, and an existing shared utility -// without loading the Android services used by the full glasses application. -Application.run({ - create: () => { - const page = new Page() - page.actionBarHidden = true - const scroll = new ScrollView() - const content = new StackLayout() - content.padding = 28 - content.verticalAlignment = 'middle' - - const label = (text: string, fontSize = 17) => { - const view = new Label() - view.text = text - view.fontSize = fontSize - view.textWrap = true - view.marginBottom = 20 - content.addChild(view) - return view - } - - label('Faceclaw', 36) - label('iOS port · First launch', 24) - label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`) - label('Glasses pairing and the Android features are still to be ported.') - const status = label('Tap below to test native input and shared TypeScript code.') - const button = new Button() - button.text = 'Run shared-code check' - button.accessibilityIdentifier = 'shared-code-check' - let checks = 0 - button.on('tap', () => { - const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77')) - const passed = result === '46616365636C6177' - checks += 1 - status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.` - console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`) - }) - content.addChild(button) - scroll.content = content - page.content = scroll - page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded')) - return page - }, -}) +Application.run({ create: createMainPage }) diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts new file mode 100644 index 00000000..c2ff1807 --- /dev/null +++ b/app/g2/ios-preview-controller.ts @@ -0,0 +1,220 @@ +import { Dialogs, type ImageSource } from '@nativescript/core' +import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app' +import { CalculatorLayer } from '../apps/calculator/calculator' +import { MathCoordinator } from '../apps/calculator/math/coordinator' +import { SurfaceCompositor } from '../graphics/surface-compositor' +import { flattenPlanes, type Plane } from '../graphics/plane' +import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image' +import { previewPixels } from '../native/ios-graphics' +import { makeInputEvent, type InputEventPayload } from '../ui/gestures' +import { noopLayerActions, type LayerActions } from '../ui/layers' +import { MenuLayer } from '../ui/menu' +import { createInProcessWindow } from '../ui/shell/in-process-window' +import { shell, type ShellWindow } from '../ui/shell/shell' +import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry' +import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged, + previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings' +import type { PhoneGesture } from '../phone-ui/phone-gestures' + +/** Local display host: shared shell/windows, no device-service initialization. */ +export class IosPreviewController { + private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT) + private renderTimer: ReturnType | null = null + private clockTimer: ReturnType | null = null + private offSettings: (() => void) | null = null + private active = false + private shellDirty = true + private inputQueue: Promise = Promise.resolve() + private lastLayout = '' + private prompting = false + private readonly actions: LayerActions = { + ...noopLayerActions, + requestRender: () => this.requestShellRender(), + startTextSettingEdit: setting => this.editSetting(setting), + startTextSettingsEdit: async (settings, _title, finished) => { + for (const setting of settings) await this.editSetting(setting) + finished?.() + }, + } + + constructor(private readonly onFrame: (image: ImageSource, focus: string) => void, + private readonly onError: (message: string) => void) { + this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' }) + shell.configure({ + actions: this.actions, + voiceInputEnabled: false, + getScreenTimeoutMs: () => null, + requestShellRender: () => this.requestShellRender(), + onWindowsChanged: () => this.requestShellRender(), + onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() }, + }) + const launcher = createLauncherWindow({ + actions: this.actions, + apps: () => [ + { appId: 'calculator', label: 'Calculator', icon: 'calculator' }, + { appId: 'display', label: 'Display', icon: 'settings' }, + ], + launchApp: id => this.launchApp(id), + uninstallApp: () => {}, + submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes), + setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() }, + }) + this.configureWindow(launcher) + shell.registerWindow(launcher) + shell.wake('window') + shell.focusWindow(launcher.windowId) + } + + resume(): void { + if (this.active) return + this.active = true + this.offSettings = onAnySettingChanged(() => { + this.relayout() + shell.foregroundWindow()?.requestRender() + this.requestShellRender() + }) + this.clockTimer = setInterval(() => this.requestShellRender(), 60_000) + this.relayout() + shell.foregroundWindow()?.requestRender() + this.requestShellRender() + console.log('[ios-preview] Main screen active') + } + pause(): void { + this.active = false + this.offSettings?.(); this.offSettings = null + if (this.clockTimer) clearInterval(this.clockTimer) + if (this.renderTimer) clearTimeout(this.renderTimer) + this.clockTimer = this.renderTimer = null + } + private configureWindow(window: ShellWindow): void { + this.compositor.configureSurface(window.surfaceId, { + ...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque', + }) + this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId) + } + private relayout(): void { + const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}` + if (layout === this.lastLayout) return + this.lastLayout = layout + for (const window of shell.getWindows()) { + this.configureWindow(window) + window.relayout?.() + } + } + private async submit(id: string, planes: Plane[]): Promise { + const image = flattenPlanes(planes) + this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }) + this.scheduleFrame() + } + private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() } + private scheduleFrame(): void { + if (!this.active || this.renderTimer !== null) return + // Coalesce window/chrome updates into one preview, capped at 30 fps. + this.renderTimer = setTimeout(() => { + this.renderTimer = null + try { + if (this.shellDirty) { + this.shellDirty = false + const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 }) + this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 }) + this.compositor.setUnderlayDim(1, shell.underlayDim()) + } + const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green') + this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) + } catch (error) { this.fail(error) } + }, 33) + } + private fail(error: unknown): void { + const message = error instanceof Error ? error.stack ?? error.message : String(error) + console.error(`[ios-preview] ${message}`) + this.onError(error instanceof Error ? error.message : String(error)) + } + gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void { + this.inputQueue = this.inputQueue.then(async () => { + if (!this.active) return + if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) { + await this.mirrorTap(nx, ny) + } else { + let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record)[gesture] ?? gesture + if (origin === 'ring' && type.startsWith('swipe-')) { + if (type === 'swipe-left' || type === 'swipe-right') return + type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down' + } + const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload) + await shell.receiveInput(event) + this.requestShellRender() + } + console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`) + }).catch(error => this.fail(error)) + } + private async mirrorTap(nx: number, ny: number): Promise { + const x = Math.max(0, Math.min(639, Math.floor(nx * 640))) + const y = Math.max(0, Math.min(479, Math.floor(ny * 480))) + const window = shell.foregroundWindow() + if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) { + const target = shell.windowAtSidebarPoint(x, y) + if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() } + return + } + if (window && !shell.hasOverlay()) { + const rect = appViewportRect(window.heightMode, window.appId) + if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return + shell.focusWindow(window.windowId) + if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return } + } + await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' })) + this.requestShellRender() + } + launchApp(id: string): void { + const existing = shell.getWindows().find(window => window.appId === id) + if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return } + const surfaceId = `window:${id}` + let render = () => {} + const actions = { ...this.actions, requestRender: () => render() } + const layer = id === 'calculator' + ? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp()) + : new MenuLayer('Display', [ + { label: 'Change screen size', onSelect: () => this.cycleDisplayMode() }, + { label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } }, + { label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } }, + ], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true }) + const app = createInProcessWindow({ + appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display', + iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings', + closeable: true, actions, baseLayer: layer, + receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined, + submitFrame: planes => this.submit(surfaceId, planes), + setSurfaceVisible: visible => { + if (layer instanceof CalculatorLayer) layer.setForeground(visible) + this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame() + }, + removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() }, + onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() }, + }) + render = app.requestRender + if (layer instanceof CalculatorLayer) layer.requestRender = render + this.configureWindow(app.window) + shell.registerWindow(app.window) + shell.focusWindow(app.window.windowId) + app.requestRender(); this.requestShellRender() + } + cycleDisplayMode(): void { + const values = DISPLAY_MODE_VALUES + displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length]) + } + get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) } + async typeIntoApp(): Promise { + if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return + this.prompting = true + try { + const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title, + message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' }) + if (result.result) shell.sendTextToForegroundWindow(result.text) + } catch (error) { this.fail(error) } + finally { this.prompting = false } + } + private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise { + const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' }) + if (result.result) setting.set(result.text) + } +} diff --git a/app/graphics/icons.ts b/app/graphics/icons.ts index 7c59d667..fc6f23d1 100644 --- a/app/graphics/icons.ts +++ b/app/graphics/icons.ts @@ -1,12 +1,10 @@ import { GrayImage } from "./image"; - -declare const com: any; -declare const global: any; +import { rasterizeSvg } from "../native/svg-rasterizer"; /** * Vector icons for window indicators (and anywhere else). SVGs are rendered - * once to a correctly sized grayscale bitmap by the Java IconRenderer (a - * small SVG subset: path/circle/rect/line/polyline) and cached here. To add + * once to a correctly sized grayscale bitmap by the platform rasterizer + * (Android IconRenderer or iOS SVGKit) and cached here. To add * an icon, drop its SVG source into ICON_SVGS — Lucide icons * (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color * Noun Project glyphs also work. @@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag if (cached !== undefined) return cached; let icon: GrayImage | null = null; - if (global.isAndroid) { - try { - const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH); - if (bytes && bytes.length >= size * size) { - icon = new GrayImage(size, size, 0); - for (let i = 0; i < size * size; i++) { - icon.pixels[i] = bytes[i] & 0xff; - } - } - } catch (error) { - console.warn(`renderIcon(${cacheName}) failed: ${error}`); - } + try { + icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH); + } catch (error) { + console.warn(`renderIcon(${cacheName}) failed: ${error}`); } cache.set(key, icon); return icon; diff --git a/app/graphics/surface-compositor.ts b/app/graphics/surface-compositor.ts new file mode 100644 index 00000000..1d8f80e7 --- /dev/null +++ b/app/graphics/surface-compositor.ts @@ -0,0 +1,75 @@ +/** Platform-independent 8bpp surface composition for the local display. */ +export type SurfaceRect = { x: number; y: number; width: number; height: number } +export type SurfaceConfiguration = SurfaceRect & { + zOrder: number + transparency: 'opaque' | 'color-key' +} +type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean } + +export class SurfaceCompositor { + private readonly surfaces = new Map() + private dimBelow = 0 + private dimFactor = 1 + private blanked = false + constructor(readonly width: number, readonly height: number) { + if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) { + throw new Error('Invalid compositor size') + } + } + configureSurface(id: string, options: SurfaceConfiguration): void { + if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger) + || options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry') + const previous = this.surfaces.get(id) + const pixels = previous?.width === options.width && previous.height === options.height + ? previous.pixels : new Uint8Array(options.width * options.height) + this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true }) + } + removeSurface(id: string): void { this.surfaces.delete(id) } + setSurfaceVisible(id: string, visible: boolean): void { + const surface = this.surfaces.get(id) + if (surface) surface.visible = visible + } + setUnderlayDim(belowZOrder: number, factor: number): void { + if (!Number.isFinite(factor)) throw new Error('Invalid dim factor') + this.dimBelow = belowZOrder + this.dimFactor = Math.max(0, Math.min(1, factor)) + } + setScreenBlanked(blanked: boolean): void { this.blanked = blanked } + submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void { + const surface = this.surfaces.get(id) + if (!surface) throw new Error(`Unknown surface: ${id}`) + if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger) + || rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) { + throw new Error('Invalid frame buffer or rectangle') + } + const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width) + const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height) + for (let y = top; y < bottom; y++) { + if (right <= left) break + const source = (y - rect.y) * rect.width + left - rect.x + surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left) + } + } + composite(): Uint8Array { + const output = new Uint8Array(this.width * this.height) + if (this.blanked) return output + const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder) + for (const surface of surfaces) { + const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width) + const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height) + const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1 + for (let y = top; y < bottom; y++) { + let source = (y - surface.y) * surface.width + left - surface.x + let target = y * this.width + left + for (let x = left; x < right; x++, source++, target++) { + const value = surface.pixels[source] + // Test the original value: opaque near-black must still cover the + // window beneath, even when dimming rounds it down to black. + if (surface.transparency === 'color-key' && value === 0) continue + output[target] = Math.round(value * dim) + } + } + } + return output + } +} diff --git a/app/native/ios-graphics.ts b/app/native/ios-graphics.ts new file mode 100644 index 00000000..6052d625 --- /dev/null +++ b/app/native/ios-graphics.ts @@ -0,0 +1,21 @@ +import { ImageSource } from '@nativescript/core' +import { GrayImage } from '../graphics/image' + +declare const FaceclawGraphics: any + +export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource { + // NSData copies the buffer; the UIImage owns its pixels after this call. + const copy = new Uint8Array(pixels) + const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) + const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green) + if (!native) throw new Error('Could not create preview image') + return new ImageSource(native) +} + +export function rasterizeSvg(svg: string, size: number): GrayImage | null { + const data = FaceclawGraphics.renderSVGSize(svg, size) + if (!data) return null + const image = new GrayImage(size, size, 0) + image.pixels.set(new Uint8Array(interop.bufferFromData(data))) + return image +} diff --git a/app/native/settings-store.ios.ts b/app/native/settings-store.ios.ts new file mode 100644 index 00000000..2d4c3dd5 --- /dev/null +++ b/app/native/settings-store.ios.ts @@ -0,0 +1,28 @@ +import { ApplicationSettings } from '@nativescript/core' + +// Main-isolate settings for the initial iOS port. Worker synchronization will +// be added when worker-hosted apps are enabled on iOS. +const listeners = new Set<(key: string) => void>() +function changed(key: string): void { + setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0) +} +export function getStringSetting(key: string, fallback: string): string { + return ApplicationSettings.getString(key, fallback) +} +export function setStringSetting(key: string, value: string): void { + if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return + ApplicationSettings.setString(key, value) + changed(key) +} +export function getBooleanSetting(key: string, fallback: boolean): boolean { + return ApplicationSettings.getBoolean(key, fallback) +} +export function setBooleanSetting(key: string, value: boolean): void { + if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return + ApplicationSettings.setBoolean(key, value) + changed(key) +} +export function onSettingsStoreChanged(listener: (key: string) => void): () => void { + listeners.add(listener) + return () => { listeners.delete(listener) } +} diff --git a/app/native/svg-rasterizer.ios.ts b/app/native/svg-rasterizer.ios.ts new file mode 100644 index 00000000..283045dd --- /dev/null +++ b/app/native/svg-rasterizer.ios.ts @@ -0,0 +1,14 @@ +import { rasterizeSvg as render } from './ios-graphics' +export function rasterizeSvg(svg: string, size: number, strokeWidth: number) { + // Shared icon sources rely on the Android renderer's white, rounded + // stroke defaults. Supply those explicitly for a standards SVG renderer. + const source = svg.replace(/]*)>/, (_tag, attributes: string) => { + const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24', + stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth), + 'stroke-linecap': 'round', 'stroke-linejoin': 'round' } + const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes)) + .map(([key, value]) => ` ${key}="${value}"`).join('') + return `` + }) + return render(source, size) +} diff --git a/app/native/svg-rasterizer.ts b/app/native/svg-rasterizer.ts new file mode 100644 index 00000000..d93ce43f --- /dev/null +++ b/app/native/svg-rasterizer.ts @@ -0,0 +1,9 @@ +import { GrayImage } from '../graphics/image' +declare const com: any +export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null { + const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth) + if (!bytes || bytes.length < size * size) return null + const image = new GrayImage(size, size, 0) + for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff + return image +} diff --git a/app/phone-ui/main-page.ios.ts b/app/phone-ui/main-page.ios.ts new file mode 100644 index 00000000..1ade0eab --- /dev/null +++ b/app/phone-ui/main-page.ios.ts @@ -0,0 +1,141 @@ +import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core' +import { IosPreviewController } from '../g2/ios-preview-controller' +import { PhoneGestureRecognizer } from './phone-gestures' +import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings' + +export function createMainPage(): Page { + const page = new Page() + page.actionBarHidden = true + const root = new GridLayout() + root.rows = 'auto,*' + page.content = root + const header = new GridLayout() + header.columns = '*,auto'; header.padding = '12 20' + const title = new Label() + title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold' + header.addChild(title) + const state = new Label() + state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle' + GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header) + const body = new GridLayout() + GridLayout.setRow(body, 1); root.addChild(body) + const mirror = new Image() + mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black') + mirror.accessibilityIdentifier = 'glasses-preview' + mirror.accessibilityLabel = 'Interactive glasses screen preview' + mirror.verticalAlignment = 'middle'; body.addChild(mirror) + const controls = new GridLayout() + controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls) + const tabs = new GridLayout() + tabs.columns = '*,*,*'; controls.addChild(tabs) + const pad = new GridLayout() + pad.borderRadius = 18; pad.backgroundColor = new Color('#202522') + pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center' + pad.accessibilityIdentifier = 'gesture-pad' + GridLayout.setRow(pad, 1); controls.addChild(pad) + const focus = new Label() + focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true + focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle' + focus.isUserInteractionEnabled = false; pad.addChild(focus) + const legend = new Label() + legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center' + legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom' + legend.isUserInteractionEnabled = false; pad.addChild(legend) + const settings = new StackLayout() + settings.verticalAlignment = 'middle' + GridLayout.setRow(settings, 1); controls.addChild(settings) + const errorLabel = new Label() + errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse' + settings.addChild(errorLabel) + let selectedTab: 'watch' | 'ring' | 'settings' = 'watch' + const controller = new IosPreviewController((image, label) => { + mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label + }, message => { + errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings') + }) + const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch')) + const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => { + const size = mirror.getActualSize() + if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height) + }) + const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => { + if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return + recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() }) + } + pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput)) + function button(text: string, action: () => void): Button { + const view = new Button() + view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action) + return view + } + const tabButtons: Button[] = [] + function selectTab(tab: typeof selectedTab): void { + padInput.cancel(); selectedTab = tab + pad.visibility = tab === 'settings' ? 'collapse' : 'visible' + settings.visibility = tab === 'settings' ? 'visible' : 'collapse' + for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5 + focus.text = tab === 'ring' ? 'Ring' : 'Watch controls' + legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu' + layout() + } + for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) { + const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab)) + GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view) + } + const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode()) + settings.addChild(sizeButton) + settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green'))) + const help = new Label() + help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu' + help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help) + const footer = new GridLayout() + footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer) + const back = button('Back', () => controller.gesture('double-tap', 'watch')) + const menu = button('Menu', () => { + controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch') + }) + const keyboard = button('Keyboard', () => { void controller.typeIntoApp() }) + for (const [index, view] of [back, menu, keyboard].entries()) { + view.padding = '16 8'; view.fontSize = 14 + GridLayout.setColumn(view, index); footer.addChild(view) + } + let lastLayout = '' + function layout(): void { + const size = root.getActualSize() + if (size.width <= 0 || size.height <= 0) return + const key = `${size.width}:${size.height}:${selectedTab}` + if (key === lastLayout) return + lastLayout = key; padInput.cancel(); mirrorInput.cancel() + const landscape = size.width > size.height + body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*' + GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0) + const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width + mirror.width = width; mirror.height = width * 3 / 4 + pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32) + pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200))) + } + // iOS emits layoutChanged inside its layout pass. Defer mutations so + // NativeScript does not clear our new layout request at the end of that pass. + let layoutTimer: ReturnType | null = null + root.on('layoutChanged', () => { + if (layoutTimer !== null) return + layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0) + }) + const pause = () => { + if (layoutTimer !== null) clearTimeout(layoutTimer) + layoutTimer = null + padInput.cancel(); mirrorInput.cancel(); controller.pause() + } + const resume = () => controller.resume() + let offSettings: (() => void) | null = null + page.on('loaded', () => { + Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume) + offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel }) + selectTab(selectedTab); controller.resume() + }) + page.on('unloaded', () => { + pause(); offSettings?.(); offSettings = null + Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume) + }) + return page +} diff --git a/app/phone-ui/phone-gestures.ts b/app/phone-ui/phone-gestures.ts new file mode 100644 index 00000000..bdc74ae9 --- /dev/null +++ b/app/phone-ui/phone-gestures.ts @@ -0,0 +1,70 @@ +/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */ +declare function setTimeout(callback: () => void, ms: number): number +declare function clearTimeout(id: number): void +export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release' + | 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right' +export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number } +export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void } +const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType) } + +export class PhoneGestureRecognizer { + private start: PhoneTouch | null = null + private hold: unknown = null + private tap: unknown = null + private lastTap: { x: number; y: number } | null = null + private holding = false + private moved = false + private multi = false + private secondTap = false + constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void, + private readonly timer: GestureClock = clock) {} + + touch(event: PhoneTouch): void { + if (event.action === 'cancel') { this.cancel(); return } + if (event.action === 'down' && !this.start) { + this.start = event + this.holding = this.moved = this.multi = false + this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36 + if (this.secondTap) { this.clearTap() } + else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) } + this.hold = this.timer.set(() => { + this.hold = null + if (!this.start || this.moved || this.multi) return + this.holding = true + this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y) + }, 500) + } + if (!this.start) return + if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() } + const dx = event.x - this.start.x, dy = event.y - this.start.y + if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() } + if (event.action !== 'up') return + // Some platforms emit an up for each finger. Complete the multi-touch + // gesture only once, and suppress the trailing one-finger up events. + this.clearHold() + const start = this.start + this.start = null + if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return } + if (this.multi) { this.emit('double-tap', start.x, start.y); return } + if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) { + this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y) + } else if (!this.moved) { + if (this.secondTap) this.emit('double-tap', start.x, start.y) + else { + this.lastTap = { x: start.x, y: start.y } + this.tap = this.timer.set(() => { + const tap = this.lastTap + this.tap = null; this.lastTap = null + if (tap) this.emit('tap', tap.x, tap.y) + }, 280) + } + } + } + cancel(): void { + this.clearHold(); this.clearTap() + if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y) + this.start = null; this.holding = false + } + private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null } + private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null } +} diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 332ff275..9495f57b 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -144,6 +144,8 @@ export type ShellWindow = { }; export type ShellConfig = { + /** Hosts without microphone support hide and reject voice entry points. */ + voiceInputEnabled?: boolean; /** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */ actions: LayerActions; getScreenTimeoutMs: () => number | null; @@ -901,7 +903,7 @@ class Shell { handsFree?: boolean; defaultTarget: "assistant" | "app"; }): void { - if (this.voiceDialogPending) return; + if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return; this.voiceDialogPending = true; void (async () => { let ready = true; @@ -1005,6 +1007,7 @@ class Shell { * the dialog finishes on click instead of long-press-release. */ startVoiceInput(): void { + if (this.config.voiceInputEnabled === false) return; if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return; // The transcript is aimed at the window whose menu requested it; the menu // entry point defaults the highlight to Type Into App. @@ -1334,7 +1337,7 @@ class Shell { ctx.stack.pop(); }, }, - { + ...(this.config.voiceInputEnabled === false ? [] : [{ label: "Voice input", onSelect: (ctx) => { // The transcript is aimed at the foreground window, so keep focus @@ -1343,7 +1346,7 @@ class Shell { ctx.stack.pop(); this.startVoiceInput(); }, - }, + }]), ); items.push({ label: "Debug", diff --git a/scripts/ios.sh b/scripts/ios.sh new file mode 100755 index 00000000..3ebb3f65 --- /dev/null +++ b/scripts/ios.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +cd "$(dirname "$0")/.." + +# Prefer Homebrew Ruby over the older macOS system Ruby when available. +for ruby_bin in /opt/homebrew/opt/ruby/bin /usr/local/opt/ruby/bin; do + if [[ -x "$ruby_bin/ruby" ]]; then + export PATH="$ruby_bin:$PATH" + break + fi +done +if command -v ruby >/dev/null 2>&1; then + gem_bin="$(ruby -e 'puts Gem.bindir')" + export PATH="$gem_bin:$PATH" +fi + +exec ns "$@" diff --git a/tests/cloud-stt.test.cjs b/tests/cloud-stt.test.cjs new file mode 100644 index 00000000..c6f0e78e --- /dev/null +++ b/tests/cloud-stt.test.cjs @@ -0,0 +1,195 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); + +function harness(provider = 'elevenlabs', continuous = true) { + const timers = new Map(); + let timerId = 0; + const sockets = []; + const events = [], statuses = [], errors = []; + const load = loader({ + setTimeout(fn, delay) { const id = ++timerId; timers.set(id, { fn, delay }); return id; }, + clearTimeout(id) { timers.delete(id); }, + console: { log() {}, warn() {} }, + com: { faceclaw: { app: { + FaceclawWebSocketListener: function(callbacks) { return callbacks; }, + FaceclawWebSocket: class { + constructor(url, listener) { Object.assign(this, { url, listener, sent: [], closed: false, reject: false }); sockets.push(this); } + sendText(text) { if (this.reject) return false; this.sent.push(text); return true; } + sendBinary(bytes) { if (this.reject) return false; this.sent.push(bytes); return true; } + close() { this.closed = true; } + }, } }, }, + }, { + './cloud-stt': { + CLOUD_STT_SAMPLE_RATE: 16000, + encodeBase64: bytes => Buffer.from(bytes).toString('base64'), + toJavaBytes: bytes => bytes, + }, + }); + const names = { elevenlabs: 'ElevenLabsSttClient', openai: 'OpenAiRealtimeSttClient', soniox: 'SonioxSttClient' }; + const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]]; + const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts'); + const client = new ReconnectingSttClient(options => new Provider(options), { + apiKey: 'test', onTranscript: event => events.push(event), + onStatus: status => statuses.push(status), onError: error => errors.push(error), + }, () => continuous); + client.start(); + return { client, sockets, events, statuses, errors, timers, + open() { sockets.at(-1).listener.onOpen(); }, + message(message, socket = sockets.at(-1)) { socket.listener.onTextMessage(JSON.stringify(message)); }, + tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); return timer.delay; }, + }; +} + +function pcm(amplitude, ms = 50) { + const bytes = new Uint8Array(ms * 32); + for (let i = 0; i < bytes.length; i += 2) { bytes[i] = amplitude & 255; bytes[i + 1] = amplitude >> 8; } + return bytes; +} + +for (const provider of ['elevenlabs', 'openai', 'soniox']) { + test(`${provider}: failure and close reconnect, stale callbacks are ignored, stop cancels retry`, () => { + const h = harness(provider); + h.open(); + const old = h.sockets[0]; + old.listener.onFailure('network unavailable'); + old.listener.onClosed(); // Both callbacks must schedule only one retry. + assert.equal(h.timers.size, 1); + assert.equal(old.closed, true); + h.client.acceptPcm(pcm(2000)); + assert.equal(h.tick(), 1000); + assert.equal(h.sockets.length, 2); + h.open(); + assert.ok(h.sockets[1].sent.length > 0); + const eventCount = h.events.length; + old.listener.onTextMessage(JSON.stringify({ message_type: 'partial_transcript', text: 'stale' })); + old.listener.onOpen(); + assert.equal(h.events.length, eventCount); + h.sockets[1].listener.onClosed(); + assert.equal(h.timers.size, 1); + h.client.stop(); + assert.equal(h.timers.size, 0); + }); + + test(`${provider}: rejected audio send triggers recovery and requeues the packet`, () => { + const h = harness(provider); + h.open(); + h.sockets[0].reject = true; + h.client.acceptPcm(pcm(2000)); + assert.equal(h.timers.size, 1); + h.tick(); h.open(); + assert.ok(h.sockets[1].sent.length > (provider === 'elevenlabs' ? 0 : 1)); + h.client.stop(); + }); + + test(`${provider}: auth failure stops instead of retrying indefinitely`, () => { + const h = harness(provider); + h.sockets[0].listener.onFailure("Expected HTTP 101 response but was '401 Unauthorized'"); + assert.equal(h.timers.size, 0); + assert.equal(h.errors.length, 1); + }); +} + +test('reconnect preserves visible partials once and does not replay already-sent audio', () => { + const h = harness(); h.open(); + h.client.acceptPcm(pcm(2000)); + h.message({ message_type: 'partial_transcript', text: 'Before outage' }); + h.sockets[0].listener.onFailure('offline'); + assert.equal(h.events.at(-1).text, 'Before outage'); + assert.equal(h.events.at(-1).isFinal, true); + h.tick(); h.open(); + assert.equal(h.sockets[1].sent.length, 0); + h.message({ message_type: 'partial_transcript', text: 'After outage' }); + assert.equal(h.events.at(-1).text, 'After outage'); + h.client.stop(); +}); + +test('pending audio is bounded to ten seconds and backoff is capped', () => { + const h = harness(); + for (let i = 0; i < 20; i++) h.client.acceptPcm(pcm(i, 1000)); + h.open(); + const sentBytes = h.sockets[0].sent.reduce((sum, text) => sum + Buffer.from(JSON.parse(text).audio_base_64, 'base64').length, 0); + assert.equal(sentBytes, 320000); + for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) { + h.sockets.at(-1).listener.onFailure('offline'); + assert.equal(h.tick(), delay); + } + h.client.stop(); +}); + +test('finishing while disconnected cancels retries; finishing on an open socket times out cleanly', () => { + const h = harness(); h.open(); + h.sockets[0].listener.onClosed(); + h.client.finish(); + assert.equal(h.timers.size, 0); + const live = harness(); live.open(); + live.client.finish(); + assert.equal(live.tick(), 5000); + assert.equal(live.sockets[0].closed, true); + live.sockets[0].listener.onFailure('late'); + assert.equal(live.timers.size, 0); +}); + +test('only continuous capture commits at pauses, keeping ElevenLabs connected', () => { + for (const continuous of [false, true]) { + const h = harness('elevenlabs', continuous); h.open(); + h.client.acceptPcm(pcm(3000)); + h.client.acceptPcm(pcm(0, 1500)); + const commits = h.sockets[0].sent.map(JSON.parse).filter(m => m.commit); + assert.equal(commits.length, continuous ? 1 : 0); + if (continuous) { + h.message({ message_type: 'committed_transcript', text: 'First.' }); + h.message({ message_type: 'committed_transcript_with_timestamps', text: 'First.' }); + assert.equal(h.events.length, 1); + assert.equal(h.events[0].paragraphBreakAfter, true); + assert.equal(h.sockets[0].closed, false); + } + h.client.stop(); + } +}); + +test('OpenAI keeps successive items separate and drains out-of-order completions in audio order', () => { + const h = harness('openai'); h.open(); + h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500)); + h.message({ type: 'input_audio_buffer.committed', item_id: 'a' }); + h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'a', delta: 'Fir' }); + h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500)); + h.message({ type: 'input_audio_buffer.committed', item_id: 'b' }); + h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'Second.' }); + assert.equal(h.events.some(e => e.isFinal), false); + h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'First.' }); + assert.deepEqual(h.events.filter(e => e.isFinal).map(e => e.text), ['First.', 'Second.']); + assert.equal(h.sockets[0].closed, false); + h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'c', delta: 'Third.' }); + assert.equal(h.events.at(-1).text, 'Third.'); + h.client.stop(); +}); + +test('Soniox speaker and pause formatting remains separate from dictation text and resets on finalize', () => { + const h = harness('soniox'); h.open(); + h.message({ tokens: [ + { text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1', is_final: true }, + { text: ' Hi.', start_ms: 550, end_ms: 800, speaker: '2', is_final: false }, + ] }); + assert.equal(h.events.at(-1).text, 'Hello. Hi.'); + assert.equal(h.events.at(-1).transcribeText, 'Hello.\nHi.'); + h.message({ tokens: [ + { text: ' Yes.', start_ms: 550, end_ms: 800, speaker: '2', is_final: true }, + { text: '', is_final: true }, + ] }); + assert.equal(h.events.find(e => e.isFinal).transcribeText, 'Hello.\nYes.'); + h.message({ tokens: [{ text: 'New.', start_ms: 3000, end_ms: 3400, speaker: '2', is_final: false }] }); + assert.equal(h.events.at(-1).text, 'New.'); + h.client.stop(); +}); + +test('an unsolicited Soniox session end finalizes visible text and reconnects', () => { + const h = harness('soniox'); h.open(); + h.message({ tokens: [{ text: 'Before expiry.', is_final: true }], finished: true }); + assert.equal(h.events.at(-1).isFinal, true); + assert.equal(h.events.at(-1).text, 'Before expiry.'); + assert.equal(h.timers.size, 1); + h.tick(); h.open(); + assert.equal(h.sockets.length, 2); + h.client.stop(); +}); diff --git a/tests/helpers/load-typescript.cjs b/tests/helpers/load-typescript.cjs new file mode 100644 index 00000000..69702bf2 --- /dev/null +++ b/tests/helpers/load-typescript.cjs @@ -0,0 +1,28 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const vm = require('node:vm'); +const ts = require('typescript'); + +// Execute production TypeScript with only the native boundary and clock stubbed. +exports.loader = (globals = {}, mocks = {}) => { + const cache = new Map(); + function load(file) { + file = path.resolve(__dirname, '../..', file); + if (cache.has(file)) return cache.get(file); + const output = {}; + cache.set(file, output); + const source = ts.transpileModule(fs.readFileSync(file, 'utf8'), { + compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 }, + }).outputText; + vm.runInNewContext(source, { + exports: output, console, Uint8Array, ...globals, + require(name) { + if (name in mocks) return mocks[name]; + if (!name.startsWith('.')) throw new Error(`Unmocked module: ${name}`); + return load(path.resolve(path.dirname(file), name + '.ts')); + }, + }, { filename: file }); + return output; + } + return load; +}; diff --git a/tests/phone-gestures.test.cjs b/tests/phone-gestures.test.cjs new file mode 100644 index 00000000..cc90129f --- /dev/null +++ b/tests/phone-gestures.test.cjs @@ -0,0 +1,56 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { PhoneGestureRecognizer } = require('../.test-build/app/phone-ui/phone-gestures.js'); +function setup() { + let now = 0, id = 0; + const timers = new Map(), events = []; + const r = new PhoneGestureRecognizer((type, x, y) => events.push({ type, x, y }), { + set(fn, ms) { timers.set(++id, { fn, at: now + ms }); return id; }, + clear(id) { timers.delete(id); }, + }); + return { + r, events, + touch: (action, x = 20, y = 30, pointers = 1) => r.touch({ action, x, y, pointers }), + advance(ms) { + now += ms; + for (const [id, timer] of [...timers]) if (timer.at <= now && timers.delete(id)) timer.fn(); + }, + types: () => events.map(e => e.type), + }; +} +test('single tap retains mirror coordinates and waits to disambiguate double tap', () => { + const s = setup(); s.touch('down'); s.touch('up'); + assert.deepEqual(s.types(), []); s.advance(280); + assert.deepEqual(s.events, [{ type: 'tap', x: 20, y: 30 }]); +}); +test('double tap emits back without accidental selection', () => { + const s = setup(); s.touch('down'); s.touch('up'); s.advance(150); + s.touch('down'); s.touch('up'); s.advance(1000); + assert.deepEqual(s.types(), ['double-tap']); +}); +test('a hold emits a press and release, never a tap', () => { + const s = setup(); s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000); + assert.deepEqual(s.types(), ['long-press', 'long-press-release']); +}); +test('tap then hold emits the app-menu gesture without an earlier selection', () => { + const s = setup(); s.touch('down'); s.touch('up'); s.advance(100); + s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000); + assert.deepEqual(s.types(), ['short-then-long-press', 'long-press-release']); +}); +test('four directional swipes cancel pending holds and taps', () => { + for (const [x, y, expected] of [[80, 30, 'swipe-right'], [-40, 30, 'swipe-left'], [20, 90, 'swipe-down'], [20, -30, 'swipe-up']]) { + const s = setup(); s.touch('down'); s.touch('move', x, y); s.advance(500); s.touch('up', x, y); s.advance(500); + assert.deepEqual(s.types(), [expected]); + } +}); +test('two fingers send back exactly once and cancel the long-press timer', () => { + const s = setup(); s.touch('down'); s.touch('down', 21, 30, 2); s.advance(500); + s.touch('up', 21, 30, 2); s.touch('up'); s.advance(1000); + assert.deepEqual(s.types(), ['double-tap']); +}); +test('cancellation prevents delayed selections and releases active holds', () => { + const s = setup(); s.touch('down'); s.touch('up'); s.r.cancel(); s.advance(1000); + assert.deepEqual(s.types(), []); + s.touch('down'); s.advance(500); s.touch('cancel'); s.advance(1000); + assert.deepEqual(s.types(), ['long-press', 'long-press-release']); +}); diff --git a/tests/surface-compositor.test.cjs b/tests/surface-compositor.test.cjs new file mode 100644 index 00000000..de5e497b --- /dev/null +++ b/tests/surface-compositor.test.cjs @@ -0,0 +1,49 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js'); +const rect = (x, y, width, height) => ({ x, y, width, height }); +function surface(c, id, box, values, zOrder = 0, transparency = 'opaque') { + c.configureSurface(id, { ...box, zOrder, transparency }); + c.submitSurfaceFrame(id, Uint8Array.from(values), rect(0, 0, box.width, box.height)); +} +test('shell color key preserves underlying text while opaque black covers it', () => { + const c = new SurfaceCompositor(3, 2); + surface(c, 'app', rect(0, 0, 3, 2), [200, 200, 200, 200, 200, 200]); + surface(c, 'shell', rect(0, 0, 3, 2), [0, 1, 255, 0, 0, 0], 1, 'color-key'); + c.setUnderlayDim(1, 0.5); + assert.deepEqual([...c.composite()], [100, 1, 255, 100, 100, 100]); + c.setSurfaceVisible('shell', false); + assert.deepEqual([...c.composite()], [100, 100, 100, 100, 100, 100]); + surface(c, 'opaque', rect(1, 0, 1, 2), [0, 0], 2); + assert.deepEqual([...c.composite()], [100, 0, 100, 100, 0, 100]); +}); +test('partial frames clip in surface coordinates, then surfaces clip to the screen', () => { + const c = new SurfaceCompositor(3, 2); + surface(c, 'app', rect(-1, 0, 3, 2), [1, 2, 3, 4, 5, 6]); + const source = Uint8Array.from([99, 10, 20, 99, 30, 40, 99]); + c.submitSurfaceFrame('app', source.subarray(1, 5), rect(1, 1, 2, 2)); + source.fill(0); + assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]); + c.submitSurfaceFrame('app', Uint8Array.from([50, 60, 70, 80]), rect(-1, -1, 2, 2)); + // The clipped update changed the offscreen column, leaving visible pixels intact. + assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]); +}); +test('moving a surface retains its frame; resizing clears it; blanking is reversible', () => { + const c = new SurfaceCompositor(3, 1); + surface(c, 'app', rect(0, 0, 2, 1), [7, 9]); + c.configureSurface('app', { ...rect(1, 0, 2, 1), zOrder: 0, transparency: 'opaque' }); + assert.deepEqual([...c.composite()], [0, 7, 9]); + c.setScreenBlanked(true); assert.deepEqual([...c.composite()], [0, 0, 0]); + c.setScreenBlanked(false); assert.deepEqual([...c.composite()], [0, 7, 9]); + c.configureSurface('app', { ...rect(0, 0, 3, 1), zOrder: 0, transparency: 'opaque' }); + assert.deepEqual([...c.composite()], [0, 0, 0]); +}); +test('layer order is independent of creation order; removal reveals lower surfaces', () => { + const c = new SurfaceCompositor(1, 1); + surface(c, 'top', rect(0, 0, 1, 1), [20], 10); + surface(c, 'bottom', rect(0, 0, 1, 1), [80], -1); + assert.equal(c.composite()[0], 20); + c.removeSurface('top'); assert.equal(c.composite()[0], 80); + assert.throws(() => c.submitSurfaceFrame('missing', new Uint8Array(1), rect(0, 0, 1, 1))); + assert.throws(() => c.submitSurfaceFrame('bottom', new Uint8Array(2), rect(0, 0, 1, 1))); +}); diff --git a/tests/transcribe.test.cjs b/tests/transcribe.test.cjs new file mode 100644 index 00000000..6ef9b8b8 --- /dev/null +++ b/tests/transcribe.test.cjs @@ -0,0 +1,94 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); +const load = loader(); +const { TranscriptModel, TranscriptScroll, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts'); +const { SpeechPauseDetector } = load('app/native/speech-pause.ts'); +const { TimedTranscript } = load('app/native/transcript-format.ts'); + +function pcm(amplitude, ms = 50) { + const bytes = new Uint8Array(ms * 16 * 2); + for (let i = 0; i < bytes.length; i += 2) { + bytes[i] = amplitude & 255; + bytes[i + 1] = (amplitude >> 8) & 255; + } + return bytes; +} + +test('partials replace, finals correct, and pause breaks survive display and saving', () => { + const m = new TranscriptModel(); + m.accept({ text: 'Hello wor', isFinal: false }); + m.accept({ text: 'Hello world.', isFinal: true, paragraphBreakAfter: true }); + m.accept({ text: 'Next', isFinal: false }); + m.accept({ text: 'Next paragraph.', isFinal: false }); + assert.equal(m.text, 'Hello world.\nNext paragraph.'); + m.accept({ text: 'Next paragraph!', isFinal: true }); + assert.equal(m.text, 'Hello world.\nNext paragraph!'); + m.accept({ text: '', isFinal: true }); + assert.equal(m.text, 'Hello world.\nNext paragraph!'); +}); + +test('onboard pauses can precede or follow finalization', () => { + for (const before of [true, false]) { + const m = new TranscriptModel(); + m.accept({ text: 'first', isFinal: false }); + if (before) m.pause(); + m.accept({ text: 'First.', isFinal: true }); + if (!before) m.pause(); + m.accept({ text: 'Second.', isFinal: false }); + assert.equal(m.text, 'First.\nSecond.'); + } +}); + +test('formatting metadata is used only by Transcribe, with revisable live speaker tokens', () => { + const final = new TimedTranscript(); + final.append({ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1' }); + const preview = final.copy(); + preview.append({ text: ' Wrong', start_ms: 600, end_ms: 900, speaker: '2' }); + assert.equal(preview.text, 'Hello.\nWrong'); + assert.equal(final.text, 'Hello.'); + final.append({ text: ' Right.', start_ms: 600, end_ms: 900, speaker: '1' }); + final.append({ text: ' Later.', start_ms: 2400, end_ms: 2800, speaker: '1' }); + assert.equal(final.text, 'Hello. Right.\nLater.'); + const event = { text: 'Hello. Right. Later.', transcribeText: final.text, isFinal: false }; + const m = new TranscriptModel(); + m.accept(event); + assert.equal(m.text, final.text); + assert.equal(event.text.includes('\n'), false); +}); + +test('wrapping preserves explicit newlines, blank paragraphs, and long unspaced words', () => { + assert.deepEqual([...wrapTranscribeText(s => s.length, 'one two\nthree\n\nabcdefghij', 5)], + ['one', 'two', 'three', '', 'abcde', 'fghij']); +}); + +test('scroll follows the tail until scrolled back, then resumes only at the bottom', () => { + const s = new TranscriptScroll(); + assert.equal(s.layout(30, 10), 20); + assert.equal(s.layout(33, 10), 23); + s.scroll(-3); + assert.equal(s.layout(50, 10), 20); + assert.equal(s.layout(51, 10), 20); + s.scroll(100); + assert.equal(s.layout(55, 10), 45); + s.scroll(-100); + assert.equal(s.layout(60, 10), 0); +}); + +test('short transcripts become scrollable without disabling automatic following', () => { + const s = new TranscriptScroll(); + assert.equal(s.layout(3, 10), 0); + s.scroll(-3); + assert.equal(s.layout(12, 10), 2); +}); + +test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores leading silence', () => { + const d = new SpeechPauseDetector(); + for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false); + d.accept(pcm(3000)); + for (let i = 0; i < 29; i++) assert.equal(d.accept(pcm(0)), false); + assert.equal(d.accept(pcm(0)), true); + for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false); + d.accept(pcm(3000)); + assert.equal(d.accept(pcm(0, 1500)), true); +}); diff --git a/tests/tsconfig.json b/tests/tsconfig.json index cba8aa69..0bf5d080 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -11,6 +11,8 @@ "outDir": "../.test-build" }, "include": [ + "../app/graphics/surface-compositor.ts", + "../app/phone-ui/phone-gestures.ts", "../app/g2/glasses-hardware-identity.ts", "../app/g2/ble-proximity.ts", "../app/g2/even-advertisement.ts",