mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
iOS porting: Main screen and compositor
This commit is contained in:
1 parent
61b3bf8b7c
commit
99793a37a7
24 files changed
+1124
-70
No files matched your search
@@ -23,6 +23,8 @@
|
||||
<key>UISupportedInterfaceOrientations</key>
|
||||
<array>
|
||||
<string>UIInterfaceOrientationPortrait</string>
|
||||
<string>UIInterfaceOrientationLandscapeLeft</string>
|
||||
<string>UIInterfaceOrientationLandscapeRight</string>
|
||||
</array>
|
||||
<key>UIRequiresFullScreen</key>
|
||||
<true/>
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
platform :ios, '15.1'
|
||||
pod 'SVGKit', '3.0.0'
|
||||
|
||||
# SVGKit 3's physical-DPI lookup asserts on arm64 simulators. Our icons use
|
||||
# only viewBox/pixel units, so that lookup is irrelevant to their rendering.
|
||||
post_install do |installer|
|
||||
installer.pods_project.targets.each do |target|
|
||||
next unless target.name == 'SVGKit'
|
||||
target.build_configurations.each do |config|
|
||||
config.build_settings['ENABLE_NS_ASSERTIONS'] = 'NO'
|
||||
end
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,6 @@
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
@interface FaceclawGraphics : NSObject
|
||||
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
|
||||
@end
|
||||
@@ -0,0 +1,56 @@
|
||||
#import "FaceclawGraphics.h"
|
||||
#import <SVGKit/SVGKit.h>
|
||||
|
||||
@implementation FaceclawGraphics
|
||||
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green {
|
||||
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||
const uint8_t *src = gray.bytes;
|
||||
uint8_t *dst = rgba.mutableBytes;
|
||||
uint8_t lut[256];
|
||||
for (int i = 0; i < 256; i++) lut[i] = (uint8_t)round(pow(i / 255.0, 0.7) * 255.0);
|
||||
for (NSInteger i = 0; i < width * height; i++) {
|
||||
uint8_t value = lut[src[i]];
|
||||
dst[i * 4] = green ? 0 : value;
|
||||
dst[i * 4 + 1] = value;
|
||||
dst[i * 4 + 2] = green ? 0 : value;
|
||||
dst[i * 4 + 3] = 255;
|
||||
}
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
|
||||
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
|
||||
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
|
||||
if (image) CGImageRelease(image);
|
||||
CGDataProviderRelease(provider);
|
||||
CGColorSpaceRelease(space);
|
||||
return result;
|
||||
}
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
|
||||
if (size <= 0 || size > 1024) return nil;
|
||||
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
|
||||
if (!image) return nil;
|
||||
image.size = CGSizeMake(size, size);
|
||||
UIGraphicsImageRendererFormat *format = [UIGraphicsImageRendererFormat defaultFormat];
|
||||
format.scale = 1;
|
||||
format.opaque = NO;
|
||||
UIGraphicsImageRenderer *renderer = [[UIGraphicsImageRenderer alloc] initWithSize:image.size format:format];
|
||||
UIImage *raster = [renderer imageWithActions:^(UIGraphicsImageRendererContext *context) {
|
||||
[image.CALayerTree renderInContext:context.CGContext];
|
||||
}];
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:size * size * 4];
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
CGContextRef context = CGBitmapContextCreate(rgba.mutableBytes, size, size, 8, size * 4, space,
|
||||
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, size, size), raster.CGImage);
|
||||
NSMutableData *gray = [NSMutableData dataWithLength:size * size];
|
||||
const uint8_t *src = rgba.bytes;
|
||||
uint8_t *dst = gray.mutableBytes;
|
||||
for (NSInteger i = 0; i < size * size; i++) {
|
||||
dst[i] = (uint8_t)round(0.2126 * src[i * 4] + 0.7152 * src[i * 4 + 1] + 0.0722 * src[i * 4 + 2]);
|
||||
}
|
||||
CGContextRelease(context);
|
||||
CGColorSpaceRelease(space);
|
||||
return gray;
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,4 @@
|
||||
module FaceclawGraphics {
|
||||
header "FaceclawGraphics.h"
|
||||
export *
|
||||
}
|
||||
@@ -82,10 +82,11 @@ export class CalculatorLayer implements Layer {
|
||||
constructor(
|
||||
private readonly coordinator: MathCoordinator,
|
||||
private readonly actions: LayerActions,
|
||||
private readonly textInput?: () => Promise<void>,
|
||||
) {
|
||||
this.coordinator.setUsesDegrees(calculatorDegreesSetting.get());
|
||||
this.coordinator.onChanged = () => this.handleCoordinatorChanged();
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
if (!this.textInput) this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => this.onTranscript(event));
|
||||
}
|
||||
|
||||
private get mode(): CalculatorMode {
|
||||
@@ -125,6 +126,7 @@ export class CalculatorLayer implements Layer {
|
||||
}
|
||||
|
||||
private refreshListening(): void {
|
||||
if (this.textInput) return;
|
||||
if (!this.foreground) return;
|
||||
if (this.listening === "continuous") {
|
||||
if (!this.continuousActive) {
|
||||
@@ -244,6 +246,7 @@ export class CalculatorLayer implements Layer {
|
||||
* that did not.
|
||||
*/
|
||||
beginDictation(): void {
|
||||
if (this.textInput) { void this.textInput(); return; }
|
||||
if (this.dictating) return;
|
||||
this.dictating = true;
|
||||
void this.actions.startVoiceCapture(true);
|
||||
@@ -513,7 +516,8 @@ export class CalculatorLayer implements Layer {
|
||||
bodyBottom: number,
|
||||
): void {
|
||||
const prompt =
|
||||
this.listening === "continuous"
|
||||
this.textInput ? "Tap to type a problem on your phone"
|
||||
: this.listening === "continuous"
|
||||
? `Say a problem, then "${modeLabel(this.mode).toLowerCase()} it"`
|
||||
: "Tap to say a problem — or use Voice input to type one";
|
||||
const lines = wrapText(font, prompt, width - 96);
|
||||
@@ -527,7 +531,7 @@ export class CalculatorLayer implements Layer {
|
||||
|
||||
private paintHeardLine(image: GrayImage, font: UiFont, width: number, y: number): void {
|
||||
if (!this.lastInput) return;
|
||||
const label = "Heard";
|
||||
const label = this.textInput ? "Entered" : "Heard";
|
||||
image.drawText(font, 24, y, label, 110);
|
||||
const x = 24 + font.measureText(label) + 10;
|
||||
image.drawText(font, x, y, truncateText(font, this.lastInput, width - x - 24), 160);
|
||||
@@ -579,7 +583,7 @@ export class CalculatorLayer implements Layer {
|
||||
this.mode === "explain" && this.coordinator.stepPosition !== null ? "steps" : "mode";
|
||||
return gestureHints([
|
||||
[GESTURE_SCROLL, scrollTarget],
|
||||
[GESTURE_CLICK, "mic"],
|
||||
[GESTURE_CLICK, this.textInput ? "type" : "mic"],
|
||||
[GESTURE_DOUBLE_CLICK, "back"],
|
||||
]);
|
||||
}
|
||||
|
||||
+3
-46
@@ -1,47 +1,4 @@
|
||||
import { Application, Button, Device, Label, Page, ScrollView, StackLayout } from '@nativescript/core'
|
||||
import { bytesToHex, hexToBytes } from './util/hex-util'
|
||||
import { Application } from '@nativescript/core'
|
||||
import { createMainPage } from './phone-ui/main-page.ios'
|
||||
|
||||
// Milestone 1: exercise native UI, TypeScript, and an existing shared utility
|
||||
// without loading the Android services used by the full glasses application.
|
||||
Application.run({
|
||||
create: () => {
|
||||
const page = new Page()
|
||||
page.actionBarHidden = true
|
||||
const scroll = new ScrollView()
|
||||
const content = new StackLayout()
|
||||
content.padding = 28
|
||||
content.verticalAlignment = 'middle'
|
||||
|
||||
const label = (text: string, fontSize = 17) => {
|
||||
const view = new Label()
|
||||
view.text = text
|
||||
view.fontSize = fontSize
|
||||
view.textWrap = true
|
||||
view.marginBottom = 20
|
||||
content.addChild(view)
|
||||
return view
|
||||
}
|
||||
|
||||
label('Faceclaw', 36)
|
||||
label('iOS port · First launch', 24)
|
||||
label(`NativeScript is running on ${Device.model} (iOS ${Device.osVersion}).`)
|
||||
label('Glasses pairing and the Android features are still to be ported.')
|
||||
const status = label('Tap below to test native input and shared TypeScript code.')
|
||||
const button = new Button()
|
||||
button.text = 'Run shared-code check'
|
||||
button.accessibilityIdentifier = 'shared-code-check'
|
||||
let checks = 0
|
||||
button.on('tap', () => {
|
||||
const result = bytesToHex(hexToBytes('46 61 63 65 63 6c 61 77'))
|
||||
const passed = result === '46616365636C6177'
|
||||
checks += 1
|
||||
status.text = `${passed ? 'Passed' : 'Failed'} · Check ${checks}\nShared hex utilities returned ${result}.`
|
||||
console.log(`[ios-bootstrap] Shared-code check ${checks}: ${passed ? 'PASS' : 'FAIL'} (${result})`)
|
||||
})
|
||||
content.addChild(button)
|
||||
scroll.content = content
|
||||
page.content = scroll
|
||||
page.on('loaded', () => console.log('[ios-bootstrap] Faceclaw launch screen loaded'))
|
||||
return page
|
||||
},
|
||||
})
|
||||
Application.run({ create: createMainPage })
|
||||
@@ -0,0 +1,220 @@
|
||||
import { Dialogs, type ImageSource } from '@nativescript/core'
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||
import { CalculatorLayer } from '../apps/calculator/calculator'
|
||||
import { MathCoordinator } from '../apps/calculator/math/coordinator'
|
||||
import { SurfaceCompositor } from '../graphics/surface-compositor'
|
||||
import { flattenPlanes, type Plane } from '../graphics/plane'
|
||||
import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image'
|
||||
import { previewPixels } from '../native/ios-graphics'
|
||||
import { makeInputEvent, type InputEventPayload } from '../ui/gestures'
|
||||
import { noopLayerActions, type LayerActions } from '../ui/layers'
|
||||
import { MenuLayer } from '../ui/menu'
|
||||
import { createInProcessWindow } from '../ui/shell/in-process-window'
|
||||
import { shell, type ShellWindow } from '../ui/shell/shell'
|
||||
import { appViewportRect, SIDEBAR_WIDTH, sidebarStripVisible } from '../ui/shell/geometry'
|
||||
import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged,
|
||||
previewColorSetting, timeFormatSetting, verticalPositionSetting } from '../ui/dashboard-settings'
|
||||
import type { PhoneGesture } from '../phone-ui/phone-gestures'
|
||||
|
||||
/** Local display host: shared shell/windows, no device-service initialization. */
|
||||
export class IosPreviewController {
|
||||
private readonly compositor = new SurfaceCompositor(G2_LENS_WIDTH, G2_LENS_HEIGHT)
|
||||
private renderTimer: ReturnType<typeof setTimeout> | null = null
|
||||
private clockTimer: ReturnType<typeof setInterval> | null = null
|
||||
private offSettings: (() => void) | null = null
|
||||
private active = false
|
||||
private shellDirty = true
|
||||
private inputQueue: Promise<void> = Promise.resolve()
|
||||
private lastLayout = ''
|
||||
private prompting = false
|
||||
private readonly actions: LayerActions = {
|
||||
...noopLayerActions,
|
||||
requestRender: () => this.requestShellRender(),
|
||||
startTextSettingEdit: setting => this.editSetting(setting),
|
||||
startTextSettingsEdit: async (settings, _title, finished) => {
|
||||
for (const setting of settings) await this.editSetting(setting)
|
||||
finished?.()
|
||||
},
|
||||
}
|
||||
|
||||
constructor(private readonly onFrame: (image: ImageSource, focus: string) => void,
|
||||
private readonly onError: (message: string) => void) {
|
||||
this.compositor.configureSurface('shell', { x: 0, y: 0, width: 640, height: 480, zOrder: 1, transparency: 'color-key' })
|
||||
shell.configure({
|
||||
actions: this.actions,
|
||||
voiceInputEnabled: false,
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => this.requestShellRender(),
|
||||
onScreenStateChanged: on => { this.compositor.setScreenBlanked(!on); this.requestShellRender() },
|
||||
})
|
||||
const launcher = createLauncherWindow({
|
||||
actions: this.actions,
|
||||
apps: () => [
|
||||
{ appId: 'calculator', label: 'Calculator', icon: 'calculator' },
|
||||
{ appId: 'display', label: 'Display', icon: 'settings' },
|
||||
],
|
||||
launchApp: id => this.launchApp(id),
|
||||
uninstallApp: () => {},
|
||||
submitFrame: planes => this.submit(LAUNCHER_SURFACE_ID, planes),
|
||||
setSurfaceVisible: visible => { this.compositor.setSurfaceVisible(LAUNCHER_SURFACE_ID, visible); this.scheduleFrame() },
|
||||
})
|
||||
this.configureWindow(launcher)
|
||||
shell.registerWindow(launcher)
|
||||
shell.wake('window')
|
||||
shell.focusWindow(launcher.windowId)
|
||||
}
|
||||
|
||||
resume(): void {
|
||||
if (this.active) return
|
||||
this.active = true
|
||||
this.offSettings = onAnySettingChanged(() => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
})
|
||||
this.clockTimer = setInterval(() => this.requestShellRender(), 60_000)
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
this.requestShellRender()
|
||||
console.log('[ios-preview] Main screen active')
|
||||
}
|
||||
pause(): void {
|
||||
this.active = false
|
||||
this.offSettings?.(); this.offSettings = null
|
||||
if (this.clockTimer) clearInterval(this.clockTimer)
|
||||
if (this.renderTimer) clearTimeout(this.renderTimer)
|
||||
this.clockTimer = this.renderTimer = null
|
||||
}
|
||||
private configureWindow(window: ShellWindow): void {
|
||||
this.compositor.configureSurface(window.surfaceId, {
|
||||
...appViewportRect(window.heightMode, window.appId), zOrder: 0, transparency: 'opaque',
|
||||
})
|
||||
this.compositor.setSurfaceVisible(window.surfaceId, shell.foregroundWindow()?.windowId === window.windowId)
|
||||
}
|
||||
private relayout(): void {
|
||||
const layout = `${displayModeSetting.get()}:${verticalPositionSetting.get()}`
|
||||
if (layout === this.lastLayout) return
|
||||
this.lastLayout = layout
|
||||
for (const window of shell.getWindows()) {
|
||||
this.configureWindow(window)
|
||||
window.relayout?.()
|
||||
}
|
||||
}
|
||||
private async submit(id: string, planes: Plane[]): Promise<void> {
|
||||
const image = flattenPlanes(planes)
|
||||
this.compositor.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height })
|
||||
this.scheduleFrame()
|
||||
}
|
||||
private requestShellRender(): void { this.shellDirty = true; this.scheduleFrame() }
|
||||
private scheduleFrame(): void {
|
||||
if (!this.active || this.renderTimer !== null) return
|
||||
// Coalesce window/chrome updates into one preview, capped at 30 fps.
|
||||
this.renderTimer = setTimeout(() => {
|
||||
this.renderTimer = null
|
||||
try {
|
||||
if (this.shellDirty) {
|
||||
this.shellDirty = false
|
||||
const image = flattenPlanes(shell.paintSurface(), { width: 640, height: 480 })
|
||||
this.compositor.submitSurfaceFrame('shell', image.pixels, { x: 0, y: 0, width: 640, height: 480 })
|
||||
this.compositor.setUnderlayDim(1, shell.underlayDim())
|
||||
}
|
||||
const image = previewPixels(this.compositor.composite(), 640, 480, previewColorSetting.get() === 'green')
|
||||
this.onFrame(image, `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`)
|
||||
} catch (error) { this.fail(error) }
|
||||
}, 33)
|
||||
}
|
||||
private fail(error: unknown): void {
|
||||
const message = error instanceof Error ? error.stack ?? error.message : String(error)
|
||||
console.error(`[ios-preview] ${message}`)
|
||||
this.onError(error instanceof Error ? error.message : String(error))
|
||||
}
|
||||
gesture(gesture: PhoneGesture, origin: 'watch' | 'ring' | 'mirror', nx = 0, ny = 0): void {
|
||||
this.inputQueue = this.inputQueue.then(async () => {
|
||||
if (!this.active) return
|
||||
if (gesture === 'tap' && origin === 'mirror' && shell.isScreenOn()) {
|
||||
await this.mirrorTap(nx, ny)
|
||||
} else {
|
||||
let type: string = ({ tap: 'click', 'double-tap': 'double-click' } as Record<string, string>)[gesture] ?? gesture
|
||||
if (origin === 'ring' && type.startsWith('swipe-')) {
|
||||
if (type === 'swipe-left' || type === 'swipe-right') return
|
||||
type = type === 'swipe-up' ? 'scroll-up' : 'scroll-down'
|
||||
}
|
||||
const event = makeInputEvent({ type, source: origin === 'ring' ? 'ring' : 'watch' } as InputEventPayload)
|
||||
await shell.receiveInput(event)
|
||||
this.requestShellRender()
|
||||
}
|
||||
console.log(`[ios-preview] ${origin} ${gesture}: ${shell.describeInputTarget()}`)
|
||||
}).catch(error => this.fail(error))
|
||||
}
|
||||
private async mirrorTap(nx: number, ny: number): Promise<void> {
|
||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||
const window = shell.foregroundWindow()
|
||||
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId) && x < SIDEBAR_WIDTH) {
|
||||
const target = shell.windowAtSidebarPoint(x, y)
|
||||
if (target) { shell.focusWindow(target.windowId); target.requestRender(); this.requestShellRender() }
|
||||
return
|
||||
}
|
||||
if (window && !shell.hasOverlay()) {
|
||||
const rect = appViewportRect(window.heightMode, window.appId)
|
||||
if (x < rect.x || x >= rect.x + rect.width || y < rect.y || y >= rect.y + rect.height) return
|
||||
shell.focusWindow(window.windowId)
|
||||
if (await window.hitTest?.(x - rect.x, y - rect.y)) { this.requestShellRender(); return }
|
||||
}
|
||||
await shell.receiveInput(makeInputEvent({ type: 'click', source: 'watch' }))
|
||||
this.requestShellRender()
|
||||
}
|
||||
launchApp(id: string): void {
|
||||
const existing = shell.getWindows().find(window => window.appId === id)
|
||||
if (existing) { shell.focusWindow(existing.windowId); existing.requestRender(); this.requestShellRender(); return }
|
||||
const surfaceId = `window:${id}`
|
||||
let render = () => {}
|
||||
const actions = { ...this.actions, requestRender: () => render() }
|
||||
const layer = id === 'calculator'
|
||||
? new CalculatorLayer(new MathCoordinator(), actions, () => this.typeIntoApp())
|
||||
: new MenuLayer('Display', [
|
||||
{ label: 'Change screen size', onSelect: () => this.cycleDisplayMode() },
|
||||
{ label: 'Toggle green / grayscale preview', onSelect: () => { previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green') } },
|
||||
{ label: 'Toggle 12 / 24 hour clock', onSelect: () => { timeFormatSetting.set(timeFormatSetting.get() === '24h' ? '12h' : '24h') } },
|
||||
], { x: 8, y: 8, width: 320, minHeight: 160, opaque: true })
|
||||
const app = createInProcessWindow({
|
||||
appId: id, windowId: id, title: id === 'calculator' ? 'Calculator' : 'Display',
|
||||
iconLetter: id === 'calculator' ? '=' : 'D', icon: id === 'calculator' ? 'calculator' : 'settings',
|
||||
closeable: true, actions, baseLayer: layer,
|
||||
receiveTextInput: id === 'calculator' ? text => { app.stack.receiveTextInput(text); app.requestRender() } : undefined,
|
||||
submitFrame: planes => this.submit(surfaceId, planes),
|
||||
setSurfaceVisible: visible => {
|
||||
if (layer instanceof CalculatorLayer) layer.setForeground(visible)
|
||||
this.compositor.setSurfaceVisible(surfaceId, visible); this.scheduleFrame()
|
||||
},
|
||||
removeSurface: () => { this.compositor.removeSurface(surfaceId); this.scheduleFrame() },
|
||||
onClosed: () => { if (layer instanceof CalculatorLayer) layer.onRemoved() },
|
||||
})
|
||||
render = app.requestRender
|
||||
if (layer instanceof CalculatorLayer) layer.requestRender = render
|
||||
this.configureWindow(app.window)
|
||||
shell.registerWindow(app.window)
|
||||
shell.focusWindow(app.window.windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
}
|
||||
cycleDisplayMode(): void {
|
||||
const values = DISPLAY_MODE_VALUES
|
||||
displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length])
|
||||
}
|
||||
get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) }
|
||||
async typeIntoApp(): Promise<void> {
|
||||
if (this.prompting || !shell.foregroundWindow()?.receiveTextInput) return
|
||||
this.prompting = true
|
||||
try {
|
||||
const result = await Dialogs.prompt({ title: 'Type into ' + shell.foregroundWindow()?.title,
|
||||
message: 'Enter an expression or follow-up.', okButtonText: 'Send', cancelButtonText: 'Cancel' })
|
||||
if (result.result) shell.sendTextToForegroundWindow(result.text)
|
||||
} catch (error) { this.fail(error) }
|
||||
finally { this.prompting = false }
|
||||
}
|
||||
private async editSetting(setting: { editorTitle: string; get(): string; set(value: string): void }): Promise<void> {
|
||||
const result = await Dialogs.prompt({ title: setting.editorTitle, defaultText: setting.get(), okButtonText: 'Save', cancelButtonText: 'Cancel' })
|
||||
if (result.result) setting.set(result.text)
|
||||
}
|
||||
}
|
||||
+7
-17
@@ -1,12 +1,10 @@
|
||||
import { GrayImage } from "./image";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
import { rasterizeSvg } from "../native/svg-rasterizer";
|
||||
|
||||
/**
|
||||
* Vector icons for window indicators (and anywhere else). SVGs are rendered
|
||||
* once to a correctly sized grayscale bitmap by the Java IconRenderer (a
|
||||
* small SVG subset: path/circle/rect/line/polyline) and cached here. To add
|
||||
* once to a correctly sized grayscale bitmap by the platform rasterizer
|
||||
* (Android IconRenderer or iOS SVGKit) and cached here. To add
|
||||
* an icon, drop its SVG source into ICON_SVGS — Lucide icons
|
||||
* (https://lucide.dev, stroked, 24px viewBox) work as-is; simple single-color
|
||||
* Noun Project glyphs also work.
|
||||
@@ -171,18 +169,10 @@ function renderSvgCached(cacheName: string, svg: string, size: number): GrayImag
|
||||
if (cached !== undefined) return cached;
|
||||
|
||||
let icon: GrayImage | null = null;
|
||||
if (global.isAndroid) {
|
||||
try {
|
||||
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, Math.round(size), ICON_STROKE_WIDTH);
|
||||
if (bytes && bytes.length >= size * size) {
|
||||
icon = new GrayImage(size, size, 0);
|
||||
for (let i = 0; i < size * size; i++) {
|
||||
icon.pixels[i] = bytes[i] & 0xff;
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
||||
}
|
||||
try {
|
||||
icon = rasterizeSvg(svg, Math.round(size), ICON_STROKE_WIDTH);
|
||||
} catch (error) {
|
||||
console.warn(`renderIcon(${cacheName}) failed: ${error}`);
|
||||
}
|
||||
cache.set(key, icon);
|
||||
return icon;
|
||||
|
||||
@@ -0,0 +1,75 @@
|
||||
/** Platform-independent 8bpp surface composition for the local display. */
|
||||
export type SurfaceRect = { x: number; y: number; width: number; height: number }
|
||||
export type SurfaceConfiguration = SurfaceRect & {
|
||||
zOrder: number
|
||||
transparency: 'opaque' | 'color-key'
|
||||
}
|
||||
type Surface = SurfaceConfiguration & { pixels: Uint8Array; visible: boolean }
|
||||
|
||||
export class SurfaceCompositor {
|
||||
private readonly surfaces = new Map<string, Surface>()
|
||||
private dimBelow = 0
|
||||
private dimFactor = 1
|
||||
private blanked = false
|
||||
constructor(readonly width: number, readonly height: number) {
|
||||
if (!Number.isInteger(width) || !Number.isInteger(height) || width <= 0 || height <= 0) {
|
||||
throw new Error('Invalid compositor size')
|
||||
}
|
||||
}
|
||||
configureSurface(id: string, options: SurfaceConfiguration): void {
|
||||
if (![options.x, options.y, options.width, options.height, options.zOrder].every(Number.isInteger)
|
||||
|| options.width <= 0 || options.height <= 0) throw new Error('Invalid surface geometry')
|
||||
const previous = this.surfaces.get(id)
|
||||
const pixels = previous?.width === options.width && previous.height === options.height
|
||||
? previous.pixels : new Uint8Array(options.width * options.height)
|
||||
this.surfaces.set(id, { ...options, pixels, visible: previous?.visible ?? true })
|
||||
}
|
||||
removeSurface(id: string): void { this.surfaces.delete(id) }
|
||||
setSurfaceVisible(id: string, visible: boolean): void {
|
||||
const surface = this.surfaces.get(id)
|
||||
if (surface) surface.visible = visible
|
||||
}
|
||||
setUnderlayDim(belowZOrder: number, factor: number): void {
|
||||
if (!Number.isFinite(factor)) throw new Error('Invalid dim factor')
|
||||
this.dimBelow = belowZOrder
|
||||
this.dimFactor = Math.max(0, Math.min(1, factor))
|
||||
}
|
||||
setScreenBlanked(blanked: boolean): void { this.blanked = blanked }
|
||||
submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect): void {
|
||||
const surface = this.surfaces.get(id)
|
||||
if (!surface) throw new Error(`Unknown surface: ${id}`)
|
||||
if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger)
|
||||
|| rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) {
|
||||
throw new Error('Invalid frame buffer or rectangle')
|
||||
}
|
||||
const left = Math.max(0, rect.x), right = Math.min(surface.width, rect.x + rect.width)
|
||||
const top = Math.max(0, rect.y), bottom = Math.min(surface.height, rect.y + rect.height)
|
||||
for (let y = top; y < bottom; y++) {
|
||||
if (right <= left) break
|
||||
const source = (y - rect.y) * rect.width + left - rect.x
|
||||
surface.pixels.set(pixels.subarray(source, source + right - left), y * surface.width + left)
|
||||
}
|
||||
}
|
||||
composite(): Uint8Array {
|
||||
const output = new Uint8Array(this.width * this.height)
|
||||
if (this.blanked) return output
|
||||
const surfaces = [...this.surfaces.values()].filter(s => s.visible).sort((a, b) => a.zOrder - b.zOrder)
|
||||
for (const surface of surfaces) {
|
||||
const left = Math.max(0, surface.x), right = Math.min(this.width, surface.x + surface.width)
|
||||
const top = Math.max(0, surface.y), bottom = Math.min(this.height, surface.y + surface.height)
|
||||
const dim = surface.zOrder < this.dimBelow ? this.dimFactor : 1
|
||||
for (let y = top; y < bottom; y++) {
|
||||
let source = (y - surface.y) * surface.width + left - surface.x
|
||||
let target = y * this.width + left
|
||||
for (let x = left; x < right; x++, source++, target++) {
|
||||
const value = surface.pixels[source]
|
||||
// Test the original value: opaque near-black must still cover the
|
||||
// window beneath, even when dimming rounds it down to black.
|
||||
if (surface.transparency === 'color-key' && value === 0) continue
|
||||
output[target] = Math.round(value * dim)
|
||||
}
|
||||
}
|
||||
}
|
||||
return output
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
import { ImageSource } from '@nativescript/core'
|
||||
import { GrayImage } from '../graphics/image'
|
||||
|
||||
declare const FaceclawGraphics: any
|
||||
|
||||
export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource {
|
||||
// NSData copies the buffer; the UIImage owns its pixels after this call.
|
||||
const copy = new Uint8Array(pixels)
|
||||
const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)
|
||||
const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green)
|
||||
if (!native) throw new Error('Could not create preview image')
|
||||
return new ImageSource(native)
|
||||
}
|
||||
|
||||
export function rasterizeSvg(svg: string, size: number): GrayImage | null {
|
||||
const data = FaceclawGraphics.renderSVGSize(svg, size)
|
||||
if (!data) return null
|
||||
const image = new GrayImage(size, size, 0)
|
||||
image.pixels.set(new Uint8Array(interop.bufferFromData(data)))
|
||||
return image
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
import { ApplicationSettings } from '@nativescript/core'
|
||||
|
||||
// Main-isolate settings for the initial iOS port. Worker synchronization will
|
||||
// be added when worker-hosted apps are enabled on iOS.
|
||||
const listeners = new Set<(key: string) => void>()
|
||||
function changed(key: string): void {
|
||||
setTimeout(() => { for (const listener of [...listeners]) listener(key) }, 0)
|
||||
}
|
||||
export function getStringSetting(key: string, fallback: string): string {
|
||||
return ApplicationSettings.getString(key, fallback)
|
||||
}
|
||||
export function setStringSetting(key: string, value: string): void {
|
||||
if (getStringSetting(key, '') === value && ApplicationSettings.hasKey(key)) return
|
||||
ApplicationSettings.setString(key, value)
|
||||
changed(key)
|
||||
}
|
||||
export function getBooleanSetting(key: string, fallback: boolean): boolean {
|
||||
return ApplicationSettings.getBoolean(key, fallback)
|
||||
}
|
||||
export function setBooleanSetting(key: string, value: boolean): void {
|
||||
if (getBooleanSetting(key, false) === value && ApplicationSettings.hasKey(key)) return
|
||||
ApplicationSettings.setBoolean(key, value)
|
||||
changed(key)
|
||||
}
|
||||
export function onSettingsStoreChanged(listener: (key: string) => void): () => void {
|
||||
listeners.add(listener)
|
||||
return () => { listeners.delete(listener) }
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
import { rasterizeSvg as render } from './ios-graphics'
|
||||
export function rasterizeSvg(svg: string, size: number, strokeWidth: number) {
|
||||
// Shared icon sources rely on the Android renderer's white, rounded
|
||||
// stroke defaults. Supply those explicitly for a standards SVG renderer.
|
||||
const source = svg.replace(/<svg\b([^>]*)>/, (_tag, attributes: string) => {
|
||||
const defaults = { xmlns: 'http://www.w3.org/2000/svg', width: '24', height: '24',
|
||||
stroke: svg.includes('fill="none"') ? '#ffffff' : 'none', fill: '#ffffff', 'stroke-width': String(strokeWidth),
|
||||
'stroke-linecap': 'round', 'stroke-linejoin': 'round' }
|
||||
const missing = Object.entries(defaults).filter(([key]) => !new RegExp(`\\b${key}=`).test(attributes))
|
||||
.map(([key, value]) => ` ${key}="${value}"`).join('')
|
||||
return `<svg${attributes}${missing}>`
|
||||
})
|
||||
return render(source, size)
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
import { GrayImage } from '../graphics/image'
|
||||
declare const com: any
|
||||
export function rasterizeSvg(svg: string, size: number, strokeWidth: number): GrayImage | null {
|
||||
const bytes = com.faceclaw.app.IconRenderer.renderSvgGray(svg, size, strokeWidth)
|
||||
if (!bytes || bytes.length < size * size) return null
|
||||
const image = new GrayImage(size, size, 0)
|
||||
for (let i = 0; i < size * size; i++) image.pixels[i] = bytes[i] & 0xff
|
||||
return image
|
||||
}
|
||||
@@ -0,0 +1,141 @@
|
||||
import { Application, Button, Color, GridLayout, Image, Label, Page, StackLayout, type TouchGestureEventData } from '@nativescript/core'
|
||||
import { IosPreviewController } from '../g2/ios-preview-controller'
|
||||
import { PhoneGestureRecognizer } from './phone-gestures'
|
||||
import { onAnySettingChanged, previewColorSetting } from '../ui/dashboard-settings'
|
||||
|
||||
export function createMainPage(): Page {
|
||||
const page = new Page()
|
||||
page.actionBarHidden = true
|
||||
const root = new GridLayout()
|
||||
root.rows = 'auto,*'
|
||||
page.content = root
|
||||
const header = new GridLayout()
|
||||
header.columns = '*,auto'; header.padding = '12 20'
|
||||
const title = new Label()
|
||||
title.text = 'Faceclaw'; title.fontSize = 25; title.fontWeight = 'bold'
|
||||
header.addChild(title)
|
||||
const state = new Label()
|
||||
state.text = 'Preview only'; state.fontSize = 14; state.verticalAlignment = 'middle'
|
||||
GridLayout.setColumn(state, 1); header.addChild(state); root.addChild(header)
|
||||
const body = new GridLayout()
|
||||
GridLayout.setRow(body, 1); root.addChild(body)
|
||||
const mirror = new Image()
|
||||
mirror.stretch = 'fill'; mirror.backgroundColor = new Color('black')
|
||||
mirror.accessibilityIdentifier = 'glasses-preview'
|
||||
mirror.accessibilityLabel = 'Interactive glasses screen preview'
|
||||
mirror.verticalAlignment = 'middle'; body.addChild(mirror)
|
||||
const controls = new GridLayout()
|
||||
controls.rows = 'auto,*,auto'; controls.padding = 16; body.addChild(controls)
|
||||
const tabs = new GridLayout()
|
||||
tabs.columns = '*,*,*'; controls.addChild(tabs)
|
||||
const pad = new GridLayout()
|
||||
pad.borderRadius = 18; pad.backgroundColor = new Color('#202522')
|
||||
pad.verticalAlignment = 'middle'; pad.horizontalAlignment = 'center'
|
||||
pad.accessibilityIdentifier = 'gesture-pad'
|
||||
GridLayout.setRow(pad, 1); controls.addChild(pad)
|
||||
const focus = new Label()
|
||||
focus.color = new Color('#ffffff'); focus.fontSize = 22; focus.textWrap = true
|
||||
focus.textAlignment = 'center'; focus.margin = 18; focus.verticalAlignment = 'middle'
|
||||
focus.isUserInteractionEnabled = false; pad.addChild(focus)
|
||||
const legend = new Label()
|
||||
legend.color = new Color('#b8c1bb'); legend.fontSize = 12; legend.textAlignment = 'center'
|
||||
legend.textWrap = true; legend.margin = 12; legend.verticalAlignment = 'bottom'
|
||||
legend.isUserInteractionEnabled = false; pad.addChild(legend)
|
||||
const settings = new StackLayout()
|
||||
settings.verticalAlignment = 'middle'
|
||||
GridLayout.setRow(settings, 1); controls.addChild(settings)
|
||||
const errorLabel = new Label()
|
||||
errorLabel.textWrap = true; errorLabel.color = new Color('#a02d2d'); errorLabel.visibility = 'collapse'
|
||||
settings.addChild(errorLabel)
|
||||
let selectedTab: 'watch' | 'ring' | 'settings' = 'watch'
|
||||
const controller = new IosPreviewController((image, label) => {
|
||||
mirror.imageSource = image; focus.text = selectedTab === 'ring' ? 'Ring' : label
|
||||
}, message => {
|
||||
errorLabel.text = message; errorLabel.visibility = 'visible'; selectTab('settings')
|
||||
})
|
||||
const padInput = new PhoneGestureRecognizer(gesture => controller.gesture(gesture, selectedTab === 'ring' ? 'ring' : 'watch'))
|
||||
const mirrorInput = new PhoneGestureRecognizer((gesture, x, y) => {
|
||||
const size = mirror.getActualSize()
|
||||
if (size.width > 0 && size.height > 0) controller.gesture(gesture, 'mirror', x / size.width, y / size.height)
|
||||
})
|
||||
const touch = (recognizer: PhoneGestureRecognizer) => (event: TouchGestureEventData) => {
|
||||
if (!['down', 'move', 'up', 'cancel'].includes(event.action)) return
|
||||
recognizer.touch({ action: event.action as 'down' | 'move' | 'up' | 'cancel', x: event.getX(), y: event.getY(), pointers: event.getPointerCount() })
|
||||
}
|
||||
pad.on('touch', touch(padInput)); mirror.on('touch', touch(mirrorInput))
|
||||
function button(text: string, action: () => void): Button {
|
||||
const view = new Button()
|
||||
view.text = text; view.fontSize = 15; view.margin = 3; view.on('tap', action)
|
||||
return view
|
||||
}
|
||||
const tabButtons: Button[] = []
|
||||
function selectTab(tab: typeof selectedTab): void {
|
||||
padInput.cancel(); selectedTab = tab
|
||||
pad.visibility = tab === 'settings' ? 'collapse' : 'visible'
|
||||
settings.visibility = tab === 'settings' ? 'visible' : 'collapse'
|
||||
for (let i = 0; i < tabButtons.length; i++) tabButtons[i].opacity = ['settings', 'watch', 'ring'][i] === tab ? 1 : 0.5
|
||||
focus.text = tab === 'ring' ? 'Ring' : 'Watch controls'
|
||||
legend.text = tab === 'ring' ? 'Swipe up/down · Tap to select\nDouble-tap: back · Hold: menu' : 'Swipe to navigate · Tap to select\nDouble-tap / two fingers: back · Hold: menu'
|
||||
layout()
|
||||
}
|
||||
for (const [index, tab] of (['settings', 'watch', 'ring'] as const).entries()) {
|
||||
const view = button(tab[0].toUpperCase() + tab.slice(1), () => selectTab(tab))
|
||||
GridLayout.setColumn(view, index); tabs.addChild(view); tabButtons.push(view)
|
||||
}
|
||||
const sizeButton = button(controller.displayModeLabel, () => controller.cycleDisplayMode())
|
||||
settings.addChild(sizeButton)
|
||||
settings.addChild(button('Toggle green / grayscale', () => previewColorSetting.set(previewColorSetting.get() === 'green' ? 'white' : 'green')))
|
||||
const help = new Label()
|
||||
help.text = 'Tap preview icons to open apps.\nHold: system menu\nTap then hold: app menu'
|
||||
help.fontSize = 13; help.textWrap = true; help.margin = 6; settings.addChild(help)
|
||||
const footer = new GridLayout()
|
||||
footer.columns = '*,*,*'; GridLayout.setRow(footer, 2); controls.addChild(footer)
|
||||
const back = button('Back', () => controller.gesture('double-tap', 'watch'))
|
||||
const menu = button('Menu', () => {
|
||||
controller.gesture('long-press', 'watch'); controller.gesture('long-press-release', 'watch')
|
||||
})
|
||||
const keyboard = button('Keyboard', () => { void controller.typeIntoApp() })
|
||||
for (const [index, view] of [back, menu, keyboard].entries()) {
|
||||
view.padding = '16 8'; view.fontSize = 14
|
||||
GridLayout.setColumn(view, index); footer.addChild(view)
|
||||
}
|
||||
let lastLayout = ''
|
||||
function layout(): void {
|
||||
const size = root.getActualSize()
|
||||
if (size.width <= 0 || size.height <= 0) return
|
||||
const key = `${size.width}:${size.height}:${selectedTab}`
|
||||
if (key === lastLayout) return
|
||||
lastLayout = key; padInput.cancel(); mirrorInput.cancel()
|
||||
const landscape = size.width > size.height
|
||||
body.rows = landscape ? '*' : 'auto,*'; body.columns = landscape ? '*,320' : '*'
|
||||
GridLayout.setRow(controls, landscape ? 0 : 1); GridLayout.setColumn(controls, landscape ? 1 : 0)
|
||||
const width = landscape ? Math.min(size.width - 320, (size.height - 64) * 4 / 3) : size.width
|
||||
mirror.width = width; mirror.height = width * 3 / 4
|
||||
pad.width = selectedTab === 'ring' ? 150 : Math.min(360, (landscape ? 320 : size.width) - 32)
|
||||
pad.height = Math.max(120, Math.min(250, size.height - (landscape ? 180 : width * 3 / 4 + 200)))
|
||||
}
|
||||
// iOS emits layoutChanged inside its layout pass. Defer mutations so
|
||||
// NativeScript does not clear our new layout request at the end of that pass.
|
||||
let layoutTimer: ReturnType<typeof setTimeout> | null = null
|
||||
root.on('layoutChanged', () => {
|
||||
if (layoutTimer !== null) return
|
||||
layoutTimer = setTimeout(() => { layoutTimer = null; layout() }, 0)
|
||||
})
|
||||
const pause = () => {
|
||||
if (layoutTimer !== null) clearTimeout(layoutTimer)
|
||||
layoutTimer = null
|
||||
padInput.cancel(); mirrorInput.cancel(); controller.pause()
|
||||
}
|
||||
const resume = () => controller.resume()
|
||||
let offSettings: (() => void) | null = null
|
||||
page.on('loaded', () => {
|
||||
Application.on(Application.suspendEvent, pause); Application.on(Application.resumeEvent, resume)
|
||||
offSettings?.(); offSettings = onAnySettingChanged(() => { sizeButton.text = controller.displayModeLabel })
|
||||
selectTab(selectedTab); controller.resume()
|
||||
})
|
||||
page.on('unloaded', () => {
|
||||
pause(); offSettings?.(); offSettings = null
|
||||
Application.off(Application.suspendEvent, pause); Application.off(Application.resumeEvent, resume)
|
||||
})
|
||||
return page
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
/** One touch recognizer for a phone pad or mirror. Coordinates are in DIP. */
|
||||
declare function setTimeout(callback: () => void, ms: number): number
|
||||
declare function clearTimeout(id: number): void
|
||||
export type PhoneGesture = 'tap' | 'double-tap' | 'long-press' | 'long-press-release'
|
||||
| 'short-then-long-press' | 'swipe-up' | 'swipe-down' | 'swipe-left' | 'swipe-right'
|
||||
export type PhoneTouch = { action: 'down' | 'move' | 'up' | 'cancel'; x: number; y: number; pointers: number }
|
||||
export type GestureClock = { set: (callback: () => void, ms: number) => unknown; clear: (id: unknown) => void }
|
||||
const clock: GestureClock = { set: (callback, ms) => setTimeout(callback, ms), clear: id => clearTimeout(id as ReturnType<typeof setTimeout>) }
|
||||
|
||||
export class PhoneGestureRecognizer {
|
||||
private start: PhoneTouch | null = null
|
||||
private hold: unknown = null
|
||||
private tap: unknown = null
|
||||
private lastTap: { x: number; y: number } | null = null
|
||||
private holding = false
|
||||
private moved = false
|
||||
private multi = false
|
||||
private secondTap = false
|
||||
constructor(private readonly emit: (gesture: PhoneGesture, x: number, y: number) => void,
|
||||
private readonly timer: GestureClock = clock) {}
|
||||
|
||||
touch(event: PhoneTouch): void {
|
||||
if (event.action === 'cancel') { this.cancel(); return }
|
||||
if (event.action === 'down' && !this.start) {
|
||||
this.start = event
|
||||
this.holding = this.moved = this.multi = false
|
||||
this.secondTap = !!this.lastTap && Math.hypot(event.x - this.lastTap.x, event.y - this.lastTap.y) < 36
|
||||
if (this.secondTap) { this.clearTap() }
|
||||
else if (this.lastTap) { const tap = this.lastTap; this.clearTap(); this.emit('tap', tap.x, tap.y) }
|
||||
this.hold = this.timer.set(() => {
|
||||
this.hold = null
|
||||
if (!this.start || this.moved || this.multi) return
|
||||
this.holding = true
|
||||
this.emit(this.secondTap ? 'short-then-long-press' : 'long-press', this.start.x, this.start.y)
|
||||
}, 500)
|
||||
}
|
||||
if (!this.start) return
|
||||
if (event.pointers > 1) { this.multi = true; this.clearHold(); this.clearTap() }
|
||||
const dx = event.x - this.start.x, dy = event.y - this.start.y
|
||||
if (Math.hypot(dx, dy) > 14) { this.moved = true; this.clearHold() }
|
||||
if (event.action !== 'up') return
|
||||
// Some platforms emit an up for each finger. Complete the multi-touch
|
||||
// gesture only once, and suppress the trailing one-finger up events.
|
||||
this.clearHold()
|
||||
const start = this.start
|
||||
this.start = null
|
||||
if (this.holding) { this.emit('long-press-release', start.x, start.y); this.holding = false; return }
|
||||
if (this.multi) { this.emit('double-tap', start.x, start.y); return }
|
||||
if (Math.max(Math.abs(dx), Math.abs(dy)) >= 30) {
|
||||
this.emit(Math.abs(dx) > Math.abs(dy) ? (dx > 0 ? 'swipe-right' : 'swipe-left') : (dy > 0 ? 'swipe-down' : 'swipe-up'), start.x, start.y)
|
||||
} else if (!this.moved) {
|
||||
if (this.secondTap) this.emit('double-tap', start.x, start.y)
|
||||
else {
|
||||
this.lastTap = { x: start.x, y: start.y }
|
||||
this.tap = this.timer.set(() => {
|
||||
const tap = this.lastTap
|
||||
this.tap = null; this.lastTap = null
|
||||
if (tap) this.emit('tap', tap.x, tap.y)
|
||||
}, 280)
|
||||
}
|
||||
}
|
||||
}
|
||||
cancel(): void {
|
||||
this.clearHold(); this.clearTap()
|
||||
if (this.holding && this.start) this.emit('long-press-release', this.start.x, this.start.y)
|
||||
this.start = null; this.holding = false
|
||||
}
|
||||
private clearHold(): void { if (this.hold !== null) this.timer.clear(this.hold); this.hold = null }
|
||||
private clearTap(): void { if (this.tap !== null) this.timer.clear(this.tap); this.tap = null; this.lastTap = null }
|
||||
}
|
||||
@@ -144,6 +144,8 @@ export type ShellWindow = {
|
||||
};
|
||||
|
||||
export type ShellConfig = {
|
||||
/** Hosts without microphone support hide and reject voice entry points. */
|
||||
voiceInputEnabled?: boolean;
|
||||
/** Actions handed to shell overlay layers; requestRender must re-render the shell surface. */
|
||||
actions: LayerActions;
|
||||
getScreenTimeoutMs: () => number | null;
|
||||
@@ -901,7 +903,7 @@ class Shell {
|
||||
handsFree?: boolean;
|
||||
defaultTarget: "assistant" | "app";
|
||||
}): void {
|
||||
if (this.voiceDialogPending) return;
|
||||
if (this.config.voiceInputEnabled === false || this.voiceDialogPending) return;
|
||||
this.voiceDialogPending = true;
|
||||
void (async () => {
|
||||
let ready = true;
|
||||
@@ -1005,6 +1007,7 @@ class Shell {
|
||||
* the dialog finishes on click instead of long-press-release.
|
||||
*/
|
||||
startVoiceInput(): void {
|
||||
if (this.config.voiceInputEnabled === false) return;
|
||||
if (!this.screenOn || this.activeVoiceLayer || !this.stack.isAtBase()) return;
|
||||
// The transcript is aimed at the window whose menu requested it; the menu
|
||||
// entry point defaults the highlight to Type Into App.
|
||||
@@ -1334,7 +1337,7 @@ class Shell {
|
||||
ctx.stack.pop();
|
||||
},
|
||||
},
|
||||
{
|
||||
...(this.config.voiceInputEnabled === false ? [] : [{
|
||||
label: "Voice input",
|
||||
onSelect: (ctx) => {
|
||||
// The transcript is aimed at the foreground window, so keep focus
|
||||
@@ -1343,7 +1346,7 @@ class Shell {
|
||||
ctx.stack.pop();
|
||||
this.startVoiceInput();
|
||||
},
|
||||
},
|
||||
}]),
|
||||
);
|
||||
items.push({
|
||||
label: "Debug",
|
||||
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
# Prefer Homebrew Ruby over the older macOS system Ruby when available.
|
||||
for ruby_bin in /opt/homebrew/opt/ruby/bin /usr/local/opt/ruby/bin; do
|
||||
if [[ -x "$ruby_bin/ruby" ]]; then
|
||||
export PATH="$ruby_bin:$PATH"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if command -v ruby >/dev/null 2>&1; then
|
||||
gem_bin="$(ruby -e 'puts Gem.bindir')"
|
||||
export PATH="$gem_bin:$PATH"
|
||||
fi
|
||||
|
||||
exec ns "$@"
|
||||
@@ -0,0 +1,195 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
|
||||
function harness(provider = 'elevenlabs', continuous = true) {
|
||||
const timers = new Map();
|
||||
let timerId = 0;
|
||||
const sockets = [];
|
||||
const events = [], statuses = [], errors = [];
|
||||
const load = loader({
|
||||
setTimeout(fn, delay) { const id = ++timerId; timers.set(id, { fn, delay }); return id; },
|
||||
clearTimeout(id) { timers.delete(id); },
|
||||
console: { log() {}, warn() {} },
|
||||
com: { faceclaw: { app: {
|
||||
FaceclawWebSocketListener: function(callbacks) { return callbacks; },
|
||||
FaceclawWebSocket: class {
|
||||
constructor(url, listener) { Object.assign(this, { url, listener, sent: [], closed: false, reject: false }); sockets.push(this); }
|
||||
sendText(text) { if (this.reject) return false; this.sent.push(text); return true; }
|
||||
sendBinary(bytes) { if (this.reject) return false; this.sent.push(bytes); return true; }
|
||||
close() { this.closed = true; }
|
||||
}, } }, },
|
||||
}, {
|
||||
'./cloud-stt': {
|
||||
CLOUD_STT_SAMPLE_RATE: 16000,
|
||||
encodeBase64: bytes => Buffer.from(bytes).toString('base64'),
|
||||
toJavaBytes: bytes => bytes,
|
||||
},
|
||||
});
|
||||
const names = { elevenlabs: 'ElevenLabsSttClient', openai: 'OpenAiRealtimeSttClient', soniox: 'SonioxSttClient' };
|
||||
const Provider = load(`app/native/${provider}-stt.ts`)[names[provider]];
|
||||
const { ReconnectingSttClient } = load('app/native/reconnecting-stt.ts');
|
||||
const client = new ReconnectingSttClient(options => new Provider(options), {
|
||||
apiKey: 'test', onTranscript: event => events.push(event),
|
||||
onStatus: status => statuses.push(status), onError: error => errors.push(error),
|
||||
}, () => continuous);
|
||||
client.start();
|
||||
return { client, sockets, events, statuses, errors, timers,
|
||||
open() { sockets.at(-1).listener.onOpen(); },
|
||||
message(message, socket = sockets.at(-1)) { socket.listener.onTextMessage(JSON.stringify(message)); },
|
||||
tick() { const [id, timer] = timers.entries().next().value; timers.delete(id); timer.fn(); return timer.delay; },
|
||||
};
|
||||
}
|
||||
|
||||
function pcm(amplitude, ms = 50) {
|
||||
const bytes = new Uint8Array(ms * 32);
|
||||
for (let i = 0; i < bytes.length; i += 2) { bytes[i] = amplitude & 255; bytes[i + 1] = amplitude >> 8; }
|
||||
return bytes;
|
||||
}
|
||||
|
||||
for (const provider of ['elevenlabs', 'openai', 'soniox']) {
|
||||
test(`${provider}: failure and close reconnect, stale callbacks are ignored, stop cancels retry`, () => {
|
||||
const h = harness(provider);
|
||||
h.open();
|
||||
const old = h.sockets[0];
|
||||
old.listener.onFailure('network unavailable');
|
||||
old.listener.onClosed(); // Both callbacks must schedule only one retry.
|
||||
assert.equal(h.timers.size, 1);
|
||||
assert.equal(old.closed, true);
|
||||
h.client.acceptPcm(pcm(2000));
|
||||
assert.equal(h.tick(), 1000);
|
||||
assert.equal(h.sockets.length, 2);
|
||||
h.open();
|
||||
assert.ok(h.sockets[1].sent.length > 0);
|
||||
const eventCount = h.events.length;
|
||||
old.listener.onTextMessage(JSON.stringify({ message_type: 'partial_transcript', text: 'stale' }));
|
||||
old.listener.onOpen();
|
||||
assert.equal(h.events.length, eventCount);
|
||||
h.sockets[1].listener.onClosed();
|
||||
assert.equal(h.timers.size, 1);
|
||||
h.client.stop();
|
||||
assert.equal(h.timers.size, 0);
|
||||
});
|
||||
|
||||
test(`${provider}: rejected audio send triggers recovery and requeues the packet`, () => {
|
||||
const h = harness(provider);
|
||||
h.open();
|
||||
h.sockets[0].reject = true;
|
||||
h.client.acceptPcm(pcm(2000));
|
||||
assert.equal(h.timers.size, 1);
|
||||
h.tick(); h.open();
|
||||
assert.ok(h.sockets[1].sent.length > (provider === 'elevenlabs' ? 0 : 1));
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test(`${provider}: auth failure stops instead of retrying indefinitely`, () => {
|
||||
const h = harness(provider);
|
||||
h.sockets[0].listener.onFailure("Expected HTTP 101 response but was '401 Unauthorized'");
|
||||
assert.equal(h.timers.size, 0);
|
||||
assert.equal(h.errors.length, 1);
|
||||
});
|
||||
}
|
||||
|
||||
test('reconnect preserves visible partials once and does not replay already-sent audio', () => {
|
||||
const h = harness(); h.open();
|
||||
h.client.acceptPcm(pcm(2000));
|
||||
h.message({ message_type: 'partial_transcript', text: 'Before outage' });
|
||||
h.sockets[0].listener.onFailure('offline');
|
||||
assert.equal(h.events.at(-1).text, 'Before outage');
|
||||
assert.equal(h.events.at(-1).isFinal, true);
|
||||
h.tick(); h.open();
|
||||
assert.equal(h.sockets[1].sent.length, 0);
|
||||
h.message({ message_type: 'partial_transcript', text: 'After outage' });
|
||||
assert.equal(h.events.at(-1).text, 'After outage');
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test('pending audio is bounded to ten seconds and backoff is capped', () => {
|
||||
const h = harness();
|
||||
for (let i = 0; i < 20; i++) h.client.acceptPcm(pcm(i, 1000));
|
||||
h.open();
|
||||
const sentBytes = h.sockets[0].sent.reduce((sum, text) => sum + Buffer.from(JSON.parse(text).audio_base_64, 'base64').length, 0);
|
||||
assert.equal(sentBytes, 320000);
|
||||
for (const delay of [1000, 2000, 4000, 8000, 16000, 30000, 30000]) {
|
||||
h.sockets.at(-1).listener.onFailure('offline');
|
||||
assert.equal(h.tick(), delay);
|
||||
}
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test('finishing while disconnected cancels retries; finishing on an open socket times out cleanly', () => {
|
||||
const h = harness(); h.open();
|
||||
h.sockets[0].listener.onClosed();
|
||||
h.client.finish();
|
||||
assert.equal(h.timers.size, 0);
|
||||
const live = harness(); live.open();
|
||||
live.client.finish();
|
||||
assert.equal(live.tick(), 5000);
|
||||
assert.equal(live.sockets[0].closed, true);
|
||||
live.sockets[0].listener.onFailure('late');
|
||||
assert.equal(live.timers.size, 0);
|
||||
});
|
||||
|
||||
test('only continuous capture commits at pauses, keeping ElevenLabs connected', () => {
|
||||
for (const continuous of [false, true]) {
|
||||
const h = harness('elevenlabs', continuous); h.open();
|
||||
h.client.acceptPcm(pcm(3000));
|
||||
h.client.acceptPcm(pcm(0, 1500));
|
||||
const commits = h.sockets[0].sent.map(JSON.parse).filter(m => m.commit);
|
||||
assert.equal(commits.length, continuous ? 1 : 0);
|
||||
if (continuous) {
|
||||
h.message({ message_type: 'committed_transcript', text: 'First.' });
|
||||
h.message({ message_type: 'committed_transcript_with_timestamps', text: 'First.' });
|
||||
assert.equal(h.events.length, 1);
|
||||
assert.equal(h.events[0].paragraphBreakAfter, true);
|
||||
assert.equal(h.sockets[0].closed, false);
|
||||
}
|
||||
h.client.stop();
|
||||
}
|
||||
});
|
||||
|
||||
test('OpenAI keeps successive items separate and drains out-of-order completions in audio order', () => {
|
||||
const h = harness('openai'); h.open();
|
||||
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
|
||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'a' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'a', delta: 'Fir' });
|
||||
h.client.acceptPcm(pcm(3000)); h.client.acceptPcm(pcm(0, 1500));
|
||||
h.message({ type: 'input_audio_buffer.committed', item_id: 'b' });
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'b', transcript: 'Second.' });
|
||||
assert.equal(h.events.some(e => e.isFinal), false);
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.completed', item_id: 'a', transcript: 'First.' });
|
||||
assert.deepEqual(h.events.filter(e => e.isFinal).map(e => e.text), ['First.', 'Second.']);
|
||||
assert.equal(h.sockets[0].closed, false);
|
||||
h.message({ type: 'conversation.item.input_audio_transcription.delta', item_id: 'c', delta: 'Third.' });
|
||||
assert.equal(h.events.at(-1).text, 'Third.');
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test('Soniox speaker and pause formatting remains separate from dictation text and resets on finalize', () => {
|
||||
const h = harness('soniox'); h.open();
|
||||
h.message({ tokens: [
|
||||
{ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1', is_final: true },
|
||||
{ text: ' Hi.', start_ms: 550, end_ms: 800, speaker: '2', is_final: false },
|
||||
] });
|
||||
assert.equal(h.events.at(-1).text, 'Hello. Hi.');
|
||||
assert.equal(h.events.at(-1).transcribeText, 'Hello.\nHi.');
|
||||
h.message({ tokens: [
|
||||
{ text: ' Yes.', start_ms: 550, end_ms: 800, speaker: '2', is_final: true },
|
||||
{ text: '<fin>', is_final: true },
|
||||
] });
|
||||
assert.equal(h.events.find(e => e.isFinal).transcribeText, 'Hello.\nYes.');
|
||||
h.message({ tokens: [{ text: 'New.', start_ms: 3000, end_ms: 3400, speaker: '2', is_final: false }] });
|
||||
assert.equal(h.events.at(-1).text, 'New.');
|
||||
h.client.stop();
|
||||
});
|
||||
|
||||
test('an unsolicited Soniox session end finalizes visible text and reconnects', () => {
|
||||
const h = harness('soniox'); h.open();
|
||||
h.message({ tokens: [{ text: 'Before expiry.', is_final: true }], finished: true });
|
||||
assert.equal(h.events.at(-1).isFinal, true);
|
||||
assert.equal(h.events.at(-1).text, 'Before expiry.');
|
||||
assert.equal(h.timers.size, 1);
|
||||
h.tick(); h.open();
|
||||
assert.equal(h.sockets.length, 2);
|
||||
h.client.stop();
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
// Execute production TypeScript with only the native boundary and clock stubbed.
|
||||
exports.loader = (globals = {}, mocks = {}) => {
|
||||
const cache = new Map();
|
||||
function load(file) {
|
||||
file = path.resolve(__dirname, '../..', file);
|
||||
if (cache.has(file)) return cache.get(file);
|
||||
const output = {};
|
||||
cache.set(file, output);
|
||||
const source = ts.transpileModule(fs.readFileSync(file, 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText;
|
||||
vm.runInNewContext(source, {
|
||||
exports: output, console, Uint8Array, ...globals,
|
||||
require(name) {
|
||||
if (name in mocks) return mocks[name];
|
||||
if (!name.startsWith('.')) throw new Error(`Unmocked module: ${name}`);
|
||||
return load(path.resolve(path.dirname(file), name + '.ts'));
|
||||
},
|
||||
}, { filename: file });
|
||||
return output;
|
||||
}
|
||||
return load;
|
||||
};
|
||||
@@ -0,0 +1,56 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { PhoneGestureRecognizer } = require('../.test-build/app/phone-ui/phone-gestures.js');
|
||||
function setup() {
|
||||
let now = 0, id = 0;
|
||||
const timers = new Map(), events = [];
|
||||
const r = new PhoneGestureRecognizer((type, x, y) => events.push({ type, x, y }), {
|
||||
set(fn, ms) { timers.set(++id, { fn, at: now + ms }); return id; },
|
||||
clear(id) { timers.delete(id); },
|
||||
});
|
||||
return {
|
||||
r, events,
|
||||
touch: (action, x = 20, y = 30, pointers = 1) => r.touch({ action, x, y, pointers }),
|
||||
advance(ms) {
|
||||
now += ms;
|
||||
for (const [id, timer] of [...timers]) if (timer.at <= now && timers.delete(id)) timer.fn();
|
||||
},
|
||||
types: () => events.map(e => e.type),
|
||||
};
|
||||
}
|
||||
test('single tap retains mirror coordinates and waits to disambiguate double tap', () => {
|
||||
const s = setup(); s.touch('down'); s.touch('up');
|
||||
assert.deepEqual(s.types(), []); s.advance(280);
|
||||
assert.deepEqual(s.events, [{ type: 'tap', x: 20, y: 30 }]);
|
||||
});
|
||||
test('double tap emits back without accidental selection', () => {
|
||||
const s = setup(); s.touch('down'); s.touch('up'); s.advance(150);
|
||||
s.touch('down'); s.touch('up'); s.advance(1000);
|
||||
assert.deepEqual(s.types(), ['double-tap']);
|
||||
});
|
||||
test('a hold emits a press and release, never a tap', () => {
|
||||
const s = setup(); s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
|
||||
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
|
||||
});
|
||||
test('tap then hold emits the app-menu gesture without an earlier selection', () => {
|
||||
const s = setup(); s.touch('down'); s.touch('up'); s.advance(100);
|
||||
s.touch('down'); s.advance(500); s.touch('up'); s.advance(1000);
|
||||
assert.deepEqual(s.types(), ['short-then-long-press', 'long-press-release']);
|
||||
});
|
||||
test('four directional swipes cancel pending holds and taps', () => {
|
||||
for (const [x, y, expected] of [[80, 30, 'swipe-right'], [-40, 30, 'swipe-left'], [20, 90, 'swipe-down'], [20, -30, 'swipe-up']]) {
|
||||
const s = setup(); s.touch('down'); s.touch('move', x, y); s.advance(500); s.touch('up', x, y); s.advance(500);
|
||||
assert.deepEqual(s.types(), [expected]);
|
||||
}
|
||||
});
|
||||
test('two fingers send back exactly once and cancel the long-press timer', () => {
|
||||
const s = setup(); s.touch('down'); s.touch('down', 21, 30, 2); s.advance(500);
|
||||
s.touch('up', 21, 30, 2); s.touch('up'); s.advance(1000);
|
||||
assert.deepEqual(s.types(), ['double-tap']);
|
||||
});
|
||||
test('cancellation prevents delayed selections and releases active holds', () => {
|
||||
const s = setup(); s.touch('down'); s.touch('up'); s.r.cancel(); s.advance(1000);
|
||||
assert.deepEqual(s.types(), []);
|
||||
s.touch('down'); s.advance(500); s.touch('cancel'); s.advance(1000);
|
||||
assert.deepEqual(s.types(), ['long-press', 'long-press-release']);
|
||||
});
|
||||
@@ -0,0 +1,49 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { SurfaceCompositor } = require('../.test-build/app/graphics/surface-compositor.js');
|
||||
const rect = (x, y, width, height) => ({ x, y, width, height });
|
||||
function surface(c, id, box, values, zOrder = 0, transparency = 'opaque') {
|
||||
c.configureSurface(id, { ...box, zOrder, transparency });
|
||||
c.submitSurfaceFrame(id, Uint8Array.from(values), rect(0, 0, box.width, box.height));
|
||||
}
|
||||
test('shell color key preserves underlying text while opaque black covers it', () => {
|
||||
const c = new SurfaceCompositor(3, 2);
|
||||
surface(c, 'app', rect(0, 0, 3, 2), [200, 200, 200, 200, 200, 200]);
|
||||
surface(c, 'shell', rect(0, 0, 3, 2), [0, 1, 255, 0, 0, 0], 1, 'color-key');
|
||||
c.setUnderlayDim(1, 0.5);
|
||||
assert.deepEqual([...c.composite()], [100, 1, 255, 100, 100, 100]);
|
||||
c.setSurfaceVisible('shell', false);
|
||||
assert.deepEqual([...c.composite()], [100, 100, 100, 100, 100, 100]);
|
||||
surface(c, 'opaque', rect(1, 0, 1, 2), [0, 0], 2);
|
||||
assert.deepEqual([...c.composite()], [100, 0, 100, 100, 0, 100]);
|
||||
});
|
||||
test('partial frames clip in surface coordinates, then surfaces clip to the screen', () => {
|
||||
const c = new SurfaceCompositor(3, 2);
|
||||
surface(c, 'app', rect(-1, 0, 3, 2), [1, 2, 3, 4, 5, 6]);
|
||||
const source = Uint8Array.from([99, 10, 20, 99, 30, 40, 99]);
|
||||
c.submitSurfaceFrame('app', source.subarray(1, 5), rect(1, 1, 2, 2));
|
||||
source.fill(0);
|
||||
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
|
||||
c.submitSurfaceFrame('app', Uint8Array.from([50, 60, 70, 80]), rect(-1, -1, 2, 2));
|
||||
// The clipped update changed the offscreen column, leaving visible pixels intact.
|
||||
assert.deepEqual([...c.composite()], [2, 3, 0, 10, 20, 0]);
|
||||
});
|
||||
test('moving a surface retains its frame; resizing clears it; blanking is reversible', () => {
|
||||
const c = new SurfaceCompositor(3, 1);
|
||||
surface(c, 'app', rect(0, 0, 2, 1), [7, 9]);
|
||||
c.configureSurface('app', { ...rect(1, 0, 2, 1), zOrder: 0, transparency: 'opaque' });
|
||||
assert.deepEqual([...c.composite()], [0, 7, 9]);
|
||||
c.setScreenBlanked(true); assert.deepEqual([...c.composite()], [0, 0, 0]);
|
||||
c.setScreenBlanked(false); assert.deepEqual([...c.composite()], [0, 7, 9]);
|
||||
c.configureSurface('app', { ...rect(0, 0, 3, 1), zOrder: 0, transparency: 'opaque' });
|
||||
assert.deepEqual([...c.composite()], [0, 0, 0]);
|
||||
});
|
||||
test('layer order is independent of creation order; removal reveals lower surfaces', () => {
|
||||
const c = new SurfaceCompositor(1, 1);
|
||||
surface(c, 'top', rect(0, 0, 1, 1), [20], 10);
|
||||
surface(c, 'bottom', rect(0, 0, 1, 1), [80], -1);
|
||||
assert.equal(c.composite()[0], 20);
|
||||
c.removeSurface('top'); assert.equal(c.composite()[0], 80);
|
||||
assert.throws(() => c.submitSurfaceFrame('missing', new Uint8Array(1), rect(0, 0, 1, 1)));
|
||||
assert.throws(() => c.submitSurfaceFrame('bottom', new Uint8Array(2), rect(0, 0, 1, 1)));
|
||||
});
|
||||
@@ -0,0 +1,94 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const load = loader();
|
||||
const { TranscriptModel, TranscriptScroll, wrapTranscribeText } = load('app/apps/transcribe/transcript-model.ts');
|
||||
const { SpeechPauseDetector } = load('app/native/speech-pause.ts');
|
||||
const { TimedTranscript } = load('app/native/transcript-format.ts');
|
||||
|
||||
function pcm(amplitude, ms = 50) {
|
||||
const bytes = new Uint8Array(ms * 16 * 2);
|
||||
for (let i = 0; i < bytes.length; i += 2) {
|
||||
bytes[i] = amplitude & 255;
|
||||
bytes[i + 1] = (amplitude >> 8) & 255;
|
||||
}
|
||||
return bytes;
|
||||
}
|
||||
|
||||
test('partials replace, finals correct, and pause breaks survive display and saving', () => {
|
||||
const m = new TranscriptModel();
|
||||
m.accept({ text: 'Hello wor', isFinal: false });
|
||||
m.accept({ text: 'Hello world.', isFinal: true, paragraphBreakAfter: true });
|
||||
m.accept({ text: 'Next', isFinal: false });
|
||||
m.accept({ text: 'Next paragraph.', isFinal: false });
|
||||
assert.equal(m.text, 'Hello world.\nNext paragraph.');
|
||||
m.accept({ text: 'Next paragraph!', isFinal: true });
|
||||
assert.equal(m.text, 'Hello world.\nNext paragraph!');
|
||||
m.accept({ text: '', isFinal: true });
|
||||
assert.equal(m.text, 'Hello world.\nNext paragraph!');
|
||||
});
|
||||
|
||||
test('onboard pauses can precede or follow finalization', () => {
|
||||
for (const before of [true, false]) {
|
||||
const m = new TranscriptModel();
|
||||
m.accept({ text: 'first', isFinal: false });
|
||||
if (before) m.pause();
|
||||
m.accept({ text: 'First.', isFinal: true });
|
||||
if (!before) m.pause();
|
||||
m.accept({ text: 'Second.', isFinal: false });
|
||||
assert.equal(m.text, 'First.\nSecond.');
|
||||
}
|
||||
});
|
||||
|
||||
test('formatting metadata is used only by Transcribe, with revisable live speaker tokens', () => {
|
||||
const final = new TimedTranscript();
|
||||
final.append({ text: 'Hello.', start_ms: 0, end_ms: 500, speaker: '1' });
|
||||
const preview = final.copy();
|
||||
preview.append({ text: ' Wrong', start_ms: 600, end_ms: 900, speaker: '2' });
|
||||
assert.equal(preview.text, 'Hello.\nWrong');
|
||||
assert.equal(final.text, 'Hello.');
|
||||
final.append({ text: ' Right.', start_ms: 600, end_ms: 900, speaker: '1' });
|
||||
final.append({ text: ' Later.', start_ms: 2400, end_ms: 2800, speaker: '1' });
|
||||
assert.equal(final.text, 'Hello. Right.\nLater.');
|
||||
const event = { text: 'Hello. Right. Later.', transcribeText: final.text, isFinal: false };
|
||||
const m = new TranscriptModel();
|
||||
m.accept(event);
|
||||
assert.equal(m.text, final.text);
|
||||
assert.equal(event.text.includes('\n'), false);
|
||||
});
|
||||
|
||||
test('wrapping preserves explicit newlines, blank paragraphs, and long unspaced words', () => {
|
||||
assert.deepEqual([...wrapTranscribeText(s => s.length, 'one two\nthree\n\nabcdefghij', 5)],
|
||||
['one', 'two', 'three', '', 'abcde', 'fghij']);
|
||||
});
|
||||
|
||||
test('scroll follows the tail until scrolled back, then resumes only at the bottom', () => {
|
||||
const s = new TranscriptScroll();
|
||||
assert.equal(s.layout(30, 10), 20);
|
||||
assert.equal(s.layout(33, 10), 23);
|
||||
s.scroll(-3);
|
||||
assert.equal(s.layout(50, 10), 20);
|
||||
assert.equal(s.layout(51, 10), 20);
|
||||
s.scroll(100);
|
||||
assert.equal(s.layout(55, 10), 45);
|
||||
s.scroll(-100);
|
||||
assert.equal(s.layout(60, 10), 0);
|
||||
});
|
||||
|
||||
test('short transcripts become scrollable without disabling automatic following', () => {
|
||||
const s = new TranscriptScroll();
|
||||
assert.equal(s.layout(3, 10), 0);
|
||||
s.scroll(-3);
|
||||
assert.equal(s.layout(12, 10), 2);
|
||||
});
|
||||
|
||||
test('pause detector uses 1.5 seconds of PCM, fires once per pause, and ignores leading silence', () => {
|
||||
const d = new SpeechPauseDetector();
|
||||
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
|
||||
d.accept(pcm(3000));
|
||||
for (let i = 0; i < 29; i++) assert.equal(d.accept(pcm(0)), false);
|
||||
assert.equal(d.accept(pcm(0)), true);
|
||||
for (let i = 0; i < 60; i++) assert.equal(d.accept(pcm(0)), false);
|
||||
d.accept(pcm(3000));
|
||||
assert.equal(d.accept(pcm(0, 1500)), true);
|
||||
});
|
||||
@@ -11,6 +11,8 @@
|
||||
"outDir": "../.test-build"
|
||||
},
|
||||
"include": [
|
||||
"../app/graphics/surface-compositor.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
"../app/g2/ble-proximity.ts",
|
||||
"../app/g2/even-advertisement.ts",
|
||||
|
||||
Reference in new issue
Block a user