diff --git a/.gitleaks.toml b/.gitleaks.toml index e87ce09b..d660879f 100644 --- a/.gitleaks.toml +++ b/.gitleaks.toml @@ -4,6 +4,10 @@ useDefault = true [[rules]] id = "generic-api-key" - [rules.allowlist] + [[rules.allowlists]] description = "SIGN_KEY in even-api.ts is a client-side constant the app needs to sign EvenHub requests, not a credential." paths = ['''app/apps/evenhub/even-api\.ts'''] + + [[rules.allowlists]] + description = "Made-up T3 Code pairing tokens used as parser test fixtures." + paths = ['''tests/t3code-pairing\.test\.cjs'''] diff --git a/.oxlintrc.json b/.oxlintrc.json new file mode 100644 index 00000000..5f63cfb3 --- /dev/null +++ b/.oxlintrc.json @@ -0,0 +1,85 @@ +{ + // Run via ./lint.sh (or `npm run lint`). Rule docs: + // https://oxc.rs/docs/guide/usage/linter/rules.html + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["eslint", "typescript", "unicorn", "oxc", "import", "promise"], + "options": { + // Type-aware rules run through oxlint-tsgolint (typescript-go), which + // reads tsconfig.json with TS7 semantics (no baseUrl, rootDir explicit). + "typeAware": true + }, + "categories": { + "correctness": "warn", + "suspicious": "warn", + "pedantic": "off", + "perf": "off", + "style": "off", + "restriction": "off", + "nursery": "off" + }, + "rules": { + // --- Opted in from non-default categories --- + "eslint/eqeqeq": ["warn", "always", { "null": "ignore" }], + "eslint/no-fallthrough": "warn", + "eslint/no-var": "warn", + "eslint/prefer-const": "warn", + "import/no-duplicates": "warn", + "typescript/no-deprecated": "warn", + "typescript/no-misused-promises": "warn", + // Rethrowing / rejecting with a caught `unknown` is propagation, not a bug. + "typescript/only-throw-error": ["warn", { "allowThrowingUnknown": true }], + "typescript/prefer-promise-reject-errors": ["warn", { "allowThrowingUnknown": true }], + "typescript/restrict-plus-operands": "warn", + "typescript/switch-exhaustiveness-check": [ + "warn", + { "considerDefaultExhaustiveForUnions": true } + ], + + // --- Turned off from the enabled categories --- + // Intentional control-character stripping and ANSI/terminal handling. + "eslint/no-control-regex": "off", + // Private-by-convention `_field` names are common here. + "eslint/no-underscore-dangle": "off", + // Side-effect imports (polyfills, registrations) are deliberate. + "import/no-unassigned-import": "off", + // Fires on `.then()` chains used for side effects. + "promise/always-return": "off", + // Worker onmessage/postMessage are NativeScript workers, not windows. + "unicorn/prefer-add-event-listener": "off", + "unicorn/require-post-message-target-origin": "off", + // In-place sort/reverse on locally owned arrays is fine. + "unicorn/no-array-sort": "off", + "unicorn/no-array-reverse": "off", + // Stylistic: prefers hoisting closures that don't capture anything. + "unicorn/consistent-function-scoping": "off", + // `for (const l of [...listeners])` snapshots before notifying. + "unicorn/no-useless-spread": "off", + // references.d.ts pulls in @nativescript/types this way. + "typescript/triple-slash-reference": "off", + // Mostly `${error}` on unknown catch values, which is fine. + "typescript/restrict-template-expressions": "off", + // Mostly String(x) on unknown-typed parsed JSON. + "typescript/no-base-to-string": "off", + // Flags exhaustive switches that tsc already checks. + "typescript/consistent-return": "off", + // The app is not strictNullChecks, so these misfire on `!` and on + // String()/Number() coercions of Java-bridged values. + "typescript/no-unnecessary-type-assertion": "off", + "typescript/no-unnecessary-type-conversion": "off", + // Too noisy for NativeScript code that casts bridged `any` values. + "typescript/no-unsafe-type-assertion": "off" + }, + // TypeScript only: scripts, tests and vendored web code are plain JS. + "ignorePatterns": [ + "**/*.js", + "**/*.cjs", + "**/*.mjs", + "**/*.jsx", + "node_modules/**", + "platforms/**", + "dist/**", + "faceclaw-extensions/dist/**", + "old/**", + "tmp/**" + ] +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt index 86f45475..4e145be7 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/AndroidSessionLink.kt @@ -9,6 +9,10 @@ class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLi override fun connect(address: String, timeoutMs: Int, autoConnect: Boolean): Boolean = bleManager.connect(address, timeoutMs, autoConnect) + override fun beginConnect(address: String, background: Boolean): Boolean = bleManager.beginConnect(address, background) + + override fun isConnected(address: String): Boolean = bleManager.isConnected(address) + override fun requestHighPriority(address: String) { bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH) } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 8c853e88..c5b6d52e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -187,6 +187,8 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre fun setListener(listener: FaceclawBleCommunicatorListener?) = core.setListener(listener) + fun setRequiredFirmwareRevision(revision: Int) = core.setRequiredFirmwareRevision(revision) + fun start() { if (core.start()) { activeInstance = this @@ -292,6 +294,10 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre fun stopG2AudioForwarding() = core.stopG2AudioForwarding() + fun addAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.addAudioMonitorListener(listener) + + fun removeAudioMonitorListener(listener: FaceclawAudioPacketListener?) = core.removeAudioMonitorListener(listener) + fun addCompassListener(listener: FaceclawCompassListener?) = core.addCompassListener(listener) fun removeCompassListener(listener: FaceclawCompassListener?) = core.removeCompassListener(listener) @@ -316,6 +322,14 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** + * Posted to the main looper after each step of an animation the preview is replaying (menu + * highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover + * frames sent to the glasses, which carry the whole animation in one display list. + */ + fun setPreviewAnimationListener(listener: Runnable?) = + core.setPreviewAnimationListener(listener?.let { { mainHandler.post(it); Unit } }) + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(): String { @@ -429,6 +443,9 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre contentFingerprint, paintMs, frameId, if (glyphs == null) null else AndroidByteReader(glyphs)) } + /** Whether submitting this full-surface content would change nothing (see [GlassesSessionCore.isSurfaceCurrent]). */ + fun isSurfaceCurrent(surfaceId: String, fingerprint: String): Boolean = core.isSurfaceCurrent(surfaceId, fingerprint) + /** Play a tone sequence via CFW load_image_z mode 5 kind 4 (payload built on the TS side). */ fun playBuzzerSequence(payload: java.nio.ByteBuffer?) { val bytes = ByteArray(if (payload == null) 0 else payload.remaining()) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt index 7ef99d9d..4ebe81ca 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt @@ -11,14 +11,13 @@ import android.bluetooth.BluetoothGattService import android.bluetooth.BluetoothManager import android.bluetooth.BluetoothProfile import android.content.Context -import android.os.Handler -import android.os.Looper -import android.os.SystemClock import android.util.Log import java.util.UUID import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.CountDownLatch +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors import java.util.concurrent.TimeUnit import java.util.concurrent.atomic.AtomicLong @@ -28,6 +27,26 @@ class FaceclawBleManager(context: Context) { private const val TAG = "FaceclawBle" private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200) + // Handed to an operation waiter whose link dropped before its completion + // callback arrived; never a real GATT status. + private const val STATUS_LINK_LOST = -1 + + // Android's Bluetooth stack has process-wide command-pipeline constraints on some + // devices, so every GATT operation in the process is serialized on this one lock, + // across manager instances too (the session and each stock-firmware flow own one). + // An operation holds it until its completion callback; waiting for a connection + // to come up, or backing off before a write retry, does not. + private val bluetoothApiLock = Any() + + // Listener callbacks run here, in arrival order, instead of on the binder thread + // that delivered them. BluetoothGatt's callback interface is oneway, so a listener + // that blocks (the session core waits for its monitor) would otherwise hold back + // every later callback for that device, including the write completion that the + // monitor's holder may itself be waiting for. + private val callbackExecutor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "FaceclawBleCallbacks").apply { isDaemon = true } + } + // Process-wide outbound-traffic totals, sampled by the phone UI's BLE // bandwidth indicator. Static so counts from every manager instance and // isolate land in the same totals. @@ -48,29 +67,35 @@ class FaceclawBleManager(context: Context) { } } - private val context: Context - private val bluetoothAdapter: BluetoothAdapter - private val mainHandler = Handler(Looper.getMainLooper()) + /** + * One BluetoothGatt client for an address: a pending connection attempt until + * [connected]. [gatt] is set once connectGatt returns; a callback can arrive before + * that, while it is still null. + */ + private class GattLink { + @Volatile var gatt: BluetoothGatt? = null + @Volatile var connected = false + val connectLatch = CountDownLatch(1) + } - private val gattClients = ConcurrentHashMap() - private val bluetoothApiLock = Any() + private enum class OpKind { MTU, SERVICES, DESCRIPTOR, WRITE } + + /** The GATT operation awaiting its completion callback (the global lock allows one at a time). */ + private class PendingOp(val address: String, val kind: OpKind) { + val latch = CountDownLatch(1) + @Volatile var status: Int? = null + } + + private val context: Context + private val bluetoothManager: BluetoothManager + private val bluetoothAdapter: BluetoothAdapter + + private val links = ConcurrentHashMap() + private val negotiatedMtus = ConcurrentHashMap() private val messageLocks = ConcurrentHashMap() - private val connectLatches = ConcurrentHashMap() - private val connectResults = ConcurrentHashMap() - - private val servicesLatches = ConcurrentHashMap() - private val servicesStatuses = ConcurrentHashMap() - - private val mtuLatches = ConcurrentHashMap() - private val negotiatedMtus = ConcurrentHashMap() - private val mtuStatuses = ConcurrentHashMap() - - private val descriptorLatches = ConcurrentHashMap() - private val descriptorStatuses = ConcurrentHashMap() - - private val writeLatches = ConcurrentHashMap() - private val writeStatuses = ConcurrentHashMap() + @Volatile + private var pendingOp: PendingOp? = null @Volatile private var listener: FaceclawBleListener? = null @@ -81,6 +106,7 @@ class FaceclawBleManager(context: Context) { if (bluetoothManager == null || bluetoothManager.adapter == null) { throw IllegalStateException("Bluetooth adapter unavailable") } + this.bluetoothManager = bluetoothManager this.bluetoothAdapter = bluetoothManager.adapter } @@ -123,9 +149,19 @@ class FaceclawBleManager(context: Context) { } } - /** Whether a GATT client for this address is open, i.e. no disconnect callback has arrived for it. */ + /** Whether a connection to this address is up (not merely pending). */ fun isConnected(address: String?): Boolean { - return address != null && gattClients.containsKey(address) + return address != null && links[address]?.connected == true + } + + /** + * Start connecting without waiting; a later [connect] waits on the same attempt. + * [background] uses Android's autoConnect: a low-duty-cycle attempt with no timeout + * that lands whenever the device next advertises (a direct attempt scans hard but + * gives up after about 30 s). Returns true while a connection exists or is pending. + */ + fun beginConnect(address: String?, background: Boolean): Boolean { + return startConnect(requireAddress(address), background) != null } fun connect(address: String?, timeoutMs: Int): Boolean = connect(address, timeoutMs, false) @@ -136,71 +172,73 @@ class FaceclawBleManager(context: Context) { * set (confirmed via its own BLE debug log, "connect() - device: ..., * auto: true"), while every other faceclaw connection (both glasses arms) * uses autoConnect=false. Direct connect (false) is faster to establish - * but Android supervises it less patiently; a live test tonight showed - * the ring-only direct connection dying almost exactly 5 seconds in, - * repeatedly, with nothing else competing for it - consistent with a - * companion device Android's stack expects to be babied via autoConnect, - * not direct-connected like the glasses. autoConnect=true can take much - * longer to actually complete (Android manages it as a background - * reconnect, not an immediate attempt), so this needs its own longer - * timeout - do not reuse CONNECT_TIMEOUT_MS/awaitLatch's short window for - * it, that would just trade one spurious failure for another. - * - * Only the connectGatt call and the timeout cleanup run under the global - * API lock; the wait for the link does not. A pending connection is not a - * GATT operation in flight, and holding the lock across it let an - * unreachable ring (20s autoConnect timeout, retried every 2s) starve every - * glasses write. + * but Android supervises it less patiently; a live test showed the + * ring-only direct connection dying almost exactly 5 seconds in, + * repeatedly, with nothing else competing for it. autoConnect=true can + * take much longer to actually complete (Android manages it as a + * background reconnect, not an immediate attempt), so this needs its own + * longer timeout. Like [connect], waits on an attempt already pending. */ fun connect(address: String?, timeoutMs: Int, autoConnect: Boolean): Boolean { - if (address == null || address.trim().isEmpty()) { - throw IllegalArgumentException("address is required") + val key = requireAddress(address) + val link = startConnect(key, autoConnect) ?: return false + if (link.connected) { + return true } - val latch = CountDownLatch(1) - val gatt: BluetoothGatt - synchronized(gattLock(address)) { - if (gattClients[address] != null) { - return true + if (!awaitLatch(link.connectLatch, timeoutMs)) { + Log.w(TAG, "connect timed out after ${timeoutMs}ms: address=$key" + + " (no connection event; the device may not be advertising, or may be connected to something else)") + dropLink(key, link) + return false + } + return link.connected + } + + /** The current client for [address], starting a connection attempt when there is none. */ + private fun startConnect(address: String, background: Boolean): GattLink? { + synchronized(bluetoothApiLock) { + val existing = links[address] + if (existing != null) { + if (!existing.connected || isOsConnected(existing)) { + return existing + } + // A client can go stale without a disconnect callback (the Bluetooth + // service drops its clients when the adapter turns off). + Log.w(TAG, "dropping stale client: address=$address (Android reports no GATT connection)") + dropLink(address, existing) } - val device = bluetoothAdapter.getRemoteDevice(address) - ?: throw IllegalArgumentException("remote device not found: $address") - - connectLatches[address] = latch - connectResults.remove(address) - gatt = device.connectGatt( + val link = GattLink() + links[address] = link + val gatt: BluetoothGatt? = device.connectGatt( context, - autoConnect, + background, gattCallback, BluetoothDevice.TRANSPORT_LE, BluetoothDevice.PHY_LE_2M or BluetoothDevice.PHY_LE_1M - ) ?: run { - connectLatches.remove(address, latch) - return false + ) + if (gatt == null) { + links.remove(address, link) + Log.w(TAG, "connectGatt returned null: address=$address") + return null } - gattClients[address] = gatt + link.gatt = gatt + Log.i(TAG, "connect started: address=$address background=$background") + return link } + } - val completed = awaitLatch(latch, timeoutMs) - connectLatches.remove(address, latch) - val connected = connectResults.remove(address) == true - if (completed && connected && gattClients[address] === gatt) { - return true + private fun isOsConnected(link: GattLink): Boolean { + val gatt = link.gatt ?: return false + return try { + bluetoothManager.getConnectionState(gatt.device, BluetoothProfile.GATT) == BluetoothProfile.STATE_CONNECTED + } catch (t: Throwable) { + true } - // Timed out, or failed. Whoever removed the client from gattClients - // (the disconnect callback, or disconnect()) also closes it. - synchronized(gattLock(address)) { - if (gattClients.remove(address, gatt)) { - negotiatedMtus.remove(address) - gatt.disconnect() - gatt.close() - } - } - return false } fun requestConnectionPriority(address: String, priority: Int): Boolean { - synchronized(gattLock(address)) { + synchronized(bluetoothApiLock) { val gatt = requireGatt(address) return gatt.requestConnectionPriority(priority) } @@ -208,7 +246,7 @@ class FaceclawBleManager(context: Context) { /** One-shot benchmark preferences; success is established by callbacks/HCI, not submission. */ fun prepareBenchmarkLink(address: String, mode: Int) { - synchronized(gattLock(address)) { + synchronized(bluetoothApiLock) { val gatt = requireGatt(address) if ((mode and 1) != 0) { val accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH) @@ -229,69 +267,27 @@ class FaceclawBleManager(context: Context) { } fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean { - synchronized(gattLock(address)) { - val latch = CountDownLatch(1) - mtuLatches[address] = latch - mtuStatuses.remove(address) - - val gatt = requireGatt(address) - val started = gatt.requestMtu(mtu) - if (!started) { - mtuLatches.remove(address) - return false - } - if (!awaitLatch(latch, timeoutMs)) { - mtuLatches.remove(address) - mtuStatuses.remove(address) - return false - } - val status = mtuStatuses.remove(address) - mtuLatches.remove(address) - return status != null && status == BluetoothGatt.GATT_SUCCESS - } + val status = runOp(address, OpKind.MTU, timeoutMs) { gatt -> gatt.requestMtu(mtu) } + return status == BluetoothGatt.GATT_SUCCESS } fun discoverServices(address: String, timeoutMs: Int): Boolean { - synchronized(gattLock(address)) { - val latch = CountDownLatch(1) - servicesLatches[address] = latch - servicesStatuses.remove(address) - - val gatt = requireGatt(address) - val started = gatt.discoverServices() - if (!started) { - servicesLatches.remove(address) - return false - } - if (!awaitLatch(latch, timeoutMs)) { - servicesLatches.remove(address) - servicesStatuses.remove(address) - return false - } - val status = servicesStatuses.remove(address) - servicesLatches.remove(address) - return status != null && status == BluetoothGatt.GATT_SUCCESS - } + val status = runOp(address, OpKind.SERVICES, timeoutMs) { gatt -> gatt.discoverServices() } + return status == BluetoothGatt.GATT_SUCCESS } fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean { - synchronized(gattLock(address)) { - val latch = CountDownLatch(1) - descriptorLatches[address] = latch - descriptorStatuses.remove(address) - + synchronized(bluetoothApiLock) { val gatt = requireGatt(address) val characteristic = requireCharacteristic(gatt, characteristicUuid) val notificationSet = gatt.setCharacteristicNotification(characteristic, enable) if (!notificationSet) { - descriptorLatches.remove(address) return false } val descriptor = characteristic.getDescriptor(java.util.UUID.fromString(BleProtocol.CCCD_UUID)) if (descriptor == null) { - descriptorLatches.remove(address) return true } @@ -299,20 +295,39 @@ class FaceclawBleManager(context: Context) { BluetoothGattDescriptor.ENABLE_NOTIFICATION_VALUE else BluetoothGattDescriptor.DISABLE_NOTIFICATION_VALUE - val started = gatt.writeDescriptor(descriptor) - if (!started) { - descriptorLatches.remove(address) - descriptorStatuses.remove(address) - return false + val status = runOp(address, OpKind.DESCRIPTOR, timeoutMs) { client -> client.writeDescriptor(descriptor) } + return status == BluetoothGatt.GATT_SUCCESS + } + } + + /** + * Start one GATT operation and wait for its completion callback, holding the global + * lock throughout. Returns the callback's status ([STATUS_LINK_LOST] when the link + * dropped first), or null when Android refused the request or it timed out. + */ + private fun runOp(address: String, kind: OpKind, timeoutMs: Int, start: (BluetoothGatt) -> Boolean): Int? { + synchronized(bluetoothApiLock) { + val op = PendingOp(address, kind) + // Published before the connected check in requireGatt, so a disconnect racing + // with this start either fails that check or finds the op to release. + pendingOp = op + try { + val gatt = requireGatt(address) + if (!start(gatt)) { + Log.w(TAG, "$kind not started: address=$address") + return null + } + if (!awaitLatch(op.latch, timeoutMs)) { + Log.w(TAG, "$kind timed out after ${timeoutMs}ms: address=$address") + return null + } + if (op.status == STATUS_LINK_LOST) { + Log.w(TAG, "$kind abandoned: link lost: address=$address") + } + return op.status + } finally { + pendingOp = null } - if (!awaitLatch(latch, timeoutMs)) { - descriptorLatches.remove(address) - descriptorStatuses.remove(address) - return false - } - val status = descriptorStatuses.remove(address) - descriptorLatches.remove(address) - return status != null && status == BluetoothGatt.GATT_SUCCESS } } @@ -339,7 +354,7 @@ class FaceclawBleManager(context: Context) { synchronized(messageLock(address)) { val gatt: BluetoothGatt val characteristic: BluetoothGattCharacteristic - synchronized(gattLock(address)) { + synchronized(bluetoothApiLock) { gatt = requireGatt(address) characteristic = requireCharacteristic(gatt, characteristicUuid) } @@ -369,37 +384,43 @@ class FaceclawBleManager(context: Context) { var retryCount = 0 while (true) { val retryReason: String - synchronized(gattLock(address)) { - if (gattClients[address] !== gatt) { - // Disconnected (and closed) since the write started. - Log.w(TAG, "writeCharacteristic abandoned: address=$address no longer connected") - return false - } - val latch = CountDownLatch(1) - writeLatches[address] = latch - writeStatuses.remove(address) - - val result = gatt.writeCharacteristic(characteristic, data!!, writeType) - if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY) { - writeLatches.remove(address, latch) - writeStatuses.remove(address) - retryReason = "busy" - } else if (result != android.bluetooth.BluetoothStatusCodes.SUCCESS) { - writeLatches.remove(address, latch) - writeStatuses.remove(address) - retryReason = "start result=$result" - } else { - if (!awaitLatch(latch, timeoutMs)) { - writeLatches.remove(address, latch) - writeStatuses.remove(address) + synchronized(bluetoothApiLock) { + val op = PendingOp(address, OpKind.WRITE) + pendingOp = op + try { + // The lock is released between frames and retries, so the client may + // have dropped (or been replaced by a reconnect) since the write started. + if (!isCurrentClient(address, gatt)) { + Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address") return false } - val status = writeStatuses.remove(address) - writeLatches.remove(address, latch) - if (status != null && status == BluetoothGatt.GATT_SUCCESS) { - return true + val result = gatt.writeCharacteristic(characteristic, data!!, writeType) + if (result != android.bluetooth.BluetoothStatusCodes.SUCCESS) { + if (!isCurrentClient(address, gatt)) { + Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address") + return false + } + retryReason = if (result == android.bluetooth.BluetoothStatusCodes.ERROR_GATT_WRITE_REQUEST_BUSY) + "busy" + else + "start result=$result" + } else { + if (!awaitLatch(op.latch, timeoutMs)) { + Log.w(TAG, "writeCharacteristic timed out after ${timeoutMs}ms waiting for completion: address=$address") + return false + } + val status = op.status + if (status == BluetoothGatt.GATT_SUCCESS) { + return true + } + if (status == STATUS_LINK_LOST) { + Log.w(TAG, "writeCharacteristic abandoned: link lost: address=$address") + return false + } + retryReason = "callback status=$status" } - retryReason = "callback status=$status" + } finally { + pendingOp = null } } // Back off outside the lock so other devices' operations proceed. @@ -425,33 +446,81 @@ class FaceclawBleManager(context: Context) { } } + /** Close this address's client, cancelling a pending connection attempt. */ fun disconnect(address: String?) { if (address == null) { return } - val gatt = gattClients.remove(address) - negotiatedMtus.remove(address) - // Wake anything waiting on this device first: a closed client never - // calls back, and the waiter may be holding the lock close() needs. - releasePendingOperations(address) - if (gatt == null) { + val link = links[address] ?: return + dropLink(address, link) + } + + fun close() { + for (address in links.keys) { + disconnect(address) + } + } + + /** + * Forget [link] if it is still current for [address], release anything waiting on it, + * then disconnect and close its client. Closing suppresses the client's remaining + * callbacks, so listeners hear nothing about a link dropped here. + */ + private fun dropLink(address: String, link: GattLink) { + if (!links.remove(address, link)) { return } - synchronized(gattLock(address)) { + link.connected = false + negotiatedMtus.remove(address) + failPendingOp(address) + link.connectLatch.countDown() + // startConnect holds the lock until link.gatt is set, so it is set by the time + // this block runs (or connectGatt failed and there is nothing to close). + synchronized(bluetoothApiLock) { + val gatt = link.gatt ?: return gatt.disconnect() gatt.close() } } - fun close() { - for (address in gattClients.keys) { - disconnect(address) + private fun failPendingOp(address: String) { + val op = pendingOp + if (op != null && op.address == address && op.latch.count > 0) { + op.status = STATUS_LINK_LOST + op.latch.countDown() } } + private fun completeOp(address: String, kind: OpKind, status: Int) { + val op = pendingOp + if (op != null && op.address == address && op.kind == kind) { + op.status = status + op.latch.countDown() + } + } + + /** Whether [gatt] is still the connected client for [address]. */ + private fun isCurrentClient(address: String, gatt: BluetoothGatt): Boolean { + val link = links[address] + return link != null && link.connected && link.gatt === gatt + } + + /** Serializes whole multi-frame messages per device; taken before [bluetoothApiLock], never after. */ + private fun messageLock(address: String): Any { + return messageLocks.getOrPut(address) { Any() } + } + + private fun requireAddress(address: String?): String { + if (address == null || address.trim().isEmpty()) { + throw IllegalArgumentException("address is required") + } + return address + } + private fun requireGatt(address: String): BluetoothGatt { - val gatt = gattClients[address] - if (gatt == null) { + val link = links[address] + val gatt = link?.gatt + if (link == null || gatt == null || !link.connected) { throw IllegalStateException("Not connected: $address") } return gatt @@ -477,74 +546,54 @@ class FaceclawBleManager(context: Context) { } } - private fun gattLock(address: String?): Any { - // Android's Bluetooth stack has process-wide command-pipeline constraints on some devices. - // Keep every BluetoothGatt API call serialized globally, even for different MAC addresses. - // Never hold it across anything unbounded or slow that isn't a GATT operation in flight - // (connection waits, retry sleeps): every glasses frame write needs it. - return bluetoothApiLock - } - - /** Serializes whole multi-frame messages per device; taken before [gattLock], never after. */ - private fun messageLock(address: String): Any { - return messageLocks.getOrPut(address) { Any() } - } - - /** - * Fail every operation still waiting on a callback from [address]. They see no status and - * return false, instead of holding the global lock until their timeouts expire. - */ - private fun releasePendingOperations(address: String) { - connectLatches.remove(address)?.countDown() - servicesLatches.remove(address)?.countDown() - mtuLatches.remove(address)?.countDown() - descriptorLatches.remove(address)?.countDown() - writeLatches.remove(address)?.countDown() - } - private val gattCallback: BluetoothGattCallback = object : BluetoothGattCallback() { override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) { - Log.i(TAG, "onConnectionStateChange: status=$status newState=$newState") val address = gatt.device.address - - if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) { - connectResults[address] = true - val latch = connectLatches.remove(address) - latch?.countDown() - dispatchConnectionState(address, true) + Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState") + val link = links[address] + val current = link?.gatt + if (link == null || (current != null && current !== gatt)) { + // A client this manager no longer tracks: make sure it is released. + synchronized(bluetoothApiLock) { + if (newState != BluetoothProfile.STATE_DISCONNECTED) gatt.disconnect() + gatt.close() + } return } - if (newState == BluetoothProfile.STATE_DISCONNECTED) { - connectResults[address] = false - val latch = connectLatches.remove(address) - latch?.countDown() - // Unpublish and wake this client's waiters before taking the lock: one of them - // may hold it, waiting for a callback that will now never come. - if (gattClients.remove(address, gatt)) { - negotiatedMtus.remove(address) - releasePendingOperations(address) - } - synchronized(gattLock(address)) { - gatt.close() - } - dispatchConnectionState(address, false) + if (newState == BluetoothProfile.STATE_CONNECTED && status == BluetoothGatt.GATT_SUCCESS) { + link.connected = true + link.connectLatch.countDown() + dispatchConnectionState(address, link, true) + return + } + if (newState != BluetoothProfile.STATE_DISCONNECTED && status == BluetoothGatt.GATT_SUCCESS) { + return + } + + // Disconnected, or a failed attempt (including a connection reported with an + // error status). Wake the waiters first: the lock below may be held by one of + // them, waiting for a callback that will now never come. + links.remove(address, link) + link.connected = false + negotiatedMtus.remove(address) + failPendingOp(address) + link.connectLatch.countDown() + dispatchConnectionState(address, link, false) + synchronized(bluetoothApiLock) { + if (newState != BluetoothProfile.STATE_DISCONNECTED) gatt.disconnect() + gatt.close() } } override fun onServicesDiscovered(gatt: BluetoothGatt, status: Int) { - val address = gatt.device.address - servicesStatuses[address] = status - val latch = servicesLatches.remove(address) - latch?.countDown() + completeOp(gatt.device.address, OpKind.SERVICES, status) } override fun onMtuChanged(gatt: BluetoothGatt, mtu: Int, status: Int) { val address = gatt.device.address if (status == BluetoothGatt.GATT_SUCCESS) negotiatedMtus[address] = mtu - mtuStatuses[address] = status - val latch = mtuLatches.remove(address) - latch?.countDown() + completeOp(address, OpKind.MTU, status) } override fun onPhyRead(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) { @@ -558,17 +607,11 @@ class FaceclawBleManager(context: Context) { } override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) { - val address = gatt.device.address - descriptorStatuses[address] = status - val latch = descriptorLatches.remove(address) - latch?.countDown() + completeOp(gatt.device.address, OpKind.DESCRIPTOR, status) } override fun onCharacteristicWrite(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, status: Int) { - val address = gatt.device.address - writeStatuses[address] = status - val latch = writeLatches.remove(address) - latch?.countDown() + completeOp(gatt.device.address, OpKind.WRITE, status) } override fun onCharacteristicChanged(gatt: BluetoothGatt, characteristic: BluetoothGattCharacteristic, value: ByteArray) { @@ -581,14 +624,31 @@ class FaceclawBleManager(context: Context) { } } - private fun dispatchConnectionState(address: String, connected: Boolean) { - val current = listener ?: return - current.onConnectionStateChange(address, connected) + private fun dispatchConnectionState(address: String, link: GattLink, connected: Boolean) { + callbackExecutor.execute { + // Drop news about a client that a newer attempt has already replaced. + val current = links[address] + if (if (connected) current !== link else current != null && current !== link) { + return@execute + } + val target = listener ?: return@execute + try { + target.onConnectionStateChange(address, connected) + } catch (t: Throwable) { + Log.w(TAG, "connection-state listener failed: address=$address", t) + } + } } private fun dispatchNotification(address: String, characteristicUuid: String, data: ByteArray?) { - val current = listener ?: return val copy = data?.clone() ?: ByteArray(0) - current.onNotification(address, characteristicUuid, copy) + callbackExecutor.execute { + val target = listener ?: return@execute + try { + target.onNotification(address, characteristicUuid, copy) + } catch (t: Throwable) { + Log.w(TAG, "notification listener failed: address=$address", t) + } + } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubChromeClient.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubChromeClient.kt new file mode 100644 index 00000000..0b03f8d9 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubChromeClient.kt @@ -0,0 +1,22 @@ +package com.faceclaw.app + +import android.util.Log +import android.webkit.ConsoleMessage +import android.webkit.WebChromeClient + +/** + * Surfaces a hosted EvenHub app's console in logcat (tag FaceclawEvenHubConsole). + * Native rather than a NativeScript-extended WebChromeClient: chatty apps log + * many lines a second, and each one used to cost a Java->JS callback plus three + * JS->Java calls back to read the message, all on the main thread. + */ +class FaceclawEvenHubChromeClient : WebChromeClient() { + override fun onConsoleMessage(message: ConsoleMessage): Boolean { + Log.i(TAG, "${message.message()} (${message.sourceId()}:${message.lineNumber()})") + return true + } + + private companion object { + const val TAG = "FaceclawEvenHubConsole" + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt index 90c8c1e7..d099d419 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt @@ -23,4 +23,10 @@ class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) { listener.onEvenAppMessage(handlerName, argsJson, callId) } } + + /** The page's timer shim has work due before the host's next tick. */ + @JavascriptInterface + fun wakeTimers() { + FaceclawEvenHubWebViewHost.getInstance().wakeTimers() + } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt index bee8e1da..7e5cad69 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt @@ -1,16 +1,27 @@ package com.faceclaw.app import android.app.Activity +import android.content.Context import android.content.MutableContextWrapper +import android.graphics.Color +import android.graphics.Typeface import android.os.Build import android.os.Handler import android.os.Looper +import android.text.TextUtils +import android.util.TypedValue +import android.view.Gravity import android.view.View import android.view.ViewGroup import android.webkit.WebView import android.widget.FrameLayout +import android.widget.LinearLayout +import android.widget.TextView +import androidx.core.view.ViewCompat +import androidx.core.view.WindowInsetsCompat import java.util.ArrayList +import java.util.HashMap /** * Keeps EvenHub app WebViews alive and rendering while the phone shows the @@ -33,15 +44,23 @@ import java.util.ArrayList * the chosen WebView to the top of the overlay); hiding sends it back behind. * All view work happens on the main thread (callers are on the NS/JS thread). * + * Layout: the activity is edge-to-edge, so the overlay tracks the window + * insets itself. A top bar (the shown app's name and a close button) sits + * under the status bar, and the WebViews fill the area between it and the + * navigation bar / keyboard. That area is the same whether or not the overlay + * is shown, so showing an app doesn't resize (and re-lay-out) its page. + * * Timer keep-alive: when the phone screen turns off, Chromium heavily throttles * the page's own setTimeout/setInterval (intensive background throttling clamps * them to ~1/sec), so timer-driven apps (e.g. snake's game loop) slow to a * crawl even though the renderer is still alive. Host-initiated * evaluateJavascript is NOT subject to that throttle, so we drive the app's * timers ourselves: a document-start shim replaces setTimeout/setInterval (and - * requestAnimationFrame) with JS queues fired by window.__fcTimerTick() / - * __fcRafTick(), and this host ticks those on the main thread at a fixed rate - * regardless of screen state. The main Looper keeps running under the + * requestAnimationFrame) with JS queues fired by window.__fcTick(), and this + * host ticks those on the main thread regardless of screen state. The tick + * returns when the page next has work, so an idle page isn't evaluated at + * 60Hz; work scheduled between ticks wakes the host through + * FaceclawEvenHubJsBridge.wakeTimers. The main Looper keeps running under the * foreground service, so the tick survives the screen turning off. (Matches the * official Even app's approach.) * @@ -68,28 +87,85 @@ import java.util.ArrayList */ class FaceclawEvenHubWebViewHost { private var overlay: FrameLayout? = null + /** Holds the WebViews, inside [overlay] below the top bar. */ + private var stack: FrameLayout? = null + private var titleView: TextView? = null private var overlayActivity: Activity? = null private var shown = false + private var shownTitle = "" + private var onClose: Runnable? = null private val mainHandler = Handler(Looper.getMainLooper()) private val webViews: MutableList = ArrayList() - private var ticking = false - private val ticker: Runnable = object : Runnable { + private val tickers: MutableMap = HashMap() + + /** + * Drives one page's timer/rAF queues (window.__fcTick). Each tick's result + * says when the page next has work; the ticker sleeps until then, or until + * the page wakes it. It also re-ticks after IDLE_POLL_MS regardless, so a + * lost evaluateJavascript callback (e.g. mid-navigation) or a page whose + * wake didn't reach us stalls its timers for at most that long. + */ + private inner class Ticker(private val web: WebView) : Runnable { + private var inFlight = false + private var wakePending = false + private var stopped = false + override fun run() { - for (i in 0 until webViews.size) { - webViews[i].evaluateJavascript( - "window.__fcTimerTick&&__fcTimerTick();window.__fcRafTick&&__fcRafTick()", null) + if (stopped) return + inFlight = true + web.evaluateJavascript(TICK_JS) { result -> onTicked(result) } + schedule(IDLE_POLL_MS) + } + + private fun onTicked(result: String?) { + inFlight = false + if (stopped) return + // -1 (nothing pending) and null (shim not installed yet) both leave + // only the idle poll; the page wakes us when it schedules work. + val next = result?.toDoubleOrNull()?.toLong() ?: -1L + val delay = when { + wakePending -> 0L + next < 0 -> IDLE_POLL_MS + else -> next.coerceIn(TICK_MS, IDLE_POLL_MS) } - if (ticking) mainHandler.postDelayed(this, TICK_MS) + wakePending = false + schedule(delay) + } + + fun wake() { + if (stopped) return + if (inFlight) wakePending = true else schedule(0) + } + + fun stop() { + stopped = true + mainHandler.removeCallbacks(this) + } + + private fun schedule(delayMs: Long) { + mainHandler.removeCallbacks(this) + mainHandler.postDelayed(this, delayMs) } } companion object { private var instance: FaceclawEvenHubWebViewHost? = null - /** How often the host fires the JS timer queue. 60Hz keeps games smooth. */ + /** Shortest interval between ticks. 60Hz keeps games smooth. */ private const val TICK_MS = 16L + /** Longest a page goes without a tick, as a backstop for missed wakes. */ + private const val IDLE_POLL_MS = 1000L + + private const val TICK_JS = "window.__fcTick?__fcTick():null" + + /** Top bar height, below the status bar (Material toolbar height). */ + private const val BAR_HEIGHT_DP = 56f + + /** Matches the NativeScript ActionBar (NativeScriptToolbarStyle). */ + private val BAR_COLOR = Color.parseColor("#424242") + @JvmStatic @Synchronized fun getInstance(): FaceclawEvenHubWebViewHost { @@ -103,36 +179,112 @@ class FaceclawEvenHubWebViewHost { val existing = overlay if (existing != null && overlayActivity === activity && existing.parent != null) return existing val content = activity.findViewById(android.R.id.content) + (existing?.parent as ViewGroup?)?.removeView(existing) val created = FrameLayout(activity) - if (existing != null) { - (existing.parent as ViewGroup?)?.removeView(existing) - for (web in webViews) { - existing.removeView(web) - rebaseContext(web, activity) - created.addView(web, FrameLayout.LayoutParams( - ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) - } + created.setBackgroundColor(Color.BLACK) + val createdStack = FrameLayout(activity) + created.addView(createdStack, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + val bar = buildTopBar(activity) + created.addView(bar, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT, Gravity.TOP)) + for (web in webViews) { + (web.parent as ViewGroup?)?.removeView(web) + rebaseContext(web, activity) + createdStack.addView(web, FrameLayout.LayoutParams( + ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + } + // Insets are passed on unconsumed: before API 30 a consuming child + // would starve its later siblings (the NativeScript UI) of them. + ViewCompat.setOnApplyWindowInsetsListener(created) { _, insets -> + applyInsets(bar, createdStack, insets) + insets } overlay = created + stack = createdStack overlayActivity = activity // index 0 = behind the NativeScript content view (raised again below if shown). content.addView(created, 0, FrameLayout.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) + ViewCompat.requestApplyInsets(created) if (shown) created.bringToFront() return created } + /** The bar across the top of a shown app: the app's name, then a close button. */ + private fun buildTopBar(activity: Activity): LinearLayout { + val bar = LinearLayout(activity) + bar.orientation = LinearLayout.HORIZONTAL + bar.gravity = Gravity.CENTER_VERTICAL + bar.setBackgroundColor(BAR_COLOR) + // Swallow touches so they don't fall through to the WebViews. + bar.isClickable = true + + val title = TextView(activity) + title.text = shownTitle + title.setTextColor(Color.WHITE) + title.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + title.typeface = Typeface.create("sans-serif-medium", Typeface.NORMAL) + title.maxLines = 1 + title.ellipsize = TextUtils.TruncateAt.END + bar.addView(title, LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f).apply { + marginStart = dp(activity, 16f) + marginEnd = dp(activity, 12f) + }) + + val close = TextView(activity) + close.text = "\u2715" + close.setTextColor(Color.WHITE) + close.setTextSize(TypedValue.COMPLEX_UNIT_SP, 20f) + close.gravity = Gravity.CENTER + close.contentDescription = "Close" + val ripple = TypedValue() + if (activity.theme.resolveAttribute(android.R.attr.selectableItemBackgroundBorderless, ripple, true)) { + close.setBackgroundResource(ripple.resourceId) + } + close.setOnClickListener { onClose?.run() } + val closeSize = dp(activity, 48f) + bar.addView(close, LinearLayout.LayoutParams(closeSize, closeSize).apply { + marginEnd = dp(activity, 4f) + }) + titleView = title + return bar + } + + /** + * Put the bar under the status bar (its background filling in behind it) + * and the WebViews between the bar and the navigation bar or keyboard. + */ + private fun applyInsets(bar: View, webStack: View, insets: WindowInsetsCompat) { + val bars = insets.getInsets( + WindowInsetsCompat.Type.systemBars() or WindowInsetsCompat.Type.displayCutout()) + val ime = insets.getInsets(WindowInsetsCompat.Type.ime()) + val barHeight = dp(bar.context, BAR_HEIGHT_DP) + bar.setPadding(bars.left, bars.top, bars.right, 0) + bar.layoutParams = (bar.layoutParams as FrameLayout.LayoutParams).apply { + height = bars.top + barHeight + } + webStack.layoutParams = (webStack.layoutParams as FrameLayout.LayoutParams).apply { + setMargins(bars.left, bars.top + barHeight, bars.right, maxOf(bars.bottom, ime.bottom)) + } + } + + private fun dp(context: Context, value: Float): Int { + return TypedValue.applyDimension( + TypedValue.COMPLEX_UNIT_DIP, value, context.resources.displayMetrics).toInt() + } + private fun rebaseContext(web: WebView, activity: Activity) { (web.context as? MutableContextWrapper)?.baseContext = activity } /** Add a WebView to the host, full-size and rendering, hidden behind the UI. */ fun attach(activity: Activity, web: WebView) { - val o = ensureOverlay(activity) + ensureOverlay(activity) rebaseContext(web, activity) web.visibility = View.VISIBLE if (web.parent == null) { - o.addView(web, FrameLayout.LayoutParams( + stack!!.addView(web, FrameLayout.LayoutParams( ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT)) } // resumeTimers() is process-global; make sure nothing left timers paused. @@ -148,15 +300,33 @@ class FaceclawEvenHubWebViewHost { web.setRendererPriorityPolicy(WebView.RENDERER_PRIORITY_IMPORTANT, false) } if (!webViews.contains(web)) webViews.add(web) - if (!ticking) { - ticking = true + if (!tickers.containsKey(web)) { + val ticker = Ticker(web) + tickers[web] = ticker mainHandler.postDelayed(ticker, TICK_MS) } } - /** Bring an app's WebView to the front of [activity] so it is visible on the phone. */ - fun showOnPhone(activity: Activity?, web: WebView) { + /** + * A page scheduled a timer or animation frame due before its next tick. + * Called from the JavaBridge thread; which page asked doesn't matter, as + * an extra tick for another page is harmless. + */ + fun wakeTimers() { + mainHandler.post { + for (ticker in tickers.values) ticker.wake() + } + } + + /** + * Bring an app's WebView to the front of [activity] so it is visible on the + * phone, titled [title]. The top bar's close button runs [onClose]. + */ + fun showOnPhone(activity: Activity?, web: WebView, title: String, onClose: Runnable) { val o = if (activity != null) ensureOverlay(activity) else overlay ?: return + shownTitle = title + this.onClose = onClose + titleView?.text = title web.bringToFront() o.bringToFront() shown = true @@ -165,6 +335,7 @@ class FaceclawEvenHubWebViewHost { /** Send the overlay back behind the NativeScript UI. */ fun hideOnPhone() { shown = false + onClose = null val o = overlay ?: return val content = o.parent as ViewGroup? if (content != null) { @@ -180,11 +351,8 @@ class FaceclawEvenHubWebViewHost { /** Remove and destroy a WebView (its app is closing). */ fun detach(web: WebView) { webViews.remove(web) - if (webViews.isEmpty() && ticking) { - ticking = false - mainHandler.removeCallbacks(ticker) - } - overlay?.removeView(web) + tickers.remove(web)?.stop() + stack?.removeView(web) web.destroy() } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFileDownload.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFileDownload.kt new file mode 100644 index 00000000..57cfd80c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawFileDownload.kt @@ -0,0 +1,90 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper + +import java.io.File +import java.io.FileOutputStream +import java.util.concurrent.TimeUnit + +import okhttp3.Call +import okhttp3.OkHttpClient +import okhttp3.Request + +/** + * One GET streamed straight to a file on a worker thread. For binary bodies + * too large to cross into JS through NativeScript's fetch(), which copies the + * Java byte[] into a Uint8Array one bridge call per byte (a 43MB EvenHub + * package ran the process out of native memory). JS reads the finished file + * back with one bulk copy (readBinaryFile). + */ +class FaceclawFileDownload( + private val url: String, + private val destPath: String, + private val timeoutMs: Long, + listener: FaceclawFileDownloadListener? +) { + private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper()) + private val listener: FaceclawFileDownloadListener = + listener ?: throw IllegalArgumentException("listener is required") + + @Volatile + private var call: Call? = null + + @Volatile + private var cancelled = false + + fun start() { + Thread({ run() }, "FaceclawFileDl").start() + } + + /** Aborts the request; no callback follows. */ + fun cancel() { + cancelled = true + call?.cancel() + } + + private fun run() { + val dest = File(destPath) + try { + dest.parentFile?.mkdirs() + val request = Request.Builder().url(url).build() + val call = getClient().newCall(request) + call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS) + this.call = call + call.execute().use { response -> + val body = response.body + val contentLength = body?.contentLength() ?: -1L + if (response.isSuccessful && body != null) { + body.byteStream().use { input -> + FileOutputStream(dest).use { output -> input.copyTo(output, 1 shl 16) } + } + } + if (!cancelled) callbackHandler.post { listener.onDone(response.code, contentLength) } + } + } catch (e: Exception) { + dest.delete() + if (!cancelled) callbackHandler.post { listener.onFailure(e.message ?: e.toString()) } + } + } + + companion object { + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + if (sharedClient == null) { + synchronized(FaceclawFileDownload::class.java) { + if (sharedClient == null) { + sharedClient = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + } + } + return sharedClient!! + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt index 7c40de56..c0f6f229 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt @@ -87,8 +87,14 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { return false } + /** + * The tray's notification icons, iconSize*iconSize gray bytes each: one + * per notification group. When keysOut is given, each icon's + * notification key (the group member it was taken from) is appended + * to it in step, so the tray can tell which notification an icon is. + */ @JvmStatic - fun getActiveNotificationIconGrays(iconSize: Int, maxIcons: Int): ByteArray { + fun getActiveNotificationIconGrays(iconSize: Int, maxIcons: Int, keysOut: MutableList?): ByteArray { val service = activeService val size = Math.max(1, Math.min(96, iconSize)) val limit = Math.max(0, maxIcons) @@ -130,6 +136,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { + " drawable=" + drawable.javaClass.simpleName + " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight) appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName) + keysOut?.add(statusBarNotification.key ?: "") if (dedupeGroupKey != null) { emittedGroupKeys.add(dedupeGroupKey) } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt index 4f4c65fa..6ae3b8ab 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt @@ -51,11 +51,13 @@ class FaceclawPreviewCompositor(context: Context) { if (activeInstance === this) { activeInstance = null } - frameListener = null + setFrameListener(null) } + /** Also notified after each step of an animation the preview is replaying (see SurfaceCompositor). */ fun setFrameListener(listener: Runnable?) { this.frameListener = listener + compositor.setPreviewAnimationListener(listener?.let { { mainHandler.post(it); Unit } }) } /** Set the compositor's output frame size. Call before configuring surfaces. */ diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt deleted file mode 100644 index b5108f48..00000000 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawSystemStatusIconProvider.kt +++ /dev/null @@ -1,113 +0,0 @@ -package com.faceclaw.app - -import android.content.Context -import android.net.ConnectivityManager -import android.net.Network -import android.net.NetworkCapabilities -import android.net.wifi.WifiInfo -import android.net.wifi.WifiManager -import android.os.Build -import android.telephony.SignalStrength -import android.telephony.TelephonyManager -import android.util.Log - -import java.io.ByteArrayOutputStream -import java.lang.reflect.Method - -/** Reads Wi-Fi/cell/hotspot state; the icon artwork is the shared StatusIconArt. */ -class FaceclawSystemStatusIconProvider private constructor() { - companion object { - private const val TAG = "FaceclawStatusIcons" - - @JvmStatic - fun getSystemStatusIconGrays(context: Context?, iconSize: Int): ByteArray { - if (context == null) { - return ByteArray(0) - } - val appContext = context.applicationContext - val size = Math.max(1, Math.min(96, iconSize)) - val out = ByteArrayOutputStream(size * size * 3) - - val wifiLevel = getWifiLevel(appContext) - if (wifiLevel >= 0) { - append(out, StatusIconArt.scale(StatusIconArt.wifi(wifiLevel), size)) - } - - val cellLevel = getCellLevel(appContext) - if (cellLevel >= 0) { - append(out, StatusIconArt.scale(StatusIconArt.cell(cellLevel), size)) - } - - if (isHotspotEnabled(appContext)) { - append(out, StatusIconArt.scale(StatusIconArt.hotspot(), size)) - } - - return out.toByteArray() - } - - private fun getWifiLevel(context: Context): Int { - try { - val connectivityManager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager? - val activeNetwork: Network? = if (connectivityManager == null) null else connectivityManager.activeNetwork - val capabilities: NetworkCapabilities? = if (activeNetwork == null) null else connectivityManager!!.getNetworkCapabilities(activeNetwork) - if (capabilities == null || !capabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI)) { - return -1 - } - val wifiManager = context.applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager? - val info: WifiInfo? = if (wifiManager == null) null else wifiManager.connectionInfo - if (info == null || !isValidRssi(info.rssi)) { - return -1 - } - return Math.max(0, Math.min(4, WifiManager.calculateSignalLevel(info.rssi, 5))) - } catch (t: Throwable) { - Log.w(TAG, "failed to read Wi-Fi status", t) - return -1 - } - } - - private fun getCellLevel(context: Context): Int { - if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P) { - return -1 - } - try { - val telephonyManager = context.getSystemService(Context.TELEPHONY_SERVICE) as TelephonyManager? - val signalStrength: SignalStrength? = if (telephonyManager == null) null else telephonyManager.signalStrength - if (signalStrength == null) { - return -1 - } - return Math.max(0, Math.min(4, signalStrength.level)) - } catch (e: SecurityException) { - // Most Android versions require READ_PHONE_STATE for this. We skip it - // rather than prompting for a broad phone permission just for an icon. - return -1 - } catch (t: Throwable) { - Log.w(TAG, "failed to read cellular status", t) - return -1 - } - } - - private fun isHotspotEnabled(context: Context): Boolean { - try { - val wifiManager = context.applicationContext.getSystemService(Context.WIFI_SERVICE) as WifiManager? - if (wifiManager == null) { - return false - } - val method: Method = wifiManager.javaClass.getDeclaredMethod("getWifiApState") - method.isAccessible = true - val result: Any? = method.invoke(wifiManager) - val state = if (result is Int) result else -1 - return state == 13 || state == 12 // WIFI_AP_STATE_ENABLED / ENABLING - } catch (ignored: Throwable) { - return false - } - } - - private fun isValidRssi(rssi: Int): Boolean { - return rssi > -127 && rssi < 0 - } - - private fun append(out: ByteArrayOutputStream, bytes: ByteArray) { - out.write(bytes, 0, bytes.size) - } - } -} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt new file mode 100644 index 00000000..382655a0 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt @@ -0,0 +1,791 @@ +package com.faceclaw.app + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaPlayer +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.Log + +import org.json.JSONObject + +import java.io.IOException +import java.nio.ByteBuffer +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantLock +import java.util.zip.CRC32 +import kotlin.concurrent.withLock + +/** + * Plays a video file onto the glasses. A background thread decodes the video track + * (MediaExtractor + MediaCodec), scales each due frame's luma to gray (VideoFrameScaler) and + * submits it straight into a rect of a window's compositor surface, through the same + * submitSurfaceFrame entry point app workers use, so no JS runs per frame. When wanted, the + * audio track plays through a MediaPlayer on the phone's default media route, and the video + * clock follows its position. + * + * The TS side (app/native/video-player.ts) owns the window: it paints the rest of the surface + * and repaints the video rect from [copyLastFrame] whenever it renders. Frames are submitted + * only while playing, and [pause] returns only once no further frame can be submitted, so a + * repaint right after it (the paused overlay) is never overwritten. + * + * Control methods are for the main thread; listener callbacks are posted to it. + */ +class FaceclawVideoPlayer(context: Context) { + interface Listener { + /** + * "playing", "paused" or "ended", with the media position. Also sent as "paused" when a + * seek made while paused has decoded its new frame (see [copyLastFrame]). + */ + fun onStateChanged(state: String?, positionMs: Long) + + /** Playback stopped on an error (unreadable file, no decoder, ...). */ + fun onError(message: String?) + } + + companion object { + private const val TAG = "FaceclawVideo" + private const val CODEC_TIMEOUT_US = 10_000L + /** Longest single sleep while waiting for a frame to come due, so commands stay responsive. */ + private const val MAX_WAIT_US = 20_000L + /** A frame this far behind the clock is dropped undrawn... */ + private const val LATE_DROP_US = 50_000L + /** ...unless nothing has been drawn for this long, so a slow phone still shows progress. */ + private const val MAX_DROP_GAP_NANOS = 250_000_000L + /** + * Decoded frames closer together than this are thinned: past ~30fps the glasses link + * could never show them, and each one still costs a scale and a composite. + */ + private const val MIN_FRAME_INTERVAL_US = 30_000L + /** Audio position is ignored this long after the clock starts (output startup latency). */ + private const val AUDIO_SETTLE_NANOS = 500_000_000L + /** Clock drift from the audio position that makes the video clock jump to it. */ + private const val AUDIO_RESYNC_US = 60_000L + /** How often the audio position is read (a binder call into the media server). */ + private const val AUDIO_CHECK_INTERVAL_NANOS = 250_000_000L + /** A seek never lands closer than this to the end, so it always shows a frame. */ + private const val SEEK_END_MARGIN_US = 500_000L + + private val AUDIO_ATTRIBUTES: AudioAttributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE) + .build() + + private fun MediaFormat.intOr(key: String, fallback: Int): Int = + if (containsKey(key)) getInteger(key) else fallback + + private fun MediaFormat.longOr(key: String, fallback: Long): Long = + if (containsKey(key)) getLong(key) else fallback + + private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int { + for (i in 0 until extractor.trackCount) { + val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith(mimePrefix)) return i + } + return -1 + } + + /** + * Describe a video file for the settings panel, as JSON: {width, height} in display + * orientation (rotation applied), rotation, durationMs, hasAudio, mime; or {error}. + */ + @JvmStatic + fun probe(path: String?): String { + val result = JSONObject() + val extractor = MediaExtractor() + try { + extractor.setDataSource(path ?: throw IOException("no path")) + val videoTrack = findTrack(extractor, "video/") + if (videoTrack < 0) { + result.put("error", "No video track") + return result.toString() + } + var durationUs = 0L + for (i in 0 until extractor.trackCount) { + durationUs = Math.max(durationUs, extractor.getTrackFormat(i).longOr(MediaFormat.KEY_DURATION, 0L)) + } + val format = extractor.getTrackFormat(videoTrack) + val rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0)) + val codedWidth = format.intOr(MediaFormat.KEY_WIDTH, 0) + val codedHeight = format.intOr(MediaFormat.KEY_HEIGHT, 0) + val swap = rotation == 90 || rotation == 270 + result.put("width", if (swap) codedHeight else codedWidth) + result.put("height", if (swap) codedWidth else codedHeight) + result.put("rotation", rotation) + result.put("durationMs", durationUs / 1000) + result.put("hasAudio", findTrack(extractor, "audio/") >= 0) + result.put("mime", format.getString(MediaFormat.KEY_MIME) ?: "") + } catch (e: Exception) { + Log.w(TAG, "probe failed for $path", e) + result.put("error", e.message ?: e.toString()) + } finally { + extractor.release() + } + return result.toString() + } + } + + /** What one start() call asked for; fixed for that playback's lifetime. */ + private class Params( + val path: String, + val surfaceId: String, + val x: Int, + val y: Int, + val width: Int, + val height: Int, + val monochrome: Boolean, + val gamma: Float, + val audio: Boolean, + ) + + private enum class Wait { DUE, INTERRUPTED, STOP } + + private val appContext: Context = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + @Volatile private var listener: Listener? = null + + private val lock = ReentrantLock() + private val wake = lock.newCondition() + + // Everything below is guarded by lock. + /** Bumped by every start/stop; a playback thread whose generation is stale exits. */ + private var generation = 0 + private var params: Params? = null + /** The user's intent: playing or paused. */ + private var playing = false + private var ended = false + /** A seek (or the initial start) is decoding toward its target; the clock waits for its frame. */ + private var seeking = false + private var pendingSeekUs = -1L + private var durationUs = 0L + private var clockRunning = false + /** Media time while the clock is stopped. */ + private var heldUs = 0L + private var clockBaseUs = 0L + private var clockBaseNanos = 0L + private var clockStartedNanos = 0L + private var lastAudioCheckNanos = 0L + private var audio: MediaPlayer? = null + private var hasAudioFocus = false + private var focusRequest: AudioFocusRequest? = null + /** The most recently decoded frame (params.width x params.height), shown or previewed. */ + private var lastFrame: ByteArray? = null + /** Checksum of the frame last submitted, so an unchanged frame is not recomposited; -1 = none. */ + private var lastSubmittedChecksum = -1L + private var submitFailures = 0 + + private val focusListener = AudioManager.OnAudioFocusChangeListener { change -> + when (change) { + AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> { + if (change == AudioManager.AUDIOFOCUS_LOSS) lock.withLock { hasAudioFocus = false } + // Another app (a call, a music player) took the audio: pause like any player would. + if (isPlaying()) { + pause() + notifyState(generationNow(), "paused", getPositionMs()) + } + } + } + } + + fun setListener(listener: Listener?) { + this.listener = listener + } + + /** + * Start playing [path] from [startMs] into the (rectX, rectY, width, height) rect of + * compositor surface [surfaceId], replacing any current playback. Decoding and audio setup + * happen on the playback thread; failures arrive as [Listener.onError]. + */ + fun start( + path: String, + surfaceId: String, + rectX: Int, + rectY: Int, + width: Int, + height: Int, + monochrome: Boolean, + gamma: Float, + withAudio: Boolean, + startMs: Long, + ) { + require(width > 0 && height > 0) { "bad video rect ${width}x$height" } + val p = Params(path, surfaceId, rectX, rectY, width, height, monochrome, gamma, withAudio) + val gen = lock.withLock { + endPlaybackLocked() + params = p + playing = true + ended = false + seeking = true + pendingSeekUs = Math.max(0L, startMs) * 1000 + heldUs = pendingSeekUs + // The old frame may not match the new settings (size, monochrome). + lastFrame = null + lastSubmittedChecksum = -1 + submitFailures = 0 + generation + } + val thread = Thread({ Playback(gen, p).run() }, "FaceclawVideo") + thread.priority = Thread.NORM_PRIORITY + 1 + thread.start() + } + + /** Pause; once this returns no further frame is submitted until [resume]. */ + fun pause() { + lock.withLock { + if (!playing) return + playing = false + holdClockLocked(clockUsLocked()) + wake.signalAll() + } + } + + fun resume() { + lock.withLock { + if (playing || ended || params == null) return + playing = true + // A fresh frame must go out even if it matches the last one: the TS side painted over + // the surface while paused. + lastSubmittedChecksum = -1 + if (!seeking) startClockLocked() + wake.signalAll() + } + } + + /** Seek to [positionMs]; while paused, the new frame is decoded and announced as "paused". */ + fun seekTo(positionMs: Long) { + lock.withLock { + if (params == null || ended) return + var targetUs = Math.max(0L, positionMs) * 1000 + if (durationUs > 0) targetUs = Math.min(targetUs, Math.max(0L, durationUs - SEEK_END_MARGIN_US)) + pendingSeekUs = targetUs + seeking = true + holdClockLocked(targetUs) + lastSubmittedChecksum = -1 + wake.signalAll() + } + } + + fun getPositionMs(): Long = lock.withLock { clockUsLocked() / 1000 } + + fun getDurationMs(): Long = lock.withLock { durationUs / 1000 } + + fun isPlaying(): Boolean = lock.withLock { playing && !ended && params != null } + + /** + * Copy the most recent frame into [out], which must hold exactly the video rect's + * width*height bytes (row-major 8bpp gray). False when there is no frame of that size yet. + */ + fun copyLastFrame(out: ByteBuffer): Boolean { + lock.withLock { + val frame = lastFrame ?: return false + if (out.capacity() != frame.size) return false + out.clear() + out.put(frame) + out.clear() + return true + } + } + + /** End playback and free the decoder and audio; the last frame stays readable. */ + fun stop() { + lock.withLock { endPlaybackLocked() } + } + + /** stop(), and drop the listener and frame. */ + fun release() { + lock.withLock { + endPlaybackLocked() + lastFrame = null + } + listener = null + } + + private fun generationNow(): Int = lock.withLock { generation } + + private fun endPlaybackLocked() { + generation++ + params = null + playing = false + seeking = false + pendingSeekUs = -1 + // Keep the position readable (an error's caller resumes from it). + heldUs = clockUsLocked() + clockRunning = false + releaseAudioLocked() + wake.signalAll() + } + + // ---- clock (all under lock) ---- + + private fun clockUsLocked(): Long = + if (!clockRunning) heldUs else clockBaseUs + (System.nanoTime() - clockBaseNanos) / 1000 + + private fun startClockLocked() { + if (clockRunning) return + clockRunning = true + clockBaseUs = heldUs + clockBaseNanos = System.nanoTime() + clockStartedNanos = clockBaseNanos + val player = audio ?: return + requestAudioFocusLocked() + try { + player.start() + } catch (e: IllegalStateException) { + Log.w(TAG, "audio start failed", e) + } + } + + private fun holdClockLocked(atUs: Long) { + heldUs = atUs + if (!clockRunning) return + clockRunning = false + try { + audio?.let { if (it.isPlaying) it.pause() } + } catch (e: IllegalStateException) { + Log.w(TAG, "audio pause failed", e) + } + } + + /** + * Keep the video clock on the audio position: output latency and seeks make the two drift, + * and a picture ahead of or behind its sound is the one sync error people notice. + */ + private fun followAudioLocked() { + val player = audio ?: return + val now = System.nanoTime() + if (!clockRunning || now - clockStartedNanos < AUDIO_SETTLE_NANOS) return + if (now - lastAudioCheckNanos < AUDIO_CHECK_INTERVAL_NANOS) return + lastAudioCheckNanos = now + val audioUs = try { + if (!player.isPlaying) return // finished (audio shorter than video) or not started + player.currentPosition * 1000L + } catch (e: IllegalStateException) { + return + } + if (Math.abs(audioUs - clockUsLocked()) > AUDIO_RESYNC_US) { + clockBaseUs = audioUs + clockBaseNanos = System.nanoTime() + } + } + + private fun seekAudioLocked(targetUs: Long) { + val player = audio ?: return + try { + if (Build.VERSION.SDK_INT >= 26) { + player.seekTo(targetUs / 1000, MediaPlayer.SEEK_CLOSEST) + } else { + player.seekTo((targetUs / 1000).toInt()) + } + } catch (e: IllegalStateException) { + Log.w(TAG, "audio seek failed", e) + } + } + + private fun requestAudioFocusLocked() { + if (hasAudioFocus) return + val result = if (Build.VERSION.SDK_INT >= 26) { + val request = focusRequest ?: AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setAudioAttributes(AUDIO_ATTRIBUTES) + .setOnAudioFocusChangeListener(focusListener, mainHandler) + .build() + .also { focusRequest = it } + audioManager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + audioManager.requestAudioFocus(focusListener, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN) + } + // Play regardless: the sound was asked for, and a refusal is transient (a call ending). + hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + } + + private fun releaseAudioLocked() { + val player = audio ?: return + audio = null + try { + player.release() + } catch (e: Exception) { + Log.w(TAG, "audio release failed", e) + } + if (hasAudioFocus) { + hasAudioFocus = false + if (Build.VERSION.SDK_INT >= 26) { + focusRequest?.let { audioManager.abandonAudioFocusRequest(it) } + } else { + @Suppress("DEPRECATION") + audioManager.abandonAudioFocus(focusListener) + } + } + } + + // ---- notifications ---- + + private fun notifyState(gen: Int, state: String, positionMs: Long) { + mainHandler.post { + if (lock.withLock { gen == generation }) listener?.onStateChanged(state, positionMs) + } + } + + private fun notifyError(gen: Int, message: String) { + mainHandler.post { + if (lock.withLock { gen == generation }) listener?.onError(message) + } + } + + // ---- frame output (under lock) ---- + + private fun submitLocked(p: Params, pixels: ByteArray, checksum: Long) { + val buffer = ByteBuffer.wrap(pixels) + // Frame id 0: per-frame FrameTimings entries at video rate would crowd every real + // input/render frame out of the timing export. + val fingerprint = "video:$generation:" + java.lang.Long.toHexString(checksum) + try { + val communicator = FaceclawBleCommunicator.getActive() + if (communicator != null) { + communicator.submitSurfaceFrame(buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null) + } else { + FaceclawPreviewCompositor.getActive()?.submitSurfaceFrame( + buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null) + } + submitFailures = 0 + } catch (e: RuntimeException) { + // The window's surface went away or shrank under us (closed, relaid out): drop the frame. + if (submitFailures++ == 0) Log.w(TAG, "video frame submit failed: ${e.message}") + } + } + + /** + * One start() call's decode thread. Owns the extractor and decoder; everything shared with + * the control methods goes through the player's lock. + */ + private inner class Playback(private val gen: Int, private val p: Params) { + private var extractor: MediaExtractor? = null + private var codec: MediaCodec? = null + private val info = MediaCodec.BufferInfo() + private var inputDone = false + private var outputDone = false + private var trackEndUs = 0L + /** Frames before this are decoded only on the way to a seek target. */ + private var skipUntilUs = 0L + private var lastDrawnPtsUs = Long.MIN_VALUE / 2 + private var lastDrawnNanos = 0L + private var rotation = 0 + private var lut = ByteArray(256) + private var scaler: VideoFrameScaler? = null + private var luma = ByteArray(0) + /** The frame being built; swapped with lastFrame when shown. */ + private var frame = ByteArray(p.width * p.height) + private val crc = CRC32() + + fun run() { + try { + open() + loop() + } catch (t: Throwable) { + Log.w(TAG, "playback failed for ${p.path}", t) + val current = lock.withLock { + val current = gen == generation + if (current) endPlaybackLocked() + current + } + // endPlaybackLocked bumped the generation; report under the new one. + if (current) notifyError(generationNow(), t.message ?: t.toString()) + } finally { + try { + codec?.stop() + } catch (ignored: Throwable) { + } + codec?.release() + extractor?.release() + } + } + + private fun current(): Boolean = lock.withLock { gen == generation } + + private fun open() { + val ex = MediaExtractor() + extractor = ex + ex.setDataSource(p.path) + val track = findTrack(ex, "video/") + if (track < 0) throw IOException("No video track") + ex.selectTrack(track) + val format = ex.getTrackFormat(track) + rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0)) + trackEndUs = format.longOr(MediaFormat.KEY_DURATION, 0L) + // Most video is limited ("TV") range; only an explicit full-range tag says otherwise. + val limited = format.intOr(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) != + MediaFormat.COLOR_RANGE_FULL + lut = VideoFrameScaler.toneLut(limited, p.gamma, p.monochrome) + val mime = format.getString(MediaFormat.KEY_MIME) ?: throw IOException("Unknown video format") + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + val decoder = try { + MediaCodec.createDecoderByType(mime) + } catch (e: Exception) { + throw IOException("No decoder for $mime") + } + codec = decoder + decoder.configure(format, null, null, 0) + decoder.start() + val player = if (p.audio && findTrack(ex, "audio/") >= 0) openAudio() else null + lock.withLock { + if (gen != generation) { + player?.release() + return + } + durationUs = trackEndUs + // Positioned by the start seek, which is the first command loop() handles. + audio = player + } + } + + /** A prepared MediaPlayer for the file's audio, or null (played silent) if it won't open. */ + private fun openAudio(): MediaPlayer? { + val player = MediaPlayer() + return try { + player.setAudioAttributes(AUDIO_ATTRIBUTES) + player.setDataSource(p.path) + player.prepare() + player + } catch (e: Exception) { + Log.w(TAG, "audio unavailable for ${p.path}; playing silent", e) + player.release() + null + } + } + + private fun loop() { + while (true) { + // Commands: stop, seek, or wait out a pause. + var seekUs = -1L + lock.withLock { + while (true) { + if (gen != generation) return + if (pendingSeekUs >= 0) { + seekUs = pendingSeekUs + pendingSeekUs = -1 + break + } + if (playing || seeking) break + wake.await() + } + } + if (seekUs >= 0) { + seek(seekUs) + continue + } + if (outputDone) { + if (awaitEnd()) return + continue + } + feedInput() + val index = codec!!.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) + if (index < 0) continue // try again later / format or buffers changed + val endOfStream = (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + if (info.size > 0) { + handleFrame(index, info.presentationTimeUs) + } else { + codec!!.releaseOutputBuffer(index, false) + } + if (endOfStream) outputDone = true + } + } + + private fun seek(targetUs: Long) { + lock.withLock { + if (gen != generation) return + seekAudioLocked(targetUs) + } + extractor!!.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC) + codec!!.flush() + inputDone = false + outputDone = false + skipUntilUs = targetUs + lastDrawnPtsUs = Long.MIN_VALUE / 2 + } + + private fun feedInput() { + if (inputDone) return + val decoder = codec!! + val ex = extractor!! + while (true) { + val index = decoder.dequeueInputBuffer(0) + if (index < 0) return + val buffer = decoder.getInputBuffer(index) ?: return + val size = ex.readSampleData(buffer, 0) + if (size < 0) { + decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + return + } + decoder.queueInputBuffer(index, 0, size, ex.sampleTime, 0) + ex.advance() + } + } + + /** Decide a decoded frame's fate (skip, preview, drop, draw); always releases it. */ + private fun handleFrame(index: Int, ptsUs: Long) { + try { + if (ptsUs < skipUntilUs) return + val landingSeek = lock.withLock { seeking } + if (landingSeek) { + landSeek(index, ptsUs) + return + } + if (ptsUs - lastDrawnPtsUs < MIN_FRAME_INTERVAL_US) return + if (waitUntilDue(ptsUs) != Wait.DUE) return + val lateUs = lock.withLock { clockUsLocked() } - ptsUs + if (lateUs > LATE_DROP_US && System.nanoTime() - lastDrawnNanos < MAX_DROP_GAP_NANOS) return + val checksum = convert(index) + present(ptsUs, checksum) + } finally { + codec!!.releaseOutputBuffer(index, false) + } + } + + /** + * The first frame at or past a seek target (or the start position): drawn and the clock + * started when playing, kept as the paused preview otherwise. + */ + private fun landSeek(index: Int, ptsUs: Long) { + val checksum = convert(index) + var previewed = false + lock.withLock { + if (gen != generation || pendingSeekUs >= 0) return + seeking = false + heldUs = ptsUs + swapInFrameLocked() + if (playing) { + submitLocked(p, lastFrame!!, checksum) + lastSubmittedChecksum = checksum + startClockLocked() + } else { + previewed = true + } + } + lastDrawnPtsUs = ptsUs + lastDrawnNanos = System.nanoTime() + notifyState(gen, if (previewed) "paused" else "playing", ptsUs / 1000) + } + + /** Show a due frame, unless a pause or seek got in first. */ + private fun present(ptsUs: Long, checksum: Long) { + lock.withLock { + if (gen != generation || !playing || pendingSeekUs >= 0) return + swapInFrameLocked() + if (checksum != lastSubmittedChecksum) { + submitLocked(p, lastFrame!!, checksum) + lastSubmittedChecksum = checksum + } + } + lastDrawnPtsUs = ptsUs + lastDrawnNanos = System.nanoTime() + } + + private fun swapInFrameLocked() { + val shown = frame + val previous = lastFrame + frame = if (previous != null && previous.size == shown.size) previous else ByteArray(shown.size) + lastFrame = shown + } + + /** Block until the clock reaches [ptsUs]; paused time doesn't count. */ + private fun waitUntilDue(ptsUs: Long): Wait { + lock.withLock { + while (true) { + if (gen != generation) return Wait.STOP + if (pendingSeekUs >= 0) return Wait.INTERRUPTED + if (!playing) { + wake.await() + continue + } + followAudioLocked() + val now = clockUsLocked() + if (now >= ptsUs) return Wait.DUE + wake.await(Math.min(ptsUs - now, MAX_WAIT_US), TimeUnit.MICROSECONDS) + } + } + } + + /** + * The decoder is drained: hold the last frame until the clock reaches the end, then + * report "ended". False when a seek interrupted the wait (playback continues). + */ + private fun awaitEnd(): Boolean { + val endUs = Math.max(trackEndUs, lastDrawnPtsUs) + val landing = lock.withLock { seeking } + if (!landing && waitUntilDue(endUs) != Wait.DUE) return !current() + lock.withLock { + if (gen != generation) return true + if (pendingSeekUs >= 0) return false + playing = false + seeking = false + ended = true + holdClockLocked(endUs) + releaseAudioLocked() + } + notifyState(gen, "ended", endUs / 1000) + return true + } + + /** Scale the output buffer's frame into [frame]; returns its checksum. */ + private fun convert(index: Int): Long { + val decoder = codec!! + val image = try { + decoder.getOutputImage(index) + } catch (e: Exception) { + null + } + if (image != null) { + try { + val plane = image.planes[0] + val crop = image.cropRect + val offset = copyLuma(plane.buffer) + crop.top * plane.rowStride + crop.left * plane.pixelStride + scaleInto(crop.width(), crop.height(), offset, plane.rowStride, plane.pixelStride) + } finally { + image.close() + } + } else { + // No Image access: every YUV420 layout starts with the Y plane at the buffer start. + val buffer = decoder.getOutputBuffer(index) ?: throw IOException("Decoder returned no frame") + buffer.position(info.offset) + buffer.limit(info.offset + info.size) + val format = decoder.outputFormat + val width = format.getInteger(MediaFormat.KEY_WIDTH) + val stride = format.intOr(MediaFormat.KEY_STRIDE, width) + val left = format.intOr("crop-left", 0) + val top = format.intOr("crop-top", 0) + val right = format.intOr("crop-right", width - 1) + val bottom = format.intOr("crop-bottom", format.getInteger(MediaFormat.KEY_HEIGHT) - 1) + copyLuma(buffer) + scaleInto(right - left + 1, bottom - top + 1, top * stride + left, stride, 1) + } + crc.reset() + crc.update(frame, 0, frame.size) + return crc.value + } + + /** Bulk-copy the plane (one memcpy; per-byte reads from a direct buffer are slow). Returns 0. */ + private fun copyLuma(buffer: ByteBuffer): Int { + val bytes = buffer.remaining() + if (luma.size < bytes) luma = ByteArray(bytes) + buffer.get(luma, 0, bytes) + return 0 + } + + private fun scaleInto(width: Int, height: Int, offset: Int, rowStride: Int, pixelStride: Int) { + var s = scaler + if (s == null || s.sourceWidth != width || s.sourceHeight != height) { + s = VideoFrameScaler(width, height, rotation, p.width, p.height) + scaler = s + } + s.scale(luma, offset, rowStride, pixelStride, lut, frame) + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt index 5be4a511..9c7b6edd 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawWearBridge.kt @@ -23,6 +23,8 @@ import java.nio.charset.StandardCharsets import java.util.ArrayDeque import java.util.ArrayList import java.util.Collections +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors /** * Phone side of the Wear OS integration: the single place that talks to the @@ -61,6 +63,8 @@ class FaceclawWearBridge private constructor(context: Context) { const val PATH_STATE_REQUEST = "/faceclaw/state/request" /** Watch -> phone: the watch's own battery; carries no seq and is never acked. */ const val PATH_WATCH_BATTERY = "/faceclaw/battery" + /** Watch -> phone: touch-down ahead of a gesture; carries no seq and is never acked. */ + const val PATH_WATCH_PRESS = "/faceclaw/press" /** Phone -> watch: answered with PATH_WATCH_BATTERY even while the watch app is closed. */ const val PATH_WATCH_BATTERY_REQUEST = "/faceclaw/battery/request" const val PATH_ACK = "/faceclaw/ack" @@ -114,6 +118,16 @@ class FaceclawWearBridge private constructor(context: Context) { private val pending = ArrayDeque() private val watchNodes: MutableList = ArrayList() private var lastPublishedState: String? = null + /** Latest state waiting for [publishExecutor]; null when none is queued. Guarded by [lock]. */ + private var queuedState: String? = null + /** + * Serializing a DataMap and handing it to Play services costs ~10-25ms per + * publish, and the switcher publishes on every foreground change, so it + * runs here instead of on the main thread. + */ + private val publishExecutor: ExecutorService = Executors.newSingleThreadExecutor { runnable -> + Thread(runnable, "FaceclawWearPublish").apply { isDaemon = true } + } private class PendingMessage( val path: String, @@ -210,12 +224,28 @@ class FaceclawWearBridge private constructor(context: Context) { publishState(json, false) } - /** As [publishState]; `force` re-sends an unchanged state. */ + /** + * As [publishState]; `force` re-sends an unchanged state. The put happens + * on [publishExecutor]; states that arrive while one is still waiting + * replace it, since only the latest matters to the watch. + */ fun publishState(json: String?, force: Boolean) { if (!available || json == null) return synchronized(lock) { if (!force && json == lastPublishedState) return lastPublishedState = json + val alreadyQueued = queuedState != null + queuedState = json + if (alreadyQueued) return + } + publishExecutor.execute { publishQueuedState() } + } + + private fun publishQueuedState() { + val json: String + synchronized(lock) { + json = queuedState ?: return + queuedState = null } try { val request = PutDataMapRequest.create(PATH_STATE) @@ -303,6 +333,9 @@ class FaceclawWearBridge private constructor(context: Context) { // dashboard to receive it, it is simply dropped (the dashboard polls // again when it starts), and the watch does not expect an ack. if (PATH_WATCH_BATTERY == path) return + // A touch-down only matters live; the gesture that follows it gets + // the failure ack below. + if (PATH_WATCH_PRESS == path) return // No JS listener yet: tell the watch so it can explain why nothing is // happening. Only state requests are kept for replay (see class doc); diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt index 54f31c42..6c98b9b0 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -30,5 +30,23 @@ class PreviewBitmapUtil private constructor() { val colors = PreviewPalette.expand(bytes, 0, width * height, brightenGamma, green) return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) } + + /** + * A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes + * alpha; black, which the glasses show as unlit, is transparent), for + * showing glasses artwork on the phone's dark action bar. + */ + @JvmStatic + fun whiteIconFromGray(gray: ByteBuffer?, width: Int, height: Int): Bitmap { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw IllegalArgumentException("invalid gray icon buffer") + } + val bytes = ByteArray(width * height) + gray.get(bytes) + val colors = IntArray(width * height) { i -> + ((bytes[i].toInt() and 0xff) shl 24) or 0x00ffffff + } + return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) + } } } diff --git a/App_Resources/iOS/src/FaceclawFontRenderer.m b/App_Resources/iOS/src/FaceclawFontRenderer.m index 012f36d4..d7598ca6 100644 --- a/App_Resources/iOS/src/FaceclawFontRenderer.m +++ b/App_Resources/iOS/src/FaceclawFontRenderer.m @@ -29,6 +29,51 @@ static NSAttributedString *Attributed(CTFontRef font, NSString *text, BOOL glyph if (glyphLayout) attrs[(__bridge id)kCTLigatureAttributeName] = @0; return [[NSAttributedString alloc] initWithString:text attributes:attrs]; } +static CTLineRef CreateRawLine(CTFontRef font, NSString *text, BOOL glyphLayout) CF_RETURNS_RETAINED { + return CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, text, glyphLayout)); +} +static BOOL IsColorRun(CTRunRef run) { + CTFontRef font = CFDictionaryGetValue(CTRunGetAttributes(run), kCTFontAttributeName); + return font && (CTFontGetSymbolicTraits(font) & kCTFontTraitColorGlyphs); +} +static BOOL HasColorRun(CTLineRef line) { + for (id run in (__bridge NSArray *)CTLineGetGlyphRuns(line)) if (IsColorRun((__bridge CTRunRef)run)) return YES; + return NO; +} +// iOS falls back to Apple Color Emoji for default-text symbols the face lacks +// (▶ U+25B6 in Roboto), where Android's minikin picks a text font; the +// alpha-only raster keeps just the emoji's silhouette (▶ becomes a solid +// rounded square). Request text presentation (U+FE0E) for each lone codepoint +// that fell back to a color font, wherever a text font can supply it. Returns +// `text` itself when nothing changes. +static NSString *PreferTextPresentation(CTFontRef font, NSString *text, CTLineRef line, BOOL glyphLayout) { + NSMutableIndexSet *color = [NSMutableIndexSet new]; + for (id run in (__bridge NSArray *)CTLineGetGlyphRuns(line)) if (IsColorRun((__bridge CTRunRef)run)) { + CFRange range = CTRunGetStringRange((__bridge CTRunRef)run); + [color addIndexesInRange:NSMakeRange(range.location, range.length)]; + } + if (!color.count) return text; + NSMutableString *out = [NSMutableString new]; + __block BOOL changed = NO; + [text enumerateSubstringsInRange:NSMakeRange(0, text.length) options:NSStringEnumerationByComposedCharacterSequences + usingBlock:^(NSString *ch, NSRange range, __unused NSRange enclosing, __unused BOOL *stop) { + [out appendString:ch]; + // Multi-scalar sequences (ZWJ, flags, keycaps, explicit selectors) stay emoji. + if (![color containsIndex:range.location] || [ch lengthOfBytesUsingEncoding:NSUTF32StringEncoding] != 4) return; + NSString *variant = [ch stringByAppendingString:@"\uFE0E"]; + CTLineRef retry = CreateRawLine(font, variant, glyphLayout); + if (!HasColorRun(retry)) { [out appendString:@"\uFE0E"]; changed = YES; } + CFRelease(retry); + }]; + return changed ? out : text; +} +static CTLineRef CreateLine(CTFontRef font, NSString *text, BOOL glyphLayout) CF_RETURNS_RETAINED { + CTLineRef line = CreateRawLine(font, text, glyphLayout); + NSString *preferred = PreferTextPresentation(font, text, line, glyphLayout); + if (preferred == text) return line; + CFRelease(line); + return CreateRawLine(font, preferred, glyphLayout); +} static BOOL ValidText(NSString *text) { return text != nil && text.length <= 100000; } static BOOL ValidRaster(NSInteger width, NSInteger height) { return width > 0 && height > 0 && width <= 65535 && height <= 65535 && width * height <= 4000000; @@ -85,7 +130,7 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do if (!ValidText(text)) return 0; CTFontRef font = CreateFont(path, size); if (!font) return 0; - CTLineRef line = CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, text, YES)); + CTLineRef line = CreateLine(font, text, YES); double width = CTLineGetTypographicBounds(line, NULL, NULL, NULL); CFRelease(line); CFRelease(font); return width; } @@ -95,7 +140,7 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do if (!font) return [NSData data]; uint32_t scalar = (uint32_t)cp; NSString *text = [[NSString alloc] initWithBytes:&scalar length:4 encoding:NSUTF32LittleEndianStringEncoding]; - CTLineRef line = CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, text, YES)); + CTLineRef line = CreateLine(font, text, YES); double advance = CTLineGetTypographicBounds(line, NULL, NULL, NULL); NSMutableData *out = [NSMutableData dataWithLength:10]; U16(out.mutableBytes, MAX(0, MIN(65535, lround(advance * 64)))); @@ -128,7 +173,7 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do if (!ValidText(text)) return [NSData data]; CTFontRef font = CreateFont(path, size); if (!font) return [NSData data]; - CTLineRef line = CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, text, NO)); + CTLineRef line = CreateLine(font, text, NO); CGRect bounds = CTLineGetImageBounds(line, NULL); BOOL ink = !CGRectIsNull(bounds) && !CGRectIsEmpty(bounds); NSInteger left = ink ? MIN(0, floor(CGRectGetMinX(bounds))) : 0; @@ -155,6 +200,9 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do NSInteger step = MAX(1, ascent + descent + (NSInteger)ceil(MAX(0, CTFontGetLeading(font)))); maxLines = MIN(maxLines, MIN(65535 / step, 4000000 / width / step)); if (maxLines <= 0) { CFRelease(font); return [NSData data]; } + CTLineRef probe = CreateRawLine(font, text, NO); + text = PreferTextPresentation(font, text, probe, NO); // Before any index math below. + CFRelease(probe); NSAttributedString *attributed = Attributed(font, text, NO); CTTypesetterRef typesetter = CTTypesetterCreateWithAttributedString((__bridge CFAttributedStringRef)attributed); NSMutableArray *lines = [NSMutableArray new]; @@ -165,8 +213,8 @@ static NSData *ImagePacket(NSData *pixels, NSInteger width, NSInteger height, do CTLineRef line; if (lines.count == (NSUInteger)maxLines - 1 && start + count < (CFIndex)text.length) { NSString *rest = [[text substringFromIndex:start] stringByReplacingOccurrencesOfString:@"\n" withString:@" "]; - CTLineRef full = CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, rest, NO)); - CTLineRef token = CTLineCreateWithAttributedString((__bridge CFAttributedStringRef)Attributed(font, @"…", NO)); + CTLineRef full = CreateRawLine(font, rest, NO); + CTLineRef token = CreateRawLine(font, @"…", NO); line = CTLineCreateTruncatedLine(full, width, kCTLineTruncationEnd, token); if (!line) line = CFRetain(token); CFRelease(full); CFRelease(token); diff --git a/CHANGELOG b/CHANGELOG index 83780177..1c45f736 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,9 +1,49 @@ # Changelog -## 0.8.1 - Future +## 0.8.3 - Future +* Fix several issues that would cause disconnection and disconnect-looping +* Add a client for T3 Code +* Weather app shows an icon for current conditions, and can add the weather to the status bar and the Glanceboard system card (Weather menu > Settings) + +## 0.8.2 - 2026-10-03 + +### User Interface Customization + +* Customizable app switcher: put it on the left, right, bottom, or make it a popup +* The status bar can be combined with the app switcher when it's on the bottom +* The display depth is configurable when the app switcher is on the bottom +* Add Dense style for battery indicators + +### Other Features and Fixes + +* Animations have configurable speed +* Fix an issue where animations on the phone preview display stopped before finishing +* Terminal app lists every configured host, including ones that are still connecting +* Music app shows its library-browsing app picker on the front page when nothing is playing +* Fix iOS failing to connect when an incompatible firmware version is installed +* Mark stock firmware 2.3.2 as validated for installing custom firmware over + +## 0.8.1 - 2026-09-30 + +### Assorted Features and Bug Fixes + +* Improve connection reliability +* Add a video player, launched by picking a video file in the Files app +* Fix some bugs impacting the Microphones app * Add config settings to disable animation categories +* Reduce phone CPU usage +* Putting on the glasses auto-wakes the screen +* Add a button to the phone screen top bar to open EvenHub app phone UI +* Glanceboard font can be configured separately from the main font +* Move Terminal and Navigate settings out of the Settings app and into their respective apps + +### EvenHub + * EvenHub compatibility uses menu animations and is closer to style matched +* EvenHub apps show an icon in the phone top bar to open their phone-side UI +* Fix a crash on install of larger EvenHub apps +* Fix line wrapping on EvenHub permissions dialog ## 0.8.0 - 2026-09-26 diff --git a/CLAUDE.md b/CLAUDE.md index 7dd039ef..128f6379 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -21,9 +21,10 @@ relevant tools and an Android device ID to install on; do this in build_paths.sh, which is created from build_paths.sh.template and not checked into git. -To lint and typecheck, prefer to use build.sh; if the user asks you to run or +To typecheck, prefer to use build.sh; if the user asks you to run or test the app, use build_and_run.sh and then use `adb logcat` to view the -results. +results. `lint.sh` (`npm run lint`) runs oxlint over the TypeScript, including +type-aware rules; rule selection is in .oxlintrc.json. If you are working on low-level communication bits, consider checking out https://github.com/Commute773/g2-kit-unofficial/ and referring to ble/docs/ diff --git a/README.md b/README.md index a8f40e87..355a206e 100644 --- a/README.md +++ b/README.md @@ -16,9 +16,10 @@ User-facing documentation lives at https://faceclaw.org/. ![App launcher](website/screenshots/launcher.png) ![Music player](website/screenshots/music-player.png) +![Customization settings](website/screenshots/settings-customization.png) ![Assistant settings](website/screenshots/settings-assistant.png) -![Display settings](website/screenshots/settings-display.png) ![Compass](website/screenshots/compass.png) +![Blocks](website/screenshots/blocks.png) ## Installation @@ -67,16 +68,14 @@ import it into the new version. your own long-running OpenClaw agent. * **Multitasking**, with an app-switcher sidebar and app launcher. * **Glanceboard**, a dashboard with widgets that you can quickly view with a - tap or tap-and-hold from the screen-off state. - * **Mostly-compatible with EvenHub apps.** + * **Mostly-compatible with EvenHub.** Install and runapps from the Even app + store. * **A lock screen**; glasses lock automatically when you take them off and unlock when you unlock your phone. * **The full display.** Full-screen apps can use the full 640x480 display area, rather than the 576x288 that EvenHub apps can use. - * **Integration with Android notifications**: a top bar that shows the same - icons your phone does, popups when notifications arrive, and menu items to - dismiss or use Android-app-provided custom actions like mark as read or - quick reply. + * **Integration with Android notifications**: shows the same notification + icons your phone does, and popups when notifications arrive. * **Terminal mirroring.** Mirror terminal apps such as Claude Code or Codex CLI with [g2mirror](https://github.com/jimrandomh/g2mirror), view them on the glasses, and send them inputs with the voice assistant. @@ -88,17 +87,12 @@ import it into the new version. * **Power management**: the glasses go to sleep properly when the screen is off, and wake when you double-tap the ring or speak the wakeword, allowing battery life similar to the stock Even app. - * **A Wear OS watch app** that complements (or replaces) the R1 ring: tap, + * **A Wear OS watch app** that replaces (and outdoes) the R1 ring: tap, swipe, hold and crown gestures, side buttons, app launching and window switching, voice or keyboard queries to the assistant with the reply on your wrist, typing into apps, and glasses status/lock/display control. - * **On-phone screen mirroring**: Shows the glasses screen contents and - provides a phone-screen touch area equivalent to the R1 ring or the Wear OS - watch app. This can also be used to try out the app without pairing or - flashing a real pair of glasses. - * **Dual-language NativeScript architecture**, with Java for the - multithreaded Android API and bluetooth stack bits, Typescript for the bits - you want to hack on. + * **On-phone screen mirroring**: Your phone screen shows the same contents as + the glasses, and you can use the phone touchscreen as an input device.. ## Watch remote (Wear OS) diff --git a/app/app.css b/app/app.css index 6e2738bf..b9ce8ecb 100644 --- a/app/app.css +++ b/app/app.css @@ -16,8 +16,9 @@ ActionItem { color: #ffffff; } -/* The main page's custom titleView: title left, connection status and the - * warning triangle right-aligned beside the overflow menu. */ +/* The main page's custom titleView: title left; the EvenHub app icon, + * connection status and the warning triangle right-aligned beside the + * overflow menu. */ .action-bar-title { color: #ffffff; font-size: 18; @@ -32,6 +33,16 @@ ActionItem { vertical-align: center; } +/* The foreground EvenHub app's icon (opens its phone UI); the padding widens + * the tap target. */ +.action-bar-app-icon { + width: 32; + height: 32; + padding: 5.5; + margin-right: 8; + vertical-align: center; +} + /* The emoji brings its own yellow; the padding widens the tap target. */ .action-bar-warning { font-size: 18; @@ -292,6 +303,10 @@ Button.pairing-link { color: #2a5bc4; font-size: 14; border-width: 0; + /* Buttons keep their Material elevation shadow even with a transparent + * background, which shows up as a stray, downward-offset outline. */ + android-elevation: 0; + android-dynamic-elevation-offset: 0; } /* diff --git a/app/app.ios.css b/app/app.ios.css index 121a6bf7..17bcf99b 100644 --- a/app/app.ios.css +++ b/app/app.ios.css @@ -174,6 +174,7 @@ Page.main-page { } .main-page .action-bar-title { font-size: 18; font-weight: bold; color: #ffffff; } .main-page .action-bar-status { font-size: 14; color: #ffffff; margin-right: 12; } +.main-page .action-bar-warning { font-size: 18; padding: 4 6; } .main-page .overflow-button { background-color: transparent; color: #ffffff; diff --git a/app/apps/ai-chat/ai-chat-app.ts b/app/apps/ai-chat/ai-chat-app.ts index 256c90af..83b176e4 100644 --- a/app/apps/ai-chat/ai-chat-app.ts +++ b/app/apps/ai-chat/ai-chat-app.ts @@ -108,6 +108,7 @@ class AiChatLayer implements Layer { return; case "long-press-release": this.draft.release(); return; case "double-click": this.draft.cancel(); shell.yieldFocusToSidebar(); return; + default: return; } } menuItems(): MenuItem[] { @@ -123,20 +124,20 @@ class AiChatLayer implements Layer { { label: "Switch session", disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Sessions", [...this.conversations.list()].reverse().map((item) => ({ label: `${item.id === record.id ? "✓ " : ""}${this.conversations.title(item)}`, - onSelect: (ctx) => { if (this.conversations.select(item.id)) ctx.stack.clearToBase(); }, + onSelect: (pickCtx) => { if (this.conversations.select(item.id)) pickCtx.stack.clearToBase(); }, }))) }, { label: `Model: ${assistantModelLabel(record.model)}`, disabled: () => external || busy(), onSelect: (ctx) => submenu(ctx, "Model", ASSISTANT_MODEL_CHOICES.map((model) => ({ label: `${model === record.model ? "✓ " : ""}${assistantModelLabel(model)}`, disabled: () => !this.conversations.available(model, record.reasoning), - onSelect: (ctx) => { if (this.conversations.configure(model, record.reasoning)) ctx.stack.clearToBase(); }, + onSelect: (pickCtx) => { if (this.conversations.configure(model, record.reasoning)) pickCtx.stack.clearToBase(); }, }))) }, { label: `Reasoning: ${record.reasoning}`, disabled: () => { const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() }); return external || busy() || !llm?.effort; }, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({ label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`, - onSelect: (ctx) => { if (this.conversations.configure(record.model, reasoning)) ctx.stack.clearToBase(); }, + onSelect: (pickCtx) => { if (this.conversations.configure(record.model, reasoning)) pickCtx.stack.clearToBase(); }, }))) }, ...(record.session?.isTurnActive() ? [{ label: "Cancel response", onSelect: (ctx: LayerContext) => { record.session?.cancel(); ctx.stack.clearToBase(); diff --git a/app/apps/all-apps.ts b/app/apps/all-apps.ts index 06f40493..f46c4005 100644 --- a/app/apps/all-apps.ts +++ b/app/apps/all-apps.ts @@ -4,6 +4,7 @@ import launcherApp from "./launcher"; import timerApp from "./timer"; import calculatorApp from "./calculator"; import terminalApp from "./terminal"; +import t3codeApp from "./t3code"; import filesApp from "./files"; import musicApp from "./music"; import nightscoutApp from "./nightscout"; @@ -38,6 +39,7 @@ export const ALL_APPS: readonly AppDefinition[] = [ timerApp, calculatorApp, terminalApp, + t3codeApp, filesApp, musicApp, nightscoutApp, diff --git a/app/apps/blocks/blocks-app.worker.ts b/app/apps/blocks/blocks-app.worker.ts index b17adaf0..2b570197 100644 --- a/app/apps/blocks/blocks-app.worker.ts +++ b/app/apps/blocks/blocks-app.worker.ts @@ -46,6 +46,12 @@ const CELL = 12; const BOARD_X = 60; const BOARD_Y = 10; const PANEL_X = 230; +/** + * Height the layout above is drawn for: a min-height window's viewport under + * the standard top bar. A taller viewport (the status bar folded into the + * switcher row, a full-panel display) centers it vertically. + */ +const LAYOUT_HEIGHT = 260; /** * Gravity starts here and speeds up with score (see dropIntervalMs). Slow @@ -284,6 +290,11 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { updateTickTimer(window); } break; + case "navigation-sensors": + case "resize-window": + case "text-input": + case "tool-call": + break; } }; @@ -349,7 +360,7 @@ function handleInput(window: BlocksWindow, event: InputEvent, frameId: number): // An open window menu owns all input (it closes itself via pop); menus are // list UIs, so watch swipes take their standard fallback meanings there. if (window.menu?.isOpen()) { - window.menu + void window.menu .handleInput(directionalFallback(event)) .catch((error) => console.error(`blocks menu input failed: ${error}`)) .then(() => renderAndSubmit(window, frameId)); @@ -596,7 +607,7 @@ function paint(window: BlocksWindow): Plane[] { } function paintContent(window: BlocksWindow): GrayImage { - const image = new GrayImage(window.viewportWidth, window.viewportHeight, 0); + const image = new GrayImage(window.viewportWidth, LAYOUT_HEIGHT, 0); image.drawRect(BOARD_X - 2, BOARD_Y - 2, COLS * CELL + 4, ROWS * CELL + 4, 120); if (window.phase === "paused") { // Hide the board so pausing can't be used to study the stack. @@ -605,9 +616,17 @@ function paintContent(window: BlocksWindow): GrayImage { drawCenteredIn(image, smallFont, BOARD_X, COLS * CELL, BOARD_Y + 150, `${GESTURE_DOUBLE_CLICK} leave`, 150); } else { paintBoard(image, window); - if (window.phase === "game-over") paintGameOver(image, window); + if (window.phase === "game-over") paintGameOver(image); } paintPanel(image, window); + return centeredInViewport(window, image); +} + +/** Place the fixed-height layout vertically centered in the window's viewport. */ +function centeredInViewport(window: BlocksWindow, layout: GrayImage): GrayImage { + if (layout.height === window.viewportHeight) return layout; + const image = new GrayImage(window.viewportWidth, window.viewportHeight, 0); + layout.composeInto(image, 0, Math.max(0, Math.floor((window.viewportHeight - layout.height) / 2))); return image; } @@ -634,10 +653,44 @@ function paintBoard(image: GrayImage, window: BlocksWindow): void { } function drawCell(image: GrayImage, x: number, y: number, shade: number): void { - image.fillRect(BOARD_X + x * CELL, BOARD_Y + y * CELL, CELL - 1, CELL - 1, shade); + drawTile(image, BOARD_X + x * CELL, BOARD_Y + y * CELL, shade); } -function paintGameOver(image: GrayImage, window: BlocksWindow): void { +/** + * One beveled tile (CELL - 1 square, leaving a 1px gap): lit top and left + * edges, shadowed bottom and right, stepping in over two rings to a raised + * face. The face keeps the piece's shade, so pieces stay distinguishable by + * brightness after 4bpp quantization. + */ +function drawTile(image: GrayImage, x: number, y: number, shade: number): void { + const size = CELL - 1; + const lit = (amount: number): number => Math.round(shade + (255 - shade) * amount); + const dim = (factor: number): number => Math.round(shade * factor); + image.fillRect(x, y, size, size, dim(0.8)); + drawBevelRing(image, x, y, size, lit(0.6), dim(0.4), shade); + drawBevelRing(image, x + 1, y + 1, size - 2, lit(0.3), dim(0.6), shade); + image.fillRect(x + 3, y + 3, size - 6, size - 6, shade); +} + +/** A square's 1px edge: light along the top and left, dark along the bottom and right, mid at the two mixed corners. */ +function drawBevelRing( + image: GrayImage, + x: number, + y: number, + size: number, + light: number, + dark: number, + mid: number, +): void { + image.fillRect(x, y, size, 1, light); + image.fillRect(x, y, 1, size, light); + image.fillRect(x, y + size - 1, size, 1, dark); + image.fillRect(x + size - 1, y, 1, size, dark); + image.setPixel(x + size - 1, y, mid); + image.setPixel(x, y + size - 1, mid); +} + +function paintGameOver(image: GrayImage): void { const boxY = BOARD_Y + 70; image.fillRect(BOARD_X, boxY, COLS * CELL, 100, 0); image.drawRect(BOARD_X + 4, boxY + 4, COLS * CELL - 8, 92, 150); @@ -660,7 +713,7 @@ function paintPanel(image: GrayImage, window: BlocksWindow): void { const offsetX = PANEL_X + Math.round((previewBoxSize - (maxX - minX + 1) * CELL) / 2); const offsetY = previewY + Math.round((previewBoxSize - (maxY - minY + 1) * CELL) / 2); for (const [cx, cy] of cells) { - image.fillRect(offsetX + (cx - minX) * CELL, offsetY + (cy - minY) * CELL, CELL - 1, CELL - 1, def.shade); + drawTile(image, offsetX + (cx - minX) * CELL, offsetY + (cy - minY) * CELL, def.shade); } const statX = PANEL_X + previewBoxSize + 40; diff --git a/app/apps/calculator/calculator-app.ts b/app/apps/calculator/calculator-app.ts index 53b68053..e71b827a 100644 --- a/app/apps/calculator/calculator-app.ts +++ b/app/apps/calculator/calculator-app.ts @@ -96,8 +96,7 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce }, ]; - let app: InProcessWindow; - app = createInProcessWindow({ + const app = createInProcessWindow({ appId: "calculator", windowId: CALCULATOR_WINDOW_ID, title: "Calculator", diff --git a/app/apps/calculator/math/advanced.ts b/app/apps/calculator/math/advanced.ts index 3ceb5112..fcba203e 100644 --- a/app/apps/calculator/math/advanced.ts +++ b/app/apps/calculator/math/advanced.ts @@ -24,7 +24,7 @@ import { findDigits, highlightedMatch, } from "./digit-search"; -import { type MathWorkload, longRunOffer as makeLongRunOffer, type LongRunOffer } from "./workload"; +import { type MathWorkload } from "./workload"; import { binomial, factorizationText, diff --git a/app/apps/calculator/math/number-theory.ts b/app/apps/calculator/math/number-theory.ts index 405b3c42..dd3f83e9 100644 --- a/app/apps/calculator/math/number-theory.ts +++ b/app/apps/calculator/math/number-theory.ts @@ -322,7 +322,7 @@ export function digitalRoot(value: number): number { export function isPalindrome(value: number): boolean { const text = String(Math.abs(value)); - return text === [...text].reverse().join(""); + return text === text.split("").reverse().join(""); } export function isPerfect(value: number): boolean { diff --git a/app/apps/calculator/math/polynomial.ts b/app/apps/calculator/math/polynomial.ts index 404ed4a7..b93e7269 100644 --- a/app/apps/calculator/math/polynomial.ts +++ b/app/apps/calculator/math/polynomial.ts @@ -123,7 +123,7 @@ export class Polynomial { */ dividingByRoot(root: MathNumber): Polynomial { if (this.degree < 1) return this; - const quotient: MathNumber[] = new Array(this.degree).fill(integer(0)); + const quotient: MathNumber[] = Array.from({ length: this.degree }, () => integer(0)); let carry: MathNumber = integer(0); for (let power = this.degree; power >= 1; power--) { const coefficient = add(this.at(power), carry); @@ -235,7 +235,7 @@ function addAt(value: MathNumber, power: number, coefficients: MathNumber[]): vo function convolve(lhs: MathNumber[], rhs: MathNumber[]): MathNumber[] { if (lhs.length === 0 || rhs.length === 0) return [integer(0)]; - const result: MathNumber[] = new Array(lhs.length + rhs.length - 1).fill(integer(0)); + const result: MathNumber[] = Array.from({ length: lhs.length + rhs.length - 1 }, () => integer(0)); for (let i = 0; i < lhs.length; i++) { if (isZero(lhs[i]!)) continue; for (let j = 0; j < rhs.length; j++) { diff --git a/app/apps/calculator/math/solver.ts b/app/apps/calculator/math/solver.ts index 385d2a44..1a34afab 100644 --- a/app/apps/calculator/math/solver.ts +++ b/app/apps/calculator/math/solver.ts @@ -19,7 +19,6 @@ import { multiplyExpr, negate, num, - numberValue, relation as relationExpr, spokenRelation, subtractExpr, diff --git a/app/apps/calculator/math/symbolic-integral.ts b/app/apps/calculator/math/symbolic-integral.ts index 6897868f..7d71367f 100644 --- a/app/apps/calculator/math/symbolic-integral.ts +++ b/app/apps/calculator/math/symbolic-integral.ts @@ -29,12 +29,12 @@ import { numberEquals, rational, reciprocal, + isZero, } from "./math-number"; import { simplify } from "./simplifier"; import { differentiate, integrateNumeric } from "./derivative"; import { evaluateValue } from "./evaluator"; import { Polynomial } from "./polynomial"; -import { isZero } from "./math-number"; /** Indefinite integral with respect to `variable`, without the constant. */ export function integrate(expression: MathExpression, variable: string): MathExpression | null { diff --git a/app/apps/calculator/math/text-renderer.ts b/app/apps/calculator/math/text-renderer.ts index 04adcb95..57dbad68 100644 --- a/app/apps/calculator/math/text-renderer.ts +++ b/app/apps/calculator/math/text-renderer.ts @@ -103,8 +103,7 @@ function render(expression: MathExpression, parentPrecedence: number): string { return wrap(`1/${inner}`, parentPrecedence > PRECEDENCE.product); } const body = - `${render(expression.base, PRECEDENCE.power + 1)}` + - `^${render(expression.exponent, PRECEDENCE.power + 1)}`; + `${render(expression.base, PRECEDENCE.power + 1)}^${render(expression.exponent, PRECEDENCE.power + 1)}`; return wrap(body, parentPrecedence > PRECEDENCE.power); } @@ -116,11 +115,7 @@ function render(expression: MathExpression, parentPrecedence: number): string { } case "relation": - return ( - `${render(expression.lhs, PRECEDENCE.relation)} ` + - `${expression.relation} ` + - `${render(expression.rhs, PRECEDENCE.relation)}` - ); + return `${render(expression.lhs, PRECEDENCE.relation)} ${expression.relation} ${render(expression.rhs, PRECEDENCE.relation)}`; } } diff --git a/app/apps/calculator/math/trigonometry.ts b/app/apps/calculator/math/trigonometry.ts index 53ba6fe7..82e19ab8 100644 --- a/app/apps/calculator/math/trigonometry.ts +++ b/app/apps/calculator/math/trigonometry.ts @@ -26,14 +26,14 @@ export type SolutionFamily = { baseExpression: MathExpression | null; }; -export function familyValues(family: SolutionFamily, lower: number, upper: number): number[] { - if (family.period <= 0) { - return family.base >= lower && family.base <= upper ? [family.base] : []; +export function familyValues(solutionFamily: SolutionFamily, lower: number, upper: number): number[] { + if (solutionFamily.period <= 0) { + return solutionFamily.base >= lower && solutionFamily.base <= upper ? [solutionFamily.base] : []; } const results: number[] = []; - let n = Math.floor((lower - family.base) / family.period); + let n = Math.floor((lower - solutionFamily.base) / solutionFamily.period); for (;;) { - const value = family.base + n * family.period; + const value = solutionFamily.base + n * solutionFamily.period; if (value > upper) break; if (value >= lower) results.push(value); n += 1; @@ -42,14 +42,14 @@ export function familyValues(family: SolutionFamily, lower: number, upper: numbe return results; } -export function familyText(family: SolutionFamily): string { - const baseText = family.baseExpression - ? plain(family.baseExpression) - : describeNumber(fromDecimal(family.base)); +export function familyText(solutionFamily: SolutionFamily): string { + const baseText = solutionFamily.baseExpression + ? plain(solutionFamily.baseExpression) + : describeNumber(fromDecimal(solutionFamily.base)); // The separator is load-bearing: "2*pi" run straight into the index reads // as "2*pin", a different and nonsensical expression to anyone scanning it // on the lens. - const periodText = describePi(family.period); + const periodText = describePi(solutionFamily.period); return `${baseText} + ${periodText}·n`; } diff --git a/app/apps/calendar/calendar-app.ts b/app/apps/calendar/calendar-app.ts index 80098e9b..39caf6ff 100644 --- a/app/apps/calendar/calendar-app.ts +++ b/app/apps/calendar/calendar-app.ts @@ -21,7 +21,6 @@ export const CALENDAR_SURFACE_ID = "window:calendar"; */ export function createCalendarAppWindow(options: InProcessAppOptions): InProcessWindow { let requesting = false; - let app: InProcessWindow; let unsubscribe = () => {}; let tick: ReturnType | null = null; @@ -37,7 +36,7 @@ export function createCalendarAppWindow(options: InProcessAppOptions): InProcess .finally(() => { requesting = false; }); }; - app = createInProcessWindow({ + const app = createInProcessWindow({ appId: "calendar", windowId: CALENDAR_WINDOW_ID, title: "Calendar", diff --git a/app/apps/compass/calibration-layer.ts b/app/apps/compass/calibration-layer.ts index 55acb41d..772fccef 100644 --- a/app/apps/compass/calibration-layer.ts +++ b/app/apps/compass/calibration-layer.ts @@ -80,7 +80,7 @@ export class CompassCalibrationLayer implements Layer { // The crosshair marks where the wearer is looking, so its horizontal // position must be the true centre of the display, not of the app - // viewport the sidebar has pushed to the right. Vertically it just sits in + // viewport a side strip has pushed off centre. Vertically it just sits in // the free space between the instructions and the readouts; the window // band's own placement on screen is the wearer's Display setting and // doesn't affect the heading being aimed. diff --git a/app/apps/compass/compass-app.ts b/app/apps/compass/compass-app.ts index 3c7b9b1c..0f546d7f 100644 --- a/app/apps/compass/compass-app.ts +++ b/app/apps/compass/compass-app.ts @@ -38,6 +38,9 @@ const TOP_PAD = 2; const BOTTOM_PAD = 4; /** Clearance between the rose's widest point and the viewport edge. */ const EDGE_PAD = 8; +/** Inset of the top-right magnetometer calibration readout from the window edges. */ +const ACCURACY_TOP_PAD = TOP_PAD + 4; +const ACCURACY_RIGHT_PAD = EDGE_PAD + 4; /** Cap on the rose's radius, so a tall window doesn't get a comical one. */ const MAX_ROSE_RADIUS = 98; @@ -161,7 +164,7 @@ class CompassLayer implements Layer { const readoutHeight = headingFont.lineHeight; // One column centred on the display's true centre, so the rose sits where - // the wearer is looking rather than 32px right of it. + // the wearer is looking rather than 32px beside it (with a side strip). const cx = screenCenterInViewportX(); // The rose is sized and placed first — it hangs off the bottom edge — and @@ -197,7 +200,7 @@ class CompassLayer implements Layer { y += smallStep; } const accuracyText = magneticAccuracyText(this.diagnostics); - image.drawText(small, width - EDGE_PAD - small.measureText(accuracyText), TOP_PAD, accuracyText, 175); + image.drawText(small, width - ACCURACY_RIGHT_PAD - small.measureText(accuracyText), ACCURACY_TOP_PAD, accuracyText, 175); return image; } @@ -211,10 +214,9 @@ class CompassLayer implements Layer { } export function createCompassAppWindow(options: InProcessAppOptions): InProcessWindow { - let app: InProcessWindow; let requestRender = () => {}; const layer = new CompassLayer(() => requestRender()); - app = createInProcessWindow({ + const app = createInProcessWindow({ appId: "compass", windowId: COMPASS_WINDOW_ID, title: "Compass", diff --git a/app/apps/developer/accelerometer-demo.ts b/app/apps/developer/accelerometer-demo.ts index 657e85cf..bdc88c0a 100644 --- a/app/apps/developer/accelerometer-demo.ts +++ b/app/apps/developer/accelerometer-demo.ts @@ -2,7 +2,7 @@ import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fon import { GrayImage } from "../../graphics/image"; import { clamp } from "../../util/numeric-util"; import { addImuListener, imuSourceLabel, setImuReportEnabled, type ImuReading } from "../../native/imu"; -import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures"; +import { type InputEvent } from "../../ui/gestures"; import { Layer, type LayerContext } from "../../ui/layers"; import { shell } from "../../ui/shell/shell"; diff --git a/app/apps/developer/buzzer-demo.ts b/app/apps/developer/buzzer-demo.ts index 1772d555..258ace7c 100644 --- a/app/apps/developer/buzzer-demo.ts +++ b/app/apps/developer/buzzer-demo.ts @@ -1,7 +1,7 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { GrayImage } from "../../graphics/image"; import { Layer, type LayerActions, type LayerContext } from "../../ui/layers"; -import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, type InputEvent } from "../../ui/gestures"; +import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures"; import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; import { centeredTextY, tightRowHeight } from "../../ui/metrics"; import { diff --git a/app/apps/developer/developer-app.ts b/app/apps/developer/developer-app.ts index 1f1c831c..42c0ef4a 100644 --- a/app/apps/developer/developer-app.ts +++ b/app/apps/developer/developer-app.ts @@ -1,4 +1,4 @@ -import { MenuLayer, drawSubmenuIndicator, type MenuItem } from "../../ui/menu"; +import { MenuLayer, submenuItem, type MenuItem } from "../../ui/menu"; import { ScreenTestLayer } from "./screen-test"; import { InputEventsLayer } from "./input-events"; import { shell } from "../../ui/shell/shell"; @@ -6,14 +6,13 @@ import { BuzzerDemoLayer } from "./buzzer-demo"; import { AccelerometerDemoLayer } from "./accelerometer-demo"; import { BandwidthBenchmarkLayer } from "./bandwidth-benchmark"; import { LightSensorDemoLayer } from "./light-sensor-demo"; +import { MicrophonesMonitorLayer } from "./microphones-monitor"; import { ResourceUsageLayer } from "./resource-usage"; import { RingStatusLayer } from "./ring-status"; import { LoadAppFromQrLayer, LoadAppFromUrlLayer } from "./load-app"; import { unicodeTestMenu } from "./unicode-test"; import { type AppContext } from "../app-definition"; -import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { appViewportSize } from "../../ui/shell/geometry"; -import { LIST_ROW_TEXT_INSET } from "../../ui/metrics"; import { createInProcessWindow, YieldAtRootLayer, @@ -36,22 +35,10 @@ const MENU_LAYOUT = { opaque: true, }; -/** A row that opens a nested page: the label plus a right-edge ">". */ -function submenuItem(label: string, onSelect: MenuItem["onSelect"]): MenuItem { - return { - label, - onSelect, - render: ({ image, x, y, width, height, selected, disabled, text }) => { - const font = getDefaultSmallFont(); - const value = disabled ? 70 : selected ? 255 : 200; - image.drawText(font, x, y + LIST_ROW_TEXT_INSET, text, value); - drawSubmenuIndicator(image, font, x, y, width, height, value); - }, - }; -} +type OpenPage = (ctx: Parameters[0]) => void; /** The diagnostic demos, one level down from the root menu. */ -function debugTestsMenu(openInputEvents: (ctx: Parameters[0]) => void): MenuLayer { +function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): MenuLayer { return new MenuLayer( "Debug tests", [ @@ -94,6 +81,12 @@ function debugTestsMenu(openInputEvents: (ctx: Parameters[ ctx.stack.push(new RingStatusLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender)); }, }, + { + label: "Microphones", + disabled: global.isIOS, + description: global.isIOS ? "Not available on iOS yet." : undefined, + onSelect: openMicrophones, + }, { label: "BLE bandwidth", onSelect: (ctx) => { @@ -116,6 +109,7 @@ function debugTestsMenu(openInputEvents: (ctx: Parameters[ */ export function createDeveloperAppWindow(appContext: AppContext, options: InProcessAppOptions): InProcessWindow { let inputEvents: InputEventsLayer | null = null; + let microphones: MicrophonesMonitorLayer | null = null; const menu = new MenuLayer( "Developer", [ @@ -150,6 +144,11 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc () => { if (inputEvents === page) inputEvents = null; }); inputEvents = page; inputCtx.stack.push(page); + }, (micCtx) => { + const page = new MicrophonesMonitorLayer(DEVELOPER_WINDOW_ID, micCtx.actions.requestRender, + () => { if (microphones === page) microphones = null; }); + microphones = page; + micCtx.stack.push(page); })); }), ], @@ -164,7 +163,7 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc icon: "wrench", closeable: true, actions: options.actions, - menuItems: () => inputEvents?.menuItems() ?? [], + menuItems: () => inputEvents?.menuItems() ?? microphones?.menuItems() ?? [], // Dictating a URL is the one thing worth speaking at in this app; the // load pages take the text and every other page ignores it. receiveTextInput: (text) => { diff --git a/app/apps/developer/microphones-monitor.ts b/app/apps/developer/microphones-monitor.ts new file mode 100644 index 00000000..44100a34 --- /dev/null +++ b/app/apps/developer/microphones-monitor.ts @@ -0,0 +1,426 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GrayImage } from "../../graphics/image"; +import { clamp } from "../../util/numeric-util"; +import { + addAudioMonitorListeners, + audioMonitorCommunicator, + isFaceclawAudioCaptureActive, + type AudioArm, +} from "../../native/audio-monitor"; +import { micArrayController } from "../microphones/mic-control"; +import { + MIC_RET_NONE, + STOCK_PACKET_BYTES, + decodeMicStatus, + decodeMicStreamFrame, + splitInterleavedPcm16, + type MicStatus, + type MicStreamFrame, +} from "../microphones/mic-protocol"; +import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures"; +import { type MenuItem } from "../../ui/menu"; +import { Layer, type LayerContext } from "../../ui/layers"; +import { shell } from "../../ui/shell/shell"; + +// Packet rate / byte rate / largest gap are measured over this trailing window. +const RATE_WINDOW_MS = 2_000; +// Activity strip: packet counts per bucket over the last HISTORY_MS. +const HISTORY_MS = 20_000; +const BUCKET_MS = 500; +const BUCKETS = HISTORY_MS / BUCKET_MS; +// Repaint cadence; also how often a replaced communicator is re-subscribed. +const TICK_MS = 250; +// Stock packet trailer (see Lc3PacketFramer.kt): SSR, angle, counter. +const STOCK_SSR_OFFSET = 200; +const STOCK_ANGLE_OFFSET = 202; +const STOCK_COUNTER_OFFSET = 204; +// Counter gaps at or beyond this are late/duplicate copies, not losses. +const STOCK_LATE_GAP = 128; +const LEVEL_FLOOR_DB = -90; + +// wallMs (JS receipt time) windows the rates; arrivalMs (the session core's +// elapsed-realtime stamp, taken on the BLE thread) measures gaps without the +// main-thread hop's jitter. +type Arrival = { wallMs: number; arrivalMs: number; bytes: number }; + +type ArmStats = { + packets: number; + stockPackets: number; + smPackets: number; + otherPackets: number; + lastOtherBytes: number; + lastArrivalWallMs: number | null; + recent: Arrival[]; + buckets: number[]; + stockCounter: number | null; + stockLost: number; + stockDuplicate: number; + stockAngle: number; + stockSsr: number; + frame: MicStreamFrame | null; + smSeq: number | null; + smSeqGaps: number; + channelDb: number[]; + status: MicStatus | null; + statusAtMs: number | null; +}; + +function emptyStats(): ArmStats { + return { + packets: 0, + stockPackets: 0, + smPackets: 0, + otherPackets: 0, + lastOtherBytes: 0, + lastArrivalWallMs: null, + recent: [], + buckets: Array.from({ length: BUCKETS }, () => 0), + stockCounter: null, + stockLost: 0, + stockDuplicate: 0, + stockAngle: 0, + stockSsr: 0, + frame: null, + smSeq: null, + smSeqGaps: 0, + channelDb: [], + status: null, + statusAtMs: null, + }; +} + +/** + * Passive microphone monitor: shows what the glasses are sending on the + * audio characteristic, per temple, without enabling anything. Open it, then + * activate the mics from another app (Microphones, Transcribe, voice input) + * and come back: collection continues while this page is hidden, so the + * counters and the 20 s activity strip show whether that activation produced + * a stream, what kind (stock mono LC3 or CFW 'SM' frames), and how healthy it + * is. CFW mic_control status records are shown when they arrive: after a + * CONFIGURE, with the periodic battery read, or from this window's "Query mic + * status" menu item (a status request only; it touches no mic hardware). + * Only the right temple can send status; each lens's debug overlay shows its + * own counters. + */ +export class MicrophonesMonitorLayer implements Layer { + private readonly stats: Record<"L" | "R", ArmStats> = { L: emptyStats(), R: emptyStats() }; + private unknownArmPackets = 0; + private bucketStartMs = Date.now(); + private communicator: any = null; + private unsubscribe: (() => void) | null = null; + private timer: ReturnType | null = null; + private removed = false; + + constructor( + private readonly windowId: string, + private readonly requestRender: () => void, + private readonly onClosed: () => void = () => {}, + ) { + this.tick(); + this.timer = setInterval(() => this.tick(), TICK_MS); + } + + onRemoved(): void { + this.removed = true; + if (this.timer !== null) { + clearInterval(this.timer); + this.timer = null; + } + this.unsubscribe?.(); + this.unsubscribe = null; + this.communicator = null; + this.onClosed(); + } + + private tick(): void { + if (this.removed) return; + const communicator = audioMonitorCommunicator(); + if (communicator !== this.communicator) { + this.unsubscribe?.(); + this.communicator = communicator; + this.unsubscribe = addAudioMonitorListeners( + communicator, + (data, arm, arrivalMs) => this.onPacket(data, arm, arrivalMs), + (body, arm) => this.onStatus(body, arm), + ); + } + this.advanceBuckets(Date.now()); + if (shell.isWindowVisible(this.windowId)) this.requestRender(); + } + + private advanceBuckets(nowMs: number): void { + while (nowMs - this.bucketStartMs >= BUCKET_MS) { + this.bucketStartMs += BUCKET_MS; + for (const stats of Object.values(this.stats)) { + stats.buckets.shift(); + stats.buckets.push(0); + } + } + } + + private onPacket(data: Uint8Array, arm: AudioArm, arrivalMs: number): void { + if (this.removed) return; + if (arm === "?") { + this.unknownArmPackets++; + return; + } + const stats = this.stats[arm]; + const nowMs = Date.now(); + this.advanceBuckets(nowMs); + stats.packets++; + stats.buckets[BUCKETS - 1]!++; + stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length }); + stats.lastArrivalWallMs = nowMs; + + const frame = decodeMicStreamFrame(data); + if (frame) { + this.onStreamFrame(stats, frame); + } else if (data.length === STOCK_PACKET_BYTES) { + this.onStockPacket(stats, data); + } else { + stats.otherPackets++; + stats.lastOtherBytes = data.length; + } + } + + private onStockPacket(stats: ArmStats, data: Uint8Array): void { + stats.stockPackets++; + const counter = data[STOCK_COUNTER_OFFSET]!; + if (stats.stockCounter !== null) { + const gap = (counter - stats.stockCounter) & 0xff; + if (gap === 0 || gap >= STOCK_LATE_GAP) { + stats.stockDuplicate++; + return; + } + stats.stockLost += gap - 1; + } + stats.stockCounter = counter; + stats.stockSsr = readS16(data, STOCK_SSR_OFFSET); + stats.stockAngle = readS16(data, STOCK_ANGLE_OFFSET); + } + + private onStreamFrame(stats: ArmStats, frame: MicStreamFrame): void { + stats.smPackets++; + if (stats.smSeq !== null && frame.sequence !== ((stats.smSeq + 1) & 0xffff)) stats.smSeqGaps++; + stats.smSeq = frame.sequence; + stats.frame = frame; + const channels = splitInterleavedPcm16(frame); + stats.channelDb = channels ? channels.map(rmsDb) : []; + } + + private onStatus(body: Uint8Array, arm: AudioArm): void { + if (this.removed || arm === "?") return; + const status = decodeMicStatus(body); + if (!status) return; + this.stats[arm].status = status; + this.stats[arm].statusAtMs = Date.now(); + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const line = font.lineHeight; + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + const nowMs = Date.now(); + + image.drawText(font, 16, 5, "Microphones (passive)", 230); + const capture = isFaceclawAudioCaptureActive() ? "Faceclaw capture: on" : "Faceclaw capture: off"; + image.drawText(font, width - 16 - font.measureText(capture), 5, capture, 170); + + if (!this.communicator) { + image.drawText(font, 16, 5 + line * 2, "Glasses not connected.", 190); + image.drawText(font, 16, height - 16, `${GESTURE_DOUBLE_CLICK} back`, 110); + return image; + } + + const gutter = 16; + const columnWidth = Math.floor((width - gutter * 3) / 2); + const stripHeight = 22; + const top = 5 + line + 6; + const bottom = height - 16 - stripHeight - 8; + this.paintArm(image, "L", "Left", gutter, top, columnWidth, bottom, nowMs); + this.paintArm(image, "R", "Right", gutter * 2 + columnWidth, top, columnWidth, bottom, nowMs); + + const stripY = height - 16 - stripHeight - 4; + const stripMax = Math.max(1, ...this.stats.L.buckets, ...this.stats.R.buckets); + drawStrip(image, this.stats.L.buckets, gutter, stripY, columnWidth, stripHeight, stripMax); + drawStrip(image, this.stats.R.buckets, gutter * 2 + columnWidth, stripY, columnWidth, stripHeight, stripMax); + + const footer = this.unknownArmPackets > 0 + ? `last ${HISTORY_MS / 1000} s unattributed packets ${this.unknownArmPackets}` + : `packets per ${BUCKET_MS} ms, last ${HISTORY_MS / 1000} s`; + image.drawText(font, gutter, height - 16, footer, 110); + const back = `${GESTURE_DOUBLE_CLICK} back tap reset`; + image.drawText(font, width - gutter - font.measureText(back), height - 16, back, 110); + return image; + } + + private paintArm( + image: GrayImage, + arm: "L" | "R", + label: string, + x: number, + top: number, + width: number, + bottom: number, + nowMs: number, + ): void { + const font = getDefaultSmallFont(); + const line = font.lineHeight; + const stats = this.stats[arm]; + const cutoff = nowMs - RATE_WINDOW_MS; + while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift(); + const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS; + const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS; + let maxGap = 0; + for (let i = 1; i < stats.recent.length; i++) { + maxGap = Math.max(maxGap, stats.recent[i]!.arrivalMs - stats.recent[i - 1]!.arrivalMs); + } + + const rows: Array<[string, number]> = []; + const streaming = stats.recent.length > 0; + rows.push([`${label}: ${streaming ? `${rate.toFixed(1)} pkt/s ${byteRate.toFixed(1)} kB/s` : "silent"}`, streaming ? 235 : 150]); + const age = stats.lastArrivalWallMs === null ? "never" : `${formatAge(nowMs - stats.lastArrivalWallMs)} ago`; + rows.push([`last ${age}${streaming && stats.recent.length > 1 ? ` max gap ${maxGap} ms` : ""}`, 170]); + rows.push([`total ${stats.packets}: stock ${stats.stockPackets} SM ${stats.smPackets} other ${stats.otherPackets}${stats.otherPackets ? ` (${stats.lastOtherBytes} B)` : ""}`, 170]); + if (stats.stockPackets > 0) { + rows.push([`stock LC3: lost ${stats.stockLost} dup ${stats.stockDuplicate}`, 190]); + rows.push([` angle ${stats.stockAngle}° ssr ${stats.stockSsr}`, 170]); + } + const frame = stats.frame; + if (frame) { + const flags = [frame.truncated ? "trunc" : "", stats.smSeqGaps ? `seq gaps ${stats.smSeqGaps}` : ""].filter(Boolean).join(" "); + rows.push([`SM: ${frame.channelCount}ch ${frame.sampleRateHz / 1000} kHz ${frame.codec} ${frame.payload.length} B ${flags}`, 190]); + rows.push([` angle ${frame.angleDegrees}° ssr ${frame.ssr}`, 170]); + } + const status = stats.status; + if (status) { + const armed = status.hardwareArmed ? "armed" : status.active ? "configured" : "inactive"; + rows.push([`CFW ${armed}: ${status.source} mask ${status.channelMask} frames ${status.framesEmitted}`, 190]); + rows.push([` ${status.codec} ${status.effectiveRateHz / 1000} kHz ${formatAge(nowMs - (stats.statusAtMs ?? nowMs))} ago`, 150]); + const diag = status.diagnostics; + if (diag) { + rows.push([` tap ${diag.tapCalls} calls last ${diag.tapLastBytes} B @slot ${diag.tapLastSlot}`, 190]); + rows.push([` skip idle ${diag.skipIdle} lease ${diag.skipLease} alloc ${diag.skipAlloc}`, 170]); + rows.push([ + ` reg ${formatRet(diag.registerResult)} unreg ${formatRet(diag.unregisterResult)} ntf ${formatRet(diag.notifyResult)} slots ${diag.slots.map(formatSlot).join("")} lease ${diag.leaseRemainingS} s`, + 170, + ]); + } else { + rows.push([" (no tap diagnostics: older firmware)", 120]); + } + } else { + rows.push(["CFW status: none seen", 120]); + } + + let y = top; + for (const [text, shade] of rows) { + if (y + line > bottom) return; + image.drawText(font, x, y, fitText(font, text, width), shade); + y += line; + } + // Per-channel RMS level bars for raw PCM frames. + for (let channel = 0; channel < stats.channelDb.length; channel++) { + if (y + line > bottom) return; + const db = stats.channelDb[channel]!; + const tag = `ch${channel} ${db <= LEVEL_FLOOR_DB ? "-inf" : db.toFixed(0)} dB`; + image.drawText(font, x, y, tag, 190); + const barX = x + font.measureText("ch0 -100 dB") + 6; + const barWidth = Math.max(0, width - (barX - x)); + const fill = Math.round(clamp((db - LEVEL_FLOOR_DB) / -LEVEL_FLOOR_DB, 0, 1) * barWidth); + image.drawRect(barX, y + 3, barWidth, line - 6, 60); + if (fill > 0) image.fillRect(barX, y + 3, fill, line - 6, db > -3 ? 255 : 170); + y += line; + } + } + + /** Window context-menu entries while this page is open. */ + menuItems(): MenuItem[] { + return [ + { + label: "Query mic status", + onSelect: (ctx) => { + micArrayController.query(); + ctx.stack.pop(); + }, + }, + { + label: "Reset counters", + onSelect: (ctx) => { + this.resetCounters(); + ctx.stack.pop(); + }, + }, + ]; + } + + private resetCounters(): void { + this.stats.L = emptyStats(); + this.stats.R = emptyStats(); + this.unknownArmPackets = 0; + this.requestRender(); + } + + async handleInput(event: InputEvent, ctx: LayerContext): Promise { + if (event.type === "double-click") { + ctx.stack.pop(); + } else if (event.type === "click") { + this.resetCounters(); + } + } +} + +function formatRet(value: number): string { + return value === MIC_RET_NONE ? "-" : String(value); +} + +/** Same letters as the firmware overlay: - empty, S another callback, T our tap. */ +function formatSlot(state: "empty" | "other" | "tap"): string { + return state === "tap" ? "T" : state === "other" ? "S" : "-"; +} + +function readS16(data: Uint8Array, offset: number): number { + const value = data[offset]! | (data[offset + 1]! << 8); + return value >= 0x8000 ? value - 0x10000 : value; +} + +function rmsDb(samples: Float32Array): number { + let sumSquares = 0; + for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!; + const rms = Math.sqrt(sumSquares / Math.max(1, samples.length)); + return rms > 0 ? Math.max(LEVEL_FLOOR_DB, 20 * Math.log10(rms)) : LEVEL_FLOOR_DB; +} + +function formatAge(ms: number): string { + if (ms < 1_000) return `${Math.max(0, Math.round(ms))} ms`; + if (ms < 60_000) return `${(ms / 1000).toFixed(1)} s`; + return `${Math.round(ms / 60_000)} min`; +} + +function fitText(font: ReturnType, text: string, width: number): string { + if (font.measureText(text) <= width) return text; + let end = text.length; + while (end > 0 && font.measureText(`${text.slice(0, end)}…`) > width) end--; + return `${text.slice(0, end)}…`; +} + +function drawStrip( + image: GrayImage, + buckets: readonly number[], + x: number, + y: number, + width: number, + height: number, + max: number, +): void { + image.drawRect(x, y, width, height, 45); + const barWidth = width / buckets.length; + for (let i = 0; i < buckets.length; i++) { + const count = buckets[i]!; + if (count <= 0) continue; + const barHeight = Math.max(1, Math.round((count / max) * (height - 2))); + const left = x + Math.floor(i * barWidth); + const right = x + Math.floor((i + 1) * barWidth) - 1; + image.fillRect(left + 1, y + height - 1 - barHeight, Math.max(1, right - left), barHeight, 200); + } +} diff --git a/app/apps/developer/screen-test.ts b/app/apps/developer/screen-test.ts index af3e7491..52cb911f 100644 --- a/app/apps/developer/screen-test.ts +++ b/app/apps/developer/screen-test.ts @@ -64,6 +64,8 @@ export class ScreenTestLayer implements Layer { case "double-click": ctx.stack.pop(); return; + default: + return; } } } diff --git a/app/apps/developer/unicode-test.ts b/app/apps/developer/unicode-test.ts index 35c3ff37..200c86d7 100644 --- a/app/apps/developer/unicode-test.ts +++ b/app/apps/developer/unicode-test.ts @@ -236,6 +236,8 @@ export class UnicodeSampleLayer implements Layer { case "double-click": ctx.stack.pop(); return; + default: + return; } } diff --git a/app/apps/evenhub/even-platform.ts b/app/apps/evenhub/even-platform.ts index bee53a2c..18283f3d 100644 --- a/app/apps/evenhub/even-platform.ts +++ b/app/apps/evenhub/even-platform.ts @@ -1,4 +1,4 @@ -import { Application } from "@nativescript/core"; +import { Utils } from "@nativescript/core"; export function hmacSha256Base64(secret: string, message: string): string { const charset = java.nio.charset.StandardCharsets.UTF_8; @@ -10,7 +10,7 @@ export function hmacSha256Base64(secret: string, message: string): string { } export function getPhoneOpenUdid(): string { - const resolver = Application.android.context?.getContentResolver(); + const resolver = Utils.android.getApplicationContext()?.getContentResolver(); const id = resolver ? String(android.provider.Settings.Secure.getString(resolver, android.provider.Settings.Secure.ANDROID_ID) ?? "") : ""; if (!id) throw new Error("Android did not provide a device ID for EvenHub authentication."); return id; diff --git a/app/apps/evenhub/installed-apps.ts b/app/apps/evenhub/installed-apps.ts index ea05c4f1..a785f34f 100644 --- a/app/apps/evenhub/installed-apps.ts +++ b/app/apps/evenhub/installed-apps.ts @@ -51,9 +51,19 @@ export function installedEvenHubPackageId(appId: string): string | null { } } +// The parsed index for the last stored value: the launcher and the watch +// state both list installed apps on every refresh, and the sort's +// localeCompare dominated the cost. +let parsedIndex: { raw: string; apps: InstalledEvenHubApp[] } | null = null; + export function getInstalledEvenHubApps(): InstalledEvenHubApp[] { const raw = getStringSetting(STORAGE_KEY, ""); if (!raw) return []; + if (parsedIndex?.raw !== raw) parsedIndex = { raw, apps: parseInstalledIndex(raw) }; + return parsedIndex.apps.map((app) => ({ ...app })); +} + +function parseInstalledIndex(raw: string): InstalledEvenHubApp[] { try { const parsed = JSON.parse(raw); if (!Array.isArray(parsed)) return []; diff --git a/app/apps/evenhub/manager.ts b/app/apps/evenhub/manager.ts index 7b3b327c..0b12dabe 100644 --- a/app/apps/evenhub/manager.ts +++ b/app/apps/evenhub/manager.ts @@ -6,11 +6,12 @@ * * Glasses-first: launching does NOT take over the phone screen. The phone * stays on the Faceclaw dashboard; an app's phone UI is shown on demand - * (window menu -> "Show phone UI"), overlaying the dashboard, and the Back - * button returns to it. Apps are kept alive until explicitly closed (no memory - * eviction yet). + * (window menu -> "Show phone UI", or the app-icon button in the phone's + * action bar while the app is foreground on the glasses), overlaying the + * dashboard, and the Back button or the overlay's close button returns to it. + * Apps are kept alive until explicitly closed (no memory eviction yet). */ -import { Application } from "@nativescript/core"; +import { Application, type ImageSource } from "@nativescript/core"; import { type AppContext } from "../app-definition"; import { appFilesDirPath, @@ -23,6 +24,8 @@ import { EvenHubSession } from "./session"; import { createEvenHubWindow } from "./evenhub-window"; import { createEvenHubWebView, type EvenHubWebView } from "./webview"; import { shell } from "../../ui/shell/shell"; +import { GrayImage } from "../../graphics/image"; +import { grayImageToWhiteIconSource } from "../../native/gray-image-preview"; import { readEvenHubPackageManifest, installedEvenHubAppId, @@ -36,12 +39,31 @@ type RunningApp = { packageId: string; session: EvenHubSession; webView: EvenHubWebView; + /** The app's icon for the phone action bar, rendered on first use. */ + phoneIcon?: ImageSource | null; }; +/** The phone action bar's button for opening the foreground app's phone UI. */ +export type PhoneUiButton = { windowId: string; title: string; icon: ImageSource | null }; + +/** Pixel size the action-bar icon is rendered at (~21dp at xxhdpi). */ +const PHONE_ICON_SIZE = 63; + const running = new Map(); let nextSerial = 1; let phoneShownWindowId: string | null = null; let backHandlerRegistered = false; +const phoneShownListeners = new Set<() => void>(); + +/** Called whenever an app's phone UI is shown or hidden. Returns an unsubscribe. */ +export function onPhoneShownChanged(listener: () => void): () => void { + phoneShownListeners.add(listener); + return () => phoneShownListeners.delete(listener); +} + +function notifyPhoneShownChanged(): void { + for (const listener of phoneShownListeners) listener(); +} /** Intercept Android Back while an app's phone UI is overlaying the dashboard. */ function ensureBackHandler(): void { @@ -112,6 +134,7 @@ async function startApp( if (phoneShownWindowId === windowId) { webView.hideOnPhone(); phoneShownWindowId = null; + notifyPhoneShownChanged(); } webView.destroy(); cleanup?.(); @@ -205,15 +228,35 @@ export function showOnPhone(windowId: string): void { const app = running.get(windowId); if (!app) return; if (phoneShownWindowId && phoneShownWindowId !== windowId) hidePhone(); - app.webView.showOnPhone(); + app.webView.showOnPhone(hidePhone); phoneShownWindowId = windowId; + notifyPhoneShownChanged(); } -/** Return the phone to the dashboard (Back, or the shown app closing). */ +/** Return the phone to the dashboard (Back, the close button, or the shown app closing). */ export function hidePhone(): void { if (!phoneShownWindowId) return; running.get(phoneShownWindowId)?.webView.hideOnPhone(); phoneShownWindowId = null; + notifyPhoneShownChanged(); +} + +/** + * The foreground glasses window, when it is an EvenHub app whose phone UI + * isn't showing: the phone action bar then offers a button to show it. + */ +export function phoneUiButton(): PhoneUiButton | null { + const window = shell.foregroundWindow(); + const app = window ? running.get(window.windowId) : undefined; + if (!window || !app || phoneShownWindowId === app.windowId) return null; + if (app.phoneIcon === undefined) { + // The same artwork as the glasses sidebar (installed icon, else the + // generic package icon), drawn white. + const image = new GrayImage(PHONE_ICON_SIZE, PHONE_ICON_SIZE, 0); + window.drawIcon(image, 0, 0, PHONE_ICON_SIZE); + app.phoneIcon = grayImageToWhiteIconSource(image); + } + return { windowId: app.windowId, title: window.title, icon: app.phoneIcon }; } export function runningAppCount(): number { diff --git a/app/apps/evenhub/permission-dialog.ts b/app/apps/evenhub/permission-dialog.ts index 07977024..3f0560ff 100644 --- a/app/apps/evenhub/permission-dialog.ts +++ b/app/apps/evenhub/permission-dialog.ts @@ -1,6 +1,6 @@ import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { GrayImage } from "../../graphics/image"; -import { truncateText } from "../../graphics/textwrap"; +import { truncateText, wrapText } from "../../graphics/textwrap"; import { type InputEvent } from "../../ui/gestures"; import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers"; import { drawListScrollbar, drawSelectionHighlight } from "../../ui/menu"; @@ -8,9 +8,9 @@ import { permissionDetail, permissionLabel, type EvenHubPermission } from "./per import { openPrivacyPolicyOnPhone } from "./privacy-policy"; import { lineStep, listRowHeight } from "../../ui/metrics"; -const DIALOG_X = 8; -const DIALOG_Y = 8; -const DIALOG_WIDTH = 272; +/** Minimum gap between the dialog and the edges of the window it's centered in. */ +const DIALOG_MARGIN = 8; +const DIALOG_WIDTH = 320; const PADDING = 10; const PERM_GAP = 4; @@ -27,6 +27,8 @@ const PERM_GAP = 4; * list on screen. */ export class EvenHubPermissionDialogLayer implements Layer { + /** Floats in front of the window content, like the context menus. */ + readonly depth = 2; /** Starts on Allow; the optional privacy-policy action is inserted after it. */ private selectedIndex = 0; /** First content row drawn; rows above it are scrolled off. */ @@ -50,10 +52,12 @@ export class EvenHubPermissionDialogLayer implements Layer { paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage { const font = getDefaultSmallFont(); - const { height: viewportHeight } = ctx.stack.getBaseSize(); + const { width: viewportWidth, height: viewportHeight } = ctx.stack.getBaseSize(); const image = paintBelow(); - const textWidth = DIALOG_WIDTH - 2 * PADDING - 4; - const left = DIALOG_X + PADDING + 2; + const dialogWidth = Math.min(DIALOG_WIDTH, viewportWidth - 2 * DIALOG_MARGIN); + const dialogX = ((viewportWidth - dialogWidth) / 2) | 0; + const textWidth = dialogWidth - 2 * PADDING - 4; + const left = dialogX + PADDING + 2; const headerStep = lineStep(font) + 2; const permLabelStep = lineStep(font) + 1; @@ -80,15 +84,17 @@ export class EvenHubPermissionDialogLayer implements Layer { for (let index = 0; index < this.permissions.length; index++) { const permission = this.permissions[index]!; const detail = permissionDetail(permission); + // One row per wrapped line, so the explanation scrolls line by line. + const detailLines = detail ? wrapText(font, detail, textWidth - 8) : []; const gapBelow = PERM_GAP + (index === this.permissions.length - 1 ? 2 : 0); rows.push({ - height: permLabelStep + (detail ? 0 : gapBelow), + height: permLabelStep + (detailLines.length ? 0 : gapBelow), draw: (top) => image.drawText(font, left, top, truncateText(font, permissionLabel(permission.name), textWidth), 220), }); - if (detail) { + for (let line = 0; line < detailLines.length; line++) { rows.push({ - height: permDetailStep + gapBelow, - draw: (top) => image.drawText(font, left + 8, top, truncateText(font, detail, textWidth - 8), 140), + height: permDetailStep + (line === detailLines.length - 1 ? gapBelow : 0), + draw: (top) => image.drawText(font, left + 8, top, detailLines[line]!, 140), }); } } @@ -98,15 +104,16 @@ export class EvenHubPermissionDialogLayer implements Layer { draw: (top) => { const selected = this.atBottom() && index === this.selectedIndex; if (selected) { - drawSelectionHighlight(image, DIALOG_X + 12, top, DIALOG_WIDTH - 24, actionRowH - 1, focused, 8); + drawSelectionHighlight(image, dialogX + 12, top, dialogWidth - 24, actionRowH - 1, focused, 8); } - image.drawText(font, DIALOG_X + 22, top + 3, actions[index]!, selected ? 255 : 200); + image.drawText(font, dialogX + 22, top + 3, actions[index]!, selected ? 255 : 200); }, }); } const contentHeight = rows.reduce((sum, row) => sum + row.height, 0); - const height = Math.min(contentHeight + 2 * PADDING, viewportHeight - 2 * DIALOG_Y); + const height = Math.min(contentHeight + 2 * PADDING, viewportHeight - 2 * DIALOG_MARGIN); + const dialogY = ((viewportHeight - height) / 2) | 0; const visibleHeight = height - 2 * PADDING; // The bottommost scroll position: the largest suffix of rows that fits. @@ -120,13 +127,13 @@ export class EvenHubPermissionDialogLayer implements Layer { this.firstRow = Math.max(0, Math.min(this.firstRow, maxFirstRow)); // Fill 1 (transparent color key is 0), outline for the dialog edge. - image.fillRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 1); - image.drawRoundedRect(DIALOG_X, DIALOG_Y, DIALOG_WIDTH, height, 72); + image.fillRoundedRect(dialogX, dialogY, dialogWidth, height, 1); + image.drawRoundedRect(dialogX, dialogY, dialogWidth, height, 72); - let y = DIALOG_Y + PADDING; + let y = dialogY + PADDING; for (let index = this.firstRow; index < rows.length; index++) { const row = rows[index]!; - if (y + row.height > DIALOG_Y + PADDING + visibleHeight) break; + if (y + row.height > dialogY + PADDING + visibleHeight) break; row.draw(y); y += row.height; } @@ -134,8 +141,8 @@ export class EvenHubPermissionDialogLayer implements Layer { if (maxFirstRow > 0) { drawListScrollbar( image, - DIALOG_X + DIALOG_WIDTH - 7, - DIALOG_Y + PADDING, + dialogX + dialogWidth - 7, + dialogY + PADDING, visibleHeight, this.firstRow, rows.length - maxFirstRow, diff --git a/app/apps/evenhub/permissions.ts b/app/apps/evenhub/permissions.ts index 31c545f0..0eeb0f59 100644 --- a/app/apps/evenhub/permissions.ts +++ b/app/apps/evenhub/permissions.ts @@ -22,7 +22,7 @@ export type EvenHubPermissionName = | "album" | "camera" | "fs" - | string; + | (string & {}); export type EvenHubPermission = { name: EvenHubPermissionName; diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 74ea59f9..95957c1d 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -110,30 +110,61 @@ export const EVENHUB_BRIDGE_INJECT_SCRIPT = ` }; })(); (function () { - // Host-driven timers. Chromium heavily throttles a page's own - // setTimeout/setInterval when the phone screen is off, which slows - // timer-driven apps to a crawl. Replace them with a queue fired by - // window.__fcTimerTick(), which the native host calls at ~60Hz regardless of + // Host-driven timers and requestAnimationFrame. Chromium heavily throttles a + // page's own setTimeout/setInterval when the phone screen is off, and pauses + // or throttles a hidden page's rAF, which slows timer-driven apps to a crawl + // and stalls canvas apps that render on a rAF loop. Replace them with queues + // fired by window.__fcTick(), which the native host calls regardless of // screen state (host-initiated evaluateJavascript escapes the throttle). - if (window.__fcTimerTick) return; + // (The glasses render via BLE, not the phone's display, so vsync alignment + // doesn't matter — a steady frame rate does.) + // + // __fcTick returns how many ms until it next has work (-1: none), and the + // host sleeps until then instead of evaluating JS at 60Hz for an idle page. + // Work added outside a tick that is due before the host's next tick wakes it + // through the bridge. + if (window.__fcTick) return; + var FRAME_MS = 16; var timers = new Map(); var nextId = 1; + var rafs = new Map(); + var nextRafId = 1; + var inTick = false; + // When the host will next call __fcTick; Infinity while it is idle. + var hostNextTickAt = Infinity; + function needTickBy(due) { + if (inTick || due >= hostNextTickAt) return; + hostNextTickAt = Date.now(); + var bridge = window.__faceclawEvenHub; + if (bridge && bridge.wakeTimers) bridge.wakeTimers(); + } window.setTimeout = function (cb, delay) { var args = Array.prototype.slice.call(arguments, 2); var id = nextId++; - timers.set(id, { cb: cb, args: args, due: Date.now() + (+delay || 0), interval: 0 }); + var due = Date.now() + (+delay || 0); + timers.set(id, { cb: cb, args: args, due: due, interval: 0 }); + needTickBy(due); return id; }; window.setInterval = function (cb, delay) { var args = Array.prototype.slice.call(arguments, 2); var id = nextId++; var d = +delay || 0; - timers.set(id, { cb: cb, args: args, due: Date.now() + d, interval: d }); + var due = Date.now() + d; + timers.set(id, { cb: cb, args: args, due: due, interval: d }); + needTickBy(due); return id; }; window.clearTimeout = function (id) { timers.delete(id); }; window.clearInterval = function (id) { timers.delete(id); }; - window.__fcTimerTick = function () { + window.requestAnimationFrame = function (cb) { + var id = nextRafId++; + rafs.set(id, cb); + needTickBy(Date.now() + FRAME_MS); + return id; + }; + window.cancelAnimationFrame = function (id) { rafs.delete(id); }; + function runTimers() { var now = Date.now(); var dueIds = []; timers.forEach(function (entry, id) { if (entry.due <= now) dueIds.push(id); }); @@ -151,24 +182,8 @@ export const EVENHUB_BRIDGE_INJECT_SCRIPT = ` } try { entry.cb.apply(null, entry.args); } catch (e) { if (window.console) console.error(e); } } - }; -})(); -(function () { - // Host-driven requestAnimationFrame, same idea as the timers above: a hidden - // page's rAF is paused/throttled with the screen off, stalling canvas apps - // that render on a rAF loop. Drive it from window.__fcRafTick(), called by the - // native ticker each frame. (The glasses render via BLE, not the phone's - // display, so vsync alignment doesn't matter — a steady frame rate does.) - if (window.__fcRafTick) return; - var rafs = new Map(); - var nextRafId = 1; - window.requestAnimationFrame = function (cb) { - var id = nextRafId++; - rafs.set(id, cb); - return id; - }; - window.cancelAnimationFrame = function (id) { rafs.delete(id); }; - window.__fcRafTick = function () { + } + function runRafs() { if (rafs.size === 0) return; var ts = window.performance && performance.now ? performance.now() : Date.now(); // Snapshot and clear: callbacks that re-request during this frame run on @@ -178,6 +193,32 @@ export const EVENHUB_BRIDGE_INJECT_SCRIPT = ` current.forEach(function (cb) { try { cb(ts); } catch (e) { if (window.console) console.error(e); } }); + } + function inTickDo(fn) { + inTick = true; + try { fn(); } finally { inTick = false; } + } + // Fixed-rate entry points for hosts that don't use __fcTick's schedule (the + // iOS host calls both every frame). + window.__fcTimerTick = function () { inTickDo(runTimers); }; + window.__fcRafTick = function () { inTickDo(runRafs); }; + window.__fcTick = function () { + inTick = true; + try { + runTimers(); + runRafs(); + } finally { + inTick = false; + } + var now = Date.now(); + var delay = rafs.size ? FRAME_MS : -1; + timers.forEach(function (entry) { + var d = entry.due - now; + if (delay < 0 || d < delay) delay = d; + }); + if (delay >= 0) delay = Math.max(FRAME_MS, Math.ceil(delay)); + hostNextTickAt = delay < 0 ? Infinity : now + delay; + return delay; }; })(); `; @@ -1136,7 +1177,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH } return out; }; - if (requested.length === 0 || requested.every((id) => this.grantedApiKeys.has(id))) { + if (requested.every((id) => this.grantedApiKeys.has(id))) { return Promise.resolve(buildGrant()); } if (!this.windowHooks) return Promise.resolve({}); diff --git a/app/apps/evenhub/store-layer.ts b/app/apps/evenhub/store-layer.ts index 966fc279..bd974dc8 100644 --- a/app/apps/evenhub/store-layer.ts +++ b/app/apps/evenhub/store-layer.ts @@ -174,7 +174,7 @@ export class EvenHubStoreLayer implements Layer { for (const [index, line] of wrapText(font, message, width - LIST_X * 2).entries()) { image.drawText(font, LIST_X, headerH + 16 + index * lineStep(font), line, 190); } - image.drawText(font, LIST_X, height - font.lineHeight - 4, `${GESTURE_CLICK} edit credentials`, 105); + image.drawText(font, LIST_X, height - font.lineHeight - 4, `${GESTURE_CLICK} log in`, 105); return image; } diff --git a/app/apps/evenhub/updates.ts b/app/apps/evenhub/updates.ts index 05139c4c..0a2016aa 100644 --- a/app/apps/evenhub/updates.ts +++ b/app/apps/evenhub/updates.ts @@ -73,7 +73,7 @@ export async function checkForEvenHubUpdates( onProgress?: (done: number, total: number) => void, ): Promise { const installed = getInstalledEvenHubApps(); - const results: EvenHubAppUpdate[] = new Array(installed.length); + const results: EvenHubAppUpdate[] = []; let next = 0; let done = 0; const worker = async () => { diff --git a/app/apps/evenhub/webview.ts b/app/apps/evenhub/webview.ts index 686682a1..655822ac 100644 --- a/app/apps/evenhub/webview.ts +++ b/app/apps/evenhub/webview.ts @@ -22,11 +22,13 @@ import { FACECLAW_VERSION } from "../../version"; declare const android: any; declare const com: any; +declare const java: any; export type EvenHubWebView = { evaluateJs: (js: string) => void; destroy: () => void; - showOnPhone: () => void; + /** Show the app's phone UI; `onClose` runs when its top bar's close button is tapped. */ + showOnPhone: (onClose: () => void) => void; hideOnPhone: () => void; /** The native android.webkit.WebView, for the host. */ native: unknown; @@ -96,16 +98,7 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { webView.addJavascriptInterface(new com.faceclaw.app.FaceclawEvenHubJsBridge(listener), "__faceclawEvenHub"); // Surface the app's console in logcat (tag FaceclawEvenHubConsole). - const ChromeClient = (android.webkit.WebChromeClient as any).extend({ - onConsoleMessage: (message: any): boolean => { - android.util.Log.i( - "FaceclawEvenHubConsole", - `${message.message()} (${message.sourceId()}:${message.lineNumber()})`, - ); - return true; - }, - }); - webView.setWebChromeClient(new ChromeClient()); + webView.setWebChromeClient(new com.faceclaw.app.FaceclawEvenHubChromeClient()); const nativeHost = com.faceclaw.app.FaceclawEvenHubWebViewHost.getInstance(); nativeHost.attach(activity, webView); @@ -127,8 +120,13 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { console.warn(`evenhub webview destroy failed: ${error}`); } }, - showOnPhone: () => - nativeHost.showOnPhone(Application.android?.foregroundActivity ?? Application.android?.startActivity ?? null, webView), + showOnPhone: (onClose: () => void) => + nativeHost.showOnPhone( + Application.android?.foregroundActivity ?? Application.android?.startActivity ?? null, + webView, + session.manifest.name, + new java.lang.Runnable({ run: onClose }), + ), hideOnPhone: () => nativeHost.hideOnPhone(), }; } diff --git a/app/apps/files/files-app.ts b/app/apps/files/files-app.ts index 615c8f9a..d8872e58 100644 --- a/app/apps/files/files-app.ts +++ b/app/apps/files/files-app.ts @@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access"; import { isFontFile } from "../../native/font-files"; import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts"; import { isDecodableImageFile } from "../../native/image-files"; +import { isPlayableVideoFile } from "../../native/video-player"; import { readEvenHubPackageManifest } from "../evenhub/installed-apps"; import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog"; import { FileBrowserLayer } from "./file-browser"; @@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & { installEhpkApp: (path: string) => Promise | void; /** Open a font file's previewer as its own shell window. */ openFontWindow: (title: string, path: string) => void; + /** Open a video file's player (settings panel first) as its own shell window. */ + openVideoWindow: (title: string, path: string) => void; }; /** * The Files app's launcher-opened window: a file browser over Places * (bookmarks and storage roots). Picking any file opens an info dialog with - * metadata plus, for viewable types (text, images), the open actions. + * metadata plus, for viewable types (text, images, video), the open actions. */ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow { let created: InProcessWindow | null = null; const browser = new FileBrowserLayer({ - isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)), + isSupportedFile: (name) => + TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || + (!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))), // The browser handles double-click itself (up a level), so it is not // wrapped in YieldAtRootLayer; it yields explicitly from the top level. onLeave: () => shell.yieldFocusToSidebar(), @@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI }, ]; } + if (isPlayableVideoFile(entry.name)) { + return [ + { + label: "Play video", + onSelect: (ctx) => { + ctx.stack.pop(); + options.openVideoWindow(entry.name, entry.path); + }, + }, + ]; + } if (isFontFile(entry.name)) { // Install copies into the app-internal fonts directory, making the face // available in the Settings font pickers. The row relabels itself with diff --git a/app/apps/files/image-viewer.ts b/app/apps/files/image-viewer.ts index 974dae06..a6bf3456 100644 --- a/app/apps/files/image-viewer.ts +++ b/app/apps/files/image-viewer.ts @@ -1,4 +1,3 @@ -import { type BdfFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { truncateText } from "../../graphics/textwrap"; import { GrayImage } from "../../graphics/image"; diff --git a/app/apps/files/index.ts b/app/apps/files/index.ts index 92301cca..65f9430f 100644 --- a/app/apps/files/index.ts +++ b/app/apps/files/index.ts @@ -13,6 +13,7 @@ import { FILES_SURFACE_ID, FILES_WINDOW_ID, } from "./files-app"; +import { createVideoPlayerWindow } from "./video-window"; // Document windows are closeable and non-singleton; the serial keeps their // windowIds unique. @@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v }); } +function openVideoWindow(ctx: AppContext, title: string, path: string): void { + const windowId = `files:doc:${nextDocumentSerial++}`; + void ctx + .launchInProcessApp(windowId, `window:${windowId}`, (options) => + createVideoPlayerWindow(windowId, title, path, options), + ) + .catch((error) => { + ctx.appendLog(`video window failed: ${error}`); + }); +} + const filesApp: AppDefinition = { appId: "files", title: "Files", @@ -78,6 +90,7 @@ const filesApp: AppDefinition = { } }, openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path), + openVideoWindow: (title, path) => openVideoWindow(ctx, title, path), }), ), // A document arriving via Android's Share intent opens as its own window. diff --git a/app/apps/files/video-layout.ts b/app/apps/files/video-layout.ts new file mode 100644 index 00000000..e8cc1f15 --- /dev/null +++ b/app/apps/files/video-layout.ts @@ -0,0 +1,86 @@ +/** + * Where a video goes in the player window, for each size choice. Pure + * geometry, so it is unit-tested without the NativeScript runtime. + */ + +export type VideoSize = "small" | "medium" | "large" | "full"; + +export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"]; + +export type Size = { width: number; height: number }; +export type Rect = Size & { x: number; y: number }; + +/** + * Bounding boxes for the fixed sizes; a video is fitted inside, keeping its + * aspect. Large and full fit the whole viewport instead (full in a tall + * window). Bigger pictures cost the glasses link more per frame, so smaller + * sizes play at a higher frame rate. + */ +const FIXED_BOXES: Partial> = { + small: { width: 160, height: 120 }, + medium: { width: 240, height: 180 }, +}; + +export function videoSizeLabel(size: VideoSize): string { + switch (size) { + case "small": + return "Small"; + case "medium": + return "Medium"; + case "large": + return "Large"; + case "full": + return "Full screen"; + } +} + +/** Full screen needs the tall window band; the rest play in the standard one. */ +export function videoHeightMode(size: VideoSize): "min" | "max" { + return size === "full" ? "max" : "min"; +} + +/** + * The largest size with the video's aspect ratio that fits in box, with even + * dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel + * columns) of at least 2. Upscales small videos too: the size is the + * viewer's choice. + */ +export function fitVideo(video: Size, box: Size): Size { + if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) }; + const scale = Math.min(box.width / video.width, box.height / video.height); + return { + width: evenFloor(Math.min(box.width, Math.round(video.width * scale))), + height: evenFloor(Math.min(box.height, Math.round(video.height * scale))), + }; +} + +/** + * The video's rect within a viewport, centered. x snaps to a multiple of 4 + * and y to a multiple of 2 so frame deltas don't straddle an extra encoder + * block at each edge. + */ +export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect { + const fixed = FIXED_BOXES[size]; + const box = fixed + ? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) } + : viewport; + const fitted = fitVideo(video, box); + return { + x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4), + y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2), + ...fitted, + }; +} + +/** "3:52", or "1:02:03" past an hour. */ +export function formatMediaTime(ms: number): string { + const total = Math.max(0, Math.floor(ms / 1000)); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const seconds = String(total % 60).padStart(2, "0"); + return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`; +} + +function evenFloor(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} diff --git a/app/apps/files/video-window.ts b/app/apps/files/video-window.ts new file mode 100644 index 00000000..2310e117 --- /dev/null +++ b/app/apps/files/video-window.ts @@ -0,0 +1,443 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GrayImage } from "../../graphics/image"; +import { truncateText, wrapText } from "../../graphics/textwrap"; +import { PHOTO_GAMMA } from "../../native/image-files"; +import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player"; +import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; +import { type InputEvent } from "../../ui/gestures"; +import { type Layer, type LayerContext } from "../../ui/layers"; +import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu"; +import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "../../ui/shell/in-process-window"; +import { appViewportSize } from "../../ui/shell/geometry"; +import { + formatMediaTime, + VIDEO_SIZES, + videoHeightMode, + videoRect, + videoSizeLabel, + type Rect, + type VideoSize, +} from "./video-layout"; + +export const videoSizeSetting = new ConfigSettingEnum({ + id: "video-size", + label: "Size", + storageKey: "video.size", + defaultValue: "large", + values: VIDEO_SIZES, + formatValue: videoSizeLabel, +}); + +export const videoAudioSetting = new ConfigSettingBoolean({ + id: "video-audio", + label: "Audio", + storageKey: "video.audio", + defaultValue: true, + description: "Play the video's sound on the phone's current audio output.", +}); + +export const videoMonochromeSetting = new ConfigSettingBoolean({ + id: "video-monochrome", + label: "Monochrome", + storageKey: "video.monochrome", + defaultValue: false, + description: "Show only black and white: far less to send per frame, so a higher frame rate.", +}); + +/** How far one ring scroll seeks. */ +const SEEK_STEP_MS = 10_000; +const MARGIN_X = 18; +const TITLE_Y = 10; +const PANEL_WIDTH = 340; + +/** + * One opened video: its probe result, the native player, and the window + * glue. The window's base layer is the settings panel; playing pushes the + * playback layer, and backing out of it (or the video ending) returns to the + * panel with the position kept for Resume. + */ +class VideoSession { + readonly probe: VideoInfo | { error: string }; + private player: VideoPlayer | null = null; + private window: InProcessWindow | null = null; + private playback: VideoPlaybackLayer | null = null; + /** Where Resume starts; 0 = from the beginning. */ + resumeMs = 0; + /** The last playback error, shown on the panel until the next Play. */ + error: string | null = null; + + constructor( + readonly path: string, + readonly title: string, + readonly surfaceId: string, + ) { + this.probe = probeVideo(path); + } + + get info(): VideoInfo | null { + return "error" in this.probe ? null : this.probe; + } + + attach(window: InProcessWindow): void { + this.window = window; + } + + isPlaying(): boolean { + return this.player?.isPlaying() ?? false; + } + + positionMs(): number { + return this.player?.positionMs() ?? this.resumeMs; + } + + /** Current rect for a size choice, in the viewport of the band that size plays in. */ + rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect { + const info = this.info; + return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport); + } + + play(ctx: LayerContext, fromStart: boolean): void { + const window = this.window; + const info = this.info; + if (!window || !info) return; + this.error = null; + const size = videoSizeSetting.get(); + window.setHeightMode(videoHeightMode(size)); + const viewport = ctx.stack.getBaseSize(); + const rect = this.rectFor(size, viewport); + const player = this.ensurePlayer(); + this.playback = new VideoPlaybackLayer(this, rect, viewport); + ctx.stack.push(this.playback); + player.start({ + path: this.path, + surfaceId: this.surfaceId, + rect, + monochrome: videoMonochromeSetting.get(), + gamma: PHOTO_GAMMA, + audio: info.hasAudio && videoAudioSetting.get(), + startMs: fromStart ? 0 : this.resumeMs, + }); + } + + togglePause(): void { + const player = this.player; + if (!player) return; + if (player.isPlaying()) player.pause(); + else player.resume(); + } + + seekBy(deltaMs: number): void { + const player = this.player; + if (!player) return; + const duration = this.info?.durationMs ?? 0; + const target = Math.max(0, player.positionMs() + deltaMs); + player.seekTo(duration > 0 ? Math.min(target, duration) : target); + } + + /** The window went to the background or the screen went off: stop, keep the place. */ + pauseForHost(): void { + if (!this.player?.isPlaying()) return; + this.player.pause(); + // Paint the paused overlay now, so the retained surface shows it on return. + this.window?.requestRender(); + } + + /** Leave the playback layer for the panel, remembering where to resume. */ + returnToPanel(resumeMs: number): void { + const player = this.player; + if (player) { + player.pause(); + player.stop(); + } + this.resumeMs = resumeMs; + const window = this.window; + const playback = this.playback; + this.playback = null; + if (window && playback) window.stack.popThrough(playback); + window?.setHeightMode("min"); + window?.requestRender(); + } + + lastFrame(rect: Rect): GrayImage | null { + return this.player?.lastFrame(rect.width, rect.height) ?? null; + } + + release(): void { + this.player?.release(); + this.player = null; + } + + private ensurePlayer(): VideoPlayer { + if (!this.player) { + this.player = new VideoPlayer({ + onState: (state, positionMs) => this.onPlayerState(state, positionMs), + onError: (message) => { + this.error = message; + this.returnToPanel(this.positionMs()); + }, + }); + } + return this.player; + } + + private onPlayerState(state: VideoPlaybackState, _positionMs: number): void { + if (state === "ended") { + this.returnToPanel(0); + return; + } + // A paused seek decoded its new frame: repaint it under the overlay. + if (state === "paused") this.window?.requestRender(); + } +} + +type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean }; + +/** + * The pre-play settings panel (the window's root): picture size, audio and + * monochrome, all remembered for the next video, then Play (or Resume plus + * Play from start once a position is kept). + */ +class VideoSettingsLayer implements Layer { + private readonly menu = new Menu({ + wrap: false, + rowGap: 1, + highlight: { radius: 6 }, + getHeight: () => listRowHeight(getDefaultSmallFont()), + draw: (args) => this.drawRow(args), + }); + constructor(private readonly session: VideoSession) {} + + private rows(): PanelRow[] { + if (!this.session.info) return []; + const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }]; + rows.push({ kind: "play", fromStart: false }); + if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true }); + return rows; + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220); + const step = lineStep(font); + let y = TITLE_Y + step; + + const info = this.session.info; + const errors: string[] = []; + if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`); + if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`); + if (info) { + const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"]; + image.drawText(font, MARGIN_X, y, facts.join(" · "), 130); + y += step; + } + for (const error of errors) { + for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) { + image.drawText(font, MARGIN_X, y, line, 200); + y += step; + } + } + y += 6; + + const rows = this.rows(); + const selected = this.menu.selectedItem; + const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1; + // Land on Resume/Play by default: the settings usually stay as they were. + const playIndex = rows.findIndex((row) => row.kind === "play"); + this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined); + const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6)); + this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) }, + ctx.stack.isFocused()); + return image; + } + + async handleInput(event: InputEvent, ctx: LayerContext): Promise { + switch (event.type) { + case "scroll-up": + case "scroll-down": + await this.menu.handleInput(event); + return; + case "click": { + const row = this.menu.selectedItem; + if (row) this.activate(row, ctx); + return; + } + default: + return; + } + } + + private activate(row: PanelRow, ctx: LayerContext): void { + switch (row.kind) { + case "size": + videoSizeSetting.set(videoSizeSetting.next()); + return; + case "audio": + if (this.session.info?.hasAudio) videoAudioSetting.toggle(); + return; + case "monochrome": + videoMonochromeSetting.toggle(); + return; + case "play": + this.session.play(ctx, row.fromStart); + return; + } + } + + private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs): void { + const font = getDefaultSmallFont(); + const textX = x + 10; + const textWidth = width - 20; + switch (item.kind) { + case "size": { + const size = videoSizeSetting.get(); + // The picture size this choice gets in its own band (full screen plays in the tall window). + const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size))); + drawRightValueMenuItem(image, font, textX, y, textWidth, "Size", + `${videoSizeLabel(size)} ${rect.width}x${rect.height}`); + return; + } + case "audio": { + const hasAudio = this.session.info?.hasAudio ?? false; + drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)", + hasAudio && videoAudioSetting.get(), selected); + return; + } + case "monochrome": + drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected); + return; + case "play": { + const resumeMs = this.session.resumeMs; + const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play"; + image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200); + return; + } + } + } +} + +function sameRow(a: PanelRow, b: PanelRow): boolean { + return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart); +} + +/** + * The playing video. The native player draws frames into the video rect on + * its own; this layer paints only when the window renders (input, pause, + * focus): black around the latest frame, plus a status box while paused. + * Click pauses/resumes, scroll seeks 10s, double-click returns to the panel. + */ +class VideoPlaybackLayer implements Layer { + private leaving = false; + + constructor( + private readonly session: VideoSession, + private readonly rect: Rect, + /** The viewport the rect was laid out in. */ + private readonly viewport: { width: number; height: number }, + ) {} + + paint(ctx: LayerContext): GrayImage { + const { width, height } = ctx.stack.getBaseSize(); + if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) { + // The window was relaid out (display mode change) and the native player's rect no + // longer fits the surface: back out to the panel, which lays out afresh on Play. + this.leaving = true; + setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0); + } + const image = new GrayImage(width, height, 0); + const frame = this.session.lastFrame(this.rect); + if (frame) image.bitBlt(frame, this.rect.x, this.rect.y); + if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null); + return image; + } + + async handleInput(event: InputEvent): Promise { + switch (event.type) { + case "click": + this.session.togglePause(); + return; + case "scroll-up": + this.session.seekBy(-SEEK_STEP_MS); + return; + case "scroll-down": + this.session.seekBy(SEEK_STEP_MS); + return; + case "double-click": + this.session.returnToPanel(this.session.positionMs()); + return; + default: + return; + } + } + + /** + * "Paused 1:23 / 3:52" over a progress bar, below the picture when there is + * room (small sizes), else over its bottom edge. + */ + private drawPausedBox(image: GrayImage, loading: boolean): void { + const font = getDefaultSmallFont(); + const duration = this.session.info?.durationMs ?? 0; + const position = this.session.positionMs(); + const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`; + const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24); + const boxHeight = font.lineHeight + 14; + const rect = this.rect; + const x = Math.round(rect.x + (rect.width - boxWidth) / 2); + const below = rect.y + rect.height + 4; + const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6); + image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6); + image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6); + image.drawText(font, x + 12, y + 5, text, 220); + if (!loading && duration > 0) { + const barWidth = boxWidth - 24; + image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40); + image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200); + } + } +} + +/** A video file's player as its own window, opened from the Files app. */ +export function createVideoPlayerWindow( + windowId: string, + title: string, + path: string, + options: InProcessAppOptions, +): InProcessWindow { + const session = new VideoSession(path, title, `window:${windowId}`); + const window = createInProcessWindow({ + appId: "files", + windowId, + title, + iconLetter: "V", + icon: "film", + closeable: true, + actions: options.actions, + baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)), + // A playing video is the thing being watched: no idle screen-off. + keepsScreenOn: () => session.isPlaying(), + onForegroundChanged: (foreground) => { + if (!foreground) session.pauseForHost(); + }, + setScreenOn: (on) => { + if (!on) session.pauseForHost(); + }, + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + reconfigureSurface: options.reconfigureSurface, + onClosed: () => { + session.release(); + options.onClosed(); + }, + }); + session.attach(window); + return window; +} diff --git a/app/apps/flappy/flappy-app.worker.ts b/app/apps/flappy/flappy-app.worker.ts index d880c239..27fae1c7 100644 --- a/app/apps/flappy/flappy-app.worker.ts +++ b/app/apps/flappy/flappy-app.worker.ts @@ -17,8 +17,9 @@ * fall back to raster (the planner needs a non-negative origin), which is * small and rare. * - * Controls (a watch swipe in either direction is the primary flap): - * ring-press flaps immediately; watch swipes/clicks also flap. Double-click pauses; + * Controls: a touch-down on the ring or watch pad (ring-press) flaps + * immediately; a click, a watch swipe in either direction or a crown turn + * without one flaps too. Double-click pauses; * tap-then-hold opens the window menu. Ready / paused / game over: click or * a swipe starts or resumes, double-click yields focus. Losing input focus * mid-flight (a shell overlay such as the system menu or a notification, @@ -45,6 +46,7 @@ import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, type InputEvent, + PressTracker, } from "../../ui/gestures"; import { clamp } from "../../util/numeric-util"; import { @@ -153,6 +155,8 @@ type FlappyWindow = { lastMinorSfxAtMs: number; soundOn: boolean; lastSubmittedFingerprint: string; + /** Which gestures already flapped on their ring-press. */ + presses: PressTracker; }; const windows = new Map(); @@ -210,6 +214,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { lastMinorSfxAtMs: 0, soundOn: loadSoundEnabled("flappy"), lastSubmittedFingerprint: "", + presses: new PressTracker(), }; resetGame(window); windows.set(message.windowId, window); @@ -282,6 +287,10 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { syncTickTimer(window); } break; + case "navigation-sensors": + case "text-input": + case "tool-call": + break; } }; @@ -374,10 +383,11 @@ function windowMenu(window: FlappyWindow): WindowMenu { // --- Input ------------------------------------------------------------------ function handleInput(window: FlappyWindow, event: InputEvent, frameId: number): void { + const followsPress = window.presses.followsPress(event); // An open window menu owns all input (it closes itself via pop); menus are // list UIs, so watch swipes take their standard fallback meanings there. if (window.menu?.isOpen()) { - window.menu + void window.menu .handleInput(directionalFallback(event)) .catch((error) => console.error(`flappy menu input failed: ${error}`)) .then(() => renderAndSubmit(window, frameId)); @@ -398,7 +408,7 @@ function handleInput(window: FlappyWindow, event: InputEvent, frameId: number): switch (window.phase) { case "ready": case "playing": - handleFlightInput(window, event, frameId); + handleFlightInput(window, event, frameId, followsPress); break; case "dying": // Let the tumble finish; a flap mashed during it must not restart. @@ -410,22 +420,24 @@ function handleInput(window: FlappyWindow, event: InputEvent, frameId: number): } } -/** Input while ready or flying: the flap gestures, plus pause. */ -function handleFlightInput(window: FlappyWindow, event: InputEvent, frameId: number): void { +/** + * Input while ready or flying: the flap gestures, plus pause. `followsPress` + * marks a gesture whose touch-down already flapped (see PressTracker). + */ +function handleFlightInput(window: FlappyWindow, event: InputEvent, frameId: number, followsPress: boolean): void { switch (event.type) { + case "ring-press": + flap(window); + break; // Either swipe direction flaps: on the watch the flap is a reflex, and // which way the thumb went shouldn't matter. case "swipe-up": case "swipe-down": - case "ring-press": - flap(window); - break; case "click": - // The ring already flapped on down; its later click must not flap again. - if (event.source !== "ring") flap(window); + if (!followsPress) flap(window); break; case "scroll-up": - if (event.source === "watch") flap(window); + if (event.source === "watch" && !followsPress) flap(window); break; case "double-click": if (window.phase === "playing") { @@ -720,6 +732,9 @@ function paintHud(image: GrayImage, window: FlappyWindow): void { `${GESTURE_CLICK} new game ${GESTURE_DOUBLE_CLICK} leave`, ]); break; + case "playing": + case "dying": + break; } } diff --git a/app/apps/freecell/freecell-app.worker.ts b/app/apps/freecell/freecell-app.worker.ts index ae4f2dc0..fb939857 100644 --- a/app/apps/freecell/freecell-app.worker.ts +++ b/app/apps/freecell/freecell-app.worker.ts @@ -58,9 +58,6 @@ function rankOf(card: number): number { function isRed(card: number): boolean { return (card & 1) === 1; } -function cardLabel(card: number): string { - return RANK_CHARS[rankOf(card)]! + SUIT_CHARS[suitOf(card)]!; -} const CARD_W = 60; const CARD_H = 36; @@ -208,6 +205,13 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { if (window.foreground) renderAndSubmit(window, 0); break; } + case "input-focus": + case "navigation-sensors": + case "resize-window": + case "screen": + case "text-input": + case "tool-call": + break; } }; @@ -285,7 +289,7 @@ function handleInput(window: FreecellWindow, event: InputEvent, frameId: number) // An open window menu owns all input (it closes itself via pop); menus are // list UIs, so watch swipes take their standard fallback meanings there. if (window.menu?.isOpen()) { - window.menu + void window.menu .handleInput(directionalFallback(event)) .catch((error) => console.error(`freecell menu input failed: ${error}`)) .then(() => renderAndSubmit(window, frameId)); @@ -792,7 +796,6 @@ function paintCard( if (selected) image.drawRoundedRect(x + 1, y + 1, CARD_W - 2, CARD_H - 2, 255, 3); const rankText: string = RANK_CHARS[rankOf(card)]! const suitText: string = SUIT_CHARS[suitOf(card)]!; - const labelText = rankText + suitText; image.drawText(labelFont, x + 5, y, rankText, 255); image.drawText(suitFont, x + 5 + labelFont.measureText(rankText), y + 3, suitText, 150); if (isRed(card)) { diff --git a/app/apps/glanceboard/configuration.ts b/app/apps/glanceboard/configuration.ts index a783df47..36c40f5a 100644 --- a/app/apps/glanceboard/configuration.ts +++ b/app/apps/glanceboard/configuration.ts @@ -1,11 +1,24 @@ -import { getStringSetting } from "../../native/settings-store"; +import { getBooleanSetting, getStringSetting } from "../../native/settings-store"; import { QUADRANT_LAYOUT, SIX_SLOT_LAYOUT } from "./layout"; -/** Configuration-only query usable by workers without loading widget/UI modules. */ -export function isTerminalWidgetConfigured(): boolean { +/** Whether the current layout has `widgetId` in any slot (configuration only, enabled or not). */ +function isGlanceWidgetConfigured(widgetId: string): boolean { const layout = getStringSetting("glanceboard.layout", "2x2") === "2x3" ? SIX_SLOT_LAYOUT : QUADRANT_LAYOUT; // Both layouts store their slots under the original quadrants key. None // of the default slots needs a worker, so absent keys cannot retain one. return layout.slots.some((_, index) => - getStringSetting(`glanceboard.${QUADRANT_LAYOUT.id}.slot.${index}`, "none") === "terminal"); + getStringSetting(`glanceboard.${QUADRANT_LAYOUT.id}.slot.${index}`, "none") === widgetId); +} + +/** Configuration-only query usable by workers without loading widget/UI modules. */ +export function isTerminalWidgetConfigured(): boolean { + return isGlanceWidgetConfigured("terminal"); +} + +/** + * Whether the T3 Code widget can show: it is in a slot and the Glanceboard is + * enabled (the T3 worker holds its server connections open only then). + */ +export function isT3CodeWidgetInUse(): boolean { + return getBooleanSetting("glanceboard.enabled", false) && isGlanceWidgetConfigured("t3code"); } diff --git a/app/apps/glanceboard/glance-font.ts b/app/apps/glanceboard/glance-font.ts new file mode 100644 index 00000000..4857110d --- /dev/null +++ b/app/apps/glanceboard/glance-font.ts @@ -0,0 +1,8 @@ +import { UiFontOverride } from "../../graphics/ui-fonts"; + +/** + * The font Glanceboard widgets paint with: the Glanceboard's own Font + * setting, or the Display UI font while that is unset. Its small, medium and + * large roles carry the same line-height guarantees as getDefault*Font. + */ +export const glanceFont = new UiFontOverride("glanceboard.font"); diff --git a/app/apps/glanceboard/glanceboard-app.ts b/app/apps/glanceboard/glanceboard-app.ts index e6bc38f7..303b02d7 100644 --- a/app/apps/glanceboard/glanceboard-app.ts +++ b/app/apps/glanceboard/glanceboard-app.ts @@ -2,6 +2,7 @@ import { GrayImage } from "../../graphics/image"; import { truncateText, wrapText } from "../../graphics/textwrap"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { uiFontOverridePickerMenuItem } from "../../ui/font-picker"; import { type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerContext } from "../../ui/layers"; import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu"; @@ -10,6 +11,7 @@ import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; import { shell } from "../../ui/shell/shell"; import { GlanceBoard } from "./board"; +import { glanceFont } from "./glance-font"; import { clearConflictingSlots, glanceboardEnabledSetting, @@ -298,7 +300,7 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void { openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current))); } -/** Settings: layout, sleep gestures, duration, slot lines, and stereo depth. */ +/** Settings: layout, sleep gestures, duration, slot lines, widget font, and stereo depth. */ function openGlanceSettings(ctx: LayerContext): void { const items: MenuItem[] = [ enumSettingMenuItem(glanceLayoutSetting), @@ -306,6 +308,11 @@ function openGlanceSettings(ctx: LayerContext): void { toggleSettingMenuItem(glanceShowOnLongPressSetting), toggleSettingMenuItem(glanceShowOnHeadTiltSetting), toggleSettingMenuItem(glanceShowLinesSetting), + uiFontOverridePickerMenuItem(glanceFont, { + title: "Glanceboard font", + description: + "Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.", + }), enumSettingMenuItem(glanceDepthSetting), ]; ctx.stack.push(new MenuLayer("Glanceboard settings", items, pageMenuLayout(ctx.stack.getBaseSize().width))); @@ -344,7 +351,6 @@ class GlancePreviewLayer implements Layer { export function createGlanceboardAppWindow(options: InProcessAppOptions): InProcessWindow { let previewVisible = false; const heightMode = () => previewVisible && glanceLayoutSetting.get() === "2x3" ? "max" as const : "medium" as const; - let unsubscribeSettings: (() => void) | undefined; const app = createInProcessWindow({ appId: "glanceboard", windowId: GLANCEBOARD_WINDOW_ID, @@ -367,10 +373,10 @@ export function createGlanceboardAppWindow(options: InProcessAppOptions): InProc removeSurface: options.removeSurface, reconfigureSurface: options.reconfigureSurface, onClosed: () => { - unsubscribeSettings?.(); + unsubscribeSettings(); options.onClosed(); }, }); - unsubscribeSettings = onAnySettingChanged(() => app.setHeightMode(heightMode())); + const unsubscribeSettings = onAnySettingChanged(() => app.setHeightMode(heightMode())); return app; } diff --git a/app/apps/glanceboard/glanceboard-settings.ts b/app/apps/glanceboard/glanceboard-settings.ts index 38b4e42f..4d729490 100644 --- a/app/apps/glanceboard/glanceboard-settings.ts +++ b/app/apps/glanceboard/glanceboard-settings.ts @@ -5,13 +5,14 @@ import { glanceWidgetSpans } from "./widgets"; export type GlanceSlotChoice = GlanceWidgetId | "none"; -const SLOT_CHOICES: readonly GlanceSlotChoice[] = ["none", "system-card", "calendar", "terminal", "nightscout", "compass", "music", "health"]; +const SLOT_CHOICES: readonly GlanceSlotChoice[] = ["none", "system-card", "calendar", "terminal", "t3code", "nightscout", "compass", "music", "health"]; const SLOT_CHOICE_LABELS: Record = { none: "Empty", "system-card": "System card", calendar: "Calendar", terminal: "Terminal", + t3code: "T3 Code", nightscout: "Nightscout", compass: "Compass", music: "Music", diff --git a/app/apps/glanceboard/widget.ts b/app/apps/glanceboard/widget.ts index d6e965b7..03c6e4b6 100644 --- a/app/apps/glanceboard/widget.ts +++ b/app/apps/glanceboard/widget.ts @@ -27,7 +27,7 @@ export interface GlanceWidget { paint(image: GrayImage): void; } -export type GlanceWidgetId = "system-card" | "nightscout" | "compass" | "music" | "calendar" | "terminal" | "health"; +export type GlanceWidgetId = "system-card" | "nightscout" | "compass" | "music" | "calendar" | "terminal" | "t3code" | "health"; export type GlanceWidgetDefinition = { id: GlanceWidgetId; diff --git a/app/apps/glanceboard/widgets/calendar-widget.ts b/app/apps/glanceboard/widgets/calendar-widget.ts index 4607cd55..99d9b9fe 100644 --- a/app/apps/glanceboard/widgets/calendar-widget.ts +++ b/app/apps/glanceboard/widgets/calendar-widget.ts @@ -1,10 +1,10 @@ import { type GrayImage, type UiFont } from "../../../graphics/image"; import { truncateText } from "../../../graphics/textwrap"; -import { getDefaultMediumFont, getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { hasCalendarPermission } from "../../../native/calendar-permissions"; import { getCalendarReadState, onCalendarChanged, readUpcomingEvents, type CalendarEvent } from "../../../native/calendar"; import { lineStep } from "../../../ui/metrics"; import { dayHeaderLabel, formatEventTime } from "../../calendar/calendar"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; const PAD = 8; @@ -38,8 +38,8 @@ export class CalendarWidget implements GlanceWidget { } paint(image: GrayImage): void { - const small = getDefaultSmallFont(); - const medium = getDefaultMediumFont(); + const small = glanceFont.small(); + const medium = glanceFont.medium(); const step = lineStep(small); const textWidth = image.width - 2 * PAD; diff --git a/app/apps/glanceboard/widgets/compass-widget.ts b/app/apps/glanceboard/widgets/compass-widget.ts index 8e6716f0..adf0eda8 100644 --- a/app/apps/glanceboard/widgets/compass-widget.ts +++ b/app/apps/glanceboard/widgets/compass-widget.ts @@ -1,11 +1,11 @@ import { type GrayImage } from "../../../graphics/image"; -import { getDefaultMediumFont, getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { hasLocationPermission } from "../../../native/location-permissions"; import { addCompassListener, COMPASS_CHANGED, setCompassEnabled } from "../../../native/compass"; import { isCompassCalibrated, normalizeHeading } from "../../compass/calibration"; import { cardinalDirection, createCompassBackground, drawCompassRose, layoutCompassRose, TICK_HEIGHT } from "../../compass/compass-rose"; import { getDeclinationAvailability, onDeclinationChanged, refreshDeclination } from "../../compass/declination"; import { getNorthReference, resolveHeading } from "../../compass/heading"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; /** Owner token for the refcounted magnetometer, distinct from the Compass app's. */ @@ -63,8 +63,8 @@ export class CompassWidget implements GlanceWidget { } paint(image: GrayImage): void { - const medium = getDefaultMediumFont(); - const small = getDefaultSmallFont(); + const medium = glanceFont.medium(); + const small = glanceFont.small(); const { width, height } = image; const heading = this.rawHeading === null ? null : resolveHeading(this.rawHeading).displayDegrees; const headingText = heading === null ? "--°" : `${Math.round(heading)}° ${cardinalDirection(heading)}`; diff --git a/app/apps/glanceboard/widgets/index.ts b/app/apps/glanceboard/widgets/index.ts index 0ebf4c52..5a7e0ce5 100644 --- a/app/apps/glanceboard/widgets/index.ts +++ b/app/apps/glanceboard/widgets/index.ts @@ -4,6 +4,7 @@ import { CompassWidget } from "./compass-widget"; import { MusicWidget } from "./music-widget"; import { NightscoutWidget } from "./nightscout-widget"; import { SystemCardWidget } from "./system-card"; +import { T3CodeWidget } from "./t3code-widget"; import { TerminalWidget } from "./terminal-widget"; import { HealthWidget } from "./health-widget"; @@ -12,13 +13,14 @@ export const GLANCE_WIDGETS: readonly GlanceWidgetDefinition[] = [ { id: "system-card", label: "System card", create: () => new SystemCardWidget() }, { id: "calendar", label: "Calendar", tall: true, create: () => new CalendarWidget() }, { id: "terminal", label: "Terminal", tall: true, create: () => new TerminalWidget() }, + { id: "t3code", label: "T3 Code", tall: true, create: () => new T3CodeWidget() }, { id: "nightscout", label: "Nightscout", create: () => new NightscoutWidget() }, { id: "compass", label: "Compass", create: () => new CompassWidget() }, { id: "music", label: "Music", create: () => new MusicWidget() }, { id: "health", label: "Health", create: () => new HealthWidget() }, ]; -export function findGlanceWidget(id: GlanceWidgetId | string): GlanceWidgetDefinition | null { +export function findGlanceWidget(id: GlanceWidgetId | (string & {})): GlanceWidgetDefinition | null { return GLANCE_WIDGETS.find((widget) => widget.id === id) ?? null; } diff --git a/app/apps/glanceboard/widgets/music-widget.ts b/app/apps/glanceboard/widgets/music-widget.ts index b142f28f..f94418e3 100644 --- a/app/apps/glanceboard/widgets/music-widget.ts +++ b/app/apps/glanceboard/widgets/music-widget.ts @@ -1,9 +1,9 @@ import { GrayImage } from "../../../graphics/image"; import { renderIcon } from "../../../graphics/icons"; -import { getDefaultMediumFont, getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { truncateText } from "../../../graphics/textwrap"; import { mediaControllerBridge, type MediaControllerState } from "../../../native/media-controller"; import { clamp } from "../../../util/numeric-util"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; const PAD = 8; @@ -75,8 +75,8 @@ export class MusicWidget implements GlanceWidget { } paint(image: GrayImage): void { - const medium = getDefaultMediumFont(); - const small = getDefaultSmallFont(); + const medium = glanceFont.medium(); + const small = glanceFont.small(); const media = mediaControllerBridge.snapshot(); this.media = media; if (!media.available || (!media.title && !media.artist)) { @@ -142,7 +142,7 @@ export class MusicWidget implements GlanceWidget { image.drawRect(dx - 1, dy - 1, this.art.width + 2, this.art.height + 2, 60); } else { image.drawRect(ART_X, artY, ART_SIZE, ART_SIZE, 60); - const font = getDefaultSmallFont(); + const font = glanceFont.small(); image.drawText(font, ART_X + Math.round((ART_SIZE - font.measureText("no art")) / 2), artY + Math.round((ART_SIZE - font.lineHeight) / 2), "no art", 90); } diff --git a/app/apps/glanceboard/widgets/nightscout-widget.ts b/app/apps/glanceboard/widgets/nightscout-widget.ts index 14139486..0b3f9d6b 100644 --- a/app/apps/glanceboard/widgets/nightscout-widget.ts +++ b/app/apps/glanceboard/widgets/nightscout-widget.ts @@ -1,5 +1,4 @@ import { GrayImage } from "../../../graphics/image"; -import { getDefaultLargeFont, getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { truncateText } from "../../../graphics/textwrap"; import { nightscoutBridge, type NightscoutState } from "../../../native/nightscout-bridge"; import { isNightscoutSettingsConfigured, loadNightscoutThresholds } from "../../../ui/dashboard-settings"; @@ -13,6 +12,7 @@ import { formatDelta, isNightscoutPointStale, } from "../../nightscout/nightscout"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; const PAD = 8; @@ -48,8 +48,8 @@ export class NightscoutWidget implements GlanceWidget { } paint(image: GrayImage): void { - const small = getDefaultSmallFont(); - const large = getDefaultLargeFont(); + const small = glanceFont.small(); + const large = glanceFont.large(); const state = this.state; const nowMs = Date.now(); const step = lineStep(small); diff --git a/app/apps/glanceboard/widgets/system-card.ts b/app/apps/glanceboard/widgets/system-card.ts index c9b2c601..b497a434 100644 --- a/app/apps/glanceboard/widgets/system-card.ts +++ b/app/apps/glanceboard/widgets/system-card.ts @@ -1,8 +1,14 @@ -import { BATTERY_ICON_WIDTH, drawBattery } from "../../../graphics/battery"; +import { + BATTERY_ICON_WIDTH, + DENSE_BATTERY_BLOCK_HEIGHT, + drawBattery, + drawDenseBatteries, + type BatteryDevice, +} from "../../../graphics/battery"; import { type GrayImage, type UiFont } from "../../../graphics/image"; -import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { onAndroidNotificationPosted, readActiveNotificationIcons } from "../../../native/notification-icons"; import { readPhoneBatteryState } from "../../../native/phone-battery"; +import { weatherBridge } from "../../../native/weather"; import { formatClockDate, formatClockTime } from "../../../ui/clock-format"; import { batteryDisplayModeSetting, @@ -20,6 +26,9 @@ import { truncateText } from "../../../graphics/textwrap"; import { timerEngine } from "../../timer/timer-engine"; import { formatCountdown, sortTimers, timerDisplayName, timerPhase, timerRemainingMs } from "../../timer/timer-model"; import { lineStep } from "../../../ui/metrics"; +import { drawWeatherReading, measureWeatherReading, weatherReading } from "../../weather/weather-icons"; +import { weatherShowOnSystemCardSetting } from "../../weather/weather-settings"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; const PAD = 8; @@ -27,15 +36,20 @@ const NOTIFICATION_ICON_SIZE = 24; const NOTIFICATION_ICON_GAP = 4; const BATTERY_ITEM_GAP = 12; const BATTERY_LABEL_VALUE = 150; +/** Gap between the battery block and the weather reading below it. */ +const WEATHER_GAP = 6; -type BatteryItem = { label: string; percent: number; charging: boolean }; +type BatteryItem = { label: string; device: BatteryDevice; percent: number; charging: boolean }; +/** A block drawn against the card's right edge: its left edge and its rows. */ +type RightBlock = { left: number; top: number; bottom: number }; /** * Date and time, the phone's notification icons, the phone/watch/G2/R1 - * battery indicators (the Wear OS watch's only while one is reachable), and + * battery indicators (the Wear OS watch's only while one is reachable), the + * current weather (when the Weather app's "Show on system card" is on), and * the Timers app's countdowns: the top bar's contents plus timers, laid out * for a card. Battery style and per-device visibility - * follow Settings > Display > Battery indicators, so the card agrees with + * follow Settings > Customization > Battery indicators, so the card agrees with * the bar. Countdowns tick once a second only while one is running. */ export class SystemCardWidget implements GlanceWidget { @@ -45,6 +59,7 @@ export class SystemCardWidget implements GlanceWidget { private unsubscribeNotifications: (() => void) | null = null; private unsubscribeSettings: (() => void) | null = null; private unsubscribeTimers: (() => void) | null = null; + private unsubscribeWeather: (() => void) | null = null; start(requestRender: () => void): void { this.requestRender = requestRender; @@ -55,6 +70,8 @@ export class SystemCardWidget implements GlanceWidget { this.syncSecondTick(); this.requestRender?.(); }); + // The Weather app's indicator keeps the bridge refreshing; this only listens. + this.unsubscribeWeather = weatherBridge.onStateChange(() => this.requestRender?.()); this.syncSecondTick(); } @@ -70,6 +87,8 @@ export class SystemCardWidget implements GlanceWidget { this.unsubscribeSettings = null; this.unsubscribeTimers?.(); this.unsubscribeTimers = null; + this.unsubscribeWeather?.(); + this.unsubscribeWeather = null; } /** Countdowns change every second; run that clock only while one is running. */ @@ -97,14 +116,20 @@ export class SystemCardWidget implements GlanceWidget { } paint(image: GrayImage): void { - const large = getDefaultLargeFont(); - const medium = getDefaultMediumFont(); - const small = getDefaultSmallFont(); + const large = glanceFont.large(); + const medium = glanceFont.medium(); + const small = glanceFont.small(); const now = new Date(); - // Clock and date down the left; batteries take the right edge. - const batteryLeft = drawBatteryBlock(image, small, image.width - PAD, PAD); - const textWidth = Math.max(0, batteryLeft - BATTERY_ITEM_GAP - PAD); + // Clock and date down the left; batteries take the right edge, with the + // weather right-aligned beneath them. Text on the left stops short of + // whichever of those shares its rows. + const battery = drawBatteryBlock(image, small, image.width - PAD, PAD); + const weatherTop = battery.bottom > battery.top ? battery.bottom + WEATHER_GAP : PAD; + const weather = drawWeatherBlock(image, medium, image.width - PAD, weatherTop, now.getTime()); + const textRight = (top: number, bottom: number): number => + weather && weather.top < bottom && weather.bottom > top ? Math.min(battery.left, weather.left) : battery.left; + const textWidth = Math.max(0, textRight(PAD, PAD + large.lineHeight) - BATTERY_ITEM_GAP - PAD); const timeText = formatClockTime(now); const timeFont = large.measureText(timeText) <= textWidth ? large : medium; let y = PAD; @@ -116,7 +141,8 @@ export class SystemCardWidget implements GlanceWidget { // Notification icons along the bottom, as many as fit; the Timers app's // countdowns take the band between the date and the icons. const iconY = image.height - PAD - NOTIFICATION_ICON_SIZE; - drawTimers(image, small, PAD, y, textWidth, iconY - 4, now.getTime()); + const timersWidth = Math.max(0, textRight(y, iconY - 4) - BATTERY_ITEM_GAP - PAD); + drawTimers(image, small, PAD, y, timersWidth, iconY - 4, now.getTime()); const maxIcons = Math.max(0, ((image.width - 2 * PAD + NOTIFICATION_ICON_GAP) / (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP)) | 0); if (maxIcons > 0) { const { icons, stale } = readActiveNotificationIcons(maxIcons, renderPassAllowsStaleData()); @@ -130,19 +156,25 @@ export class SystemCardWidget implements GlanceWidget { function collectBatteryItems(): BatteryItem[] { const items: BatteryItem[] = []; - const push = (visibility: BatteryIndicatorVisibility, label: string, percent: number | null, charging: boolean | null) => { + const push = ( + visibility: BatteryIndicatorVisibility, + label: string, + device: BatteryDevice, + percent: number | null, + charging: boolean | null, + ) => { if (percent === null || !Number.isFinite(percent)) return; const clamped = Math.max(0, Math.min(100, Math.round(percent))); if (!batteryIndicatorVisible(visibility, clamped)) return; - items.push({ label, percent: clamped, charging: Boolean(charging) }); + items.push({ label, device, percent: clamped, charging: Boolean(charging) }); }; const phone = readPhoneBatteryState(); - push(phoneBatteryVisibilitySetting.get(), "Phone", phone.battery, phone.charging); + push(phoneBatteryVisibilitySetting.get(), "Phone", "phone", phone.battery, phone.charging); const levels = shell.getBatteryLevels(); - push(watchBatteryVisibilitySetting.get(), "Watch", levels.watch, levels.watchCharging); - push(glassesBatteryVisibilitySetting.get(), "G2", levels.headset, levels.headsetCharging); + push(watchBatteryVisibilitySetting.get(), "Watch", "watch", levels.watch, levels.watchCharging); + push(glassesBatteryVisibilitySetting.get(), "G2", "glasses", levels.headset, levels.headsetCharging); if (levels.ring !== null && Number.isInteger(levels.ring) && levels.ring >= 0 && levels.ring <= 100) { - push(ringBatteryVisibilitySetting.get(), "R1", levels.ring, levels.ringCharging); + push(ringBatteryVisibilitySetting.get(), "R1", "ring", levels.ring, levels.ringCharging); } return items; } @@ -150,16 +182,22 @@ function collectBatteryItems(): BatteryItem[] { /** * The battery indicators right-aligned at `right`, in the configured style. * Side-by-side styles put the label beside the gauge (or percentage) on one - * line; stacked styles centre the label above it. Returns the block's left - * edge (or `right` when nothing is shown). + * line; stacked styles centre the label above it; the dense style swaps + * labels for device icons and stacks two indicators per column, as in the + * top bar. Returns the block's extent (zero-height at `right` when nothing + * is shown). */ -function drawBatteryBlock(image: GrayImage, font: UiFont, right: number, top: number): number { +function drawBatteryBlock(image: GrayImage, font: UiFont, right: number, top: number): RightBlock { const items = collectBatteryItems(); - if (!items.length) return right; + if (!items.length) return { left: right, top, bottom: top }; const mode = batteryDisplayModeSetting.get(); + if (mode === "dense") { + return { left: drawDenseBatteries(image, items, right, top), top, bottom: top + DENSE_BATTERY_BLOCK_HEIGHT }; + } const percentage = mode === "percentage" || mode === "stacked-percentage"; const stacked = mode === "stacked" || mode === "stacked-percentage"; const gaugeHeight = drawBattery(0, false).height; + const valueHeight = percentage ? font.lineHeight : Math.max(font.lineHeight, gaugeHeight); let x = right; for (let index = items.length - 1; index >= 0; index--) { const item = items[index]!; @@ -178,7 +216,23 @@ function drawBatteryBlock(image: GrayImage, font: UiFont, right: number, top: nu } x -= BATTERY_ITEM_GAP; } - return x + BATTERY_ITEM_GAP; + return { left: x + BATTERY_ITEM_GAP, top, bottom: top + (stacked ? font.lineHeight + 2 : 0) + valueHeight }; +} + +/** + * The current weather's condition icon and temperature, right-aligned at + * `right` from `top`, while the system-card weather setting is on and a + * fresh reading exists. Returns the block's extent, or null when not drawn. + */ +function drawWeatherBlock(image: GrayImage, font: UiFont, right: number, top: number, nowMs: number): RightBlock | null { + if (!weatherShowOnSystemCardSetting.get()) return null; + const reading = weatherReading(weatherBridge.snapshot(), nowMs); + if (!reading) return null; + const iconSize = font.lineHeight + 4; + const height = Math.max(iconSize, font.lineHeight); + const left = right - Math.ceil(measureWeatherReading(reading, font, iconSize)); + drawWeatherReading(image, reading, font, iconSize, left, top, height, 200); + return { left, top, bottom: top + height }; } function drawBatteryValue( diff --git a/app/apps/glanceboard/widgets/t3code-widget.ts b/app/apps/glanceboard/widgets/t3code-widget.ts new file mode 100644 index 00000000..19ea3797 --- /dev/null +++ b/app/apps/glanceboard/widgets/t3code-widget.ts @@ -0,0 +1,122 @@ +import { type GrayImage, type UiFont } from "../../../graphics/image"; +import { truncateText } from "../../../graphics/textwrap"; +import { lineStep } from "../../../ui/metrics"; +import { onWorkerStateChanged, readWorkerState } from "../../../ui/shell/worker-state"; +import { loadEnvironments } from "../../t3code/t3-environments"; +import { glanceSummary, T3_GLANCE_STATE_KEY, type T3GlanceRow, type T3GlanceSnapshot } from "../../t3code/t3-glance"; +import { formatAge, threadMarker } from "../../t3code/t3-model"; +import { glanceFont } from "../glance-font"; +import { type GlanceWidget } from "../widget"; + +const PAD = 8; +/** A row fits while its glyph line clears this margin above the bottom edge. */ +const BOTTOM_MARGIN = 2; +const MINUTE_MS = 60_000; + +/** + * T3 Code's agent threads at a glance: how many need you and how many are + * working, then the threads themselves in the app's order (needs-you, then + * working, then recent) with the app's status marks. Approvals and questions + * say so at the right edge; other rows show how long ago they were active. + * Painted from the snapshot the T3 worker publishes; the app's boot hook + * keeps that worker connected while this widget is on an enabled board. + */ +export class T3CodeWidget implements GlanceWidget { + private snapshot = readWorkerState(T3_GLANCE_STATE_KEY) as T3GlanceSnapshot | undefined; + private unsubscribe: (() => void) | null = null; + private tick: ReturnType | null = null; + + start(requestRender: () => void): void { + this.stop(); + this.snapshot = readWorkerState(T3_GLANCE_STATE_KEY) as T3GlanceSnapshot | undefined; + this.unsubscribe = onWorkerStateChanged(T3_GLANCE_STATE_KEY, (snapshot) => { + this.snapshot = snapshot as T3GlanceSnapshot | undefined; + requestRender(); + }); + // Keeps the ages honest while the board stays up. + this.tick = setInterval(requestRender, MINUTE_MS); + } + + stop(): void { + this.unsubscribe?.(); + this.unsubscribe = null; + if (this.tick !== null) clearInterval(this.tick); + this.tick = null; + } + + paint(image: GrayImage): void { + const font = glanceFont.small(); + const step = lineStep(font); + const textWidth = image.width - 2 * PAD; + const snapshot = this.snapshot; + let y = PAD; + + const title = "T3 Code"; + image.drawText(font, PAD, y, title, 150); + const summary = snapshot ? glanceSummary(snapshot) : ""; + if (summary) { + const text = truncateText(font, summary, textWidth - font.measureText(title) - 12); + image.drawText(font, image.width - PAD - font.measureText(text), y, text, snapshot!.needsYou ? 230 : 140); + } + y += step; + + const message = !snapshot + ? noWorkerMessage() + : !snapshot.configured + ? "Pair a computer in the T3 Code app" + : snapshot.status || (!snapshot.rows.length ? "No active threads" : ""); + if (message) { + image.drawText(font, PAD, y, truncateText(font, message, textWidth), 120); + return; + } + + const rows = snapshot!.rows; + // Rows are placed a line step apart but only need their line height to fit. + const capacity = Math.max(0, Math.floor((image.height - BOTTOM_MARGIN - y - font.lineHeight) / step) + 1); + const hidden = rows.length + snapshot!.more - capacity; + const shown = hidden > 0 ? rows.slice(0, Math.max(0, capacity - 1)) : rows; + const nowMs = Date.now(); + const markerWidth = font.measureText("W") + 7; + for (const row of shown) { + paintRow(image, font, row, PAD, y, textWidth, markerWidth, nowMs); + y += step; + } + if (hidden > 0) { + image.drawText(font, PAD + markerWidth, y, `+${rows.length + snapshot!.more - shown.length} more`, 110); + } + } +} + +/** Before the T3 worker publishes anything (or after it stopped for lack of anything to connect to). */ +function noWorkerMessage(): string { + const environments = loadEnvironments(); + if (!environments.length) return "Pair a computer in the T3 Code app"; + if (!environments.some((environment) => environment.enabled)) return "Disconnected in the T3 Code app"; + return "Connecting…"; +} + +function rowRightText(row: T3GlanceRow, nowMs: number): string { + if (row.status === "approval") return row.requestKind === "file-change" ? "approve edit" : "approve"; + if (row.status === "input") return "question"; + return formatAge(row.activityMs, nowMs); +} + +function paintRow( + image: GrayImage, + font: UiFont, + row: T3GlanceRow, + x: number, + y: number, + width: number, + markerWidth: number, + nowMs: number, +): void { + const { marker, value: markerValue } = threadMarker(row.status, { unread: row.unread }); + if (marker) image.drawText(font, x, y, marker, markerValue); + const needsYou = row.status === "approval" || row.status === "input"; + const right = rowRightText(row, nowMs); + const rightWidth = right ? font.measureText(right) + 8 : 0; + if (right) image.drawText(font, x + width - font.measureText(right), y, right, needsYou ? 230 : 110); + const value = needsYou ? 255 : row.status === "working" || row.unread ? 215 : 165; + image.drawText(font, x + markerWidth, y, truncateText(font, row.title, Math.max(10, width - markerWidth - rightWidth)), value); +} diff --git a/app/apps/glanceboard/widgets/terminal-widget.ts b/app/apps/glanceboard/widgets/terminal-widget.ts index b48b8543..e3e55728 100644 --- a/app/apps/glanceboard/widgets/terminal-widget.ts +++ b/app/apps/glanceboard/widgets/terminal-widget.ts @@ -1,6 +1,5 @@ import { type GrayImage } from "../../../graphics/image"; import { truncateText } from "../../../graphics/textwrap"; -import { getDefaultSmallFont } from "../../../graphics/ui-fonts"; import { lineStep } from "../../../ui/metrics"; import { onWorkerStateChanged, readWorkerState } from "../../../ui/shell/worker-state"; import { @@ -10,6 +9,7 @@ import { TERMINAL_SESSIONS_STATE_KEY, type TerminalSessionsSnapshot, } from "../../terminal/session-list"; +import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; const PAD = 8; @@ -26,7 +26,7 @@ const BOTTOM_MARGIN = 2; * once the snapshot is in. */ export class TerminalWidget implements GlanceWidget { - private snapshot: TerminalSessionsSnapshot | undefined = readWorkerState(TERMINAL_SESSIONS_STATE_KEY); + private snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined; private unsubscribe: (() => void) | null = null; private requestRender: (() => void) | null = null; private phase = 0; @@ -34,9 +34,9 @@ export class TerminalWidget implements GlanceWidget { start(requestRender: () => void): void { this.requestRender = requestRender; - this.snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY); - this.unsubscribe = onWorkerStateChanged(TERMINAL_SESSIONS_STATE_KEY, (snapshot) => { - this.snapshot = snapshot; + this.snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined; + this.unsubscribe = onWorkerStateChanged(TERMINAL_SESSIONS_STATE_KEY, (snapshot) => { + this.snapshot = snapshot as TerminalSessionsSnapshot; this.syncAnimation(); requestRender(); }); @@ -72,7 +72,7 @@ export class TerminalWidget implements GlanceWidget { } paint(image: GrayImage): void { - const font = getDefaultSmallFont(); + const font = glanceFont.small(); const step = lineStep(font); const textWidth = image.width - 2 * PAD; const snapshot = this.snapshot; diff --git a/app/apps/launcher/launcher-app.ts b/app/apps/launcher/launcher-app.ts index 28ba2a90..d7cf2ccf 100644 --- a/app/apps/launcher/launcher-app.ts +++ b/app/apps/launcher/launcher-app.ts @@ -1,11 +1,10 @@ -import { type BdfFont } from "../../graphics/bdffont"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { truncateText } from "../../graphics/textwrap"; import { GrayImage } from "../../graphics/image"; import { type Plane } from "../../graphics/plane"; import { renderIcon, type IconName } from "../../graphics/icons"; import { InputEvent } from "../../ui/gestures"; -import { IconGrid, ICON_GRID_ICON_SIZE } from "../../ui/icon-grid"; +import { IconGrid, ICON_GRID_BAND_INSET_X, ICON_GRID_HIGHLIGHT_INSET_Y, ICON_GRID_ICON_SIZE } from "../../ui/icon-grid"; import { Layer, LayerActions, LayerContext } from "../../ui/layers"; import { MenuLayer, type MenuItem } from "../../ui/menu"; import { WINDOW_MENU_LAYOUT } from "../../ui/window-menu"; @@ -45,6 +44,11 @@ export const LAUNCHER_WINDOW_ID = "launcher"; export const LAUNCHER_SURFACE_ID = "window:launcher"; const GRID_TOP = 6; +/** + * The root grid's top: its first row's highlight band then sits as far + * below the window's top edge as it is from the side edges. + */ +const ROOT_GRID_TOP = ICON_GRID_BAND_INSET_X - ICON_GRID_HIGHLIGHT_INSET_Y; /** One cell of the grid: an app, or a folder holding some of the apps. */ type LauncherGridEntry = @@ -259,12 +263,13 @@ class LauncherGridLayer implements Layer { const { width, height } = ctx.stack.getBaseSize(); const image = new GrayImage(width, height, 0); // The folder-name header band scales with the font. - const gridTop = this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : GRID_TOP; + const gridTop = this.currentFolder !== null ? GRID_TOP + font.lineHeight + 4 : ROOT_GRID_TOP; if (this.currentFolder !== null) { image.drawText(font, 8, GRID_TOP - 2, truncateText(font, this.currentFolder, width - 16), 160); } - const gridBottom = height - 4; - this.grid.paint(image, { x: 0, y: gridTop, width, height: gridBottom - gridTop }, ctx.stack.isFocused()); + // The grid runs to the window's bottom edge, so a row cut off there for + // scrolling meets the window's border rather than a blank strip. + this.grid.paint(image, { x: 0, y: gridTop, width, height: height - gridTop }, ctx.stack.isFocused()); return image; } diff --git a/app/apps/microphones/mic-control.ts b/app/apps/microphones/mic-control.ts index 49b46251..fbb6a7b6 100644 --- a/app/apps/microphones/mic-control.ts +++ b/app/apps/microphones/mic-control.ts @@ -57,7 +57,7 @@ export function loadMicConfig(): MicConfig { return { ...defaults, ...parsed, - hostMics: { ...defaults.hostMics, ...(parsed.hostMics ?? {}) }, + hostMics: { ...defaults.hostMics, ...parsed.hostMics }, }; } catch { return defaultMicConfig(); @@ -72,8 +72,8 @@ class MicArrayController { private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>(); private statuses: MicArrayStatuses = { left: null, right: null }; // The Java listener proxy must stay referenced or it gets GC'd. - private statusProxy: any | null = null; - private proxyCommunicator: any | null = null; + private statusProxy: any = null; + private proxyCommunicator: any = null; private leaseTimer: ReturnType | null = null; private armed = false; diff --git a/app/apps/microphones/mic-protocol.ts b/app/apps/microphones/mic-protocol.ts index 02cd9788..aaa4061c 100644 --- a/app/apps/microphones/mic-protocol.ts +++ b/app/apps/microphones/mic-protocol.ts @@ -12,8 +12,8 @@ * * Per-temple: each arm is its own endpoint and its own 2-mic array (front * mic near the hinge = chanMask bit0, rear mic near the touchpad = bit1); - * the same config goes to both temples. Stereo payloads arrive as two - * CONCATENATED channel blocks (front then rear), not interleaved. Sessions + * the same config goes to both temples. Stereo payloads are INTERLEAVED PCM16 + * (ch0, ch1, ch0, ch1, ...), the capture dispatch buffer verbatim. Sessions * hold a 90 s lease; renew with op RENEW while streaming. * * Pure module: byte codecs only, no platform imports (the BLE plumbing lives @@ -54,7 +54,7 @@ export type MicConfig = { /** * Host-side per-microphone selection. Each temple streams its full * front+rear stereo pair over BLE (wire channelMask stays 0b11); the phone - * splits the concatenated channels and this mask decides which of the four + * deinterleaves the channels and this mask decides which of the four * mics feed levels-for-use, beamforming, captions, and recording. A temple * with both mics off is sent STOP so its stream doesn't waste bandwidth. */ @@ -168,8 +168,42 @@ export type MicStatus = { framesEmitted: number; /** The rate actually in effect (16 kHz on current firmware). */ effectiveRateHz: number; + /** Tap diagnostics; absent from firmware that sends only the 21-byte body. */ + diagnostics: MicStatusDiagnostics | null; }; +/** Result of a firmware call the temple has not made yet. */ +export const MIC_RET_NONE = 127; + +/** Occupant of one stock PCM app slot. */ +export type MicSlotState = "empty" | "other" | "tap"; + +/** + * Counters from the status extension; they reset on each CONFIGURE. The + * firmware debug overlay shows the same values on each lens. + */ +export type MicStatusDiagnostics = { + /** Calls into the PCM tap (any outcome). */ + tapCalls: number; + tapLastBytes: number; + tapLastSlot: number; + /** Tap calls dropped because the session was inactive or unarmed. */ + skipIdle: number; + /** Tap calls dropped because the streaming lease had lapsed. */ + skipLease: number; + /** Tap calls dropped because the frame buffer allocation failed. */ + skipAlloc: number; + /** Signed firmware results, or MIC_RET_NONE when not called yet. */ + registerResult: number; + unregisterResult: number; + notifyResult: number; + /** Stock PCM slot table, read live: [slot 0, slot 1]. */ + slots: [MicSlotState, MicSlotState]; + leaseRemainingS: number; +}; + +const MIC_STATUS_DIAGNOSTICS_BYTES = 40; + function le16(data: Uint8Array, offset: number): number { return data[offset]! | (data[offset + 1]! << 8); } @@ -210,6 +244,27 @@ export function decodeMicStatus(body: Uint8Array): MicStatus | null { side: body[14] === 2 ? "left" : "right", framesEmitted: le32(body, 15), effectiveRateHz: le16(body, 19) * 100, + diagnostics: body.length >= MIC_STATUS_DIAGNOSTICS_BYTES ? decodeMicDiagnostics(body) : null, + }; +} + +const SLOT_STATES: MicSlotState[] = ["empty", "other", "tap", "other"]; + +function decodeMicDiagnostics(body: Uint8Array): MicStatusDiagnostics { + const s8 = (offset: number) => (body[offset]! >= 0x80 ? body[offset]! - 0x100 : body[offset]!); + const table = body[37]!; + return { + tapCalls: le32(body, 21), + tapLastBytes: le16(body, 25), + tapLastSlot: body[27]!, + skipIdle: le16(body, 28), + skipLease: le16(body, 30), + skipAlloc: le16(body, 32), + registerResult: s8(34), + unregisterResult: s8(35), + notifyResult: s8(36), + slots: [SLOT_STATES[table & 3]!, SLOT_STATES[(table >> 2) & 3]!], + leaseRemainingS: le16(body, 38), }; } @@ -260,25 +315,26 @@ export function decodeMicStreamFrame(data: Uint8Array): MicStreamFrame | null { /** * Split a raw PCM16 SM payload into per-channel Float32Arrays in [-1, 1]. - * Channels arrive as CONCATENATED blocks (front then rear), matching the - * firmware's stereo callback, which dispatches two back-to-back extraction - * results rather than interleaving. + * Channels arrive INTERLEAVED: the firmware forwards the stock capture + * dispatch buffer, which the stock algo reads as ch0 = pcm[2i], ch1 = + * pcm[2i+1] (in samples). Channel 0 is assumed to be the front mic; that + * mapping is not yet confirmed on hardware. */ -export function splitConcatenatedPcm16(frame: MicStreamFrame): Float32Array[] | null { +export function splitInterleavedPcm16(frame: MicStreamFrame): Float32Array[] | null { if (frame.codec !== "raw" || frame.format !== "pcm16" || frame.channelCount < 1) { return null; } const channels = frame.channelCount; - const bytesPerChannel = Math.floor(frame.payload.length / channels) & ~1; - const samplesPerChannel = bytesPerChannel / 2; + const frameBytes = channels * 2; + const samplesPerChannel = Math.floor(frame.payload.length / frameBytes); if (samplesPerChannel <= 0) return null; const out: Float32Array[] = []; for (let channel = 0; channel < channels; channel++) { - const base = channel * bytesPerChannel; const samples = new Float32Array(samplesPerChannel); for (let index = 0; index < samplesPerChannel; index++) { - const lo = frame.payload[base + index * 2]!; - const hi = frame.payload[base + index * 2 + 1]!; + const offset = index * frameBytes + channel * 2; + const lo = frame.payload[offset]!; + const hi = frame.payload[offset + 1]!; let value = lo | (hi << 8); if (value >= 0x8000) value -= 0x10000; samples[index] = value / 32768; diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index 5f019ab8..f6ee7a54 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -16,7 +16,6 @@ import { type HeadOrientation, } from "./dsp"; import { micArrayController, micControlSupported, loadMicConfig, saveMicConfig } from "./mic-control"; -import type { MicConfig } from "./mic-protocol"; import { micModelPath } from "./mic-models"; import { MIC_CHANNEL_KEYS, @@ -24,9 +23,10 @@ import { isStockAudioPacket, micChannelKey, micChannelLabel, - splitConcatenatedPcm16, + splitInterleavedPcm16, templeActive, type MicChannelKey, + type MicConfig, type MicSide, } from "./mic-protocol"; import { @@ -158,8 +158,8 @@ class MicSession { // Captions state. private captionLines: CaptionLine[] = []; - private captionEngine: any | null = null; - private captionListenerProxy: any | null = null; + private captionEngine: any = null; + private captionListenerProxy: any = null; private captionsActive = false; private engineStartWallMs = 0; private sessionRowId = -1; @@ -168,7 +168,7 @@ class MicSession { private conversationLang = "und"; // Recording state. - private recorder: any | null = null; + private recorder: any = null; private recordingPath = ""; // Stock path subscriptions. @@ -177,7 +177,7 @@ class MicSession { private offCompass: (() => void) | null = null; // Extended path. - private forwardingProxy: any | null = null; + private forwardingProxy: any = null; private temples = new Map(); private templeAngles: { left: number | null; right: number | null } = { left: null, right: null }; private templeSsr: { left: number; right: number } = { left: 0, right: 0 }; @@ -193,7 +193,7 @@ class MicSession { private offVoiceActivity: (() => void) | null = null; // Phone-side spectral noise suppressor (FaceclawNoiseSuppressor). - private suppressor: any | null = null; + private suppressor: any = null; private suppressorEngine = ""; // True between the caption engine's speech-start event and the utterance @@ -586,7 +586,7 @@ class MicSession { return; } const side: MicSide = arm === "L" ? "left" : "right"; - const channels = splitConcatenatedPcm16(frame); + const channels = splitInterleavedPcm16(frame); if (!channels || channels.length === 0) return; // Meter every arriving channel — a host-disabled mic still shows its diff --git a/app/apps/microphones/microphones-app.ts b/app/apps/microphones/microphones-app.ts index ef0d491e..c8f6a6ec 100644 --- a/app/apps/microphones/microphones-app.ts +++ b/app/apps/microphones/microphones-app.ts @@ -1,14 +1,13 @@ import { imageFromAsciiArt } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { appViewportSize } from "../../ui/shell/geometry"; -import { LIST_ROW_TEXT_INSET } from "../../ui/metrics"; import { MenuLayer, TextPageLayer, drawRightValueMenuItem, - drawSubmenuIndicator, drawToggleMenuItem, openModalMenu, + submenuItem, type MenuItem, } from "../../ui/menu"; import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings"; @@ -60,7 +59,7 @@ const TRAY_ICON = imageFromAsciiArt( " # # ", " ### ### ", ], - 220, + { "#": 220 }, ); const MENU_LAYOUT = { @@ -73,19 +72,6 @@ const MENU_LAYOUT = { opaque: true, }; -function submenuItem(label: string, onSelect: MenuItem["onSelect"]): MenuItem { - return { - label, - onSelect, - render: ({ image, x, y, width, height, selected, disabled, text }) => { - const font = getDefaultSmallFont(); - const value = disabled ? 70 : selected ? 255 : 200; - image.drawText(font, x, y + LIST_ROW_TEXT_INSET, text, value); - drawSubmenuIndicator(image, font, x, y, width, height, value); - }, - }; -} - /** Persist a config change and re-apply it live when the array is streaming. */ function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void { const updated = mutate(loadMicConfig()); diff --git a/app/apps/microphones/translate.ts b/app/apps/microphones/translate.ts index 8ddb1a2d..d27bbfa4 100644 --- a/app/apps/microphones/translate.ts +++ b/app/apps/microphones/translate.ts @@ -8,7 +8,7 @@ declare const java: any; * translation to the phone's default language beneath it. */ -let languageIdentifier: any | null = null; +let languageIdentifier: any = null; // Translators are expensive to build (model download on first use); cache by // "src>dst" and keep the proxies referenced so they aren't GC'd mid-flight. const translators = new Map(); @@ -59,7 +59,7 @@ export function identifyLanguage(text: string): Promise { }); } -function translatorFor(sourceLang: string, targetLang: string): any | null { +function translatorFor(sourceLang: string, targetLang: string): any { const key = `${sourceLang}>${targetLang}`; const cached = translators.get(key); if (cached) return cached; diff --git a/app/apps/minesweeper/minesweeper-app.worker.ts b/app/apps/minesweeper/minesweeper-app.worker.ts index ebc22c9f..a4320d83 100644 --- a/app/apps/minesweeper/minesweeper-app.worker.ts +++ b/app/apps/minesweeper/minesweeper-app.worker.ts @@ -250,6 +250,11 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { syncClock(window); } break; + case "navigation-sensors": + case "resize-window": + case "text-input": + case "tool-call": + break; } }; @@ -350,7 +355,7 @@ function handleInput(window: MinesweeperWindow, event: InputEvent, frameId: numb // An open window menu owns all input (it closes itself via pop); menus are // list UIs, so watch swipes take their standard fallback meanings there. if (window.menu?.isOpen()) { - window.menu + void window.menu .handleInput(directionalFallback(event)) .catch((error) => console.error(`minesweeper menu input failed: ${error}`)) .then(() => renderAndSubmit(window, frameId)); diff --git a/app/apps/music/music-app.ts b/app/apps/music/music-app.ts index 7c08867a..5d7e24ff 100644 --- a/app/apps/music/music-app.ts +++ b/app/apps/music/music-app.ts @@ -1,5 +1,5 @@ import { getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts"; -import { GrayImage } from "../../graphics/image"; +import { GrayImage, type UiFont } from "../../graphics/image"; import { wrapText, truncateText } from "../../graphics/textwrap"; import { clamp } from "../../util/numeric-util"; import { directionalFallback, GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_SCROLL_DOWN, GESTURE_SCROLL_UP, type InputEvent, isDirectionalInput } from "../../ui/gestures"; @@ -20,8 +20,9 @@ const ART_SIZE = 112; const ART_X = 8; // left margin matches the top margin const ART_Y = 8; const META_X = ART_X + ART_SIZE + 14; -const PROGRESS_TIME_Y = ART_Y + 100; const PROGRESS_BAR_Y = ART_Y + 116; +/** Clear rows between the time labels' baseline (digits have no descenders) and the bar. */ +const PROGRESS_TIME_GAP = 3; const LIST_TOP = 136; /** Gap between the menus' last row band and the viewport bottom. */ const LIST_BOTTOM_MARGIN = 4; @@ -47,6 +48,8 @@ type FocusColumn = "actions" | "playlist"; * Music controller app: metadata + album art + transport controls for the * active Android media session (any player that publishes one), plus the * player's queue when it exposes one (scroll to a track, click to jump). + * With no session, the front page lists the apps whose library can be + * browsed (click one to open its MediaBrowseLayer). */ class MusicAppLayer implements Layer { // Watch swipes skip tracks: right = next, left = previous. @@ -66,6 +69,19 @@ class MusicAppLayer implements Layer { getHeight: () => tightRowHeight(getDefaultSmallFont()), draw: (args) => this.drawQueueItem(args), }); + /** The no-session front page's picker of apps whose library can be browsed. */ + private readonly libraryMenu = new Menu({ + wrap: false, + rowGap: 1, + highlight: { radius: 4 }, + getHeight: () => tightRowHeight(getDefaultSmallFont()), + draw: (args) => this.drawLibraryApp(args), + }); + /** + * Whether the browsable-app list has been re-queried since the no-session + * page last appeared (apps installed since the cached query then show up). + */ + private libraryAppsFresh = false; private art: GrayImage | null = null; private artKey = ""; @@ -93,16 +109,31 @@ class MusicAppLayer implements Layer { } if (!media.available) { + const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh); + this.libraryAppsFresh = true; + this.libraryMenu.setItems(apps); image.drawText(font, 24, 16, "No active media session.", 180); - if (mediaBrowserBridge.listBrowsableApps().length) { - image.drawText(font, 24, 22 + lineStep(font), "Click to browse a music app's library,", 150); - image.drawText(font, 24, 22 + 2 * lineStep(font), "or start playback on the phone.", 150); - image.drawText(font, 20, height - 16, `${GESTURE_CLICK} browse`, 110); - } else { - image.drawText(font, 24, 22 + lineStep(font), "Start playback in another app on the phone.", 150); + const prompt = apps.length + ? "Browse a music app's library, or start playback on the phone." + : "Start playback in another app on the phone."; + let y = 22 + lineStep(font); + for (const line of wrapText(font, prompt, width - 48)) { + image.drawText(font, 24, y, line, 150); + y += lineStep(font); + } + if (apps.length) { + const listTop = y + 6; + const listHeight = height - listTop - LIST_BOTTOM_MARGIN; + this.libraryMenu.paint( + image, + { x: 24 - LIST_TEXT_INSET, y: listTop - 1, width: width - 48 + 2 * LIST_TEXT_INSET - 4, height: listHeight }, + ctx.stack.isFocused(), + ); + this.libraryMenu.drawScrollbar(image, width - 12, listTop, listHeight - 4); } return image; } + this.libraryAppsFresh = false; this.drawArt(image, media); @@ -113,7 +144,7 @@ class MusicAppLayer implements Layer { // (title, second title line, artist, album, player app); when the font is // too large for all five, the title gets one truncated line instead of // wrapping to two, and the rest shift up a slot. - const maxLines = Math.floor((PROGRESS_TIME_Y - metaTop - 4 - font.lineHeight) / metaStep) + 1; + const maxLines = Math.floor((progressTimeY(font) - metaTop - 4 - font.lineHeight) / metaStep) + 1; const titleSlots = maxLines >= 5 ? 2 : 1; const title = media.title || "Unknown title"; const titleLines = @@ -171,6 +202,13 @@ class MusicAppLayer implements Layer { } } + private drawLibraryApp({ image, item: app, x, y, width, height, selected }: MenuDrawArgs): void { + const font = getDefaultSmallFont(); + const value = selected ? 255 : 200; + image.drawText(font, x + LIST_TEXT_INSET, centeredTextY(font, y, height), truncateText(font, app.appName, width - 2 * LIST_TEXT_INSET - 16), value); + drawSubmenuIndicator(image, font, x, y, width, height, value); + } + private drawQueueItem({ image, item, x, y, width, height, selected }: MenuDrawArgs): void { const font = getDefaultSmallFont(); const label = `${item.active ? "> " : " "}${item.title || "(untitled)"}`; @@ -197,8 +235,12 @@ class MusicAppLayer implements Layer { return; } if (!media.available) { - if (event.type === "click") { - this.openBrowse(ctx); + this.libraryMenu.setItems(mediaBrowserBridge.listBrowsableApps()); + if (event.type === "scroll-up" || event.type === "scroll-down") { + await this.libraryMenu.handleInput(event); + } else if (event.type === "click") { + const app = this.libraryMenu.selectedItem; + if (app) this.pushLibraryBrowser(ctx, app); } return; } @@ -252,23 +294,15 @@ class MusicAppLayer implements Layer { /** * Open the media-library browser (the Android Auto MediaBrowserService - * path): with one browsable player installed go straight to its library, - * otherwise offer a picker first. + * path) from the now-playing page: with one browsable player installed go + * straight to its library, otherwise offer a picker first. (With no session + * the picker is the front page itself.) */ private openBrowse(ctx: LayerContext): void { const apps = mediaBrowserBridge.listBrowsableApps(true); if (!apps.length) return; - const pushBrowser = (target: LayerContext, app: MediaBrowserApp) => { - target.stack.push( - new MediaBrowseLayer({ - app, - onPlayed: (browseCtx) => browseCtx.stack.pop(), - onLeave: (browseCtx) => browseCtx.stack.pop(), - }), - ); - }; if (apps.length === 1) { - pushBrowser(ctx, apps[0]!); + this.pushLibraryBrowser(ctx, apps[0]!); return; } ctx.stack.push( @@ -278,13 +312,23 @@ class MusicAppLayer implements Layer { label: app.appName, onSelect: (menuCtx: LayerContext) => { menuCtx.stack.pop(); - pushBrowser(menuCtx, app); + this.pushLibraryBrowser(menuCtx, app); }, })), ), ); } + private pushLibraryBrowser(ctx: LayerContext, app: MediaBrowserApp): void { + ctx.stack.push( + new MediaBrowseLayer({ + app, + onPlayed: (browseCtx) => browseCtx.stack.pop(), + onLeave: (browseCtx) => browseCtx.stack.pop(), + }), + ); + } + private buildActions(media: MediaControllerState, queue: MediaQueueItem[]): MusicAction[] { const volume = mediaControllerBridge.getMediaVolumePercent(); return [ @@ -316,8 +360,9 @@ class MusicAppLayer implements Layer { const font = getDefaultSmallFont(); const elapsed = formatMediaTime(media.positionMs); const duration = formatMediaTime(media.durationMs); - image.drawText(font, META_X, PROGRESS_TIME_Y, elapsed, 140); - image.drawText(font, META_X + width - font.measureText(duration), PROGRESS_TIME_Y, duration, 140); + const timeY = progressTimeY(font); + image.drawText(font, META_X, timeY, elapsed, 140); + image.drawText(font, META_X + width - font.measureText(duration), timeY, duration, 140); const barY = PROGRESS_BAR_Y; image.drawRect(META_X, barY, width, 5, 55); @@ -481,6 +526,15 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin return app; } +/** + * Line top of the elapsed/duration labels: their baseline sits a fixed gap + * above the progress bar, so the labels hug the bar whatever the font's + * height (a taller face grows upward into the metadata's line budget). + */ +function progressTimeY(font: UiFont): number { + return PROGRESS_BAR_Y - PROGRESS_TIME_GAP - font.ascent; +} + function formatMediaTime(milliseconds: number): string { const totalSeconds = Math.max(0, Math.floor(milliseconds / 1_000)); const seconds = totalSeconds % 60; diff --git a/app/apps/navigate/destinations.ts b/app/apps/navigate/destinations.ts index 03849521..689e1783 100644 --- a/app/apps/navigate/destinations.ts +++ b/app/apps/navigate/destinations.ts @@ -1,6 +1,6 @@ /** * Saved and recent destinations for the Navigate app: Home and Work (plain - * address settings, also editable in the Settings app), custom named + * address settings), custom named * destinations, and a recency list of resolved places. Pure settings-store * accessors, usable from the worker and the main thread alike. */ diff --git a/app/apps/navigate/navigate-app.worker.ts b/app/apps/navigate/navigate-app.worker.ts index 14cdd830..db34f057 100644 --- a/app/apps/navigate/navigate-app.worker.ts +++ b/app/apps/navigate/navigate-app.worker.ts @@ -19,6 +19,7 @@ import { drawRightValueMenuItem, drawSubmenuIndicator, drawToggleMenuItem, + submenuItem, type MenuItem, } from "../../ui/menu"; import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; @@ -63,7 +64,16 @@ import { GESTURE_SHORT_THEN_LONG_PRESS, type InputEvent, } from "../../ui/gestures"; -import { LocationTracker, type TrackedLocation } from "./navigation-sensors"; +import { + addCompassListener, + COMPASS_CHANGED, + handleNavigationSensorEvent, + LocationTracker, + magneticDeclinationDegrees, + setCompassEnabled, + type CompassEvent, + type TrackedLocation, +} from "./navigation-sensors"; import { fetchRoute, fetchStaticMapGray, @@ -75,14 +85,11 @@ import { type RouteProfile, type StaticMapCamera, } from "../../native/mapbox"; -import { addCompassListener, COMPASS_CHANGED, setCompassEnabled, type CompassEvent } from "./navigation-sensors"; -import { magneticDeclinationDegrees, handleNavigationSensorEvent } from "./navigation-sensors"; import { calibrateHeading, normalizeHeading } from "../compass/calibration"; import { bearingDegrees, haversineMeters, RouteFollower, type RouteProgress } from "./route-follower"; import { drawManeuverGlyph } from "./maneuver-icons"; declare const global: any; -declare const com: any; const MAP_SIZE = 260; const PANEL_X = MAP_SIZE + 10; @@ -256,8 +263,8 @@ function post(message: WorkerAppReply): void { global.postMessage(message); } -// Destinations edited on the phone (the text editor, or the Settings app's -// Home/Work rows) repaint the idle list / the edit screen live. +// Destinations edited on the phone (the text editor) repaint the idle list / +// the edit screen live. onSettingsStoreChanged((key) => { // The token key repaints too: the phone editor writes it live during // Edit token, and the idle page changes shape once one is set. @@ -376,6 +383,8 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { ); break; } + case "input-focus": + break; } }; @@ -914,7 +923,7 @@ function describeRouteStatus(): string { * display and saved-destination settings (Home, Work, custom named places), and the * recent-destinations preference. */ -function windowMenuItems(win: NavWindow): MenuItem[] { +function windowMenuItems(): MenuItem[] { const items: MenuItem[] = []; if (phase !== "idle") { items.push({ @@ -930,16 +939,9 @@ function windowMenuItems(win: NavWindow): MenuItem[] { enumSettingMenuItem(navigateDisplayModeSetting), enumSettingMenuItem(navigateVerticalPositionSetting), ); - items.push({ - label: "Saved destinations", - onSelect: (ctx) => { - ctx.stack.push(new WindowMenuLayer("Saved destinations", savedDestinationMenuItems())); - }, - render: ({ image, x, y, width, height, text }) => { - image.drawText(smallFont, x, y + LIST_ROW_TEXT_INSET, text, 200); - drawSubmenuIndicator(image, smallFont, x - 10, y, width + 20, height, 150); - }, - }); + items.push(submenuItem("Saved destinations", (ctx) => { + ctx.stack.push(new WindowMenuLayer("Saved destinations", savedDestinationMenuItems())); + })); items.push({ label: navigateRememberRecentSetting.label, onSelect: () => { @@ -993,15 +995,15 @@ function savedDestinationMenuItems(): MenuItem[] { const work = saved.find((d) => d.id === WORK_DESTINATION_ID); items.push(addressMenuItem("Home", HOME_DESTINATION_ID, home?.address ?? "")); items.push(addressMenuItem("Work", WORK_DESTINATION_ID, work?.address ?? "")); - for (const destination of saved) { - if (destination.builtin) continue; + for (const dest of saved) { + if (dest.builtin) continue; items.push({ - label: destination.name, + label: dest.name, onSelect: (ctx) => { - ctx.stack.push(new WindowMenuLayer(destination.name, customDestinationMenuItems(destination))); + ctx.stack.push(new WindowMenuLayer(dest.name, customDestinationMenuItems(dest))); }, render: ({ image, x, y, width, height }) => { - drawAddressRow(image, x, y, width, destination.name, destination.address, true); + drawAddressRow(image, x, y, width, dest.name, dest.address, true); drawSubmenuIndicator(image, smallFont, x - 10, y, width + 20, height, 150); }, }); @@ -1028,14 +1030,14 @@ function addressMenuItem(label: string, id: string, address: string): MenuItem { }; } -function customDestinationMenuItems(destination: SavedDestination): MenuItem[] { +function customDestinationMenuItems(dest: SavedDestination): MenuItem[] { return [ { label: "Navigate there", - disabled: !destination.address, + disabled: !dest.address, onSelect: (ctx) => { closeMenusAnd(ctx, () => { - void startNavigationTo({ kind: "query", query: destination.address, label: destination.name }, profile).catch( + void startNavigationTo({ kind: "query", query: dest.address, label: dest.name }, profile).catch( () => {}, ); }); @@ -1044,16 +1046,16 @@ function customDestinationMenuItems(destination: SavedDestination): MenuItem[] { { label: "Set address", onSelect: (ctx) => { - closeMenusAnd(ctx, () => beginAddressEdit(destination.id, destination.name, destination.address)); + closeMenusAnd(ctx, () => beginAddressEdit(dest.id, dest.name, dest.address)); }, }, { label: "Rename", onSelect: (ctx) => { closeMenusAnd(ctx, () => { - navigateDestinationNameDraftSetting.set(destination.name); - beginEdit(navigateDestinationNameDraftSetting, `Rename ${destination.name}`, (name) => { - if (name) updateCustomDestination(destination.id, { name }); + navigateDestinationNameDraftSetting.set(dest.name); + beginEdit(navigateDestinationNameDraftSetting, `Rename ${dest.name}`, (name) => { + if (name) updateCustomDestination(dest.id, { name }); }); }); }, @@ -1062,7 +1064,7 @@ function customDestinationMenuItems(destination: SavedDestination): MenuItem[] { label: "Remove", onSelect: (ctx) => { closeMenusAnd(ctx, () => { - removeCustomDestination(destination.id); + removeCustomDestination(dest.id); syncIdleList(); }); }, @@ -1195,10 +1197,10 @@ function idleEntries(): IdleEntry[] { run: () => { void startMap(); }, }]; entries.push(...loadSavedDestinations() - .filter((destination) => destination.address) - .map((destination): IdleEntry => ({ kind: "saved", destination }))); - for (const destination of loadRecentDestinations()) { - entries.push({ kind: "recent", destination }); + .filter((dest) => dest.address) + .map((dest): IdleEntry => ({ kind: "saved", destination: dest }))); + for (const dest of loadRecentDestinations()) { + entries.push({ kind: "recent", destination: dest }); } return entries; } @@ -1250,7 +1252,7 @@ function windowMenu(win: NavWindow): WindowMenu { windowId: win.windowId, post, title: () => win.title, - items: () => windowMenuItems(win), + items: () => windowMenuItems(), size: { width: win.viewportWidth, height: win.viewportHeight }, paintBase: () => paintContent(win), isFocused: () => win.focused, @@ -1261,7 +1263,7 @@ function windowMenu(win: NavWindow): WindowMenu { function handleInput(win: NavWindow, event: InputEvent, frameId: number): void { if (win.menu?.isOpen()) { - win.menu + void win.menu .handleInput(event) .catch((error) => console.error(`navigate menu input failed: ${error}`)) .then(() => renderAndSubmit(win, frameId)); diff --git a/app/apps/navigate/route-follower.ts b/app/apps/navigate/route-follower.ts index e40fe8c1..455f83ba 100644 --- a/app/apps/navigate/route-follower.ts +++ b/app/apps/navigate/route-follower.ts @@ -46,7 +46,7 @@ export class RouteFollower { constructor(route: Route) { this.route = route; - this.vertexMeters = new Array(route.coordinates.length); + this.vertexMeters = Array.from({ length: route.coordinates.length }, () => 0); let along = 0; this.vertexMeters[0] = 0; for (let i = 1; i < route.coordinates.length; i++) { @@ -58,7 +58,7 @@ export class RouteFollower { // Each step's maneuver happens where the previous step's travel ends, so // the maneuver point of step k sits at the sum of steps 0..k-1 distances. // (Slight drift vs the geometry is fine at guidance granularity.) - this.stepStartMeters = new Array(route.steps.length); + this.stepStartMeters = Array.from({ length: route.steps.length }, () => 0); let stepAlong = 0; for (let i = 0; i < route.steps.length; i++) { this.stepStartMeters[i] = Math.min(stepAlong, this.totalMeters); diff --git a/app/apps/pinball/pinball-app.worker.ts b/app/apps/pinball/pinball-app.worker.ts index 80c4b1b4..9881d602 100644 --- a/app/apps/pinball/pinball-app.worker.ts +++ b/app/apps/pinball/pinball-app.worker.ts @@ -12,7 +12,8 @@ * keep the BLE payload down to the dirty region around the ball. * * Controls (ball ready): scroll sets launch power, click launches. In play: - * ring-press flips both flippers for 300 ms; click launches from the plunger, + * a touch-down (ring-press) flips both flippers for 300 ms, as does a click + * without one; click launches from the plunger, * double-click pauses. Swipes no longer trigger individual flippers; * up/down raise and lower launch power at the plunger. * Paused/game over: click resumes or starts a new game, double-click yields @@ -42,6 +43,7 @@ import { GESTURE_DOUBLE_CLICK, GESTURE_SCROLL, type InputEvent, + PressTracker, } from "../../ui/gestures"; import { clamp } from "../../util/numeric-util"; @@ -61,6 +63,12 @@ const TOP = 4; const BOTTOM = 256; const LANE_X = 232; const PANEL_X = 310; +/** + * Height the table and panel are drawn for: a min-height window's viewport + * under the standard top bar. A taller viewport (the status bar folded into + * the switcher row, a full-panel display) centers the layout vertically. + */ +const LAYOUT_HEIGHT = 260; const BALL_R = 6; /** Ball resting spot on the plunger, and where launches start. */ @@ -273,6 +281,8 @@ type PinballWindow = { lastMinorSfxAtMs: number; soundOn: boolean; lastSubmittedFingerprint: string; + /** Which gestures already flipped on their ring-press. */ + presses: PressTracker; }; const windows = new Map(); @@ -336,6 +346,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { lastMinorSfxAtMs: 0, soundOn: loadSoundEnabled("pinball"), lastSubmittedFingerprint: "", + presses: new PressTracker(), }; windows.set(message.windowId, window); break; @@ -400,6 +411,11 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { syncTickTimer(window); } break; + case "navigation-sensors": + case "resize-window": + case "text-input": + case "tool-call": + break; } }; @@ -491,10 +507,11 @@ function windowMenu(window: PinballWindow): WindowMenu { } function handleInput(window: PinballWindow, event: InputEvent, frameId: number): void { + const followsPress = window.presses.followsPress(event); // An open window menu owns all input (it closes itself via pop); menus are // list UIs, so watch swipes take their standard fallback meanings there. if (window.menu?.isOpen()) { - window.menu + void window.menu .handleInput(directionalFallback(event)) .catch((error) => console.error(`pinball menu input failed: ${error}`)) .then(() => renderAndSubmit(window, frameId)); @@ -502,13 +519,14 @@ function handleInput(window: PinballWindow, event: InputEvent, frameId: number): } if (window.phase === "playing") { - handlePlayingInput(window, event, frameId); + handlePlayingInput(window, event, frameId, followsPress); } else { handleIdleInput(window, event, frameId); } } -function handlePlayingInput(window: PinballWindow, event: InputEvent, frameId: number): void { +/** `followsPress` marks a gesture whose touch-down already flipped (see PressTracker). */ +function handlePlayingInput(window: PinballWindow, event: InputEvent, frameId: number, followsPress: boolean): void { const ready = window.ballState === "ready"; switch (event.type) { case "scroll-up": @@ -539,7 +557,7 @@ function handlePlayingInput(window: PinballWindow, event: InputEvent, frameId: n case "click": if (ready) { launchBall(window); - } else if (event.source !== "ring") { + } else if (!followsPress) { flip(window, window.flippers[0]!); flip(window, window.flippers[1]!); } @@ -737,6 +755,8 @@ function stepFlippers(window: PinballWindow, dt: number): void { flipper.state = "rest"; } break; + case "rest": + break; } } } @@ -880,8 +900,8 @@ let staticBackground: GrayImage | null = null; /** Everything that never changes: walls, guides, slings, outlines, labels. */ function getStaticBackground(window: PinballWindow): GrayImage { - if (staticBackground) return staticBackground; - const image = new GrayImage(window.viewportWidth, window.viewportHeight, 0); + if (staticBackground?.width === window.viewportWidth) return staticBackground; + const image = new GrayImage(window.viewportWidth, LAYOUT_HEIGHT, 0); for (const segment of SEGMENTS) { image.drawLine(segment.x0, segment.y0, segment.x1, segment.y1, segment.shade); } @@ -981,6 +1001,14 @@ function paintContent(window: PinballWindow): GrayImage { `${GESTURE_CLICK} new game`, ]); } + return centeredInViewport(window, image); +} + +/** Place the fixed-height layout vertically centered in the window's viewport. */ +function centeredInViewport(window: PinballWindow, layout: GrayImage): GrayImage { + if (layout.height === window.viewportHeight) return layout; + const image = new GrayImage(window.viewportWidth, window.viewportHeight, 0); + layout.composeInto(image, 0, Math.max(0, Math.floor((window.viewportHeight - layout.height) / 2))); return image; } diff --git a/app/apps/roam/roam-api.ts b/app/apps/roam/roam-api.ts index a42782d5..ef33022b 100644 --- a/app/apps/roam/roam-api.ts +++ b/app/apps/roam/roam-api.ts @@ -121,7 +121,7 @@ async function roamPost(endpoint: "q" | "pull" | "write", body: object): Promise base = API_BASE; continue; } - throw new Error(`Roam API request failed: ${(error as Error)?.message ?? error}`); + throw new Error(`Roam API request failed: ${(error as Error)?.message ?? error}`, { cause: error }); } if (response.status >= 300 && response.status < 400) { const location = response.headers?.get?.("Location") ?? response.headers?.get?.("location"); diff --git a/app/apps/roam/roam-app.worker.ts b/app/apps/roam/roam-app.worker.ts index 5ec556eb..5575154b 100644 --- a/app/apps/roam/roam-app.worker.ts +++ b/app/apps/roam/roam-app.worker.ts @@ -52,7 +52,6 @@ import { } from "./roam-doc"; declare const global: any; -declare const com: any; const HEADER_HEIGHT = 30; const BOTTOM_MARGIN = 6; @@ -252,6 +251,10 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { ); break; } + case "input-focus": + case "navigation-sensors": + case "resize-window": + break; } }; @@ -339,7 +342,7 @@ function windowMenu(win: RoamWindow): WindowMenu { windowId: win.windowId, post, title: () => win.title, - items: () => menuItems(win), + items: () => menuItems(), size: { width: win.viewportWidth, height: win.viewportHeight }, paintBase: () => paintContent(win), isFocused: () => win.focused, @@ -348,7 +351,7 @@ function windowMenu(win: RoamWindow): WindowMenu { return win.menu; } -function menuItems(win: RoamWindow): MenuItem[] { +function menuItems(): MenuItem[] { const items: MenuItem[] = []; const today = todayRef(); if (currentRef?.uid !== today.uid) { @@ -415,7 +418,7 @@ function endSettingEdit(): void { function handleInput(win: RoamWindow, event: InputEvent, frameId: number): void { if (win.menu?.isOpen()) { - win.menu + void win.menu .handleInput(event) .catch((error) => console.error(`roam menu input failed: ${error}`)) .then(() => renderAndSubmit(win, frameId)); diff --git a/app/apps/settings/settings-app.ts b/app/apps/settings/settings-app.ts index 99436d6b..56761d45 100644 --- a/app/apps/settings/settings-app.ts +++ b/app/apps/settings/settings-app.ts @@ -22,7 +22,7 @@ export type SettingsAppWindow = { /** The full in-process window record, for hosts that manage window lifecycle. */ inProcess: InProcessWindow; requestRender: () => void; - /** Select a section in the left column by label (e.g. "Terminal"). */ + /** Select a section in the left column by label (e.g. "Voice"). */ focusSection: (label: string) => void; /** Whether the glasses-side text-setting editor is the top layer. */ isTextEditorOnTop: () => boolean; diff --git a/app/apps/t3code/background.ts b/app/apps/t3code/background.ts new file mode 100644 index 00000000..f72712e2 --- /dev/null +++ b/app/apps/t3code/background.ts @@ -0,0 +1,11 @@ +import { isT3CodeWidgetInUse } from "../glanceboard/configuration"; +import { loadEnvironments } from "./t3-environments"; + +/** + * Keep the T3 worker (and its server connections) running with no window + * open, for the Glanceboard widget. The app's boot hook spawns the worker + * whenever this turns true; the worker reports idle once it turns false. + */ +export function hasT3BackgroundWork(): boolean { + return isT3CodeWidgetInUse() && loadEnvironments().some((environment) => environment.enabled); +} diff --git a/app/apps/t3code/index.ts b/app/apps/t3code/index.ts new file mode 100644 index 00000000..3c7fa43d --- /dev/null +++ b/app/apps/t3code/index.ts @@ -0,0 +1,42 @@ +import { launchWorkerAppWindow, type AppDefinition } from "../app-definition"; +import { onSettingsStoreChanged } from "../../native/settings-store"; +import { hasT3BackgroundWork } from "./background"; +import { T3_ENVIRONMENTS_KEY } from "./t3-environments"; + +/** The one place the worker is constructed (string-literal path for the webpack worker loader). */ +const createWorker = () => new Worker("./t3code-app.worker"); + +/** The boot hook's settings listener; replaced if a controller boots again. */ +let stopWatchingSettings: (() => void) | null = null; + +const t3codeApp: AppDefinition = { + appId: "t3code", + title: "T3 Code", + icon: "t3code", + launch: (ctx) => + launchWorkerAppWindow(ctx, { + createWorker, + windowId: "t3code:main", + title: "T3 Code", + iconLetter: "T3", + icon: "t3code", + }), + /** + * The Glanceboard's T3 Code widget shows what this app's worker knows, so + * start the worker (windowless) whenever the widget is in use: at startup, + * and when the widget, the board, or the paired environments change. The + * worker reports idle by itself once nothing needs it. + */ + boot: (ctx) => { + const ensureWorker = () => { + if (hasT3BackgroundWork()) ctx.ensureWorkerHost(createWorker); + }; + ensureWorker(); + stopWatchingSettings?.(); + stopWatchingSettings = onSettingsStoreChanged((key) => { + if (key.startsWith("glanceboard.") || key === T3_ENVIRONMENTS_KEY) ensureWorker(); + }); + }, +}; + +export default t3codeApp; diff --git a/app/apps/t3code/t3-auth.ts b/app/apps/t3code/t3-auth.ts new file mode 100644 index 00000000..908525f5 --- /dev/null +++ b/app/apps/t3code/t3-auth.ts @@ -0,0 +1,174 @@ +/** + * HTTP half of connecting to a T3 Code environment: the unauthenticated + * environment descriptor, the one-time pairing-token exchange (RFC 8693, form + * encoded), and short-lived WebSocket tickets. + * + * Direct pairings use a plain Bearer token: 30 days, no refresh. When it + * expires (or is revoked) the ticket request fails with 401 and the user has + * to pair again with a fresh `t3 pair` link. + */ + +/** The orchestration protocol this client speaks (the server answers 426 to any other). */ +export const T3_ORCHESTRATION_PROTOCOL = 2; + +/** What every ordinary pairing grants; asking for a scope the link doesn't grant burns the token. */ +const STANDARD_SCOPES = "orchestration:read orchestration:operate terminal:operate review:write relay:read"; + +const HTTP_TIMEOUT_MS = 12_000; + +/** The subset of fetch these calls need, so tests and Node scripts can pass their own. */ +export type T3Fetch = ( + url: string, + init: { method: string; headers: Record; body?: string }, +) => Promise<{ status: number; ok: boolean; text(): Promise }>; + +export type EnvironmentDescriptor = { + environmentId: string; + label: string; + serverVersion: string; + orchestrationProtocolVersion: number; + capabilities: Record; +}; + +export type PairedCredential = { + accessToken: string; + /** Epoch ms. */ + expiresAtMs: number; +}; + +/** An HTTP failure; `authFailed` means the stored token is no good (re-pair). */ +export class T3HttpError extends Error { + constructor(message: string, readonly status: number, readonly authFailed = false) { + super(message); + this.name = "T3HttpError"; + } +} + +/** Form/query encoding (not URLSearchParams, which the NativeScript runtime mishandles). */ +function buildQuery(params: Record): string { + return Object.keys(params) + .map((key) => `${encodeURIComponent(key)}=${encodeURIComponent(params[key]!)}`) + .join("&"); +} + +async function request( + fetchImpl: T3Fetch, + url: string, + init: { method: string; headers: Record; body?: string }, +): Promise<{ status: number; ok: boolean; body: string }> { + let timer: ReturnType | null = null; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new T3HttpError("Timed out reaching the server.", 0)), HTTP_TIMEOUT_MS); + }); + try { + const response = await Promise.race([fetchImpl(url, init), timeout]); + const body = await Promise.race([response.text(), timeout]); + return { status: response.status, ok: response.ok, body }; + } catch (error) { + if (error instanceof T3HttpError) throw error; + throw new T3HttpError(`Can't reach the server: ${(error as Error)?.message ?? error}`, 0); + } finally { + if (timer) clearTimeout(timer); + } +} + +function parseJson(body: string): any { + try { + return JSON.parse(body); + } catch { + return null; + } +} + +/** Human message for T3's error envelope ({_tag, code, reason, traceId}). */ +function describeFailure(status: number, body: string, what: string): string { + const parsed = parseJson(body); + const reason = typeof parsed?.reason === "string" ? parsed.reason : typeof parsed?.code === "string" ? parsed.code : ""; + switch (reason) { + case "invalid_credential": + return what === "pair" + ? "That pairing link was already used or has expired. Run `t3 pair` for a new one." + : "The server no longer accepts this device's sign-in. Pair it again."; + case "missing_credential": + return "The server asked for a sign-in. Pair this device again."; + case "scope_not_granted": + return "That pairing link doesn't grant enough access. Run `t3 pair` for a new one."; + } + if (status === 426 || parsed?.code === "orchestration_protocol_incompatible") { + return "This T3 Code server speaks a different protocol version. Update T3 Code and Faceclaw."; + } + return `Server error (HTTP ${status}${reason ? `, ${reason}` : ""}).`; +} + +export async function fetchEnvironmentDescriptor(fetchImpl: T3Fetch, httpBaseUrl: string): Promise { + const response = await request(fetchImpl, `${httpBaseUrl}.well-known/t3/environment`, { method: "GET", headers: {} }); + const parsed = parseJson(response.body); + if (!response.ok || !parsed || typeof parsed.environmentId !== "string") { + throw new T3HttpError( + response.ok ? "That address doesn't look like a T3 Code server." : describeFailure(response.status, response.body, "descriptor"), + response.status, + ); + } + return { + environmentId: parsed.environmentId, + label: typeof parsed.label === "string" ? parsed.label : "", + serverVersion: typeof parsed.serverVersion === "string" ? parsed.serverVersion : "", + orchestrationProtocolVersion: typeof parsed.orchestrationProtocolVersion === "number" ? parsed.orchestrationProtocolVersion : 1, + capabilities: parsed.capabilities && typeof parsed.capabilities === "object" ? parsed.capabilities : {}, + }; +} + +/** Trade a one-time pairing credential for a durable bearer token. */ +export async function exchangePairingCredential( + fetchImpl: T3Fetch, + httpBaseUrl: string, + credential: string, + client: { label: string; os: string }, +): Promise { + const body = buildQuery({ + grant_type: "urn:ietf:params:oauth:grant-type:token-exchange", + subject_token: credential, + subject_token_type: "urn:t3:params:oauth:token-type:environment-bootstrap", + requested_token_type: "urn:ietf:params:oauth:token-type:access_token", + scope: STANDARD_SCOPES, + client_label: client.label, + client_device_type: "mobile", + client_os: client.os, + }); + const response = await request(fetchImpl, `${httpBaseUrl}oauth/token`, { + method: "POST", + headers: { "Content-Type": "application/x-www-form-urlencoded" }, + body, + }); + const parsed = parseJson(response.body); + if (!response.ok || typeof parsed?.access_token !== "string") { + throw new T3HttpError(describeFailure(response.status, response.body, "pair"), response.status, response.status === 401); + } + const expiresIn = typeof parsed.expires_in === "number" ? parsed.expires_in : 30 * 24 * 3600; + return { accessToken: parsed.access_token, expiresAtMs: Date.now() + expiresIn * 1000 }; +} + +/** A WebSocket ticket (valid ~5 minutes); fetch a fresh one for every connect. */ +export async function fetchWebSocketTicket(fetchImpl: T3Fetch, httpBaseUrl: string, accessToken: string): Promise { + const response = await request(fetchImpl, `${httpBaseUrl}api/auth/websocket-ticket`, { + method: "POST", + headers: { Authorization: `Bearer ${accessToken}` }, + }); + const parsed = parseJson(response.body); + if (!response.ok || typeof parsed?.ticket !== "string") { + throw new T3HttpError(describeFailure(response.status, response.body, "ticket"), response.status, response.status === 401); + } + return parsed.ticket; +} + +/** The /ws URL for a ticket. */ +export function webSocketUrl(wsBaseUrl: string, ticket: string, clientOs: string): string { + return `${wsBaseUrl}ws?${buildQuery({ + wsTicket: ticket, + clientSurface: "mobile", + clientDeviceType: "phone", + clientOs, + connectionMethod: "direct", + orchestrationProtocol: String(T3_ORCHESTRATION_PROTOCOL), + })}`; +} diff --git a/app/apps/t3code/t3-client.ts b/app/apps/t3code/t3-client.ts new file mode 100644 index 00000000..776475ed --- /dev/null +++ b/app/apps/t3code/t3-client.ts @@ -0,0 +1,425 @@ +/** + * One live connection to a paired T3 Code environment: fetches a WebSocket + * ticket with the stored bearer token, opens the RPC socket, waits for the + * server-config snapshot (checking it is the environment we paired with), + * then keeps the thread list (subscribeShell) and any open threads + * (subscribeThread) live. Owns its own retry loop: dropped connections come + * back with jittered exponential backoff and resume every stream from its + * last sequence. A rejected token stops retrying until the user pairs again. + * + * Transport and HTTP are injected, so this runs unchanged under Node. + */ +import { + fetchWebSocketTicket, + T3HttpError, + webSocketUrl, + type T3Fetch, +} from "./t3-auth"; +import { ShellModel, ThreadDetailModel, type T3ServerProvider } from "./t3-model"; +import { T3RpcConnection, T3RpcError, type RpcSocketFactory, type T3Subscription } from "./t3-rpc"; + +/** A paired environment as stored on the phone. */ +export type StoredEnvironment = { + /** Local id (stable across re-pairing). */ + id: string; + /** The server's own id, checked on every connect. */ + environmentId: string; + /** The server's label (its machine name). */ + label: string; + httpBaseUrl: string; + wsBaseUrl: string; + /** Bearer token from pairing (30-day lifetime, no refresh). */ + accessToken: string; + expiresAtMs: number; + /** Connect while the app is open. */ + enabled: boolean; +}; + +export type EnvironmentPhase = + | "idle" + | "connecting" + | "connected" + /** Waiting to try again after a failure. */ + | "retrying" + /** The token was rejected or expired: pair again. */ + | "auth-failed"; + +export type T3ClientDeps = { + fetch: T3Fetch; + openSocket: RpcSocketFactory; + /** Reported to the server as the client OS ("Android", "iOS"). */ + clientOs: string; + log?: (message: string) => void; +}; + +const RECONNECT_MIN_DELAY_MS = 1_000; +const RECONNECT_MAX_DELAY_MS = 60_000; +/** How long the socket may stay open without delivering the config snapshot. */ +const READY_TIMEOUT_MS = 20_000; +/** Resubscribe delay after a stream fails on a healthy connection (e.g. a server buffer overflow). */ +const STREAM_RETRY_MS = 250; + +export type ThreadHandle = { + readonly model: ThreadDetailModel; + /** The subscription failed for a reason other than the connection dropping. */ + readonly error: () => string; + close(): void; +}; + +type ThreadEntry = { + model: ThreadDetailModel; + subscription: T3Subscription | null; + retryTimer: ReturnType | null; + error: string; + listeners: Set<() => void>; + refs: number; +}; + +export class T3EnvironmentClient { + readonly shell = new ShellModel(); + providers: T3ServerProvider[] = []; + settings: Record | null = null; + private phaseValue: EnvironmentPhase = "idle"; + private statusValue = ""; + private connection: T3RpcConnection | null = null; + private shellSubscription: T3Subscription | null = null; + private shellRetryTimer: ReturnType | null = null; + private readyTimer: ReturnType | null = null; + private reconnectTimer: ReturnType | null = null; + private reconnectDelayMs = RECONNECT_MIN_DELAY_MS; + /** Bumped on every connect attempt and stop, so stale async work can tell it lost. */ + private generation = 0; + private running = false; + private readonly listeners = new Set<() => void>(); + private readonly threads = new Map(); + + constructor( + public config: StoredEnvironment, + private readonly deps: T3ClientDeps, + ) {} + + get phase(): EnvironmentPhase { + return this.phaseValue; + } + + /** Human-readable connection status ("" when connected and healthy). */ + get status(): string { + return this.statusValue; + } + + /** Connected, with the thread list loaded. */ + get ready(): boolean { + return this.phaseValue === "connected" && this.shell.sequence !== null; + } + + get retryScheduled(): boolean { + return this.reconnectTimer !== null; + } + + /** Any change worth repainting for: phase, status, thread list, config. */ + onChange(listener: () => void): () => void { + this.listeners.add(listener); + return () => this.listeners.delete(listener); + } + + /** Start connecting (no-op while already running). */ + start(): void { + if (this.running) return; + this.running = true; + this.reconnectDelayMs = RECONNECT_MIN_DELAY_MS; + void this.connect(); + } + + /** Disconnect and stop retrying. Open thread handles stay registered and resume on the next start. */ + stop(): void { + this.running = false; + this.generation++; + this.clearTimers(); + this.teardownConnection("Disconnected."); + this.setPhase("idle", ""); + } + + /** Retry right away (manual reconnect), resetting backoff. */ + retryNow(): void { + this.stop(); + this.start(); + } + + /** Replace the stored config (e.g. re-paired); reconnects if anything connection-relevant changed. */ + updateConfig(config: StoredEnvironment): void { + const reconnect = + config.accessToken !== this.config.accessToken || + config.httpBaseUrl !== this.config.httpBaseUrl || + config.environmentId !== this.config.environmentId; + this.config = config; + if (reconnect && this.running) this.retryNow(); + else this.emit(); + } + + /** + * Subscribe to a thread's detail stream. Handles are reference counted: + * every openThread needs its own close(). + */ + openThread(threadId: string, onChange: () => void): ThreadHandle { + let entry = this.threads.get(threadId); + if (!entry) { + entry = { model: new ThreadDetailModel(threadId), subscription: null, retryTimer: null, error: "", listeners: new Set(), refs: 0 }; + this.threads.set(threadId, entry); + if (this.phaseValue === "connected") this.subscribeThread(threadId, entry); + } + const current = entry; + current.refs++; + current.listeners.add(onChange); + let closed = false; + return { + model: current.model, + error: () => current.error, + close: () => { + if (closed) return; + closed = true; + current.listeners.delete(onChange); + if (--current.refs > 0) return; + current.subscription?.cancel(); + if (current.retryTimer) clearTimeout(current.retryTimer); + this.threads.delete(threadId); + }, + }; + } + + /** orchestration.dispatchCommand; resolves to the committed event sequence. */ + async dispatch(command: Record): Promise { + const connection = this.requireConnection(); + const result = await connection.call<{ sequence?: number }>("orchestration.dispatchCommand", command); + return typeof result?.sequence === "number" ? result.sequence : 0; + } + + /** orchestration.launchThread; resolves to the new thread's id. */ + async launchThread(payload: Record): Promise { + const connection = this.requireConnection(); + const result = await connection.call<{ threadId?: string }>("orchestration.launchThread", payload); + return String(result?.threadId ?? payload.threadId); + } + + private requireConnection(): T3RpcConnection { + if (this.phaseValue !== "connected" || !this.connection) { + throw new T3RpcError("disconnected", `Not connected to ${this.config.label || "the server"}.`); + } + return this.connection; + } + + // ------------------------------------------------------------------------- + // Connection lifecycle + + private async connect(): Promise { + const generation = ++this.generation; + this.clearTimers(); + this.teardownConnection("Reconnecting."); + if (this.config.expiresAtMs && this.config.expiresAtMs <= Date.now()) { + this.setPhase("auth-failed", "Sign-in expired. Pair this device again."); + return; + } + this.setPhase("connecting", "Connecting..."); + let ticket: string; + try { + ticket = await fetchWebSocketTicket(this.deps.fetch, this.config.httpBaseUrl, this.config.accessToken); + } catch (error) { + if (generation !== this.generation) return; + if (error instanceof T3HttpError && error.authFailed) { + this.setPhase("auth-failed", error.message); + return; + } + this.scheduleReconnect(errorMessage(error)); + return; + } + if (generation !== this.generation) return; + + const connection = new T3RpcConnection({ + url: webSocketUrl(this.config.wsBaseUrl, ticket, this.deps.clientOs), + openSocket: this.deps.openSocket, + }); + this.connection = connection; + connection.onStateChange((state, reason) => { + if (state !== "closed" || generation !== this.generation) return; + this.handleConnectionLost(reason); + }); + this.readyTimer = setTimeout(() => { + if (generation === this.generation) connection.close("The server didn't finish connecting."); + }, READY_TIMEOUT_MS); + + let ready = false; + connection.subscribe( + "subscribeServerConfig", + {}, + (values) => { + if (generation !== this.generation) return; + for (const value of values) { + const event = value as Record; + if (event?.type === "snapshot") { + const config = event.config ?? {}; + const environmentId = config.environment?.environmentId; + if (this.config.environmentId && environmentId && environmentId !== this.config.environmentId) { + this.generation++; + this.teardownConnection("Wrong server."); + this.setPhase("auth-failed", "A different T3 Code server answered at this address. Pair it again."); + return; + } + this.providers = Array.isArray(config.providers) ? config.providers : []; + this.settings = config.settings ?? null; + if (typeof config.environment?.label === "string" && config.environment.label && config.environment.label !== this.config.label) { + this.config = { ...this.config, label: config.environment.label }; + } + if (!ready) { + ready = true; + this.handleReady(generation); + } + } else if (event?.type === "providerStatuses" && Array.isArray(event.payload?.providers)) { + this.providers = event.payload.providers; + } else if (event?.type === "settingsUpdated" && event.payload?.settings) { + this.settings = event.payload.settings; + } + } + this.emit(); + }, + (error) => { + // The session is unusable without its config stream (T3's own client + // treats this as the session closing). + if (generation !== this.generation || error?.kind === "disconnected") return; + connection.close(`Server config stream ended${error ? `: ${error.message}` : "."}`); + }, + ); + } + + private handleReady(generation: number): void { + if (this.readyTimer) clearTimeout(this.readyTimer); + this.readyTimer = null; + this.reconnectDelayMs = RECONNECT_MIN_DELAY_MS; + this.setPhase("connected", ""); + this.deps.log?.(`t3code: connected to ${this.config.label} (${this.config.httpBaseUrl})`); + this.subscribeShell(generation); + for (const [threadId, entry] of this.threads) this.subscribeThread(threadId, entry); + } + + private subscribeShell(generation: number): void { + const connection = this.connection; + if (!connection || generation !== this.generation) return; + this.shell.synchronized = false; + this.shellSubscription?.cancel(); + const payload: Record = { requestCompletionMarker: true }; + if (this.shell.sequence !== null) payload.afterSequence = this.shell.sequence; + this.shellSubscription = connection.subscribe( + "orchestration.subscribeShell", + payload, + (values) => { + if (generation !== this.generation) return; + let changed = false; + for (const value of values) changed = this.shell.apply(value) || changed; + if (changed) this.emit(); + }, + (error) => { + this.shellSubscription = null; + if (generation !== this.generation || error?.kind === "disconnected") return; + this.deps.log?.(`t3code: shell stream ended: ${error?.message ?? "completed"}`); + this.shellRetryTimer = setTimeout(() => { + this.shellRetryTimer = null; + this.subscribeShell(generation); + }, STREAM_RETRY_MS); + }, + ); + } + + private subscribeThread(threadId: string, entry: ThreadEntry): void { + const connection = this.connection; + if (!connection || this.phaseValue !== "connected") return; + const generation = this.generation; + entry.subscription?.cancel(); + entry.model.synchronized = false; + const payload: Record = { threadId, requestCompletionMarker: true, acceptBoundedSnapshot: true }; + if (entry.model.sequence !== null) payload.afterSequence = entry.model.sequence; + const notify = () => { + for (const listener of [...entry.listeners]) listener(); + }; + const subscription = connection.subscribe( + "orchestration.subscribeThread", + payload, + (values) => { + if (generation !== this.generation || this.threads.get(threadId) !== entry) return; + let changed = false; + for (const value of values) changed = entry.model.apply(value) || changed; + if (entry.error) { + entry.error = ""; + changed = true; + } + if (changed) notify(); + }, + (error) => { + if (entry.subscription === subscription) entry.subscription = null; + if (generation !== this.generation || this.threads.get(threadId) !== entry || error?.kind === "disconnected") return; + entry.error = error ? error.message : "The thread stream ended."; + notify(); + if (error?.kind === "fail" || error?.kind === "interrupt") { + entry.retryTimer = setTimeout(() => { + entry.retryTimer = null; + if (this.threads.get(threadId) === entry) this.subscribeThread(threadId, entry); + }, STREAM_RETRY_MS * 4); + } + }, + ); + entry.subscription = subscription; + } + + private handleConnectionLost(reason: string): void { + this.connection = null; + this.shellSubscription = null; + for (const entry of this.threads.values()) entry.subscription = null; + if (!this.running) return; + this.scheduleReconnect(reason); + } + + private scheduleReconnect(reason: string): void { + if (!this.running) return; + const delay = Math.round(this.reconnectDelayMs * (0.75 + Math.random() * 0.5)); + this.reconnectDelayMs = Math.min(this.reconnectDelayMs * 2, RECONNECT_MAX_DELAY_MS); + this.setPhase("retrying", reason); + this.deps.log?.(`t3code: ${this.config.label}: ${reason} (retrying in ${delay}ms)`); + const generation = this.generation; + this.reconnectTimer = setTimeout(() => { + this.reconnectTimer = null; + if (generation === this.generation && this.running) void this.connect(); + }, delay); + } + + private teardownConnection(reason: string): void { + const connection = this.connection; + this.connection = null; + this.shellSubscription = null; + for (const entry of this.threads.values()) entry.subscription = null; + connection?.close(reason); + } + + private clearTimers(): void { + for (const timer of [this.readyTimer, this.shellRetryTimer]) if (timer) clearTimeout(timer); + if (this.reconnectTimer) clearTimeout(this.reconnectTimer); + this.readyTimer = null; + this.shellRetryTimer = null; + this.reconnectTimer = null; + for (const entry of this.threads.values()) { + if (entry.retryTimer) clearTimeout(entry.retryTimer); + entry.retryTimer = null; + } + } + + private setPhase(phase: EnvironmentPhase, status: string): void { + if (this.phaseValue === phase && this.statusValue === status) return; + this.phaseValue = phase; + this.statusValue = status; + this.emit(); + } + + private emit(): void { + for (const listener of [...this.listeners]) listener(); + } +} + +export function errorMessage(error: unknown): string { + return String((error as Error)?.message ?? error); +} diff --git a/app/apps/t3code/t3-environments.ts b/app/apps/t3code/t3-environments.ts new file mode 100644 index 00000000..48ede549 --- /dev/null +++ b/app/apps/t3code/t3-environments.ts @@ -0,0 +1,49 @@ +import { getStringSetting, setStringSetting } from "../../native/settings-store"; +import type { StoredEnvironment } from "./t3-client"; + +/** + * Paired T3 Code environments, stored as JSON under one settings key (like + * the Terminal app's connections). Each entry holds the bearer token issued + * at pairing; it lasts 30 days and is never shown on screen. + */ +export const T3_ENVIRONMENTS_KEY = "t3code.environments"; + +export function loadEnvironments(): StoredEnvironment[] { + const raw = getStringSetting(T3_ENVIRONMENTS_KEY, ""); + if (!raw) return []; + try { + const parsed = JSON.parse(raw); + if (!Array.isArray(parsed)) return []; + return parsed + .map((item: any): StoredEnvironment => ({ + id: String(item?.id ?? ""), + environmentId: String(item?.environmentId ?? ""), + label: String(item?.label ?? ""), + httpBaseUrl: String(item?.httpBaseUrl ?? ""), + wsBaseUrl: String(item?.wsBaseUrl ?? ""), + accessToken: String(item?.accessToken ?? ""), + expiresAtMs: Number(item?.expiresAtMs) || 0, + enabled: item?.enabled !== false, + })) + .filter((environment) => environment.id && environment.httpBaseUrl && environment.accessToken); + } catch { + return []; + } +} + +export function saveEnvironments(environments: StoredEnvironment[]): void { + setStringSetting(T3_ENVIRONMENTS_KEY, JSON.stringify(environments)); +} + +/** Insert or replace (by local id). */ +export function upsertEnvironment(environment: StoredEnvironment): void { + const environments = loadEnvironments(); + const index = environments.findIndex((candidate) => candidate.id === environment.id); + if (index >= 0) environments[index] = environment; + else environments.push(environment); + saveEnvironments(environments); +} + +export function removeEnvironment(id: string): void { + saveEnvironments(loadEnvironments().filter((environment) => environment.id !== id)); +} diff --git a/app/apps/t3code/t3-glance.ts b/app/apps/t3code/t3-glance.ts new file mode 100644 index 00000000..05ba312e --- /dev/null +++ b/app/apps/t3code/t3-glance.ts @@ -0,0 +1,103 @@ +/** + * The T3 Code thread list as the Glanceboard shows it: the T3 worker builds + * this snapshot from its live connections and publishes it through the + * host's worker-state channel; the T3 Code widget paints it on the main + * thread. Plain JSON (it crosses postMessage) with timestamps rather than + * formatted ages, so the widget's "5m" is right whenever it paints. + * + * No NativeScript imports: this module is bundled into the worker as well. + */ +import { + isThreadUnread, + sectionThreads, + threadActivityMs, + threadStatus, + type T3ThreadShell, + type T3ThreadStatus, +} from "./t3-model"; + +/** Worker-state key the T3 worker publishes its snapshot under. */ +export const T3_GLANCE_STATE_KEY = "t3code:threads"; + +/** More than a double-height slot can show; the widget says how many it left out. */ +const MAX_ROWS = 24; + +export type T3GlanceRow = { + /** Environment id + thread id. */ + key: string; + title: string; + status: T3ThreadStatus; + /** A run finished since the thread was last visited (on any T3 client). */ + unread: boolean; + /** For approvals: what kind ("command", "file-change", ...). */ + requestKind: string | null; + /** Last activity, epoch ms (server clock). */ + activityMs: number; +}; + +export type T3GlanceSnapshot = { + /** Any environment is paired. */ + configured: boolean; + /** Why there is nothing to show when no environment is connected ("Connecting…", ...); "" otherwise. */ + status: string; + needsYou: number; + working: number; + /** Needs-you first, then working, then recent; settled and snoozed threads left out. */ + rows: T3GlanceRow[]; + /** Listed threads beyond `rows`. */ + more: number; +}; + +export type T3GlanceEnvironment = { + id: string; + ready: boolean; + /** "connecting", "retrying", "auth-failed", ... (T3EnvironmentClient phase). */ + phase: string; + enabled: boolean; + threads: Iterable; +}; + +export function buildGlanceSnapshot(environments: readonly T3GlanceEnvironment[], nowMs: number): T3GlanceSnapshot { + const owner = new Map(); + for (const environment of environments) { + if (!environment.ready) continue; + for (const thread of environment.threads) owner.set(thread, environment.id); + } + const sections = sectionThreads(owner.keys(), nowMs); + const listed = [...sections.attention, ...sections.working, ...sections.recent]; + const rows = listed.slice(0, MAX_ROWS).map((thread): T3GlanceRow => { + const status = threadStatus(thread); + return { + key: `${owner.get(thread)}:${thread.id}`, + title: thread.title || "Untitled", + status, + unread: isThreadUnread(thread), + requestKind: status === "approval" ? (thread.pendingRuntimeRequest?.kind ?? null) : null, + activityMs: threadActivityMs(thread), + }; + }); + return { + configured: environments.length > 0, + status: environments.some((environment) => environment.ready) ? "" : environmentsStatus(environments), + needsYou: sections.attention.length, + working: sections.working.filter((thread) => threadStatus(thread) === "working").length, + rows, + more: listed.length - rows.length, + }; +} + +function environmentsStatus(environments: readonly T3GlanceEnvironment[]): string { + if (!environments.length) return ""; + if (environments.every((environment) => !environment.enabled)) return "Disconnected"; + if (environments.some((environment) => environment.phase === "connecting" || environment.phase === "connected")) return "Connecting…"; + if (environments.every((environment) => !environment.enabled || environment.phase === "auth-failed")) return "Pair again in the T3 Code app"; + return "Can't reach T3 Code"; +} + +/** "1 needs you · 2 working" (or "" when neither). */ +export function glanceSummary(snapshot: Pick): string { + return [ + snapshot.needsYou ? `${snapshot.needsYou} need${snapshot.needsYou === 1 ? "s" : ""} you` : "", + snapshot.working ? `${snapshot.working} working` : "", + ].filter(Boolean).join(" · "); +} diff --git a/app/apps/t3code/t3-model.ts b/app/apps/t3code/t3-model.ts new file mode 100644 index 00000000..63ecab1e --- /dev/null +++ b/app/apps/t3code/t3-model.ts @@ -0,0 +1,924 @@ +/** + * Client-side state for T3 Code's orchestration (v2) streams, independent of + * transport and rendering: the shell (projects + thread list) reducer, the + * per-thread detail reducer, thread status / ordering for the glasses list, + * one-line summaries of timeline items, and the dispatch-command builders. + * + * Mirrors T3's own client runtime (packages/client-runtime/src/state/ + * shellReducer.ts, orchestrationV2Projection.ts, models.ts; apps/mobile's + * threadListV2.ts and threadActivity.ts) in reduced form. Wire objects are + * typed loosely: only the fields read here are declared, and everything is + * treated as possibly missing so newer/older servers degrade gracefully. + */ + +// --------------------------------------------------------------------------- +// Wire shapes (subset) + +export type T3ProjectShell = { + id: string; + title: string; + workspaceRoot?: string; + defaultModelSelection?: T3ModelSelection | null; +}; + +export type T3ModelSelection = { instanceId: string; model: string; options?: unknown[] }; + +export type T3ThreadShell = { + id: string; + projectId: string; + title: string; + providerInstanceId?: string | null; + modelSelection?: T3ModelSelection | null; + status?: string; + activityRunStatus?: string | null; + activeRunId?: string | null; + latestRunId?: string | null; + activeProviderThreadId?: string | null; + latestRunRequestedAt?: string | null; + latestRunCompletedAt?: string | null; + lastVisitedAt?: string | null; + pendingRuntimeRequest?: { id: string; kind: string; createdAt?: string } | null; + pendingBackgroundTasks?: Array<{ kind?: string }>; + lastError?: string | null; + lastErrorClass?: string | null; + usageLimitResetAt?: string | null; + latestUserMessageAt?: string | null; + latestUserAuthoredMessageAt?: string | null; + hasActionableProposedPlan?: boolean; + createdAt?: string; + updatedAt?: string; + archivedAt?: string | null; + deletedAt?: string | null; + settledOverride?: string | null; + settledAt?: string | null; + snoozedUntil?: string | null; + snoozedAt?: string | null; + lineage?: { relationshipToParent?: string | null } | null; +}; + +export type T3Run = { + id: string; + status: string; + userMessageId?: string | null; + requestedAt?: string | null; + queuePosition?: number | null; +}; + +export type T3RunAttempt = { id: string; runId: string; rootNodeId?: string | null; status: string }; + +export type T3RuntimeRequest = { + id: string; + kind: string; + status: string; + responseCapability?: { type: string } | null; +}; + +export type T3ConversationMessage = { + id: string; + role?: string; + text?: string; + delegatedCompletion?: unknown; + notification?: unknown; +}; + +export type T3PlanStep = { id?: string; text: string; status: string }; + +export type T3Plan = { + id: string; + kind: string; + runId?: string | null; + status?: string; + steps?: T3PlanStep[]; +}; + +export type T3ApprovalOption = { decision: string; label: string; warning?: string }; + +export type T3Question = { + id: string; + header?: string; + question: string; + options?: Array<{ label: string; description?: string; value?: string }>; + multiSelect?: boolean; + allowCustomAnswer?: boolean; + required?: boolean; +}; + +/** A timeline row. Fields beyond the common ones depend on `type`. */ +export type T3TurnItem = { + id: string; + threadId: string; + runId: string | null; + nodeId?: string | null; + ordinal: number; + status: string; + title?: string | null; + type: string; + // user_message / assistant_message / reasoning + text?: string; + streaming?: boolean; + inputIntent?: string; + // command_execution + input?: unknown; + exitCode?: number; + outputIndicatesFailure?: boolean; + // file_change + fileName?: string; + additions?: number; + deletions?: number; + changes?: unknown[]; + // file_search / web_search + pattern?: string; + patterns?: string[]; + // dynamic_tool + toolName?: string | null; + // approval_request / user_input_request + requestId?: string; + requestKind?: string; + prompt?: string; + appName?: string; + options?: T3ApprovalOption[]; + questions?: T3Question[]; + responseMode?: string; + // notification / system_notice / run_interrupt_result + summary?: string; + detail?: string; + message?: string; + // error + failure?: { class?: string; message?: string; code?: string | null } | null; + // compaction + beforeTokenCount?: number; + afterTokenCount?: number; + // handoff + toModel?: string; + // proposed_plan + markdown?: string; +}; + +type ProjectedTurnItem = { visibility?: string; item: T3TurnItem }; + +// --------------------------------------------------------------------------- +// Shell (projects + thread list) + +export class ShellModel { + readonly projects = new Map(); + readonly threads = new Map(); + /** Last applied sequence (the resume cursor), or null before the first snapshot. */ + sequence: number | null = null; + /** The server finished replaying and the stream is live. */ + synchronized = false; + + /** Apply one subscribeShell stream item; returns whether anything changed. */ + apply(raw: unknown): boolean { + const item = raw as Record; + switch (item?.kind) { + case "synchronized": + this.synchronized = true; + return true; + case "snapshot": { + const snapshot = item.snapshot ?? {}; + if (Array.isArray(item.resolvedRepositoryIdentityRoots)) { + // Metadata-only refresh (carries threads: []): never a replacement. + return false; + } + this.projects.clear(); + this.threads.clear(); + for (const project of snapshot.projects ?? []) this.projects.set(project.id, project); + for (const thread of snapshot.threads ?? []) this.threads.set(thread.id, thread); + this.sequence = typeof snapshot.snapshotSequence === "number" ? snapshot.snapshotSequence : 0; + return true; + } + case "project.updated": + case "project.removed": + case "thread.updated": + case "thread.removed": { + const sequence = typeof item.sequence === "number" ? item.sequence : null; + if (sequence !== null && this.sequence !== null && sequence <= this.sequence) return false; + if (item.kind === "project.updated" && item.project) this.projects.set(item.project.id, item.project); + else if (item.kind === "project.removed") this.projects.delete(item.projectId); + else if (item.kind === "thread.updated" && item.thread) { + if (item.location === "archive") this.threads.delete(item.thread.id); + else this.threads.set(item.thread.id, item.thread); + } else if (item.kind === "thread.removed") this.threads.delete(item.threadId); + if (sequence !== null) this.sequence = sequence; + return true; + } + default: + return false; + } + } +} + +// --------------------------------------------------------------------------- +// Thread status and list ordering (as T3's mobile thread list derives them) + +export type T3ThreadStatus = + /** A provider approval is waiting on the user. */ + | "approval" + /** The provider asked the user a question. */ + | "input" + | "working" + /** Parked on background work (subagent, monitor) that will wake it. */ + | "waiting" + | "failed" + /** Failed on a usage limit. */ + | "limited" + | "ready"; + +const ACTIVE_RUN_STATUSES = new Set(["preparing", "queued", "starting", "running", "waiting"]); +const WAKING_BACKGROUND_KINDS = new Set(["subagent", "monitor", "background_task"]); + +export function threadStatus(thread: T3ThreadShell): T3ThreadStatus { + const request = thread.pendingRuntimeRequest; + if (request && request.kind !== "user_input" && request.kind !== "auth_refresh") return "approval"; + if (request?.kind === "user_input") return "input"; + if (thread.latestRunId == null && thread.activeProviderThreadId == null) return "ready"; + const background = + thread.status !== "failed" && + (thread.pendingBackgroundTasks ?? []).some((task) => WAKING_BACKGROUND_KINDS.has(task.kind ?? "")); + const runStatus = background ? "idle" : (thread.activityRunStatus ?? thread.status ?? "idle"); + if (ACTIVE_RUN_STATUSES.has(runStatus)) return "working"; + if (runStatus === "idle") return "waiting"; + if (runStatus === "failed") return thread.lastErrorClass === "usage_limit" ? "limited" : "failed"; + return "ready"; +} + +export function threadStatusLabel(status: T3ThreadStatus): string { + switch (status) { + case "approval": + return "Needs approval"; + case "input": + return "Has a question"; + case "working": + return "Working"; + case "waiting": + return "Waiting"; + case "failed": + return "Failed"; + case "limited": + return "Usage limit"; + case "ready": + return "Ready"; + } +} + +const STATUS_MARKERS: Record = { + approval: { marker: "!", value: 255 }, + input: { marker: "?", value: 255 }, + working: { marker: "…", value: 210 }, + waiting: { marker: "…", value: 130 }, + failed: { marker: "x", value: 200 }, + limited: { marker: "x", value: 160 }, + ready: { marker: "", value: 0 }, +}; + +/** + * The one-glyph status mark drawn left of a thread title (in the app's list + * and on the Glanceboard), with its gray level: ! approval, ? question, + * … working, x failed, ● finished since last visited, ✓ settled. + */ +export function threadMarker(status: T3ThreadStatus, flags: { settled?: boolean; unread?: boolean } = {}): { marker: string; value: number } { + if (status === "ready" && flags.settled) return { marker: "✓", value: 110 }; + if (status === "ready" && flags.unread) return { marker: "●", value: 220 }; + return STATUS_MARKERS[status]; +} + +function timeMs(value: string | null | undefined): number { + if (!value) return 0; + const parsed = Date.parse(value); + return Number.isFinite(parsed) ? parsed : 0; +} + +/** A run finished since the user (on any client) last looked at the thread. */ +export function isThreadUnread(thread: T3ThreadShell): boolean { + const completed = timeMs(thread.latestRunCompletedAt); + const visited = timeMs(thread.lastVisitedAt); + return completed > 0 && visited > 0 && completed > visited; +} + +export function isThreadSettled(thread: T3ThreadShell): boolean { + return thread.settledOverride === "settled"; +} + +export function isThreadSnoozed(thread: T3ThreadShell, nowMs: number): boolean { + if (timeMs(thread.snoozedUntil) <= nowMs) return false; + // A raised hand (question, approval, failure, or a run finishing after the + // snooze) brings it back. + const status = threadStatus(thread); + if (status === "approval" || status === "input" || status === "failed") return false; + return !(timeMs(thread.latestRunCompletedAt) > timeMs(thread.snoozedAt)); +} + +/** Archived, deleted and provider-run subagent threads stay out of the list. */ +export function isThreadListed(thread: T3ThreadShell): boolean { + return !thread.archivedAt && !thread.deletedAt && thread.lineage?.relationshipToParent !== "subagent"; +} + +/** When the thread last saw activity worth sorting by. */ +export function threadActivityMs(thread: T3ThreadShell): number { + return timeMs(thread.latestUserMessageAt) || timeMs(thread.updatedAt) || timeMs(thread.createdAt); +} + +function settledMs(thread: T3ThreadShell): number { + return timeMs(thread.settledAt) || threadActivityMs(thread); +} + +export type ThreadSection = "attention" | "working" | "recent" | "snoozed" | "settled"; + +export type ThreadSections = Record; + +/** + * The glasses list: threads that need the user first, then running ones + * (ordered by when the user last wrote, so finishing doesn't reshuffle + * them), then everything else by recent activity; snoozed and settled + * threads are kept apart. + */ +export function sectionThreads(threads: Iterable, nowMs: number): ThreadSections { + const sections: ThreadSections = { attention: [], working: [], recent: [], snoozed: [], settled: [] }; + for (const thread of threads) { + if (!isThreadListed(thread)) continue; + const status = threadStatus(thread); + if (status === "approval" || status === "input") sections.attention.push(thread); + else if (isThreadSnoozed(thread, nowMs)) sections.snoozed.push(thread); + else if (isThreadSettled(thread)) sections.settled.push(thread); + else if (status === "working" || status === "waiting") sections.working.push(thread); + else sections.recent.push(thread); + } + const byId = (a: T3ThreadShell, b: T3ThreadShell) => a.id.localeCompare(b.id); + sections.attention.sort((a, b) => timeMs(b.pendingRuntimeRequest?.createdAt) - timeMs(a.pendingRuntimeRequest?.createdAt) || byId(a, b)); + const sentMs = (thread: T3ThreadShell) => timeMs(thread.latestUserAuthoredMessageAt) || timeMs(thread.latestRunRequestedAt); + sections.working.sort((a, b) => sentMs(b) - sentMs(a) || byId(a, b)); + sections.recent.sort((a, b) => threadActivityMs(b) - threadActivityMs(a) || byId(a, b)); + sections.snoozed.sort((a, b) => timeMs(a.snoozedUntil) - timeMs(b.snoozedUntil) || byId(a, b)); + sections.settled.sort((a, b) => settledMs(b) - settledMs(a) || byId(a, b)); + return sections; +} + +/** "now", "5m", "3h", "2d", "Mar 4" style age. */ +export function formatAge(thenMs: number, nowMs: number): string { + if (!thenMs) return ""; + const seconds = Math.max(0, Math.round((nowMs - thenMs) / 1000)); + if (seconds < 60) return "now"; + const minutes = Math.round(seconds / 60); + if (minutes < 60) return `${minutes}m`; + const hours = Math.round(minutes / 60); + if (hours < 24) return `${hours}h`; + const days = Math.round(hours / 24); + if (days < 14) return `${days}d`; + const date = new Date(thenMs); + const months = ["Jan", "Feb", "Mar", "Apr", "May", "Jun", "Jul", "Aug", "Sep", "Oct", "Nov", "Dec"]; + return `${months[date.getMonth()]} ${date.getDate()}`; +} + +// --------------------------------------------------------------------------- +// Thread detail (subscribeThread) + +const MAX_ITEMS_KEPT = 400; + +export class ThreadDetailModel { + /** Last applied sequence (the resume cursor), or null before the first snapshot. */ + sequence: number | null = null; + synchronized = false; + deleted = false; + thread: Record | null = null; + readonly runs = new Map(); + readonly attempts = new Map(); + readonly runtimeRequests = new Map(); + readonly messages = new Map(); + readonly plans = new Map(); + /** This thread's own timeline rows, by id. */ + readonly items = new Map(); + /** Rows inherited from a forked-from thread: a frozen prefix. */ + private inherited: T3TurnItem[] = []; + /** Bounded-window bookkeeping: older history exists that we didn't load. */ + hasMoreHistory = false; + private historyCursor: string | null = null; + private latestLocalTurnOrdinal: number | null = null; + + constructor(readonly threadId: string) {} + + /** Apply one subscribeThread stream item; returns whether anything changed. */ + apply(raw: unknown): boolean { + const item = raw as Record; + switch (item?.kind) { + case "synchronized": + this.synchronized = true; + return true; + case "snapshot": + this.applySnapshot(item); + return true; + case "event": { + const sequence = typeof item.sequence === "number" ? item.sequence : null; + if (sequence !== null && this.sequence !== null && sequence <= this.sequence) return false; + if (sequence !== null) this.sequence = sequence; + return this.applyEvent(item.event ?? {}); + } + default: + return false; + } + } + + private applySnapshot(item: Record): void { + const projection = item.projection ?? {}; + this.sequence = typeof item.snapshotSequence === "number" ? item.snapshotSequence : 0; + this.thread = projection.thread ?? null; + this.deleted = Boolean(this.thread?.deletedAt); + const fill = (map: Map, values: T[] | undefined) => { + map.clear(); + for (const value of values ?? []) map.set(value.id, value); + }; + fill(this.runs, projection.runs); + fill(this.attempts, projection.attempts); + fill(this.runtimeRequests, projection.runtimeRequests); + fill(this.messages, projection.messages); + fill(this.plans, projection.plans); + this.items.clear(); + this.inherited = []; + for (const row of (projection.visibleTurnItems ?? []) as ProjectedTurnItem[]) { + if (!row?.item) continue; + if (row.visibility === "local") this.items.set(row.item.id, row.item); + else this.inherited.push(row.item); + } + this.hasMoreHistory = Boolean(item.hasMoreHistory); + this.historyCursor = typeof item.historyCursor === "string" ? item.historyCursor : null; + this.latestLocalTurnOrdinal = typeof item.latestLocalTurnOrdinal === "number" ? item.latestLocalTurnOrdinal : null; + } + + private applyEvent(event: Record): boolean { + const payload = event.payload; + const type = String(event.type ?? ""); + if (type === "thread.deleted") { + this.deleted = true; + return true; + } + if (type.startsWith("thread.")) { + if (payload && typeof payload === "object") this.thread = payload; + return true; + } + switch (type) { + case "run.created": + case "run.updated": + this.runs.set(payload.id, payload); + return true; + case "run-attempt.created": + case "run-attempt.updated": + this.attempts.set(payload.id, payload); + return true; + case "runtime-request.updated": + this.runtimeRequests.set(payload.id, payload); + return true; + case "message.updated": + this.messages.set(payload.id, payload); + return true; + case "plan.updated": + this.plans.set(payload.id, payload); + return true; + case "turn-item.updated": + return this.upsertItem(payload as T3TurnItem); + default: + // Unknown and uninteresting events still advance the cursor. + return false; + } + } + + private upsertItem(item: T3TurnItem): boolean { + if (!item?.id || item.threadId !== this.threadId) return false; + const partial = this.hasMoreHistory || this.historyCursor !== null; + if (!this.items.has(item.id) && partial) { + // Updates to rows from history we never loaded. + if (this.latestLocalTurnOrdinal !== null && item.ordinal <= this.latestLocalTurnOrdinal) return false; + const oldest = this.oldestLocalOrdinal(); + if (oldest !== null && item.ordinal < oldest) return false; + } + this.items.set(item.id, item); + if (partial && (this.latestLocalTurnOrdinal === null || item.ordinal > this.latestLocalTurnOrdinal)) { + this.latestLocalTurnOrdinal = item.ordinal; + } + if (this.items.size > MAX_ITEMS_KEPT) this.trimOldest(); + return true; + } + + private oldestLocalOrdinal(): number | null { + let oldest: number | null = null; + for (const item of this.items.values()) if (oldest === null || item.ordinal < oldest) oldest = item.ordinal; + return oldest; + } + + /** A long-lived subscription keeps growing; drop the oldest rows (they're history now). */ + private trimOldest(): void { + const sorted = [...this.items.values()].sort(compareItems); + for (const item of sorted.slice(0, sorted.length - MAX_ITEMS_KEPT)) this.items.delete(item.id); + this.hasMoreHistory = true; + this.inherited = []; + } + + /** Visible rows in display order (inherited fork history first). */ + timeline(): T3TurnItem[] { + const local = [...this.items.values()].filter((item) => this.isVisible(item)).sort(compareItems); + return [...this.inherited, ...local]; + } + + private isVisible(item: T3TurnItem): boolean { + const run = item.runId ? this.runs.get(item.runId) : undefined; + if (run?.status === "rolled_back") return false; + if (item.type === "user_message" && item.inputIntent === "queued_turn" && run?.status === "cancelled") return false; + if (item.type === "run_interrupt_result" && item.runId && item.nodeId) { + const superseded = [...this.attempts.values()].some( + (attempt) => attempt.runId === item.runId && attempt.rootNodeId === item.nodeId && attempt.status === "superseded", + ); + if (superseded) { + const hasRequest = [...this.items.values()].some( + (candidate) => candidate.type === "run_interrupt_request" && candidate.runId === item.runId, + ); + if (!hasRequest) return false; + } + } + return true; + } + + /** The run a Stop should interrupt, if any. */ + activeRunId(): string | null { + let active: T3Run | null = null; + for (const run of this.runs.values()) { + if (["preparing", "starting", "running", "waiting"].includes(run.status)) { + if (!active || timeMs(run.requestedAt) >= timeMs(active.requestedAt)) active = run; + } + } + return active?.id ?? null; + } + + /** Pending approvals and questions, oldest first, paired with their timeline rows. */ + pendingRequests(): PendingRequest[] { + const result: PendingRequest[] = []; + const items = [...this.items.values()].sort(compareItems); + for (const request of this.runtimeRequests.values()) { + if (request.status !== "pending") continue; + if (request.kind === "auth_refresh" || request.kind === "dynamic_tool_call") continue; + const type = request.kind === "user_input" ? "user_input_request" : "approval_request"; + const item = [...items].reverse().find((candidate) => candidate.type === type && candidate.requestId === request.id); + if (!item) continue; + const capability = request.responseCapability?.type ?? "live"; + result.push({ + request, + item, + kind: request.kind === "user_input" ? "question" : "approval", + answerable: request.kind === "user_input" ? capability !== "not_resumable" : capability === "live", + messageMode: item.responseMode === "message" || capability === "message", + }); + } + return result.sort((a, b) => compareItems(a.item, b.item)); + } + + /** Follow-ups waiting for the active run to finish (they have no timeline row yet). */ + queuedMessages(): string[] { + return [...this.runs.values()] + .filter((run) => run.status === "queued" && run.userMessageId) + .sort((a, b) => (a.queuePosition ?? 0) - (b.queuePosition ?? 0)) + .map((run) => this.messages.get(run.userMessageId!)) + .filter((message): message is T3ConversationMessage => Boolean(message && !message.delegatedCompletion && !message.notification)) + .map((message) => message.text ?? ""); + } + + /** The current todo list (the latest run's, else the newest one). */ + todoList(): T3Plan | null { + const todos = [...this.plans.values()].filter((plan) => plan.kind === "todo_list" && plan.steps?.length); + if (!todos.length) return null; + const latestRunId = this.thread?.latestRunId ?? null; + return todos.find((plan) => plan.runId === latestRunId) ?? todos[todos.length - 1]!; + } +} + +export type PendingRequest = { + request: T3RuntimeRequest; + item: T3TurnItem; + kind: "approval" | "question"; + /** False when the provider session can't take an answer any more (interrupt instead). */ + answerable: boolean; + /** Message-mode questions: answers must be non-empty strings. */ + messageMode: boolean; +}; + +function compareItems(a: T3TurnItem, b: T3TurnItem): number { + return a.ordinal - b.ordinal || String(a.id).localeCompare(String(b.id)); +} + +// --------------------------------------------------------------------------- +// Timeline presentation + +export type TimelineEntry = + | { kind: "user"; id: string; text: string; queued?: boolean } + | { kind: "assistant"; id: string; text: string; streaming: boolean } + | { kind: "activity"; id: string; label: string; detail: string; state: ActivityState } + | { kind: "request"; id: string; label: string; detail: string; resolved: boolean }; + +export type ActivityState = "running" | "done" | "failed" | "stopped"; + +function activityState(item: T3TurnItem): ActivityState { + switch (item.status) { + case "pending": + case "running": + case "waiting": + return "running"; + case "failed": + return "failed"; + case "cancelled": + case "interrupted": + return "stopped"; + default: + return "done"; + } +} + +function capitalize(text: string): string { + return text ? text[0]!.toUpperCase() + text.slice(1) : text; +} + +function oneLine(text: string | null | undefined): string { + return (text ?? "").replace(/\s+/g, " ").trim(); +} + +function baseName(path: string): string { + return path.split(/[\\/]/).filter(Boolean).pop() ?? path; +} + +/** `bash -lc '