mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into ring-health-fixes
This commit is contained in:
commit
fcea0b369a
218 files changed
+14438
-2259
No files matched your search
@@ -18,12 +18,18 @@ jobs:
|
||||
- run: npm ci
|
||||
- run: npm test
|
||||
|
||||
# The shared Kotlin tests (tests/kotlin, on the JVM host). The runner image
|
||||
# already carries the JDK 17 and Android SDK the build needs.
|
||||
# The shared Kotlin tests (tests/kotlin, on the JVM host). Pinned to
|
||||
# ubuntu-24.04 for the Android SDK its image carries; the JDK is pinned
|
||||
# explicitly because Gradle 8.14 can't run on the JDK 25 that is the default
|
||||
# on ubuntu-26.04 (now ubuntu-latest), failing with just "25.0.5".
|
||||
test-kotlin:
|
||||
runs-on: ubuntu-latest
|
||||
runs-on: ubuntu-24.04
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-java@v4
|
||||
with:
|
||||
distribution: temurin
|
||||
java-version: '17'
|
||||
- run: npm ci
|
||||
- run: npm run test:kotlin
|
||||
|
||||
|
||||
+2
-2
@@ -44,9 +44,9 @@ __pycache__
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
|
||||
# pull_config.sh output: the phone's live settings, including API keys
|
||||
# pull_config*.sh output: the phone's live settings, including API keys
|
||||
faceclaw_settings.xml
|
||||
faceclaw_settings.ios.json
|
||||
faceclaw_settings*.json
|
||||
|
||||
# Developer configs
|
||||
build_paths.sh
|
||||
|
||||
@@ -13,6 +13,13 @@ and others who contributed to documenting the G2's bluetooth protocol and firmwa
|
||||
|
||||
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
|
||||
|
||||
iPhone notification icons are Pebble's notification and timeline icons, from
|
||||
https://github.com/pebble-dev/iconography: the original Pebble Technology icons
|
||||
and Rebble community additions. They are licensed under the Apache License 2.0;
|
||||
see app/graphics/pebble-icons.LICENSE. Faceclaw's table of the iPhone apps they
|
||||
stand for builds on the one in the PebbleOS firmware,
|
||||
https://github.com/coredevices/PebbleOS (also Apache License 2.0).
|
||||
|
||||
Weather data outside the US is by Open-Meteo.com: https://open-meteo.com/. It is
|
||||
licensed under CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/).
|
||||
|
||||
|
||||
@@ -17,6 +17,10 @@ class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLi
|
||||
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
|
||||
}
|
||||
|
||||
override fun requestLe2mPhy(address: String) = bleManager.requestLe2mPhy(address)
|
||||
|
||||
override fun describePhy(address: String): String? = bleManager.describePhy(address)
|
||||
|
||||
override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean =
|
||||
bleManager.requestMtu(address, mtu, timeoutMs)
|
||||
|
||||
|
||||
@@ -18,6 +18,8 @@ import java.util.concurrent.ConcurrentHashMap
|
||||
import java.util.concurrent.CountDownLatch
|
||||
import java.util.concurrent.ExecutorService
|
||||
import java.util.concurrent.Executors
|
||||
import java.util.concurrent.ScheduledExecutorService
|
||||
import java.util.concurrent.ScheduledFuture
|
||||
import java.util.concurrent.TimeUnit
|
||||
import java.util.concurrent.atomic.AtomicLong
|
||||
|
||||
@@ -27,6 +29,10 @@ class FaceclawBleManager(context: Context) {
|
||||
private const val TAG = "FaceclawBle"
|
||||
private val WRITE_RETRY_DELAYS_MS = intArrayOf(1, 1, 1, 2, 4, 8, 12, 20, 35, 100, 200)
|
||||
|
||||
// Gap after each LE 2M PHY request before checking whether it landed and asking
|
||||
// again; the request gives up once these run out (see requestLe2mPhy).
|
||||
private val PHY_RETRY_DELAYS_MS = longArrayOf(1_000, 2_000, 4_000, 8_000, 15_000)
|
||||
|
||||
// Handed to an operation waiter whose link dropped before its completion
|
||||
// callback arrived; never a real GATT status.
|
||||
private const val STATUS_LINK_LOST = -1
|
||||
@@ -47,6 +53,19 @@ class FaceclawBleManager(context: Context) {
|
||||
Thread(runnable, "FaceclawBleCallbacks").apply { isDaemon = true }
|
||||
}
|
||||
|
||||
// Runs the LE 2M PHY request loops. Each step takes the global lock, so it must not
|
||||
// run on the binder thread or the callback executor.
|
||||
private val phyScheduler: ScheduledExecutorService = Executors.newSingleThreadScheduledExecutor { runnable ->
|
||||
Thread(runnable, "FaceclawBlePhy").apply { isDaemon = true }
|
||||
}
|
||||
|
||||
private fun phyName(phy: Int): String = when (phy) {
|
||||
BluetoothDevice.PHY_LE_1M -> "1M"
|
||||
BluetoothDevice.PHY_LE_2M -> "2M"
|
||||
BluetoothDevice.PHY_LE_CODED -> "Coded"
|
||||
else -> "?"
|
||||
}
|
||||
|
||||
// Process-wide outbound-traffic totals, sampled by the phone UI's BLE
|
||||
// bandwidth indicator. Static so counts from every manager instance and
|
||||
// isolate land in the same totals.
|
||||
@@ -76,6 +95,16 @@ class FaceclawBleManager(context: Context) {
|
||||
@Volatile var gatt: BluetoothGatt? = null
|
||||
@Volatile var connected = false
|
||||
val connectLatch = CountDownLatch(1)
|
||||
|
||||
// The PHY pair Android last reported for this connection (BluetoothDevice.PHY_LE_*),
|
||||
// 0 until a report arrives.
|
||||
@Volatile var txPhy = 0
|
||||
@Volatile var rxPhy = 0
|
||||
|
||||
// The 2M request loop's next step; only touched on phyScheduler.
|
||||
var phyStep: ScheduledFuture<*>? = null
|
||||
|
||||
fun on2mPhy(): Boolean = txPhy == BluetoothDevice.PHY_LE_2M && rxPhy == BluetoothDevice.PHY_LE_2M
|
||||
}
|
||||
|
||||
private enum class OpKind { MTU, SERVICES, DESCRIPTOR, WRITE }
|
||||
@@ -252,13 +281,72 @@ class FaceclawBleManager(context: Context) {
|
||||
val accepted = gatt.requestConnectionPriority(BluetoothGatt.CONNECTION_PRIORITY_HIGH)
|
||||
Log.i(TAG, "benchmark HIGH address=$address submitted=$accepted")
|
||||
}
|
||||
if ((mode and 2) != 0) {
|
||||
Log.i(TAG, "benchmark request 2M address=$address")
|
||||
gatt.setPreferredPhy(BluetoothDevice.PHY_LE_2M_MASK,
|
||||
BluetoothDevice.PHY_LE_2M_MASK, BluetoothDevice.PHY_OPTION_NO_PREFERRED)
|
||||
}
|
||||
gatt.readPhy()
|
||||
}
|
||||
if ((mode and 2) != 0) {
|
||||
Log.i(TAG, "benchmark request 2M address=$address")
|
||||
requestLe2mPhy(address)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask for the LE 2M PHY in both directions, then keep checking and re-asking until it
|
||||
* lands or [PHY_RETRY_DELAYS_MS] runs out. A single request is unreliable: right after
|
||||
* connecting, Android can reject it as unsupported before it has read the peer's feature
|
||||
* bits, and an on-air PHY update can lose a collision with the peer's own
|
||||
* connection-parameter request. Our preference stays 2M-only, so a granted 2M link
|
||||
* shouldn't drift back; the loop ends with the connection.
|
||||
*/
|
||||
fun requestLe2mPhy(address: String) {
|
||||
val link = links[address] ?: return
|
||||
phyScheduler.execute {
|
||||
link.phyStep?.cancel(false)
|
||||
runPhyStep(address, link, 0)
|
||||
}
|
||||
}
|
||||
|
||||
private fun runPhyStep(address: String, link: GattLink, attempt: Int) {
|
||||
link.phyStep = null
|
||||
if (links[address] !== link || !link.connected) {
|
||||
return
|
||||
}
|
||||
if (link.on2mPhy()) {
|
||||
Log.i(TAG, "LE 2M PHY active: address=$address requests=$attempt")
|
||||
return
|
||||
}
|
||||
if (attempt >= PHY_RETRY_DELAYS_MS.size) {
|
||||
Log.w(TAG, "LE 2M PHY not granted after $attempt requests: address=$address"
|
||||
+ " tx=${phyName(link.txPhy)} rx=${phyName(link.rxPhy)}")
|
||||
return
|
||||
}
|
||||
synchronized(bluetoothApiLock) {
|
||||
val gatt = link.gatt
|
||||
if (links[address] !== link || !link.connected || gatt == null) {
|
||||
return
|
||||
}
|
||||
Log.i(TAG, "requesting LE 2M PHY: address=$address attempt=${attempt + 1}")
|
||||
gatt.setPreferredPhy(BluetoothDevice.PHY_LE_2M_MASK,
|
||||
BluetoothDevice.PHY_LE_2M_MASK, BluetoothDevice.PHY_OPTION_NO_PREFERRED)
|
||||
// The update-complete event can be skipped when nothing changes; a read always
|
||||
// reports, so the next step sees the PHY even then.
|
||||
gatt.readPhy()
|
||||
}
|
||||
link.phyStep = phyScheduler.schedule(
|
||||
{ runPhyStep(address, link, attempt + 1) },
|
||||
PHY_RETRY_DELAYS_MS[attempt],
|
||||
TimeUnit.MILLISECONDS
|
||||
)
|
||||
}
|
||||
|
||||
/** The link's PHY for diagnostics: "2M", "1M", "tx/rx" when they differ, "?" before a report; null when not connected. */
|
||||
fun describePhy(address: String): String? {
|
||||
val link = links[address]
|
||||
if (link == null || !link.connected) {
|
||||
return null
|
||||
}
|
||||
val tx = link.txPhy
|
||||
val rx = link.rxPhy
|
||||
return if (tx == rx) phyName(tx) else phyName(tx) + "/" + phyName(rx)
|
||||
}
|
||||
|
||||
fun getNegotiatedMtu(address: String): Int {
|
||||
@@ -600,11 +688,23 @@ class FaceclawBleManager(context: Context) {
|
||||
override fun onPhyRead(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) {
|
||||
Log.i(TAG, "onPhyRead: address=" + gatt.device.address
|
||||
+ " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status)
|
||||
recordPhy(gatt, txPhy, rxPhy, status)
|
||||
}
|
||||
|
||||
override fun onPhyUpdate(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) {
|
||||
Log.i(TAG, "onPhyUpdate: address=" + gatt.device.address
|
||||
+ " txPhy=" + txPhy + " rxPhy=" + rxPhy + " status=" + status)
|
||||
recordPhy(gatt, txPhy, rxPhy, status)
|
||||
}
|
||||
|
||||
private fun recordPhy(gatt: BluetoothGatt, txPhy: Int, rxPhy: Int, status: Int) {
|
||||
// A failed request reports the PHYs it asked for, not the ones in use.
|
||||
if (status != BluetoothGatt.GATT_SUCCESS) return
|
||||
val link = links[gatt.device.address] ?: return
|
||||
val current = link.gatt
|
||||
if (current != null && current !== gatt) return
|
||||
link.txPhy = txPhy
|
||||
link.rxPhy = rxPhy
|
||||
}
|
||||
|
||||
override fun onDescriptorWrite(gatt: BluetoothGatt, descriptor: BluetoothGattDescriptor, status: Int) {
|
||||
|
||||
+84
-1
@@ -35,6 +35,8 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawNotify"
|
||||
private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName"
|
||||
/** MessagingStyle's conversation icon (Notification.EXTRA_CONVERSATION_ICON, hidden). */
|
||||
private const val EXTRA_CONVERSATION_ICON = "android.conversationIcon"
|
||||
/**
|
||||
* How long notification access can be granted with no listener
|
||||
* connected before the watchdog forces a rebind: longer than a
|
||||
@@ -47,6 +49,8 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
private const val LISTENER_STUCK_MS = 30_000L
|
||||
/** Notifications posted while disconnected alert on reconnect only if this recent. */
|
||||
private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L
|
||||
/** Large icons are drawn at no more than this many pixels a side before scaling down. */
|
||||
private const val LARGE_ICON_MAX_SOURCE = 512
|
||||
|
||||
/** The connected listener; null until onListenerConnected and after onListenerDisconnected. */
|
||||
@Volatile
|
||||
@@ -366,6 +370,48 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground)
|
||||
}
|
||||
|
||||
/**
|
||||
* One active notification's large icon (largeIconOf: a sender's avatar, a site's logo)
|
||||
* scaled to fit size x size as a gray packet, a light background shown as its negative
|
||||
* like a picture's; empty when the notification or its icon is unavailable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getNotificationLargeIconGray(key: String?, size: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
val service = activeService
|
||||
if (service == null || size <= 0) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val statusBarNotification = findActiveNotificationByKey(service, key)
|
||||
val icon = statusBarNotification?.notification?.let { largeIconOf(it) } ?: return ByteArray(0)
|
||||
try {
|
||||
val drawable = icon.loadDrawable(service) ?: return ByteArray(0)
|
||||
val width = if (drawable.intrinsicWidth > 0) Math.min(drawable.intrinsicWidth, LARGE_ICON_MAX_SOURCE) else size * 4
|
||||
val height = if (drawable.intrinsicHeight > 0) Math.min(drawable.intrinsicHeight, LARGE_ICON_MAX_SOURCE) else size * 4
|
||||
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
||||
drawable.setBounds(0, 0, width, height)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
return ImageFileLoader.bitmapToGrayPacket(halveToward(bitmap, size, size), size, size, gamma, dither, true)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load large icon for " + statusBarNotification.packageName, t)
|
||||
return ByteArray(0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* One active notification's custom layout (hasCustomView in its JSON) drawn to fit
|
||||
* maxWidth x maxHeight as a gray packet; see NotificationCustomViews.renderGray. Empty
|
||||
* when the notification or its layout is unavailable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getNotificationCustomViewGray(key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
val service = activeService ?: return ByteArray(0)
|
||||
val statusBarNotification = findActiveNotificationByKey(service, key)
|
||||
val notification = statusBarNotification?.notification ?: return ByteArray(0)
|
||||
val hasPicture = notification.extras?.let { hasNotificationPicture(it) } ?: false
|
||||
return NotificationCustomViews.renderGray(
|
||||
service, statusBarNotification, !hasPicture, maxWidth, maxHeight, gamma, dither)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun getActiveNotificationsJson(maxNotifications: Int): String {
|
||||
val service = activeService
|
||||
@@ -672,6 +718,32 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return null
|
||||
}
|
||||
|
||||
/**
|
||||
* The picture a notification shows beside its text: its large icon (Gmail's sender
|
||||
* avatar or logo), else from Android 11 its conversation's icon, else the icon of its
|
||||
* latest message's sender.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
private fun largeIconOf(notification: Notification): Icon? {
|
||||
notification.getLargeIcon()?.let { return it }
|
||||
val extras = notification.extras
|
||||
if (extras == null || Build.VERSION.SDK_INT < Build.VERSION_CODES.R) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
val conversationIcon: Icon? = extras.getParcelable(EXTRA_CONVERSATION_ICON)
|
||||
if (conversationIcon != null) {
|
||||
return conversationIcon
|
||||
}
|
||||
val messages = extras.getParcelableArray(Notification.EXTRA_MESSAGES) ?: return null
|
||||
return Notification.MessagingStyle.Message.getMessagesFromBundleArray(messages)
|
||||
.lastOrNull()?.senderPerson?.icon
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to read conversation icons", t)
|
||||
return null
|
||||
}
|
||||
}
|
||||
|
||||
/** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */
|
||||
private fun softwareBitmap(bitmap: Bitmap): Bitmap {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) {
|
||||
@@ -687,7 +759,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
* (NotificationIconLoader.renderArgb), one filtered pass from a much larger source
|
||||
* samples too sparsely: a graph's thin lines break up into dots.
|
||||
*/
|
||||
private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
|
||||
internal fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
|
||||
var bitmap = source
|
||||
while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
@@ -759,6 +831,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
out.put("lines", JSONArray())
|
||||
out.put("hasPicture", false)
|
||||
}
|
||||
out.put("hasLargeIcon", largeIconOf(notification) != null)
|
||||
val hasCustomView = NotificationCustomViews.hasCustomView(notification)
|
||||
out.put("hasCustomView", hasCustomView)
|
||||
val customText = JSONArray()
|
||||
// Only needed (and only worth inflating the layout for) when there's no title or text to show instead.
|
||||
if (hasCustomView && out.optString("title").isEmpty() && out.optString("text").isEmpty()) {
|
||||
for (text in NotificationCustomViews.texts(service, statusBarNotification)) {
|
||||
customText.put(text)
|
||||
}
|
||||
}
|
||||
out.put("customText", customText)
|
||||
|
||||
val actionsJson = JSONArray()
|
||||
val actions: Array<Notification.Action?>? = notification.actions
|
||||
|
||||
@@ -13,7 +13,7 @@ class FaceclawResourceUsage private constructor() {
|
||||
companion object {
|
||||
/**
|
||||
* Returns elapsedMs, processCpuMs, rssKb, nativeHeapKb, javaHeapKb,
|
||||
* javaHeapCommittedKb, and threadCount. Reading /proc/self/status avoids
|
||||
* javaHeapCommittedKb, threadCount, and peakRssKb. Reading /proc/self/status avoids
|
||||
* the considerably heavier full Debug.MemoryInfo/PSS collection once per
|
||||
* second while still reporting the RSS that Android records at exit.
|
||||
*/
|
||||
@@ -30,30 +30,34 @@ class FaceclawResourceUsage private constructor() {
|
||||
javaUsedBytes / 1024L,
|
||||
runtime.totalMemory() / 1024L,
|
||||
status[1],
|
||||
status[2],
|
||||
)
|
||||
}
|
||||
|
||||
/** Returns {VmRSS in KiB, thread count}; unavailable values are zero. */
|
||||
/** Returns {VmRSS in KiB, thread count, VmHWM in KiB}; unavailable values are zero. */
|
||||
private fun readProcessStatus(): LongArray {
|
||||
var rssKb = 0L
|
||||
var threadCount = 0L
|
||||
var peakRssKb = 0L
|
||||
try {
|
||||
BufferedReader(FileReader("/proc/self/status")).use { reader ->
|
||||
var line: String?
|
||||
while (reader.readLine().also { line = it } != null) {
|
||||
val current = line!!
|
||||
if (current.startsWith("VmRSS:")) {
|
||||
if (current.startsWith("VmHWM:")) {
|
||||
peakRssKb = firstNumber(current)
|
||||
} else if (current.startsWith("VmRSS:")) {
|
||||
rssKb = firstNumber(current)
|
||||
} else if (current.startsWith("Threads:")) {
|
||||
threadCount = firstNumber(current)
|
||||
}
|
||||
if (rssKb > 0 && threadCount > 0) break
|
||||
if (rssKb > 0 && threadCount > 0 && peakRssKb > 0) break
|
||||
}
|
||||
}
|
||||
} catch (ignored: IOException) {
|
||||
// The graph remains useful with the runtime heap counters alone.
|
||||
}
|
||||
return longArrayOf(rssKb, threadCount)
|
||||
return longArrayOf(rssKb, threadCount, peakRssKb)
|
||||
}
|
||||
|
||||
private fun firstNumber(line: String): Long {
|
||||
|
||||
@@ -25,13 +25,23 @@ class FaceclawSettings private constructor(context: Context) {
|
||||
private val prefs: SharedPreferences = context.applicationContext
|
||||
.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
|
||||
private val hub = SettingsChangeHub(object : SettingsStorage {
|
||||
override fun getString(key: String?, defaultValue: String?): String? = prefs.getString(key, defaultValue)
|
||||
// An imported or hand-edited config can store a setting as a different type than the
|
||||
// app reads it as (iOS is lenient about this); read it as text rather than crash.
|
||||
override fun getString(key: String?, defaultValue: String?): String? = try {
|
||||
prefs.getString(key, defaultValue)
|
||||
} catch (_: ClassCastException) {
|
||||
prefs.all[key]?.toString() ?: defaultValue
|
||||
}
|
||||
|
||||
override fun putString(key: String?, value: String?) {
|
||||
prefs.edit().putString(key, value).apply()
|
||||
}
|
||||
|
||||
override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = prefs.getBoolean(key, defaultValue)
|
||||
override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = try {
|
||||
prefs.getBoolean(key, defaultValue)
|
||||
} catch (_: ClassCastException) {
|
||||
prefs.all[key]?.toString()?.toBooleanStrictOrNull() ?: defaultValue
|
||||
}
|
||||
|
||||
override fun putBoolean(key: String?, value: Boolean) {
|
||||
prefs.edit().putBoolean(key, value).apply()
|
||||
|
||||
+146
-81
@@ -3,52 +3,68 @@ package com.faceclaw.app
|
||||
import android.content.BroadcastReceiver
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.content.SharedPreferences
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.AtomicFile
|
||||
import android.util.Log
|
||||
import android.util.Xml
|
||||
|
||||
import org.xmlpull.v1.XmlPullParser
|
||||
|
||||
import java.io.File
|
||||
import java.io.FileInputStream
|
||||
import java.io.FileOutputStream
|
||||
import java.io.IOException
|
||||
import java.io.InputStream
|
||||
import java.io.OutputStream
|
||||
|
||||
/**
|
||||
* adb-triggered export/import of the faceclaw_settings SharedPreferences
|
||||
* file, so configuration (API keys, device addresses, ...) can be moved off a
|
||||
* release build, where run-as is unavailable. See scripts/pull_config.sh and
|
||||
* scripts/push_config.sh, which use run-as on debuggable builds and fall back
|
||||
* to this receiver otherwise.
|
||||
* adb-triggered export/import of Faceclaw's settings as the JSON document shared with iOS
|
||||
* (SettingsTransfer; see scripts/config-transfer.md). scripts/pull_config.sh and
|
||||
* scripts/push_config.sh drive it; it works on release builds, where run-as is unavailable.
|
||||
*
|
||||
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
|
||||
* -a com.faceclaw.app.SETTINGS_EXPORT
|
||||
* adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.xml
|
||||
* adb shell am broadcast --include-stopped-packages \
|
||||
* -n com.faceclaw.app/.FaceclawSettingsPortReceiver -a com.faceclaw.app.SETTINGS_EXPORT
|
||||
* adb pull /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-export.json
|
||||
*
|
||||
* adb push settings.xml /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.xml
|
||||
* adb shell am broadcast -n com.faceclaw.app/.FaceclawSettingsPortReceiver \
|
||||
* -a com.faceclaw.app.SETTINGS_IMPORT
|
||||
* adb push settings.json /sdcard/Android/data/com.faceclaw.app/files/faceclaw-settings-import.json
|
||||
* adb shell am broadcast --include-stopped-packages \
|
||||
* -n com.faceclaw.app/.FaceclawSettingsPortReceiver -a com.faceclaw.app.SETTINGS_IMPORT
|
||||
*
|
||||
* Security: the settings file contains API tokens, so the manifest guards
|
||||
* this receiver with android.permission.DUMP — a development permission the
|
||||
* adb shell holds but ordinary apps cannot obtain without adb's help, keeping
|
||||
* the trigger reachable from adb only. The transfer file lives in
|
||||
* getExternalFilesDir(), which other apps cannot read on Android 11+ (and
|
||||
* which the scripts delete as soon as the transfer completes); on the
|
||||
* Android 7-10 devices minSdk still admits, apps holding READ_EXTERNAL_STORAGE
|
||||
* could read it during that window.
|
||||
* Settings live in two SharedPreferences files: FaceclawSettings' own, and NativeScript's
|
||||
* ApplicationSettings file (prefs.db), which still holds the pairing, onboarding and EvenHub
|
||||
* localStorage keys. Both are exported, and each imported key is written to the file the app
|
||||
* reads it from. An import merges: settings missing from the document are left alone.
|
||||
* It writes through the process's live SharedPreferences, so a running app cannot flush stale
|
||||
* values over it, then exits so the next launch starts from the imported settings.
|
||||
*
|
||||
* Security: the settings contain API tokens, so the manifest guards this receiver with
|
||||
* android.permission.DUMP — a development permission the adb shell holds but ordinary apps
|
||||
* cannot obtain without adb's help, keeping the trigger reachable from adb only. The transfer
|
||||
* file lives in getExternalFilesDir(), which other apps cannot read on Android 11+ (and which
|
||||
* the scripts delete as soon as the transfer completes); on the Android 7-10 devices minSdk
|
||||
* still admits, apps holding READ_EXTERNAL_STORAGE could read it during that window.
|
||||
*/
|
||||
class FaceclawSettingsPortReceiver : BroadcastReceiver() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawSettingsPort"
|
||||
const val ACTION_EXPORT = "com.faceclaw.app.SETTINGS_EXPORT"
|
||||
const val ACTION_IMPORT = "com.faceclaw.app.SETTINGS_IMPORT"
|
||||
private const val PREFS_FILE = "faceclaw_settings.xml"
|
||||
private const val EXPORT_NAME = "faceclaw-settings-export.xml"
|
||||
private const val IMPORT_NAME = "faceclaw-settings-import.xml"
|
||||
private const val PLATFORM = "android"
|
||||
private const val SETTINGS_PREFS = "faceclaw_settings" // FaceclawSettings
|
||||
private const val NATIVESCRIPT_PREFS = "prefs.db" // NativeScript ApplicationSettings
|
||||
private const val EXPORT_NAME = "faceclaw-settings-export.json"
|
||||
private const val IMPORT_NAME = "faceclaw-settings-import.json"
|
||||
private const val BACKUP_NAME = "faceclaw-settings-previous.json"
|
||||
|
||||
private enum class StoredAs { STRING, BOOLEAN, NUMBER }
|
||||
|
||||
/**
|
||||
* Keys the app reads through NativeScript's ApplicationSettings rather than FaceclawSettings,
|
||||
* and the getter it uses (device-addresses.ts, onboarding-state.ts, evenhub/session.ts;
|
||||
* tests/settings-transfer.test.cjs keeps this list in step with them). prefs.db also holds
|
||||
* settings abandoned when FaceclawSettings took over, so nothing else is read from it.
|
||||
*/
|
||||
private fun nativeScriptKey(key: String): StoredAs? = when {
|
||||
key == "onboarding.complete" || key == "onboarding.previewOnly" -> StoredAs.BOOLEAN
|
||||
key == "deviceIdentity.pairedAtMs" -> StoredAs.NUMBER
|
||||
key.startsWith("deviceAddress.") || key.startsWith("deviceIdentity.") || key.startsWith("evenhub:") -> StoredAs.STRING
|
||||
else -> null
|
||||
}
|
||||
}
|
||||
|
||||
override fun onReceive(context: Context, intent: Intent?) {
|
||||
@@ -57,11 +73,10 @@ class FaceclawSettingsPortReceiver : BroadcastReceiver() {
|
||||
if (ACTION_EXPORT == action) {
|
||||
resultData = exportSettings(context)
|
||||
} else if (ACTION_IMPORT == action) {
|
||||
resultData = importSettings(context)
|
||||
// Exit once the broadcast result has been delivered: this
|
||||
// process may hold stale in-memory SharedPreferences that a
|
||||
// later apply() would flush over the imported file, and the
|
||||
// next launch must re-read from disk.
|
||||
val result = importSettings(context)
|
||||
resultData = result
|
||||
if (!result.startsWith("imported:")) return
|
||||
// Restart cleanly: running code may hold settings it read before the import.
|
||||
Handler(Looper.getMainLooper()).postDelayed({
|
||||
Log.i(TAG, "Exiting after settings import")
|
||||
System.exit(0)
|
||||
@@ -69,13 +84,15 @@ class FaceclawSettingsPortReceiver : BroadcastReceiver() {
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
Log.e(TAG, "Settings $action failed", e)
|
||||
resultData = "error: $e"
|
||||
resultData = "error: ${e.message ?: e}"
|
||||
}
|
||||
}
|
||||
|
||||
private fun prefsFile(context: Context): File {
|
||||
return File(context.applicationInfo.dataDir, "shared_prefs/$PREFS_FILE")
|
||||
}
|
||||
private fun settingsPrefs(context: Context): SharedPreferences =
|
||||
context.getSharedPreferences(SETTINGS_PREFS, Context.MODE_PRIVATE)
|
||||
|
||||
private fun nativeScriptPrefs(context: Context): SharedPreferences =
|
||||
context.getSharedPreferences(NATIVESCRIPT_PREFS, Context.MODE_PRIVATE)
|
||||
|
||||
@Throws(IOException::class)
|
||||
private fun externalFile(context: Context, name: String): File {
|
||||
@@ -84,63 +101,111 @@ class FaceclawSettingsPortReceiver : BroadcastReceiver() {
|
||||
return File(dir, name)
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
private fun exportSettings(context: Context): String {
|
||||
val prefs = prefsFile(context)
|
||||
if (!prefs.exists()) {
|
||||
return "error: no settings file yet ($prefs)"
|
||||
private fun appVersion(context: Context): String? =
|
||||
runCatching { context.packageManager.getPackageInfo(context.packageName, 0).versionName }.getOrNull()
|
||||
|
||||
/** Every setting, from the file the app reads it from, as SettingsTransfer values. */
|
||||
private fun currentSettings(context: Context): Map<String, Any?> {
|
||||
val values = linkedMapOf<String, Any?>()
|
||||
for ((key, value) in settingsPrefs(context).all) {
|
||||
if (key == null || nativeScriptKey(key) != null) continue
|
||||
val exported = exportable(value)
|
||||
if (exported != null) values[key] = exported else Log.w(TAG, "Not exporting $key: unsupported type")
|
||||
}
|
||||
for ((key, value) in nativeScriptPrefs(context).all) {
|
||||
val storedAs = nativeScriptKey(key ?: continue) ?: continue
|
||||
// NativeScript stores numbers as the raw bits of a double in a long.
|
||||
val exported = if (storedAs == StoredAs.NUMBER && value is Long) {
|
||||
Double.fromBits(value).takeIf { it.isFinite() }?.let { JsonNumber.of(it) }
|
||||
} else {
|
||||
exportable(value)
|
||||
}
|
||||
if (exported != null) values[key] = exported else Log.w(TAG, "Not exporting $key: unsupported value")
|
||||
}
|
||||
return values
|
||||
}
|
||||
|
||||
private fun exportable(value: Any?): Any? = when (value) {
|
||||
is String, is Boolean -> value
|
||||
is Int, is Long -> JsonNumber(value.toString())
|
||||
is Float -> value.toDouble().takeIf { it.isFinite() }?.let { JsonNumber.of(it) }
|
||||
else -> null // string sets are unused
|
||||
}
|
||||
|
||||
private fun exportSettings(context: Context): String {
|
||||
val document = SettingsTransfer.export(currentSettings(context), PLATFORM, appVersion(context))
|
||||
val out = externalFile(context, EXPORT_NAME)
|
||||
copy(prefs, out)
|
||||
writeAtomically(out, document)
|
||||
return "exported: $out"
|
||||
}
|
||||
|
||||
@Throws(Exception::class)
|
||||
private fun importSettings(context: Context): String {
|
||||
val staged = externalFile(context, IMPORT_NAME)
|
||||
if (!staged.exists()) {
|
||||
return "error: nothing staged at $staged"
|
||||
}
|
||||
validatePrefsXml(staged)
|
||||
val prefs = prefsFile(context)
|
||||
prefs.parentFile?.mkdirs()
|
||||
if (prefs.exists()) {
|
||||
copy(prefs, File(prefs.path + ".bak"))
|
||||
val plan = try {
|
||||
SettingsTransfer.planImport(staged.readText(), PLATFORM)
|
||||
} finally {
|
||||
staged.delete() // holds API tokens
|
||||
}
|
||||
copy(staged, prefs)
|
||||
staged.delete()
|
||||
return "imported: $prefs (previous settings in $PREFS_FILE.bak)"
|
||||
}
|
||||
|
||||
/** Reject files that are not a SharedPreferences <map> document. */
|
||||
@Throws(Exception::class)
|
||||
private fun validatePrefsXml(file: File) {
|
||||
FileInputStream(file).use { input ->
|
||||
val parser = Xml.newPullParser()
|
||||
parser.setInput(input, null)
|
||||
var event = parser.next()
|
||||
while (event != XmlPullParser.START_TAG && event != XmlPullParser.END_DOCUMENT) {
|
||||
event = parser.next()
|
||||
}
|
||||
if (event != XmlPullParser.START_TAG || "map" != parser.name) {
|
||||
throw IOException("not a SharedPreferences <map> document")
|
||||
}
|
||||
// Walk the rest so malformed XML is caught before it replaces
|
||||
// the live settings file.
|
||||
while (parser.next() != XmlPullParser.END_DOCUMENT) { }
|
||||
}
|
||||
}
|
||||
|
||||
@Throws(IOException::class)
|
||||
private fun copy(from: File, to: File) {
|
||||
FileInputStream(from).use { input: InputStream ->
|
||||
FileOutputStream(to).use { out: OutputStream ->
|
||||
val buffer = ByteArray(8192)
|
||||
var n: Int
|
||||
while (input.read(buffer).also { n = it } > 0) {
|
||||
out.write(buffer, 0, n)
|
||||
// Convert everything before writing anything, so a bad value leaves settings untouched.
|
||||
val settingsWrites = ArrayList<(SharedPreferences.Editor) -> Unit>()
|
||||
val nativeScriptWrites = ArrayList<(SharedPreferences.Editor) -> Unit>()
|
||||
for ((key, value) in plan.values) {
|
||||
when (nativeScriptKey(key)) {
|
||||
// FaceclawSettings only has string and boolean getters.
|
||||
null -> if (value is Boolean) {
|
||||
settingsWrites.add { e -> e.putBoolean(key, value) }
|
||||
} else {
|
||||
settingsWrites.add { e -> e.putString(key, text(value)) }
|
||||
}
|
||||
StoredAs.STRING -> nativeScriptWrites.add { e -> e.putString(key, text(value)) }
|
||||
StoredAs.BOOLEAN -> {
|
||||
val flag = value as? Boolean ?: (value as? String)?.toBooleanStrictOrNull()
|
||||
?: throw IllegalArgumentException("Setting $key must be a boolean")
|
||||
nativeScriptWrites.add { e -> e.putBoolean(key, flag) }
|
||||
}
|
||||
StoredAs.NUMBER -> {
|
||||
val number = text(value).toDoubleOrNull()?.takeIf { it.isFinite() }
|
||||
?: throw IllegalArgumentException("Setting $key must be a number")
|
||||
nativeScriptWrites.add { e -> e.putLong(key, number.toRawBits()) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
val backup = File(context.filesDir, BACKUP_NAME)
|
||||
writeAtomically(backup, SettingsTransfer.export(currentSettings(context), PLATFORM, appVersion(context)))
|
||||
commit(settingsPrefs(context), settingsWrites)
|
||||
commit(nativeScriptPrefs(context), nativeScriptWrites)
|
||||
|
||||
val skipped = if (plan.skipped.isEmpty()) "" else
|
||||
" (left out ${plan.skipped.size} ${plan.sourcePlatform ?: "other-platform"}-only settings)"
|
||||
return "imported: ${plan.values.size} settings$skipped; previous settings saved in $backup"
|
||||
}
|
||||
|
||||
private fun text(value: Any): String = when (value) {
|
||||
is JsonNumber -> value.text
|
||||
else -> value.toString()
|
||||
}
|
||||
|
||||
private fun commit(prefs: SharedPreferences, writes: List<(SharedPreferences.Editor) -> Unit>) {
|
||||
if (writes.isEmpty()) return
|
||||
val editor = prefs.edit()
|
||||
for (write in writes) write(editor)
|
||||
if (!editor.commit()) throw IOException("Could not save settings")
|
||||
}
|
||||
|
||||
private fun writeAtomically(file: File, text: String) {
|
||||
val atomic = AtomicFile(file)
|
||||
val stream = atomic.startWrite()
|
||||
try {
|
||||
stream.write(text.toByteArray(Charsets.UTF_8))
|
||||
atomic.finishWrite(stream)
|
||||
} catch (e: IOException) {
|
||||
atomic.failWrite(stream)
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,194 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.ActivityNotFoundException
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.net.Uri
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Base64
|
||||
import android.util.Log
|
||||
import java.io.ByteArrayOutputStream
|
||||
import java.io.IOException
|
||||
import java.net.BindException
|
||||
import java.net.InetAddress
|
||||
import java.net.InetSocketAddress
|
||||
import java.net.ServerSocket
|
||||
import java.net.Socket
|
||||
import java.security.MessageDigest
|
||||
import java.security.SecureRandom
|
||||
|
||||
/**
|
||||
* OAuth sign-in in the phone's browser with a loopback redirect
|
||||
* (http://127.0.0.1:<port><path>, the redirect providers such as Spotify
|
||||
* accept from native apps): Android's counterpart of the iOS
|
||||
* FaceclawWebSignIn / FaceclawLoopbackRedirect. A receiver on the loopback
|
||||
* interface answers the browser's request for the path with a short "signed
|
||||
* in" page and hands back the full redirect URL, query included; any other
|
||||
* path (a favicon) gets a 404.
|
||||
*
|
||||
* Android blocks activity starts from the background, so the TypeScript side
|
||||
* (app/native/spotify.ts) calls start(), then openBrowser() from Faceclaw's
|
||||
* activity once it is in front. The code exchange needs no foreground, so the
|
||||
* user can stay in the browser afterwards.
|
||||
*/
|
||||
class FaceclawWebSignIn private constructor() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawWebSignIn"
|
||||
/** Request heads beyond this are not a browser's redirect; stop reading. */
|
||||
private const val MAX_REQUEST_BYTES = 16384
|
||||
private const val CLIENT_TIMEOUT_MS = 3000
|
||||
private const val DONE_PAGE =
|
||||
"<!doctype html><html><head><meta charset=\"utf-8\">" +
|
||||
"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>Faceclaw</title></head>" +
|
||||
"<body style=\"font-family: sans-serif; text-align: center; padding: 64px 24px\">" +
|
||||
"<h2>Signed in</h2><p>You can close this page and return to Faceclaw.</p></body></html>"
|
||||
|
||||
private val INSTANCE = FaceclawWebSignIn()
|
||||
private val random = SecureRandom()
|
||||
|
||||
@JvmStatic fun shared(): FaceclawWebSignIn = INSTANCE
|
||||
|
||||
/** byteCount random bytes, base64url without padding (PKCE verifiers, OAuth state). */
|
||||
@JvmStatic
|
||||
fun randomBase64Url(byteCount: Int): String {
|
||||
val bytes = ByteArray(byteCount.coerceIn(1, 256))
|
||||
random.nextBytes(bytes)
|
||||
return base64Url(bytes)
|
||||
}
|
||||
|
||||
/** SHA-256 of the string's UTF-8 bytes, base64url without padding (a PKCE S256 challenge). */
|
||||
@JvmStatic
|
||||
fun sha256Base64Url(value: String): String =
|
||||
base64Url(MessageDigest.getInstance("SHA-256").digest(value.toByteArray(Charsets.UTF_8)))
|
||||
|
||||
/** The target ("/path?query") of a GET request head; null for anything else. */
|
||||
@JvmStatic
|
||||
fun requestTarget(head: String): String? {
|
||||
val parts = head.substringBefore("\r\n").split(' ')
|
||||
return if (parts.size >= 3 && parts[0] == "GET" && parts[1].startsWith("/")) parts[1] else null
|
||||
}
|
||||
|
||||
private fun base64Url(bytes: ByteArray): String =
|
||||
Base64.encodeToString(bytes, Base64.URL_SAFE or Base64.NO_PADDING or Base64.NO_WRAP)
|
||||
}
|
||||
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val lock = Any()
|
||||
/** The receiver of the sign-in in progress; null when none is. */
|
||||
private var server: ServerSocket? = null
|
||||
private var listener: FaceclawWebSignInListener? = null
|
||||
|
||||
/**
|
||||
* Listen for the redirect to http://127.0.0.1:<port><path>, ending any
|
||||
* sign-in in progress. Returns null, or why the receiver couldn't start
|
||||
* (and then the listener never hears). Otherwise the listener hears
|
||||
* exactly once, on the main thread.
|
||||
*/
|
||||
fun start(port: Int, path: String, listener: FaceclawWebSignInListener): String? {
|
||||
cancel()
|
||||
if (port <= 0 || port > 65535 || !path.startsWith("/")) return "Invalid redirect address"
|
||||
val socket = ServerSocket()
|
||||
try {
|
||||
socket.reuseAddress = true
|
||||
socket.bind(InetSocketAddress(InetAddress.getByName("127.0.0.1"), port), 8)
|
||||
} catch (e: IOException) {
|
||||
try { socket.close() } catch (ignored: IOException) {}
|
||||
return if (e is BindException) "Another app is using port $port; close it and try again"
|
||||
else "Could not listen on port $port: ${e.message}"
|
||||
}
|
||||
synchronized(lock) {
|
||||
server = socket
|
||||
this.listener = listener
|
||||
}
|
||||
val thread = Thread({ acceptLoop(socket, port, path) }, "FaceclawWebSignIn")
|
||||
thread.isDaemon = true
|
||||
thread.start()
|
||||
return null
|
||||
}
|
||||
|
||||
/** Open the sign-in page in the phone's browser; null, or why not. Call while Faceclaw is in front. */
|
||||
fun openBrowser(context: Context, url: String): String? {
|
||||
val uri = Uri.parse(url)
|
||||
if (uri.scheme != "https") return "Invalid sign-in address"
|
||||
val intent = Intent(Intent.ACTION_VIEW, uri)
|
||||
.addCategory(Intent.CATEGORY_BROWSABLE)
|
||||
.addFlags(Intent.FLAG_ACTIVITY_NEW_TASK)
|
||||
return try {
|
||||
context.startActivity(intent)
|
||||
null
|
||||
} catch (e: ActivityNotFoundException) {
|
||||
"No browser on this phone can open the sign-in page"
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "could not open the sign-in page", e)
|
||||
"Could not open the sign-in page: ${e.message}"
|
||||
}
|
||||
}
|
||||
|
||||
/** End the sign-in in progress, if any; its listener hears "cancelled". */
|
||||
fun cancel() = fail("cancelled")
|
||||
|
||||
/** End the sign-in in progress, if any, with this error. */
|
||||
fun fail(error: String) = finish(null, null, error)
|
||||
|
||||
private fun acceptLoop(socket: ServerSocket, port: Int, path: String) {
|
||||
while (true) {
|
||||
// finish() closes the socket, which ends accept() with an exception.
|
||||
val client = try { socket.accept() } catch (e: IOException) { return }
|
||||
val target = try {
|
||||
serve(client, path)
|
||||
} catch (e: IOException) {
|
||||
null
|
||||
} finally {
|
||||
try { client.close() } catch (ignored: IOException) {}
|
||||
}
|
||||
if (target != null) {
|
||||
finish(socket, "http://127.0.0.1:$port$target", null)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Read one request head and answer it; returns its target when it is the redirect. */
|
||||
private fun serve(client: Socket, path: String): String? {
|
||||
client.soTimeout = CLIENT_TIMEOUT_MS
|
||||
val input = client.getInputStream()
|
||||
val head = ByteArrayOutputStream()
|
||||
val buffer = ByteArray(2048)
|
||||
while (head.size() < MAX_REQUEST_BYTES) {
|
||||
val count = input.read(buffer)
|
||||
if (count <= 0) break
|
||||
head.write(buffer, 0, count)
|
||||
if (head.toString("ISO-8859-1").contains("\r\n\r\n")) break
|
||||
}
|
||||
val target = requestTarget(head.toString("UTF-8"))
|
||||
val matched = target != null && target.substringBefore('?') == path
|
||||
val body = (if (matched) DONE_PAGE else "Not found").toByteArray(Charsets.UTF_8)
|
||||
val headers = "HTTP/1.1 ${if (matched) "200 OK" else "404 Not Found"}\r\n" +
|
||||
"Content-Type: text/html; charset=utf-8\r\nContent-Length: ${body.size}\r\n" +
|
||||
"Cache-Control: no-store\r\nConnection: close\r\n\r\n"
|
||||
val output = client.getOutputStream()
|
||||
output.write(headers.toByteArray(Charsets.US_ASCII))
|
||||
output.write(body)
|
||||
output.flush()
|
||||
return if (matched) target else null
|
||||
}
|
||||
|
||||
/**
|
||||
* Report the outcome once and stop listening. `socket` limits it to that
|
||||
* receiver's sign-in (a late redirect after a newer start() is dropped);
|
||||
* null ends whichever is in progress.
|
||||
*/
|
||||
private fun finish(socket: ServerSocket?, url: String?, error: String?) {
|
||||
val ended = synchronized(lock) {
|
||||
val current = server
|
||||
if (current == null || (socket != null && socket !== current)) return
|
||||
val pending = listener
|
||||
server = null
|
||||
listener = null
|
||||
try { current.close() } catch (ignored: IOException) {}
|
||||
pending
|
||||
} ?: return
|
||||
mainHandler.post { ended.onFinished(url, if (url != null) null else error) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.app.Notification
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Picture
|
||||
import android.os.Build
|
||||
import android.service.notification.StatusBarNotification
|
||||
import android.util.Log
|
||||
import android.view.View
|
||||
import android.view.ViewGroup
|
||||
import android.widget.FrameLayout
|
||||
import android.widget.RemoteViews
|
||||
import android.widget.TextView
|
||||
|
||||
import kotlin.math.ceil
|
||||
import kotlin.math.max
|
||||
import kotlin.math.min
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
* A notification's custom layout (RemoteViews), which can carry what its standard fields leave
|
||||
* out: Dexcom's G7 notification has no title or text at all, just a layout with the glucose
|
||||
* reading and trend arrow. Inflated here the way the shade does it, for its text (a stand-in
|
||||
* title in lists) and to draw it as a picture.
|
||||
*/
|
||||
internal object NotificationCustomViews {
|
||||
private const val TAG = "FaceclawNotifyView"
|
||||
/** Set by Notification.Builder when the app gave it a custom layout (Notification.EXTRA_CONTAINS_CUSTOM_VIEW, hidden). */
|
||||
private const val EXTRA_CONTAINS_CUSTOM_VIEW = "android.contains.customView"
|
||||
private const val VIEW_CACHE_MAX_ENTRIES = 8
|
||||
private const val TEXT_CACHE_MAX_ENTRIES = 64
|
||||
/** Layouts are cut off below this height, in dp; the shade caps an expanded notification at 256. */
|
||||
private const val MAX_HEIGHT_DP = 320
|
||||
|
||||
/** Inflated layouts by notification post and layout; null when inflating failed, so it isn't retried every paint. */
|
||||
private val views = Lru<View?>(VIEW_CACHE_MAX_ENTRIES)
|
||||
private val texts = Lru<List<String>>(TEXT_CACHE_MAX_ENTRIES)
|
||||
|
||||
/**
|
||||
* Whether the app gave the notification a custom layout. The flag tells one apart from the
|
||||
* standard template that Notification.Builder stores in contentView for apps targeting
|
||||
* Android 6 and older, which only repeats the title and text.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
fun hasCustomView(notification: Notification): Boolean {
|
||||
val extras = notification.extras ?: return false
|
||||
return extras.getBoolean(EXTRA_CONTAINS_CUSTOM_VIEW)
|
||||
&& (notification.contentView != null || notification.bigContentView != null || notification.headsUpContentView != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* The text of the custom layout as collapsed in the shade, one entry per visible TextView in
|
||||
* layout order; empty without a custom layout. Cached per post of the notification.
|
||||
*/
|
||||
@Synchronized
|
||||
fun texts(context: Context, statusBarNotification: StatusBarNotification): List<String> {
|
||||
val notification = statusBarNotification.notification ?: return emptyList()
|
||||
val (name, layout) = layoutFor(notification, expanded = false) ?: return emptyList()
|
||||
return texts.getOrPut(postKey(statusBarNotification, name)) {
|
||||
inflate(context, statusBarNotification, name, layout)?.let { collectTexts(it, ArrayList()) } ?: emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The custom layout drawn as a gray packet fitting maxWidth x maxHeight; empty without one
|
||||
* or when it can't be drawn. It is laid out maxWidth dp wide and shown at no more than a
|
||||
* glasses pixel per dp, where the shade's text sizes land near the glasses' UI font, then
|
||||
* trimmed to what it draws. The collapsed layout is used when the notification has a
|
||||
* picture to show under it (an expanded layout tends to repeat the picture, as Dexcom's
|
||||
* does), else the expanded one. Ink drawn for a light background comes out light on dark.
|
||||
*/
|
||||
@Synchronized
|
||||
fun renderGray(
|
||||
context: Context, statusBarNotification: StatusBarNotification, expanded: Boolean,
|
||||
maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean,
|
||||
): ByteArray {
|
||||
val notification = statusBarNotification.notification ?: return ByteArray(0)
|
||||
if (maxWidth <= 0 || maxHeight <= 0) return ByteArray(0)
|
||||
val (name, layout) = layoutFor(notification, expanded) ?: return ByteArray(0)
|
||||
val view = inflate(context, statusBarNotification, name, layout) ?: return ByteArray(0)
|
||||
try {
|
||||
val density = context.resources.displayMetrics.density
|
||||
val widthPx = max(1, (maxWidth * density).roundToInt())
|
||||
val maxHeightPx = max(1, (MAX_HEIGHT_DP * density).roundToInt())
|
||||
view.measure(
|
||||
View.MeasureSpec.makeMeasureSpec(widthPx, View.MeasureSpec.EXACTLY),
|
||||
View.MeasureSpec.makeMeasureSpec(0, View.MeasureSpec.UNSPECIFIED))
|
||||
if (view.measuredHeight > maxHeightPx) {
|
||||
view.measure(
|
||||
View.MeasureSpec.makeMeasureSpec(widthPx, View.MeasureSpec.EXACTLY),
|
||||
View.MeasureSpec.makeMeasureSpec(maxHeightPx, View.MeasureSpec.AT_MOST))
|
||||
}
|
||||
val heightPx = min(view.measuredHeight, maxHeightPx)
|
||||
if (heightPx <= 0) return ByteArray(0)
|
||||
view.layout(0, 0, widthPx, heightPx)
|
||||
|
||||
val drawn = drawView(view, widthPx, heightPx)
|
||||
val pixels = IntArray(widthPx * heightPx)
|
||||
drawn.getPixels(pixels, 0, widthPx, 0, 0, widthPx, heightPx)
|
||||
drawn.recycle()
|
||||
val bounds = GrayPacket.inkBounds(pixels, widthPx, heightPx) ?: return ByteArray(0)
|
||||
val cropWidth = bounds[2] - bounds[0]
|
||||
val cropHeight = bounds[3] - bounds[1]
|
||||
val cropped = IntArray(cropWidth * cropHeight)
|
||||
for (y in 0 until cropHeight) {
|
||||
System.arraycopy(pixels, (bounds[1] + y) * widthPx + bounds[0], cropped, y * cropWidth, cropWidth)
|
||||
}
|
||||
GrayPacket.lightenDarkInk(cropped, cropWidth, cropHeight)
|
||||
val art = Bitmap.createBitmap(cropped, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
|
||||
// A dp per glasses pixel at most: bitmapToGrayPacket otherwise blows a narrow layout up to the box.
|
||||
val fitWidth = min(maxWidth, ceil(cropWidth / density).toInt())
|
||||
val fitHeight = min(maxHeight, ceil(cropHeight / density).toInt())
|
||||
Log.i(TAG, "rendered $name layout of ${statusBarNotification.packageName}: ${cropWidth}x$cropHeight px"
|
||||
+ " into ${fitWidth}x$fitHeight")
|
||||
return ImageFileLoader.bitmapToGrayPacket(
|
||||
FaceclawMediaNotificationListenerService.halveToward(art, fitWidth, fitHeight),
|
||||
fitWidth, fitHeight, gamma, dither, true)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to draw the $name layout of ${statusBarNotification.packageName}", t)
|
||||
return ByteArray(0)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The custom layout to use, with the field it came from: the collapsed one (contentView)
|
||||
* or the expanded one (bigContentView), falling back on the other, then on the heads-up one.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
private fun layoutFor(notification: Notification, expanded: Boolean): Pair<String, RemoteViews>? {
|
||||
if (!hasCustomView(notification)) return null
|
||||
val collapsed = notification.contentView?.let { "content" to it }
|
||||
val big = notification.bigContentView?.let { "big" to it }
|
||||
val first = if (expanded) big ?: collapsed else collapsed ?: big
|
||||
return first ?: notification.headsUpContentView?.let { "headsUp" to it }
|
||||
}
|
||||
|
||||
/** A notification is reposted (with a new postTime) whenever its app updates it. */
|
||||
private fun postKey(statusBarNotification: StatusBarNotification, layoutName: String): String =
|
||||
"${statusBarNotification.key}|${statusBarNotification.postTime}|$layoutName"
|
||||
|
||||
private fun inflate(context: Context, statusBarNotification: StatusBarNotification, name: String, layout: RemoteViews): View? =
|
||||
views.getOrPut(postKey(statusBarNotification, name)) {
|
||||
try {
|
||||
// RemoteViews loads the layout and its resources from the posting app's package.
|
||||
layout.apply(context, FrameLayout(context))
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to inflate the $name layout of ${statusBarNotification.packageName}", t)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
private fun collectTexts(view: View, out: MutableList<String>): List<String> {
|
||||
if (view.visibility != View.VISIBLE) return out
|
||||
if (view is TextView) {
|
||||
val text = view.text?.toString()?.replace(Regex("\\s+"), " ")?.trim() ?: ""
|
||||
if (text.isNotEmpty()) out.add(text)
|
||||
} else if (view is ViewGroup) {
|
||||
for (index in 0 until view.childCount) {
|
||||
collectTexts(view.getChildAt(index), out)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
/**
|
||||
* Draws a laid-out view to a software bitmap. From Android 9 this goes through a Picture,
|
||||
* which (unlike a bitmap's Canvas) accepts the hardware bitmaps a layout's images can
|
||||
* arrive as; Bitmap.createBitmap then renders it on the GPU and reads it back if it has to.
|
||||
*/
|
||||
private fun drawView(view: View, width: Int, height: Int): Bitmap {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.P) {
|
||||
val picture = Picture()
|
||||
view.draw(picture.beginRecording(width, height))
|
||||
picture.endRecording()
|
||||
return Bitmap.createBitmap(picture, width, height, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
val bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888)
|
||||
view.draw(Canvas(bitmap))
|
||||
return bitmap
|
||||
}
|
||||
|
||||
/** A small LRU map; values may be null. Callers hold the object's lock. */
|
||||
private class Lru<V>(private val maxEntries: Int) {
|
||||
private val entries = object : LinkedHashMap<String, V>(16, 0.75f, true) {
|
||||
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, V>): Boolean = size > maxEntries
|
||||
}
|
||||
|
||||
fun getOrPut(key: String, compute: () -> V): V {
|
||||
if (entries.containsKey(key)) {
|
||||
@Suppress("UNCHECKED_CAST")
|
||||
return entries[key] as V
|
||||
}
|
||||
val value = compute()
|
||||
entries[key] = value
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- History icon for the keyboard-input panel's recent-inputs button.
|
||||
Material Design "history" glyph (Apache 2.0); base fill is overridden by
|
||||
tint-color in app.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:pathData="M13 3c-4.97 0-9 4.03-9 9H1l3.89 3.89.07.14L9 12H6c0-3.87 3.13-7 7-7s7 3.13 7 7-3.13 7-7 7c-1.93 0-3.68-.79-4.94-2.06l-1.42 1.42C8.27 19.99 10.51 21 13 21c4.97 0 9-4.03 9-9s-4.03-9-9-9zm-1 5v5l4.28 2.54.72-1.21-3.5-2.08V8H12z" />
|
||||
</vector>
|
||||
@@ -0,0 +1,17 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Glasses half of the keyboard-input panel's visibility button (dimmed
|
||||
while the dialog is hidden on the glasses): two lens frames, bridge and
|
||||
temple stubs. Base fill is overridden by tint-color in app.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:fillType="evenOdd"
|
||||
android:pathData="M4.5 8h3a3 3 0 0 1 3 3v1.5a3 3 0 0 1-3 3h-3a3 3 0 0 1-3-3v-1.5a3 3 0 0 1 3-3zM4.75 9.75h2.5a1.5 1.5 0 0 1 1.5 1.5v1a1.5 1.5 0 0 1-1.5 1.5h-2.5a1.5 1.5 0 0 1-1.5-1.5v-1a1.5 1.5 0 0 1 1.5-1.5zM16.5 8h3a3 3 0 0 1 3 3v1.5a3 3 0 0 1-3 3h-3a3 3 0 0 1-3-3v-1.5a3 3 0 0 1 3-3zM16.75 9.75h2.5a1.5 1.5 0 0 1 1.5 1.5v1a1.5 1.5 0 0 1-1.5 1.5h-2.5a1.5 1.5 0 0 1-1.5-1.5v-1a1.5 1.5 0 0 1 1.5-1.5z" />
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:pathData="M10 10.25Q12 8.25 14 10.25V12Q12 10 10 12zM0 8h3v1.75H0zM21 8h3v1.75h-3z" />
|
||||
</vector>
|
||||
@@ -0,0 +1,13 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<!-- Phone half of the keyboard-input panel's visibility button (dimmed while
|
||||
the dialog is hidden on the phone). Material Design "smartphone" glyph
|
||||
(Apache 2.0); base fill is overridden by tint-color in app.css. -->
|
||||
<vector xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
android:width="24dp"
|
||||
android:height="24dp"
|
||||
android:viewportWidth="24"
|
||||
android:viewportHeight="24">
|
||||
<path
|
||||
android:fillColor="#FF333333"
|
||||
android:pathData="M17 1.01L7 1c-1.1 0-2 .9-2 2v18c0 1.1.9 2 2 2h10c1.1 0 2-.9 2-2V3c0-1.1-.9-1.99-2-1.99zM17 19H7V5h10v14z" />
|
||||
</vector>
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_input_history@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 2.0 KiB |
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_show_glasses@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 1.0 KiB |
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"images": [
|
||||
{
|
||||
"idiom": "universal",
|
||||
"filename": "ic_show_phone@3x.png",
|
||||
"scale": "3x"
|
||||
}
|
||||
],
|
||||
"info": {
|
||||
"author": "xcode",
|
||||
"version": 1
|
||||
}
|
||||
}
|
||||
BIN
Binary file not shown.
|
After Width: | Height: | Size: 715 B |
@@ -13,7 +13,7 @@
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Faceclaw</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>0.8.4</string>
|
||||
<string>0.8.5</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
|
||||
@@ -3,7 +3,9 @@ TARGETED_DEVICE_FAMILY = 1;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer
|
||||
// FoundationModels (the on-phone LLM, FaceclawSystemLanguageModel.swift) only exists on iOS 26+,
|
||||
// so it is weak-linked: the app still launches on older iOS and reports the model unavailable.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer -framework AuthenticationServices -framework Security -weak_framework FoundationModels
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
#import "FaceclawAppleMusic.h"
|
||||
#import "FaceclawGraphics.h"
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <MediaPlayer/MediaPlayer.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
@@ -219,30 +220,7 @@ static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *sub
|
||||
- (NSData *)albumArt:(NSInteger)maxSize {
|
||||
if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil;
|
||||
MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork;
|
||||
UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)];
|
||||
CGImageRef source = image.CGImage;
|
||||
if (!source) return nil;
|
||||
double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
|
||||
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
|
||||
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
|
||||
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
|
||||
uint8_t *pixels = bytes + 4;
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
|
||||
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) return nil;
|
||||
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), source);
|
||||
CGContextRelease(context);
|
||||
// Composited over black (the display's see-through background); now opaque.
|
||||
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
|
||||
return packet;
|
||||
return [FaceclawGraphics bgraPacket:[artwork imageWithSize:CGSizeMake(maxSize, maxSize)].CGImage maxSize:maxSize];
|
||||
}
|
||||
|
||||
#pragma mark - Queue
|
||||
|
||||
@@ -1,6 +1,17 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
/** Developer config transfer through the app container, before workers start. */
|
||||
NS_ASSUME_NONNULL_BEGIN
|
||||
/** Converts between this phone's settings and the transfer document (Kotlin SettingsTransfer). */
|
||||
@protocol FaceclawConfigCodec <NSObject>
|
||||
/** valuesJson is a JSON object of the settings; nil when they cannot be exported. */
|
||||
- (nullable NSString *)exportValuesJson:(NSString *)valuesJson platform:(NSString *)platform appVersion:(nullable NSString *)appVersion;
|
||||
/** A JSON object: {"ok":true,"settings":{...},"skipped":[...],"sourcePlatform":...} or {"ok":false,"message":...}. */
|
||||
- (NSString *)importPlanDocument:(NSString *)document platform:(NSString *)platform;
|
||||
@end
|
||||
|
||||
/** Developer config transfer through the app container, before workers start (scripts/config-transfer.md). */
|
||||
@interface FaceclawConfigPort : NSObject
|
||||
+ (void)processPendingRequest;
|
||||
+ (void)processDirectory:(NSString *)directory defaults:(NSUserDefaults *)defaults domain:(NSString *)domain;
|
||||
+ (void)processDirectory:(NSString *)directory defaults:(NSUserDefaults *)defaults domain:(NSString *)domain
|
||||
codec:(id<FaceclawConfigCodec>)codec appVersion:(nullable NSString *)appVersion;
|
||||
@end
|
||||
NS_ASSUME_NONNULL_END
|
||||
@@ -1,57 +1,112 @@
|
||||
#import "FaceclawConfigPort.h"
|
||||
#include <math.h>
|
||||
// The app links FaceclawKit; tests/ios-config-native.cjs builds this file for macOS without it
|
||||
// and supplies its own codec.
|
||||
#if __has_include(<FaceclawKit/FaceclawKit.h>)
|
||||
#import <FaceclawKit/FaceclawKit.h>
|
||||
@interface FaceclawKitConfigCodec : NSObject <FaceclawConfigCodec>
|
||||
@end
|
||||
@implementation FaceclawKitConfigCodec
|
||||
- (NSString *)exportValuesJson:(NSString *)valuesJson platform:(NSString *)platform appVersion:(NSString *)appVersion {
|
||||
return [FaceclawKitSettingsTransfer.shared exportValuesJson:valuesJson platform:platform appVersion:appVersion];
|
||||
}
|
||||
- (NSString *)importPlanDocument:(NSString *)document platform:(NSString *)platform {
|
||||
return [FaceclawKitSettingsTransfer.shared importPlanDocument:document platform:platform];
|
||||
}
|
||||
@end
|
||||
#define FACECLAW_CONFIG_CODEC [FaceclawKitConfigCodec new]
|
||||
#endif
|
||||
|
||||
static NSString *const kPlatform = @"ios";
|
||||
static NSString *const kChangeToken = @"ios.settings.changeToken";
|
||||
|
||||
@implementation FaceclawConfigPort
|
||||
+ (void)processPendingRequest {
|
||||
#ifdef FACECLAW_CONFIG_CODEC
|
||||
NSString *library = NSSearchPathForDirectoriesInDomains(NSLibraryDirectory, NSUserDomainMask, YES).firstObject;
|
||||
[self processDirectory:[library stringByAppendingPathComponent:@"FaceclawConfigPort"]
|
||||
defaults:NSUserDefaults.standardUserDefaults domain:NSBundle.mainBundle.bundleIdentifier];
|
||||
defaults:NSUserDefaults.standardUserDefaults domain:NSBundle.mainBundle.bundleIdentifier
|
||||
codec:FACECLAW_CONFIG_CODEC
|
||||
appVersion:NSBundle.mainBundle.infoDictionary[@"CFBundleShortVersionString"]];
|
||||
#endif
|
||||
}
|
||||
+ (BOOL)validSettings:(id)settings {
|
||||
if (![settings isKindOfClass:NSDictionary.class] || [settings count] > 4096) return NO;
|
||||
for (id key in settings) {
|
||||
// Typed locals, not messages to id: FaceclawKit declares other `count`/`length` properties.
|
||||
+ (BOOL)validSettings:(id)candidate {
|
||||
if (![candidate isKindOfClass:NSDictionary.class]) return NO;
|
||||
NSDictionary *settings = candidate;
|
||||
if (settings.count > 4096) return NO;
|
||||
for (id candidateKey in settings) {
|
||||
if (![candidateKey isKindOfClass:NSString.class]) return NO;
|
||||
NSString *key = candidateKey;
|
||||
id value = settings[key];
|
||||
if (![key isKindOfClass:NSString.class] || ![key length] || [key length] > 512 ||
|
||||
if (!key.length || key.length > 512 ||
|
||||
!([value isKindOfClass:NSString.class] || [value isKindOfClass:NSNumber.class])) return NO;
|
||||
}
|
||||
return YES;
|
||||
}
|
||||
+ (void)processDirectory:(NSString *)directory defaults:(NSUserDefaults *)defaults domain:(NSString *)domain {
|
||||
/** The app's settings: strings and finite numbers (booleans included) in its persistent domain. */
|
||||
+ (NSDictionary *)exportableSettings:(NSDictionary *)domain {
|
||||
NSMutableDictionary *settings = [NSMutableDictionary new];
|
||||
for (NSString *key in domain) {
|
||||
id value = domain[key];
|
||||
if ([value isKindOfClass:NSString.class] ||
|
||||
([value isKindOfClass:NSNumber.class] && isfinite([value doubleValue]))) settings[key] = value;
|
||||
}
|
||||
[settings removeObjectForKey:kChangeToken];
|
||||
return settings;
|
||||
}
|
||||
+ (void)processDirectory:(NSString *)directory defaults:(NSUserDefaults *)defaults domain:(NSString *)domain
|
||||
codec:(id<FaceclawConfigCodec>)codec appVersion:(NSString *)appVersion {
|
||||
NSString *requestPath = [directory stringByAppendingPathComponent:@"request.json"];
|
||||
NSData *data = [NSData dataWithContentsOfFile:requestPath];
|
||||
if (!data) return;
|
||||
NSFileManager *fm = NSFileManager.defaultManager;
|
||||
// Consume once. A failed import must never be replayed at a later launch.
|
||||
[fm removeItemAtPath:requestPath error:nil];
|
||||
id request = data.length <= 2 * 1024 * 1024 ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;
|
||||
// A push embeds a document of up to 2 MiB as a JSON string, which escaping can enlarge.
|
||||
id request = data.length <= 8 * 1024 * 1024 ? [NSJSONSerialization JSONObjectWithData:data options:0 error:nil] : nil;
|
||||
BOOL valid = [request isKindOfClass:NSDictionary.class];
|
||||
NSString *identifier = valid && [request[@"id"] isKindOfClass:NSString.class] ? request[@"id"] : @"";
|
||||
NSMutableDictionary *reply = [@{@"id":identifier, @"ok":@NO, @"message":@"Invalid config request"} mutableCopy];
|
||||
[defaults synchronize];
|
||||
NSDictionary *before = [defaults persistentDomainForName:domain] ?: @{};
|
||||
if (valid && [request[@"operation"] isEqual:@"pull"]) {
|
||||
NSMutableDictionary *settings = [NSMutableDictionary new];
|
||||
for (NSString *key in before) {
|
||||
id value = before[key];
|
||||
if ([value isKindOfClass:NSString.class] || [value isKindOfClass:NSNumber.class]) settings[key] = value;
|
||||
NSDictionary *settings = [self exportableSettings:before];
|
||||
NSData *values = [NSJSONSerialization dataWithJSONObject:settings options:0 error:nil];
|
||||
NSString *document = values ? [codec exportValuesJson:[[NSString alloc] initWithData:values encoding:NSUTF8StringEncoding]
|
||||
platform:kPlatform appVersion:appVersion] : nil;
|
||||
if (document) {
|
||||
reply[@"ok"] = @YES; reply[@"message"] = @"Config exported";
|
||||
reply[@"document"] = document; reply[@"count"] = @(settings.count);
|
||||
} else reply[@"message"] = @"Could not encode settings";
|
||||
} else if (valid && [request[@"operation"] isEqual:@"push"] && [request[@"document"] isKindOfClass:NSString.class]) {
|
||||
NSData *planData = [[codec importPlanDocument:request[@"document"] platform:kPlatform] dataUsingEncoding:NSUTF8StringEncoding];
|
||||
id plan = planData ? [NSJSONSerialization JSONObjectWithData:planData options:0 error:nil] : nil;
|
||||
NSDictionary *settings = [plan isKindOfClass:NSDictionary.class] ? plan[@"settings"] : nil;
|
||||
if (![plan isKindOfClass:NSDictionary.class] || ![plan[@"ok"] isEqual:@YES]) {
|
||||
id message = [plan isKindOfClass:NSDictionary.class] ? plan[@"message"] : nil;
|
||||
reply[@"message"] = [message isKindOfClass:NSString.class] ? message : @"Invalid config";
|
||||
} else if (![self validSettings:settings]) {
|
||||
reply[@"message"] = @"Invalid config";
|
||||
} else {
|
||||
NSError *error = nil;
|
||||
NSData *backup = [NSPropertyListSerialization dataWithPropertyList:before format:NSPropertyListBinaryFormat_v1_0 options:0 error:&error];
|
||||
if (backup && [backup writeToFile:[directory stringByAppendingPathComponent:@"previous.plist"] options:NSDataWritingAtomic error:&error]) {
|
||||
NSMutableDictionary *next = [before mutableCopy];
|
||||
[next addEntriesFromDictionary:settings];
|
||||
next[kChangeToken] = NSUUID.UUID.UUIDString;
|
||||
[defaults setPersistentDomain:next forName:domain];
|
||||
if ([defaults synchronize]) {
|
||||
reply[@"ok"] = @YES; reply[@"message"] = @"Config imported";
|
||||
reply[@"count"] = @(settings.count);
|
||||
if ([plan[@"skipped"] isKindOfClass:NSArray.class]) reply[@"skipped"] = plan[@"skipped"];
|
||||
if ([plan[@"sourcePlatform"] isKindOfClass:NSString.class]) reply[@"sourcePlatform"] = plan[@"sourcePlatform"];
|
||||
} else {
|
||||
[defaults setPersistentDomain:before forName:domain]; [defaults synchronize];
|
||||
reply[@"message"] = @"Could not persist config; restored previous settings";
|
||||
}
|
||||
} else reply[@"message"] = @"Could not back up existing settings; import skipped";
|
||||
}
|
||||
[settings removeObjectForKey:@"ios.settings.changeToken"];
|
||||
reply[@"config"] = @{@"schema":@1, @"settings":settings}; reply[@"ok"] = @YES;
|
||||
reply[@"message"] = @"Config exported";
|
||||
} else if (valid && [request[@"operation"] isEqual:@"push"] && [self validSettings:request[@"settings"]]) {
|
||||
NSError *error = nil;
|
||||
NSData *backup = [NSPropertyListSerialization dataWithPropertyList:before format:NSPropertyListBinaryFormat_v1_0 options:0 error:&error];
|
||||
if (backup && [backup writeToFile:[directory stringByAppendingPathComponent:@"previous.plist"] options:NSDataWritingAtomic error:&error]) {
|
||||
NSMutableDictionary *next = [before mutableCopy];
|
||||
[next addEntriesFromDictionary:request[@"settings"]];
|
||||
next[@"ios.settings.changeToken"] = NSUUID.UUID.UUIDString;
|
||||
[defaults setPersistentDomain:next forName:domain];
|
||||
if ([defaults synchronize]) {
|
||||
reply[@"ok"] = @YES; reply[@"message"] = @"Config imported";
|
||||
reply[@"count"] = @([request[@"settings"] count]);
|
||||
} else {
|
||||
[defaults setPersistentDomain:before forName:domain]; [defaults synchronize];
|
||||
reply[@"message"] = @"Could not persist config; restored previous settings";
|
||||
}
|
||||
} else reply[@"message"] = @"Could not back up existing settings; import skipped";
|
||||
}
|
||||
NSData *response = [NSJSONSerialization dataWithJSONObject:reply options:NSJSONWritingPrettyPrinted error:nil];
|
||||
[response writeToFile:[directory stringByAppendingPathComponent:@"response.json"] options:NSDataWritingAtomic error:nil];
|
||||
|
||||
@@ -2,4 +2,8 @@
|
||||
@interface FaceclawCrypto : NSObject
|
||||
+ (NSString *)hmacSha256:(NSString *)secret message:(NSString *)message;
|
||||
+ (NSString *)sha256:(NSData *)data;
|
||||
/** byteCount bytes from the system CSPRNG, base64url without padding (an OAuth state or PKCE verifier). */
|
||||
+ (NSString *)randomBase64Url:(NSInteger)byteCount;
|
||||
/** SHA-256 of the text's UTF-8, base64url without padding (a PKCE S256 code challenge). */
|
||||
+ (NSString *)sha256Base64Url:(NSString *)text;
|
||||
@end
|
||||
@@ -1,6 +1,15 @@
|
||||
#import "FaceclawCrypto.h"
|
||||
#import <CommonCrypto/CommonHMAC.h>
|
||||
#import <CommonCrypto/CommonDigest.h>
|
||||
#import <Security/Security.h>
|
||||
|
||||
static NSString *Base64Url(NSData *data) {
|
||||
NSString *base64 = [data base64EncodedStringWithOptions:0];
|
||||
base64 = [base64 stringByReplacingOccurrencesOfString:@"+" withString:@"-"];
|
||||
base64 = [base64 stringByReplacingOccurrencesOfString:@"/" withString:@"_"];
|
||||
return [base64 stringByReplacingOccurrencesOfString:@"=" withString:@""];
|
||||
}
|
||||
|
||||
@implementation FaceclawCrypto
|
||||
+ (NSString *)hmacSha256:(NSString *)secret message:(NSString *)message {
|
||||
NSData *key = [secret dataUsingEncoding:NSUTF8StringEncoding];
|
||||
@@ -16,4 +25,16 @@
|
||||
for (NSUInteger i = 0; i < sizeof(digest); i++) [hex appendFormat:@"%02x", digest[i]];
|
||||
return hex;
|
||||
}
|
||||
+ (NSString *)randomBase64Url:(NSInteger)byteCount {
|
||||
if (byteCount <= 0 || byteCount > 1024) return nil;
|
||||
NSMutableData *bytes = [NSMutableData dataWithLength:(NSUInteger)byteCount];
|
||||
if (SecRandomCopyBytes(kSecRandomDefault, bytes.length, bytes.mutableBytes) != errSecSuccess) return nil;
|
||||
return Base64Url(bytes);
|
||||
}
|
||||
+ (NSString *)sha256Base64Url:(NSString *)text {
|
||||
NSData *data = [text dataUsingEncoding:NSUTF8StringEncoding];
|
||||
unsigned char digest[CC_SHA256_DIGEST_LENGTH];
|
||||
CC_SHA256(data.bytes, (CC_LONG)data.length, digest);
|
||||
return Base64Url([NSData dataWithBytes:digest length:sizeof(digest)]);
|
||||
}
|
||||
@end
|
||||
@@ -7,4 +7,11 @@
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
|
||||
+ (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height;
|
||||
+ (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight;
|
||||
/**
|
||||
* [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled),
|
||||
* composited over black; the input of the shared Kotlin gray tone curve and dither.
|
||||
*/
|
||||
+ (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize;
|
||||
/** bgraPacket of encoded image bytes (JPEG, PNG, ...); nil when undecodable. */
|
||||
+ (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize;
|
||||
@end
|
||||
@@ -120,4 +120,42 @@
|
||||
CGColorSpaceRelease(space);
|
||||
return gray;
|
||||
}
|
||||
+ (NSData *)bgraPacket:(CGImageRef)image maxSize:(NSInteger)maxSize {
|
||||
if (!image || maxSize <= 0 || maxSize > 1024) return nil;
|
||||
double sourceWidth = CGImageGetWidth(image), sourceHeight = CGImageGetHeight(image);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
|
||||
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
|
||||
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
|
||||
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
|
||||
uint8_t *pixels = bytes + 4;
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
|
||||
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) return nil;
|
||||
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), image);
|
||||
CGContextRelease(context);
|
||||
// Composited over black (the display's see-through background); now opaque.
|
||||
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
|
||||
return packet;
|
||||
}
|
||||
+ (NSData *)bgraPacketFromImageData:(NSData *)data maxSize:(NSInteger)maxSize {
|
||||
if (!data.length || maxSize <= 0 || maxSize > 1024) return nil;
|
||||
CGImageSourceRef source = CGImageSourceCreateWithData((__bridge CFDataRef)data, NULL);
|
||||
if (!source) return nil;
|
||||
NSDictionary *options = @{(id)kCGImageSourceCreateThumbnailFromImageAlways: @YES,
|
||||
(id)kCGImageSourceCreateThumbnailWithTransform: @YES,
|
||||
(id)kCGImageSourceThumbnailMaxPixelSize: @(maxSize)};
|
||||
CGImageRef image = CGImageSourceCreateThumbnailAtIndex(source, 0, (__bridge CFDictionaryRef)options);
|
||||
CFRelease(source);
|
||||
if (!image) return nil;
|
||||
NSData *packet = [self bgraPacket:image maxSize:maxSize];
|
||||
CGImageRelease(image);
|
||||
return packet;
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,16 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* Catches an OAuth redirect to http://127.0.0.1:<port><path> (the loopback redirect that
|
||||
* providers such as Spotify accept from native apps): listens on the loopback interface
|
||||
* only, answers the browser's request for the path with a short "return to Faceclaw"
|
||||
* page and hands back the full redirect URL, query included. Any other path (a favicon)
|
||||
* gets a 404. Foundation-only, so the macOS test builds it as is.
|
||||
*/
|
||||
@interface FaceclawLoopbackRedirect : NSObject
|
||||
/** Called on the main queue with each redirect to the path. */
|
||||
@property(nonatomic, copy) void (^redirectHandler)(NSString *url);
|
||||
/** Start listening; nil on success, otherwise why not (e.g. another app holds the port). */
|
||||
- (NSString *)startOnPort:(NSInteger)port path:(NSString *)path;
|
||||
- (void)stop;
|
||||
@end
|
||||
@@ -0,0 +1,128 @@
|
||||
#import "FaceclawLoopbackRedirect.h"
|
||||
#import <arpa/inet.h>
|
||||
#import <errno.h>
|
||||
#import <fcntl.h>
|
||||
#import <netinet/in.h>
|
||||
#import <sys/socket.h>
|
||||
#import <unistd.h>
|
||||
|
||||
/** Request heads beyond this are not a browser's redirect; stop reading. */
|
||||
static const NSUInteger kMaxRequestBytes = 16384;
|
||||
|
||||
static NSString *const kDonePage =
|
||||
@"<!doctype html><html><head><meta charset=\"utf-8\">"
|
||||
@"<meta name=\"viewport\" content=\"width=device-width, initial-scale=1\"><title>Faceclaw</title></head>"
|
||||
@"<body style=\"font-family: -apple-system, sans-serif; text-align: center; padding: 64px 24px\">"
|
||||
@"<h2>Signed in</h2><p>You can return to Faceclaw.</p></body></html>";
|
||||
|
||||
@interface FaceclawLoopbackRedirect ()
|
||||
@property(nonatomic, strong) dispatch_queue_t queue;
|
||||
@property(nonatomic, strong) dispatch_source_t acceptSource;
|
||||
@property(nonatomic) NSInteger port;
|
||||
@property(nonatomic, copy) NSString *path;
|
||||
@end
|
||||
|
||||
@implementation FaceclawLoopbackRedirect
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) {
|
||||
_queue = dispatch_queue_create("com.faceclaw.loopback-redirect", DISPATCH_QUEUE_SERIAL);
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
- (void)dealloc {
|
||||
[self stop];
|
||||
}
|
||||
|
||||
- (NSString *)startOnPort:(NSInteger)port path:(NSString *)path {
|
||||
[self stop];
|
||||
if (port <= 0 || port > 65535 || ![path hasPrefix:@"/"]) return @"Invalid redirect address";
|
||||
int fd = socket(AF_INET, SOCK_STREAM, 0);
|
||||
if (fd < 0) return [NSString stringWithFormat:@"Could not open a socket (errno %d)", errno];
|
||||
int yes = 1;
|
||||
setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes));
|
||||
struct sockaddr_in address = {0};
|
||||
address.sin_len = sizeof(address);
|
||||
address.sin_family = AF_INET;
|
||||
address.sin_port = htons((uint16_t)port);
|
||||
address.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
if (bind(fd, (struct sockaddr *)&address, sizeof(address)) != 0 || listen(fd, 8) != 0) {
|
||||
int error = errno;
|
||||
close(fd);
|
||||
return error == EADDRINUSE
|
||||
? [NSString stringWithFormat:@"Another app is using port %ld; close it and try again", (long)port]
|
||||
: [NSString stringWithFormat:@"Could not listen on port %ld (errno %d)", (long)port, error];
|
||||
}
|
||||
fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | O_NONBLOCK);
|
||||
self.port = port;
|
||||
self.path = path;
|
||||
dispatch_source_t source = dispatch_source_create(DISPATCH_SOURCE_TYPE_READ, (uintptr_t)fd, 0, self.queue);
|
||||
__weak typeof(self) weakSelf = self;
|
||||
dispatch_source_set_event_handler(source, ^{
|
||||
int client = accept(fd, NULL, NULL);
|
||||
if (client < 0) return;
|
||||
FaceclawLoopbackRedirect *strongSelf = weakSelf;
|
||||
if (strongSelf) [strongSelf serve:client];
|
||||
else close(client);
|
||||
});
|
||||
dispatch_source_set_cancel_handler(source, ^{ close(fd); });
|
||||
self.acceptSource = source;
|
||||
dispatch_resume(source);
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
if (self.acceptSource) dispatch_source_cancel(self.acceptSource);
|
||||
self.acceptSource = nil;
|
||||
}
|
||||
|
||||
/** Read one request head and answer it; runs on the receiver's queue (blocking, with timeouts). */
|
||||
- (void)serve:(int)client {
|
||||
// BSD sockets inherit O_NONBLOCK from the listener.
|
||||
fcntl(client, F_SETFL, fcntl(client, F_GETFL) & ~O_NONBLOCK);
|
||||
int yes = 1;
|
||||
setsockopt(client, SOL_SOCKET, SO_NOSIGPIPE, &yes, sizeof(yes));
|
||||
struct timeval timeout = { .tv_sec = 3, .tv_usec = 0 };
|
||||
setsockopt(client, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout));
|
||||
setsockopt(client, SOL_SOCKET, SO_SNDTIMEO, &timeout, sizeof(timeout));
|
||||
|
||||
NSMutableData *head = [NSMutableData data];
|
||||
NSData *terminator = [@"\r\n\r\n" dataUsingEncoding:NSASCIIStringEncoding];
|
||||
char buffer[2048];
|
||||
while (head.length < kMaxRequestBytes) {
|
||||
ssize_t count = recv(client, buffer, sizeof(buffer), 0);
|
||||
if (count <= 0) break;
|
||||
[head appendBytes:buffer length:(NSUInteger)count];
|
||||
if ([head rangeOfData:terminator options:0 range:NSMakeRange(0, head.length)].location != NSNotFound) break;
|
||||
}
|
||||
NSString *text = [[NSString alloc] initWithData:head encoding:NSUTF8StringEncoding] ?: @"";
|
||||
NSArray<NSString *> *requestLine = [[text componentsSeparatedByString:@"\r\n"].firstObject componentsSeparatedByString:@" "];
|
||||
NSString *target = requestLine.count >= 3 && [requestLine[0] isEqualToString:@"GET"] ? requestLine[1] : nil;
|
||||
BOOL matched = target && [[target componentsSeparatedByString:@"?"].firstObject isEqualToString:self.path];
|
||||
[self respond:client status:matched ? @"200 OK" : @"404 Not Found" body:matched ? kDonePage : @"Not found"];
|
||||
close(client);
|
||||
if (!matched) return;
|
||||
|
||||
NSString *url = [NSString stringWithFormat:@"http://127.0.0.1:%ld%@", (long)self.port, target];
|
||||
void (^handler)(NSString *) = self.redirectHandler;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (handler) handler(url);
|
||||
});
|
||||
}
|
||||
|
||||
- (void)respond:(int)client status:(NSString *)status body:(NSString *)body {
|
||||
NSData *bodyData = [body dataUsingEncoding:NSUTF8StringEncoding];
|
||||
NSString *headers = [NSString stringWithFormat:
|
||||
@"HTTP/1.1 %@\r\nContent-Type: text/html; charset=utf-8\r\nContent-Length: %lu\r\n"
|
||||
@"Cache-Control: no-store\r\nConnection: close\r\n\r\n", status, (unsigned long)bodyData.length];
|
||||
NSMutableData *response = [[headers dataUsingEncoding:NSASCIIStringEncoding] mutableCopy];
|
||||
[response appendData:bodyData];
|
||||
const uint8_t *bytes = response.bytes;
|
||||
NSUInteger sent = 0;
|
||||
while (sent < response.length) {
|
||||
ssize_t count = send(client, bytes + sent, response.length - sent, 0);
|
||||
if (count <= 0) break;
|
||||
sent += (NSUInteger)count;
|
||||
}
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,16 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/** Lightweight process counters for the on-glasses resource-usage monitor. */
|
||||
@interface FaceclawResourceUsage : NSObject
|
||||
/**
|
||||
* One sample as a JSON object: atMs (monotonic clock, counting sleep), cpuMs
|
||||
* (user+system time of every thread, alive or exited), footprintKb (the
|
||||
* phys_footprint that jetsam limits and Xcode's memory gauge shows),
|
||||
* footprintPeakKb (lifetime peak), residentKb, mallocKb (bytes in use across
|
||||
* malloc zones), availableKb (headroom before the jetsam limit; 0 when there
|
||||
* is none, as on the simulator), threads, thermalState
|
||||
* (NSProcessInfoThermalState) and lowPower. Fields whose query fails are
|
||||
* omitted.
|
||||
*/
|
||||
+ (NSString *)sample;
|
||||
@end
|
||||
@@ -0,0 +1,60 @@
|
||||
#import "FaceclawResourceUsage.h"
|
||||
#import <TargetConditionals.h>
|
||||
#import <mach/mach.h>
|
||||
#import <malloc/malloc.h>
|
||||
#import <sys/resource.h>
|
||||
#import <time.h>
|
||||
#if TARGET_OS_IPHONE
|
||||
#import <os/proc.h>
|
||||
#endif
|
||||
|
||||
static NSNumber *kib(uint64_t bytes) { return @(bytes / 1024); }
|
||||
|
||||
@implementation FaceclawResourceUsage
|
||||
+ (NSString *)sample {
|
||||
NSMutableDictionary *out = [NSMutableDictionary new];
|
||||
out[@"atMs"] = @(clock_gettime_nsec_np(CLOCK_MONOTONIC) / 1000000ULL);
|
||||
|
||||
struct rusage usage;
|
||||
if (getrusage(RUSAGE_SELF, &usage) == 0) {
|
||||
double seconds = (double)usage.ru_utime.tv_sec + (double)usage.ru_stime.tv_sec;
|
||||
double micros = (double)usage.ru_utime.tv_usec + (double)usage.ru_stime.tv_usec;
|
||||
out[@"cpuMs"] = @(seconds * 1000.0 + micros / 1000.0);
|
||||
}
|
||||
|
||||
// The kernel fills as much of the struct as it knows and shrinks count to
|
||||
// match, so later-revision fields are read only when count covers them.
|
||||
task_vm_info_data_t vm;
|
||||
mach_msg_type_number_t count = TASK_VM_INFO_COUNT;
|
||||
if (task_info(mach_task_self(), TASK_VM_INFO, (task_info_t)&vm, &count) == KERN_SUCCESS) {
|
||||
out[@"residentKb"] = kib(vm.resident_size);
|
||||
if (count >= TASK_VM_INFO_REV1_COUNT) out[@"footprintKb"] = kib(vm.phys_footprint);
|
||||
if (count >= TASK_VM_INFO_REV3_COUNT && vm.ledger_phys_footprint_peak > 0) {
|
||||
out[@"footprintPeakKb"] = kib((uint64_t)vm.ledger_phys_footprint_peak);
|
||||
}
|
||||
}
|
||||
|
||||
malloc_statistics_t heap = {0};
|
||||
malloc_zone_statistics(NULL, &heap);
|
||||
out[@"mallocKb"] = kib(heap.size_in_use);
|
||||
|
||||
#if TARGET_OS_IPHONE
|
||||
out[@"availableKb"] = kib(os_proc_available_memory());
|
||||
#endif
|
||||
|
||||
thread_act_array_t threads = NULL;
|
||||
mach_msg_type_number_t threadCount = 0;
|
||||
if (task_threads(mach_task_self(), &threads, &threadCount) == KERN_SUCCESS) {
|
||||
for (mach_msg_type_number_t i = 0; i < threadCount; i++) mach_port_deallocate(mach_task_self(), threads[i]);
|
||||
vm_deallocate(mach_task_self(), (vm_address_t)threads, threadCount * sizeof(thread_act_t));
|
||||
out[@"threads"] = @(threadCount);
|
||||
}
|
||||
|
||||
NSProcessInfo *process = NSProcessInfo.processInfo;
|
||||
out[@"thermalState"] = @(process.thermalState);
|
||||
out[@"lowPower"] = @(process.lowPowerModeEnabled);
|
||||
|
||||
NSData *json = [NSJSONSerialization dataWithJSONObject:out options:0 error:nil];
|
||||
return json ? [[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding] : @"{}";
|
||||
}
|
||||
@end
|
||||
@@ -0,0 +1,273 @@
|
||||
@preconcurrency import AVFoundation
|
||||
import CoreMedia
|
||||
import Foundation
|
||||
import Speech
|
||||
|
||||
/// Long-form on-device transcription with SpeechAnalyzer and SpeechTranscriber
|
||||
/// (iOS 26+), for continuous capture (Transcribe, Teleprompter, Calculator):
|
||||
/// FaceclawSpeech's SFSpeechRecognizer only handles short dictations. It takes
|
||||
/// the same 16 kHz mono PCM16LE the cloud providers get (see
|
||||
/// app/native/speech-analyzer-stt.ios.ts) and reports JSON events on the main
|
||||
/// queue through eventHandler:
|
||||
///
|
||||
/// {"kind":"status","message":String}
|
||||
/// {"kind":"ready","locale":String} audio is accepted from here on
|
||||
/// {"kind":"transcript","text":String,"final":Bool,
|
||||
/// "start":Double?,"end":Double?, the result's audio range
|
||||
/// "speechEnd":Double?} when its last word ends
|
||||
/// {"kind":"finished"} after finish(), once the last result is in
|
||||
/// {"kind":"error","message":String}
|
||||
///
|
||||
/// Times are seconds of audio since start. Volatile results (final: false)
|
||||
/// cover all audio not finalized yet, replacing the previous volatile one;
|
||||
/// their word times are only as fine as the result's range. A final's first
|
||||
/// word reaches back into the silence before it, so only word ends are
|
||||
/// reported.
|
||||
///
|
||||
/// There is no way to settle text at a pause: on macOS 26.4 any mid-stream
|
||||
/// SpeechAnalyzer.finalize(through:) left the next phrase's final as dots,
|
||||
/// while the model splits its finals at pauses on its own.
|
||||
/// Called from JavaScript, so on the main thread.
|
||||
@MainActor
|
||||
@objc(FaceclawSpeechAnalyzer)
|
||||
@objcMembers
|
||||
public class FaceclawSpeechAnalyzer: NSObject {
|
||||
public var eventHandler: ((String) -> Void)?
|
||||
// An AnalyzerEngine, typed loosely so the class itself loads before iOS 26.
|
||||
private var engine: AnyObject?
|
||||
|
||||
/// Whether this device can transcribe with SpeechTranscriber (iOS 26 and supported hardware).
|
||||
public static func isSupported() -> Bool {
|
||||
if #available(iOS 26.0, *) { return SpeechTranscriber.isAvailable }
|
||||
return false
|
||||
}
|
||||
|
||||
/// `language` is a locale identifier ("en", "en-US"), or "" for the device's preferred language.
|
||||
public func start(language: String) {
|
||||
cancel()
|
||||
guard #available(iOS 26.0, *) else {
|
||||
emit(["kind": "error", "message": "On-device transcription needs iOS 26 or later."])
|
||||
return
|
||||
}
|
||||
let engine = AnalyzerEngine { [weak self] event in self?.emit(event) }
|
||||
self.engine = engine
|
||||
engine.start(language: language)
|
||||
}
|
||||
|
||||
/// 16 kHz mono PCM16LE; ignored until "ready".
|
||||
public func acceptPcm(_ pcm: Data) {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.accept(pcm) }
|
||||
}
|
||||
|
||||
/// End of audio: finalize the rest, then report "finished".
|
||||
public func finish() {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.finish() }
|
||||
}
|
||||
|
||||
/// Abandon the session without further events.
|
||||
public func cancel() {
|
||||
if #available(iOS 26.0, *) { (engine as? AnalyzerEngine)?.cancel() }
|
||||
engine = nil
|
||||
}
|
||||
|
||||
private func emit(_ event: [String: Any]) {
|
||||
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
||||
guard JSONSerialization.isValidJSONObject(event),
|
||||
let data = try? JSONSerialization.data(withJSONObject: event),
|
||||
let json = String(data: data, encoding: .utf8) else { return }
|
||||
eventHandler?(json)
|
||||
}
|
||||
}
|
||||
|
||||
@available(iOS 26.0, *)
|
||||
@MainActor
|
||||
private final class AnalyzerEngine {
|
||||
private static let sampleRate = 16000.0
|
||||
private let emit: ([String: Any]) -> Void
|
||||
private let pcmFormat = AVAudioFormat(commonFormat: .pcmFormatInt16, sampleRate: AnalyzerEngine.sampleRate, channels: 1, interleaved: true)!
|
||||
private var analyzer: SpeechAnalyzer?
|
||||
private var input: AsyncStream<AnalyzerInput>.Continuation?
|
||||
private var analyzerFormat: AVAudioFormat?
|
||||
// Nil when the analyzer takes the input format as it is.
|
||||
private var converter: AVAudioConverter?
|
||||
private var ready = false
|
||||
private var finishing = false
|
||||
private var closed = false
|
||||
private var setup: Task<Void, Never>?
|
||||
private var results: Task<Void, Never>?
|
||||
|
||||
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
||||
|
||||
func start(language: String) {
|
||||
setup = Task { [weak self] in await self?.prepare(language: language) }
|
||||
}
|
||||
|
||||
/// Events stop once the session is cancelled, even from work still winding down.
|
||||
private func send(_ event: [String: Any]) {
|
||||
if !closed { emit(event) }
|
||||
}
|
||||
|
||||
private func prepare(language: String) async {
|
||||
send(["kind": "status", "message": "Preparing on-device transcription…"])
|
||||
let identifier = language.isEmpty ? (Locale.preferredLanguages.first ?? "en-US") : language
|
||||
guard let locale = await SpeechTranscriber.supportedLocale(equivalentTo: Locale(identifier: identifier)) else {
|
||||
fail("On-device transcription doesn't support \(identifier). Choose a cloud or Whisper server transcription provider (Settings > Voice).")
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
let transcriber = SpeechTranscriber(locale: locale, transcriptionOptions: [],
|
||||
reportingOptions: [.volatileResults], attributeOptions: [.audioTimeRange])
|
||||
do {
|
||||
// Keep the model installed between sessions (best effort).
|
||||
if !(await AssetInventory.reservedLocales).contains(locale) {
|
||||
_ = try? await AssetInventory.reserve(locale: locale)
|
||||
}
|
||||
if closed { return }
|
||||
if let request = try await AssetInventory.assetInstallationRequest(supporting: [transcriber]) {
|
||||
if closed { return }
|
||||
try await download(request)
|
||||
}
|
||||
if closed { return }
|
||||
guard let format = await SpeechAnalyzer.bestAvailableAudioFormat(compatibleWith: [transcriber]) else {
|
||||
fail("On-device transcription has no usable audio format.")
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
if format != pcmFormat {
|
||||
guard let converter = AVAudioConverter(from: pcmFormat, to: format) else {
|
||||
fail("On-device transcription can't convert the microphone audio.")
|
||||
return
|
||||
}
|
||||
self.converter = converter
|
||||
}
|
||||
analyzerFormat = format
|
||||
let analyzer = SpeechAnalyzer(modules: [transcriber])
|
||||
let (stream, input) = AsyncStream<AnalyzerInput>.makeStream()
|
||||
self.analyzer = analyzer
|
||||
self.input = input
|
||||
results = Task { [weak self] in await self?.collect(transcriber) }
|
||||
try await analyzer.prepareToAnalyze(in: format)
|
||||
try await analyzer.start(inputSequence: stream)
|
||||
if closed { return }
|
||||
ready = true
|
||||
send(["kind": "ready", "locale": locale.identifier])
|
||||
if finishing { await finishAnalysis() }
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch the locale's model, reporting progress while it downloads.
|
||||
private func download(_ request: AssetInstallationRequest) async throws {
|
||||
let progress = request.progress
|
||||
let reporter = Task { [weak self] in
|
||||
while !Task.isCancelled {
|
||||
let percent = Int((progress.fractionCompleted * 100).rounded())
|
||||
self?.send(["kind": "status", "message": "Downloading on-device speech model… \(percent)%"])
|
||||
try? await Task.sleep(nanoseconds: 1_000_000_000)
|
||||
}
|
||||
}
|
||||
defer { reporter.cancel() }
|
||||
try await request.downloadAndInstall()
|
||||
}
|
||||
|
||||
func accept(_ pcm: Data) {
|
||||
guard ready, !finishing, !closed, let input, let analyzerFormat else { return }
|
||||
let frames = AVAudioFrameCount(pcm.count / 2)
|
||||
guard frames > 0, let buffer = AVAudioPCMBuffer(pcmFormat: pcmFormat, frameCapacity: frames),
|
||||
let channel = buffer.int16ChannelData else { return }
|
||||
buffer.frameLength = frames
|
||||
pcm.withUnsafeBytes { raw in
|
||||
if let base = raw.baseAddress { memcpy(channel[0], base, Int(frames) * 2) }
|
||||
}
|
||||
guard let converter else {
|
||||
input.yield(AnalyzerInput(buffer: buffer))
|
||||
return
|
||||
}
|
||||
let capacity = AVAudioFrameCount((Double(frames) * analyzerFormat.sampleRate / AnalyzerEngine.sampleRate).rounded(.up)) + 32
|
||||
guard let converted = AVAudioPCMBuffer(pcmFormat: analyzerFormat, frameCapacity: capacity) else { return }
|
||||
var consumed = false
|
||||
var error: NSError?
|
||||
// noDataNow, not endOfStream: the converter keeps its resampling state for the next chunk.
|
||||
converter.convert(to: converted, error: &error) { _, status in
|
||||
if consumed {
|
||||
status.pointee = .noDataNow
|
||||
return nil
|
||||
}
|
||||
consumed = true
|
||||
status.pointee = .haveData
|
||||
return buffer
|
||||
}
|
||||
if error == nil && converted.frameLength > 0 { input.yield(AnalyzerInput(buffer: converted)) }
|
||||
}
|
||||
|
||||
func finish() {
|
||||
guard !finishing, !closed else { return }
|
||||
finishing = true
|
||||
// Before "ready", prepare() finishes once the analyzer is running.
|
||||
if ready { Task { await finishAnalysis() } }
|
||||
}
|
||||
|
||||
private func finishAnalysis() async {
|
||||
input?.finish()
|
||||
input = nil
|
||||
do {
|
||||
try await analyzer?.finalizeAndFinishThroughEndOfInput()
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
// collect() reports "finished" once the results sequence ends.
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
guard !closed else { return }
|
||||
closed = true
|
||||
input?.finish()
|
||||
input = nil
|
||||
setup?.cancel()
|
||||
results?.cancel()
|
||||
if let analyzer { Task { await analyzer.cancelAndFinishNow() } }
|
||||
analyzer = nil
|
||||
}
|
||||
|
||||
private func collect(_ transcriber: SpeechTranscriber) async {
|
||||
do {
|
||||
for try await result in transcriber.results {
|
||||
if closed { return }
|
||||
send(event(for: result))
|
||||
}
|
||||
send(["kind": "finished"])
|
||||
} catch {
|
||||
fail("On-device transcription: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
private func event(for result: SpeechTranscriber.Result) -> [String: Any] {
|
||||
let text = result.text
|
||||
var speechEnd: Double?
|
||||
for run in text.runs {
|
||||
guard let range = run.audioTimeRange,
|
||||
!String(text[run.range].characters).trimmingCharacters(in: .whitespacesAndNewlines).isEmpty else { continue }
|
||||
speechEnd = finite(range.end) ?? speechEnd
|
||||
}
|
||||
return [
|
||||
"kind": "transcript",
|
||||
"text": String(text.characters),
|
||||
"final": result.isFinal,
|
||||
"start": finite(result.range.start) ?? NSNull(),
|
||||
"end": finite(result.range.end) ?? NSNull(),
|
||||
"speechEnd": speechEnd ?? NSNull(),
|
||||
]
|
||||
}
|
||||
|
||||
private func finite(_ time: CMTime) -> Double? {
|
||||
let seconds = time.seconds
|
||||
return seconds.isFinite ? seconds : nil
|
||||
}
|
||||
|
||||
private func fail(_ message: String) {
|
||||
guard !closed else { return }
|
||||
emit(["kind": "error", "message": message])
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,694 @@
|
||||
import Foundation
|
||||
#if canImport(FoundationModels)
|
||||
import FoundationModels
|
||||
#endif
|
||||
|
||||
/// The on-phone LLM on iOS: Apple's Foundation Models framework (the Apple
|
||||
/// Intelligence system model, iOS 26+, eligible hardware). It runs out of
|
||||
/// process on the Neural Engine, so it costs the app no model download or
|
||||
/// memory, and unlike Metal work it is allowed (within a system budget) while
|
||||
/// Faceclaw is in the background driving the glasses.
|
||||
///
|
||||
/// JavaScript (app/native/local-llm.ios.ts) drives the same provider-neutral
|
||||
/// agent loop as the cloud models: each start() is one loop iteration. The
|
||||
/// request is JSON:
|
||||
///
|
||||
/// {"system":String,
|
||||
/// "tools":[{"name","description","parameters":<JSON Schema>}],
|
||||
/// "history":[{"type":"prompt"|"response","text"} |
|
||||
/// {"type":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]} |
|
||||
/// {"type":"toolOutput","id","name","text"}],
|
||||
/// "prompt":String, "maxTokens":Int, "permissive":Bool}
|
||||
///
|
||||
/// and events arrive on the main queue through eventHandler:
|
||||
///
|
||||
/// {"kind":"delta","text":String} streamed reply text
|
||||
/// {"kind":"toolCalls","calls":[{"id","name","arguments":<JSON string>}]}
|
||||
/// {"kind":"done"}
|
||||
/// {"kind":"error","code":String,"message":String}
|
||||
/// {"kind":"log","message":String}
|
||||
///
|
||||
/// The framework runs tools itself, mid-response. When the model calls one,
|
||||
/// the call is reported as "toolCalls" and the generation stays suspended
|
||||
/// inside the tool until resolveToolCalls() hands back the results that
|
||||
/// JavaScript's tool loop produced; the reply then continues on the same
|
||||
/// instance.
|
||||
///
|
||||
/// The model's context window is small (4096 tokens on iOS 26, shared by input
|
||||
/// and reply) and the framework spends ~50 tokens on each tool before its
|
||||
/// parameters, so the assistant's full tool list (30+ tools, ~5000 tokens) can't
|
||||
/// be offered at once. When the tools don't fit their share of the window, a
|
||||
/// quick schema-constrained routing request first asks the model which few
|
||||
/// tools the request needs. History is then trimmed to fit before each start().
|
||||
/// Called from JavaScript, so on the main thread.
|
||||
@MainActor
|
||||
@objc(FaceclawSystemLanguageModel)
|
||||
@objcMembers
|
||||
public class FaceclawSystemLanguageModel: NSObject {
|
||||
public var eventHandler: ((String) -> Void)?
|
||||
// A LanguageModelEngine, typed loosely so the class itself loads before iOS 26.
|
||||
private var engine: AnyObject?
|
||||
|
||||
/// "available", or why not: "unsupportedOS", "deviceNotEligible",
|
||||
/// "appleIntelligenceNotEnabled", "modelNotReady" (still downloading), "unavailable".
|
||||
/// Nonisolated: worker isolates load the model-choice list too, off the main thread.
|
||||
nonisolated public static func availability() -> String {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
switch SystemLanguageModel.default.availability {
|
||||
case .available:
|
||||
return "available"
|
||||
case .unavailable(let reason):
|
||||
switch reason {
|
||||
case .deviceNotEligible: return "deviceNotEligible"
|
||||
case .appleIntelligenceNotEnabled: return "appleIntelligenceNotEnabled"
|
||||
case .modelNotReady: return "modelNotReady"
|
||||
@unknown default: return "unavailable"
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return "unsupportedOS"
|
||||
}
|
||||
|
||||
/// The model's context window in tokens (input and output share it), or 0 when unavailable.
|
||||
nonisolated public static func contextSize() -> Int {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) { return SystemLanguageModel.default.contextSize }
|
||||
#endif
|
||||
return 0
|
||||
}
|
||||
|
||||
public func start(_ requestJson: String) {
|
||||
cancel()
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
guard let data = requestJson.data(using: .utf8),
|
||||
let request = (try? JSONSerialization.jsonObject(with: data)) as? [String: Any] else {
|
||||
emit(["kind": "error", "code": "other", "message": "Invalid on-device model request"])
|
||||
return
|
||||
}
|
||||
let engine = LanguageModelEngine { [weak self] event in self?.emit(event) }
|
||||
self.engine = engine
|
||||
engine.start(request)
|
||||
return
|
||||
}
|
||||
#endif
|
||||
emit(["kind": "error", "code": "unavailable", "message": "The on-device model needs iOS 26 or later."])
|
||||
}
|
||||
|
||||
/// `[{"id","content","isError"}]` for the calls of the last "toolCalls" event.
|
||||
public func resolveToolCalls(_ resultsJson: String) {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) {
|
||||
let data = resultsJson.data(using: .utf8) ?? Data()
|
||||
let results = ((try? JSONSerialization.jsonObject(with: data)) as? [[String: Any]]) ?? []
|
||||
(engine as? LanguageModelEngine)?.resolve(results)
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/// Abandon the generation (and any suspended tool calls) without further events.
|
||||
public func cancel() {
|
||||
#if canImport(FoundationModels)
|
||||
if #available(iOS 26.0, *) { (engine as? LanguageModelEngine)?.cancel() }
|
||||
#endif
|
||||
engine = nil
|
||||
}
|
||||
|
||||
private func emit(_ event: [String: Any]) {
|
||||
// JSONSerialization raises (not throws) on NaN and other invalid values.
|
||||
guard JSONSerialization.isValidJSONObject(event),
|
||||
let data = try? JSONSerialization.data(withJSONObject: event),
|
||||
let json = String(data: data, encoding: .utf8) else { return }
|
||||
eventHandler?(json)
|
||||
}
|
||||
}
|
||||
|
||||
#if canImport(FoundationModels)
|
||||
|
||||
/// A JavaScript-side tool: the framework calls it mid-response, and the
|
||||
/// engine parks the call until JavaScript resolves it.
|
||||
@available(iOS 26.0, *)
|
||||
private struct BridgedTool: Tool {
|
||||
typealias Arguments = GeneratedContent
|
||||
typealias Output = String
|
||||
|
||||
let name: String
|
||||
let description: String
|
||||
let parameters: GenerationSchema
|
||||
let handler: @Sendable (String, String) async throws -> String
|
||||
|
||||
func call(arguments: GeneratedContent) async throws -> String {
|
||||
try await handler(name, arguments.jsonString)
|
||||
}
|
||||
}
|
||||
|
||||
/// One tool from the request, convertible at two levels of verbosity.
|
||||
@available(iOS 26.0, *)
|
||||
private struct ToolSpec {
|
||||
let name: String
|
||||
let description: String
|
||||
let parameters: [String: Any]
|
||||
}
|
||||
|
||||
@available(iOS 26.0, *)
|
||||
@MainActor
|
||||
final class LanguageModelEngine {
|
||||
private struct PendingCall {
|
||||
let id: String
|
||||
let name: String
|
||||
let arguments: String
|
||||
let continuation: CheckedContinuation<String, Error>
|
||||
}
|
||||
|
||||
/// How long a call waits for any parallel calls before they're reported together.
|
||||
private static let toolBatchDelayNs: UInt64 = 30_000_000
|
||||
/// Tokens kept free beyond the reply budget: the framework's own framing and rounding.
|
||||
private static let contextMargin = 64
|
||||
/// Share of the context window tool definitions may take before tools are routed.
|
||||
private static let toolBudgetFraction = 0.4
|
||||
/// Most tools a routing request picks.
|
||||
private static let maxRoutedTools = 5
|
||||
/// Tools used in the latest exchanges stay offered, for follow-ups ("make it ten minutes").
|
||||
private static let keptRecentTools = 2
|
||||
|
||||
private let emit: ([String: Any]) -> Void
|
||||
private var task: Task<Void, Never>?
|
||||
// Calls not yet reported, then reported calls awaiting results.
|
||||
private var unreported: [PendingCall] = []
|
||||
private var awaiting: [String: PendingCall] = [:]
|
||||
private var flushScheduled = false
|
||||
private var closed = false
|
||||
|
||||
init(emit: @escaping ([String: Any]) -> Void) { self.emit = emit }
|
||||
|
||||
func start(_ request: [String: Any]) {
|
||||
task = Task { [weak self] in await self?.run(request) }
|
||||
}
|
||||
|
||||
func resolve(_ results: [[String: Any]]) {
|
||||
for result in results {
|
||||
guard let id = result["id"] as? String, let call = awaiting.removeValue(forKey: id) else { continue }
|
||||
let content = (result["content"] as? String) ?? ""
|
||||
let isError = (result["isError"] as? Bool) ?? false
|
||||
// An error goes back to the model as text: a thrown error would end the whole response.
|
||||
call.continuation.resume(returning: isError ? "Error: \(content)" : (content.isEmpty ? "(no output)" : content))
|
||||
}
|
||||
// A call JavaScript didn't answer can't be left hanging.
|
||||
for (_, call) in awaiting {
|
||||
call.continuation.resume(returning: "Error: the tool returned no result")
|
||||
}
|
||||
awaiting.removeAll()
|
||||
}
|
||||
|
||||
func cancel() {
|
||||
closed = true
|
||||
task?.cancel()
|
||||
task = nil
|
||||
for call in unreported { call.continuation.resume(throwing: CancellationError()) }
|
||||
for (_, call) in awaiting { call.continuation.resume(throwing: CancellationError()) }
|
||||
unreported.removeAll()
|
||||
awaiting.removeAll()
|
||||
}
|
||||
|
||||
/// Events stop once the session is cancelled, even from work still winding down.
|
||||
private func send(_ event: [String: Any]) {
|
||||
if !closed { emit(event) }
|
||||
}
|
||||
|
||||
private func run(_ request: [String: Any]) async {
|
||||
let system = (request["system"] as? String) ?? ""
|
||||
let prompt = (request["prompt"] as? String) ?? ""
|
||||
let maxTokens = (request["maxTokens"] as? NSNumber)?.intValue ?? 512
|
||||
let permissive = (request["permissive"] as? Bool) ?? false
|
||||
let toolSpecs: [ToolSpec] = ((request["tools"] as? [[String: Any]]) ?? []).compactMap { tool in
|
||||
guard let name = tool["name"] as? String else { return nil }
|
||||
return ToolSpec(name: name, description: (tool["description"] as? String) ?? "",
|
||||
parameters: (tool["parameters"] as? [String: Any]) ?? [:])
|
||||
}
|
||||
let history = (request["history"] as? [[String: Any]]) ?? []
|
||||
|
||||
// Transformations of user text (translate, summarize) get the permissive
|
||||
// guardrails; they apply to plain string replies, which tool-free requests are.
|
||||
let model = SystemLanguageModel(useCase: .general,
|
||||
guardrails: permissive ? .permissiveContentTransformations : .default)
|
||||
var offered = toolSpecs
|
||||
if !toolSpecs.isEmpty {
|
||||
switch await selectTools(model: model, specs: toolSpecs, history: history, prompt: prompt) {
|
||||
case .tools(let specs): offered = specs
|
||||
case .failed(let event):
|
||||
send(event)
|
||||
return
|
||||
}
|
||||
if closed { return }
|
||||
}
|
||||
let fitted = await fit(model: model, system: system, tools: offered, history: history,
|
||||
prompt: prompt, maxTokens: maxTokens)
|
||||
if closed { return }
|
||||
|
||||
let handler: @Sendable (String, String) async throws -> String = { [weak self] name, arguments in
|
||||
guard let self = self else { throw CancellationError() }
|
||||
return try await self.toolInvoked(name: name, arguments: arguments)
|
||||
}
|
||||
let tools: [any Tool] = fitted.tools.map {
|
||||
BridgedTool(name: $0.name, description: $0.description, parameters: $0.schema, handler: handler)
|
||||
}
|
||||
let session = LanguageModelSession(model: model, tools: tools, transcript: Transcript(entries: fitted.entries))
|
||||
let options = GenerationOptions(maximumResponseTokens: maxTokens)
|
||||
var last = ""
|
||||
do {
|
||||
for try await snapshot in session.streamResponse(to: prompt, options: options) {
|
||||
if closed { return }
|
||||
let text = snapshot.content
|
||||
// Snapshots are cumulative; one that doesn't extend the last is a new segment.
|
||||
let delta = text.hasPrefix(last) ? String(text.dropFirst(last.count)) : text
|
||||
last = text
|
||||
if !delta.isEmpty { send(["kind": "delta", "text": delta]) }
|
||||
}
|
||||
send(["kind": "done"])
|
||||
} catch {
|
||||
if closed || error is CancellationError { return }
|
||||
send(describe(error))
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Tool calls
|
||||
|
||||
private func toolInvoked(name: String, arguments: String) async throws -> String {
|
||||
if closed { throw CancellationError() }
|
||||
return try await withCheckedThrowingContinuation { continuation in
|
||||
if closed {
|
||||
continuation.resume(throwing: CancellationError())
|
||||
return
|
||||
}
|
||||
let id = "fm_" + UUID().uuidString.replacingOccurrences(of: "-", with: "").prefix(16).lowercased()
|
||||
unreported.append(PendingCall(id: id, name: name, arguments: arguments, continuation: continuation))
|
||||
scheduleFlush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Report calls in one event, after a moment for any parallel calls to arrive.
|
||||
private func scheduleFlush() {
|
||||
if flushScheduled { return }
|
||||
flushScheduled = true
|
||||
Task { [weak self] in
|
||||
try? await Task.sleep(nanoseconds: LanguageModelEngine.toolBatchDelayNs)
|
||||
self?.flush()
|
||||
}
|
||||
}
|
||||
|
||||
private func flush() {
|
||||
flushScheduled = false
|
||||
if closed || unreported.isEmpty { return }
|
||||
let calls = unreported
|
||||
unreported.removeAll()
|
||||
for call in calls { awaiting[call.id] = call }
|
||||
send(["kind": "toolCalls",
|
||||
"calls": calls.map { ["id": $0.id, "name": $0.name, "arguments": $0.arguments] }])
|
||||
}
|
||||
|
||||
// MARK: - Tool routing
|
||||
|
||||
private enum Selection {
|
||||
case tools([ToolSpec])
|
||||
case failed([String: Any])
|
||||
}
|
||||
|
||||
/// All tools when their definitions fit their share of the context window;
|
||||
/// otherwise the few the model picks for this request, plus recently used ones.
|
||||
private func selectTools(model: SystemLanguageModel, specs: [ToolSpec], history: [[String: Any]],
|
||||
prompt: String) async -> Selection {
|
||||
let budget = Int(Double(model.contextSize) * LanguageModelEngine.toolBudgetFraction)
|
||||
let cost = await toolTokens(model, buildTools(specs, compact: false))
|
||||
if cost <= budget { return .tools(specs) }
|
||||
|
||||
let names = specs.map { $0.name }
|
||||
let catalog = specs.map { "- \($0.name): \(SchemaBuilder.shorten($0.description, sentences: 1, limit: 100))" }
|
||||
.joined(separator: "\n")
|
||||
let instructions = "You pick the tools a voice assistant needs to handle the user's latest request. " +
|
||||
"Choose at most \(LanguageModelEngine.maxRoutedTools) tools from this list that the assistant should be able to use, " +
|
||||
"most important first. Choose none for small talk or questions it can answer from general knowledge.\n\n" + catalog
|
||||
var query = prompt
|
||||
if let previous = lastPromptText(history) {
|
||||
query = "Previous request: \(SchemaBuilder.shorten(previous, sentences: 3, limit: 300))\n\nLatest request: \(prompt)"
|
||||
}
|
||||
do {
|
||||
let schema = try GenerationSchema(root: DynamicGenerationSchema(name: "ToolSelection", properties: [
|
||||
DynamicGenerationSchema.Property(
|
||||
name: "tools", description: "Names of the tools needed, most important first.",
|
||||
schema: DynamicGenerationSchema(arrayOf: DynamicGenerationSchema(type: String.self, guides: [.anyOf(names)]),
|
||||
minimumElements: 0,
|
||||
maximumElements: LanguageModelEngine.maxRoutedTools)),
|
||||
]), dependencies: [])
|
||||
let session = LanguageModelSession(model: model, instructions: instructions)
|
||||
let response = try await session.respond(to: query, schema: schema,
|
||||
options: GenerationOptions(sampling: .greedy, maximumResponseTokens: 100))
|
||||
var chosen = Set(try response.content.value([String].self, forProperty: "tools"))
|
||||
let recent = recentToolNames(history).prefix(LanguageModelEngine.keptRecentTools)
|
||||
chosen.formUnion(recent)
|
||||
let selected = specs.filter { chosen.contains($0.name) }
|
||||
send(["kind": "log", "message": "routed \(specs.count) tools (\(cost) tokens) to: " +
|
||||
(selected.isEmpty ? "none" : selected.map { $0.name }.joined(separator: ", "))])
|
||||
return .tools(selected)
|
||||
} catch {
|
||||
if let generationError = error as? LanguageModelSession.GenerationError {
|
||||
switch generationError {
|
||||
case .rateLimited, .concurrentRequests, .assetsUnavailable, .guardrailViolation, .refusal,
|
||||
.unsupportedLanguageOrLocale:
|
||||
// The real request would fail the same way.
|
||||
return .failed(describe(error))
|
||||
default:
|
||||
break
|
||||
}
|
||||
}
|
||||
send(["kind": "log", "message": "tool routing failed (\(error.localizedDescription)); offering tools in order"])
|
||||
return .tools(specs)
|
||||
}
|
||||
}
|
||||
|
||||
private func toolTokens(_ model: SystemLanguageModel, _ tools: [BuiltTool]) async -> Int {
|
||||
let definitions = tools.map {
|
||||
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||
}
|
||||
let instructions = Transcript.Entry.instructions(Transcript.Instructions(segments: [], toolDefinitions: definitions))
|
||||
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||
return await tokenCount(model, [instructions], extraCharacters: schemaCharacters + 150 * tools.count)
|
||||
}
|
||||
|
||||
private func lastPromptText(_ history: [[String: Any]]) -> String? {
|
||||
for item in history.reversed() where item["type"] as? String == "prompt" {
|
||||
if let text = item["text"] as? String, !text.isEmpty { return text }
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
/// Tool names from the newest tool calls back, without repeats.
|
||||
private func recentToolNames(_ history: [[String: Any]]) -> [String] {
|
||||
var names: [String] = []
|
||||
for item in history.reversed() where item["type"] as? String == "toolCalls" {
|
||||
for call in (item["calls"] as? [[String: Any]]) ?? [] {
|
||||
if let name = call["name"] as? String, !names.contains(name) { names.append(name) }
|
||||
}
|
||||
}
|
||||
return names
|
||||
}
|
||||
|
||||
// MARK: - Context fitting
|
||||
|
||||
private struct BuiltTool {
|
||||
let name: String
|
||||
let description: String
|
||||
let schema: GenerationSchema
|
||||
}
|
||||
|
||||
private struct Fitted {
|
||||
let tools: [BuiltTool]
|
||||
let entries: [Transcript.Entry]
|
||||
}
|
||||
|
||||
/// Instructions + tool definitions + history + prompt must leave the reply
|
||||
/// budget free. In order, until it fits: shorten tool descriptions, drop the
|
||||
/// oldest exchanges, then drop the last-registered tools.
|
||||
private func fit(model: SystemLanguageModel, system: String, tools specs: [ToolSpec],
|
||||
history: [[String: Any]], prompt: String, maxTokens: Int) async -> Fitted {
|
||||
let contextSize = model.contextSize
|
||||
let reserve = min(maxTokens, contextSize / 3) + LanguageModelEngine.contextMargin
|
||||
var tools = buildTools(specs, compact: false)
|
||||
var past = transcriptEntries(history)
|
||||
let promptEntry = Transcript.Entry.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: prompt))]))
|
||||
|
||||
func entries() -> [Transcript.Entry] {
|
||||
let definitions = tools.map {
|
||||
Transcript.ToolDefinition(name: $0.name, description: $0.description, parameters: $0.schema)
|
||||
}
|
||||
let instructions = Transcript.Instructions(segments: [.text(Transcript.TextSegment(content: system))],
|
||||
toolDefinitions: definitions)
|
||||
return [.instructions(instructions)] + past
|
||||
}
|
||||
func overBudget() async -> Bool {
|
||||
// The tokenizer can't see tool schemas' size from a ToolDefinition; the fallback estimate adds them.
|
||||
let schemaCharacters = tools.reduce(0) { total, tool in total + tool.schema.debugDescription.count }
|
||||
let used = await tokenCount(model, entries() + [promptEntry], extraCharacters: schemaCharacters)
|
||||
return used + reserve > contextSize
|
||||
}
|
||||
|
||||
if !tools.isEmpty, await overBudget() {
|
||||
tools = buildTools(specs, compact: true)
|
||||
send(["kind": "log", "message": "shortened tool descriptions to fit the context window"])
|
||||
}
|
||||
var droppedEntries = 0
|
||||
while !past.isEmpty, await overBudget() {
|
||||
// Drop the oldest exchange: through to the next user prompt.
|
||||
past.removeFirst()
|
||||
droppedEntries += 1
|
||||
while let first = past.first, !isPrompt(first) {
|
||||
past.removeFirst()
|
||||
droppedEntries += 1
|
||||
}
|
||||
}
|
||||
if droppedEntries > 0 {
|
||||
send(["kind": "log", "message": "dropped \(droppedEntries) history entries to fit the context window"])
|
||||
}
|
||||
var droppedTools: [String] = []
|
||||
while !tools.isEmpty, await overBudget() {
|
||||
droppedTools.append(tools.removeLast().name)
|
||||
}
|
||||
if !droppedTools.isEmpty {
|
||||
send(["kind": "log", "message": "dropped tools to fit the context window: \(droppedTools.joined(separator: ", "))"])
|
||||
}
|
||||
return Fitted(tools: tools, entries: entries())
|
||||
}
|
||||
|
||||
private func tokenCount(_ model: SystemLanguageModel, _ entries: [Transcript.Entry], extraCharacters: Int) async -> Int {
|
||||
if #available(iOS 26.4, macOS 26.4, *) {
|
||||
if let count = try? await model.tokenCount(for: entries) { return count }
|
||||
}
|
||||
// Before iOS 26.4 there's no tokenizer API: estimate conservatively.
|
||||
var characters = extraCharacters
|
||||
for entry in entries { characters += estimateCharacters(entry) + 16 }
|
||||
return characters / 3
|
||||
}
|
||||
|
||||
private func estimateCharacters(_ entry: Transcript.Entry) -> Int {
|
||||
func segments(_ list: [Transcript.Segment]) -> Int {
|
||||
list.reduce(0) { total, segment in
|
||||
switch segment {
|
||||
case .text(let text): return total + text.content.count
|
||||
case .structure(let structure): return total + structure.content.jsonString.count
|
||||
@unknown default: return total
|
||||
}
|
||||
}
|
||||
}
|
||||
switch entry {
|
||||
case .instructions(let instructions):
|
||||
var total = segments(instructions.segments)
|
||||
for definition in instructions.toolDefinitions {
|
||||
total += definition.name.count + definition.description.count
|
||||
}
|
||||
return total
|
||||
case .prompt(let prompt): return segments(prompt.segments)
|
||||
case .response(let response): return segments(response.segments)
|
||||
case .toolCalls(let calls): return calls.reduce(0) { $0 + $1.toolName.count + $1.arguments.jsonString.count }
|
||||
case .toolOutput(let output): return segments(output.segments)
|
||||
@unknown default: return 0
|
||||
}
|
||||
}
|
||||
|
||||
private func isPrompt(_ entry: Transcript.Entry) -> Bool {
|
||||
if case .prompt = entry { return true }
|
||||
return false
|
||||
}
|
||||
|
||||
// MARK: - Request conversion
|
||||
|
||||
private func transcriptEntries(_ history: [[String: Any]]) -> [Transcript.Entry] {
|
||||
var entries: [Transcript.Entry] = []
|
||||
for item in history {
|
||||
let text = (item["text"] as? String) ?? ""
|
||||
switch item["type"] as? String {
|
||||
case "prompt":
|
||||
entries.append(.prompt(Transcript.Prompt(segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
case "response":
|
||||
entries.append(.response(Transcript.Response(assetIDs: [], segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
case "toolCalls":
|
||||
let calls: [Transcript.ToolCall] = ((item["calls"] as? [[String: Any]]) ?? []).compactMap { call in
|
||||
guard let id = call["id"] as? String, let name = call["name"] as? String else { return nil }
|
||||
let arguments = (try? GeneratedContent(json: (call["arguments"] as? String) ?? "{}"))
|
||||
?? GeneratedContent(properties: [:])
|
||||
return Transcript.ToolCall(id: id, toolName: name, arguments: arguments)
|
||||
}
|
||||
if !calls.isEmpty { entries.append(.toolCalls(Transcript.ToolCalls(calls))) }
|
||||
case "toolOutput":
|
||||
guard let id = item["id"] as? String else { continue }
|
||||
entries.append(.toolOutput(Transcript.ToolOutput(
|
||||
id: id, toolName: (item["name"] as? String) ?? "tool",
|
||||
segments: [.text(Transcript.TextSegment(content: text))])))
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return entries
|
||||
}
|
||||
|
||||
private func buildTools(_ specs: [ToolSpec], compact: Bool) -> [BuiltTool] {
|
||||
specs.map { spec in
|
||||
var builder = SchemaBuilder(compact: compact)
|
||||
let schema: GenerationSchema
|
||||
do {
|
||||
schema = try GenerationSchema(root: builder.object(spec.parameters, name: "\(spec.name)_arguments"),
|
||||
dependencies: [])
|
||||
} catch {
|
||||
send(["kind": "log", "message": "tool \(spec.name): unusable parameter schema (\(error)); offering it without parameters"])
|
||||
schema = (try? GenerationSchema(root: DynamicGenerationSchema(name: "\(spec.name)_arguments", properties: []),
|
||||
dependencies: []))
|
||||
?? GenerationSchema(type: GeneratedContent.self, properties: [])
|
||||
}
|
||||
let description = compact ? SchemaBuilder.shorten(spec.description, sentences: 1, limit: 160) : spec.description
|
||||
return BuiltTool(name: spec.name, description: description, schema: schema)
|
||||
}
|
||||
}
|
||||
|
||||
// MARK: - Errors
|
||||
|
||||
private func describe(_ error: Error) -> [String: Any] {
|
||||
if let error = error as? LanguageModelSession.GenerationError {
|
||||
switch error {
|
||||
case .exceededContextWindowSize:
|
||||
return failure("context", "That conversation is too long for the on-device model. Start a new conversation and try again.")
|
||||
case .assetsUnavailable:
|
||||
return failure("unavailable", "The Apple Intelligence model isn't available right now. Check that Apple Intelligence is on and finished downloading.")
|
||||
case .guardrailViolation, .refusal:
|
||||
return failure("refusal", "The on-device model declined this request.")
|
||||
case .unsupportedLanguageOrLocale:
|
||||
return failure("language", "The on-device model doesn't support this language.")
|
||||
case .rateLimited:
|
||||
return failure("rateLimited", "iOS is limiting the on-device model while Faceclaw is in the background. Try again in a moment, or with the phone unlocked.")
|
||||
case .concurrentRequests:
|
||||
return failure("busy", "The on-device model is busy with another request. Try again in a moment.")
|
||||
case .decodingFailure, .unsupportedGuide:
|
||||
return failure("other", "The on-device model produced an unreadable reply: \(error.localizedDescription)")
|
||||
@unknown default:
|
||||
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
if let error = error as? LanguageModelSession.ToolCallError {
|
||||
return failure("other", "On-device model tool call failed: \(error.underlyingError.localizedDescription)")
|
||||
}
|
||||
return failure("other", "On-device model failed: \(error.localizedDescription)")
|
||||
}
|
||||
|
||||
private func failure(_ code: String, _ message: String) -> [String: Any] {
|
||||
["kind": "error", "code": code, "message": message]
|
||||
}
|
||||
}
|
||||
|
||||
/// JSON Schema (the subset our tool definitions use: objects of string, number,
|
||||
/// integer, boolean, enum and array properties) to a DynamicGenerationSchema, so
|
||||
/// the framework constrains tool arguments to valid values while decoding.
|
||||
/// Anything it doesn't understand becomes a string rather than failing.
|
||||
@available(iOS 26.0, *)
|
||||
private struct SchemaBuilder {
|
||||
let compact: Bool
|
||||
private var names = Set<String>()
|
||||
|
||||
init(compact: Bool) { self.compact = compact }
|
||||
|
||||
mutating func object(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||
let properties = (json["properties"] as? [String: Any]) ?? [:]
|
||||
let required = Set((json["required"] as? [String]) ?? [])
|
||||
// JSON objects lose declaration order; keep required properties first, then by name.
|
||||
let keys = properties.keys.sorted { a, b in
|
||||
let ra = required.contains(a), rb = required.contains(b)
|
||||
return ra != rb ? ra : a < b
|
||||
}
|
||||
let members: [DynamicGenerationSchema.Property] = keys.compactMap { key in
|
||||
guard let property = properties[key] as? [String: Any] else { return nil }
|
||||
return DynamicGenerationSchema.Property(
|
||||
name: key,
|
||||
description: describe(property),
|
||||
schema: value(property, name: "\(name)_\(key)"),
|
||||
isOptional: !required.contains(key))
|
||||
}
|
||||
return DynamicGenerationSchema(name: unique(name), description: nil, properties: members)
|
||||
}
|
||||
|
||||
private mutating func value(_ json: [String: Any], name: String) -> DynamicGenerationSchema {
|
||||
if let choices = json["enum"] as? [Any], !choices.isEmpty {
|
||||
let strings = choices.compactMap { $0 as? String }
|
||||
if strings.count == choices.count {
|
||||
// A guide renders as one compact JSON Schema enum; an anyOf schema lists
|
||||
// every choice as its own subschema, at over twice the tokens.
|
||||
return DynamicGenerationSchema(type: String.self, guides: [.anyOf(strings)])
|
||||
}
|
||||
}
|
||||
switch schemaType(json) {
|
||||
case "integer":
|
||||
var guides: [GenerationGuide<Int>] = []
|
||||
if let minimum = (json["minimum"] as? NSNumber)?.intValue { guides.append(.minimum(minimum)) }
|
||||
if let maximum = (json["maximum"] as? NSNumber)?.intValue { guides.append(.maximum(maximum)) }
|
||||
return DynamicGenerationSchema(type: Int.self, guides: guides)
|
||||
case "number":
|
||||
var guides: [GenerationGuide<Double>] = []
|
||||
if let minimum = (json["minimum"] as? NSNumber)?.doubleValue { guides.append(.minimum(minimum)) }
|
||||
if let maximum = (json["maximum"] as? NSNumber)?.doubleValue { guides.append(.maximum(maximum)) }
|
||||
return DynamicGenerationSchema(type: Double.self, guides: guides)
|
||||
case "boolean":
|
||||
return DynamicGenerationSchema(type: Bool.self)
|
||||
case "array":
|
||||
let items = (json["items"] as? [String: Any]) ?? ["type": "string"]
|
||||
return DynamicGenerationSchema(arrayOf: value(items, name: "\(name)_item"),
|
||||
minimumElements: (json["minItems"] as? NSNumber)?.intValue,
|
||||
maximumElements: (json["maxItems"] as? NSNumber)?.intValue)
|
||||
case "object":
|
||||
return object(json, name: name)
|
||||
default:
|
||||
return DynamicGenerationSchema(type: String.self)
|
||||
}
|
||||
}
|
||||
|
||||
private func schemaType(_ json: [String: Any]) -> String {
|
||||
if let type = json["type"] as? String { return type }
|
||||
if let types = json["type"] as? [String], let type = types.first(where: { $0 != "null" }) { return type }
|
||||
return json["properties"] != nil ? "object" : "string"
|
||||
}
|
||||
|
||||
private func describe(_ json: [String: Any]) -> String? {
|
||||
guard let text = json["description"] as? String, !text.isEmpty else { return nil }
|
||||
return compact ? SchemaBuilder.shorten(text, sentences: 1, limit: 80) : text
|
||||
}
|
||||
|
||||
/// Schema names must be unique within one GenerationSchema.
|
||||
private mutating func unique(_ base: String) -> String {
|
||||
var name = base
|
||||
var suffix = 2
|
||||
while names.contains(name) {
|
||||
name = "\(base)_\(suffix)"
|
||||
suffix += 1
|
||||
}
|
||||
names.insert(name)
|
||||
return name
|
||||
}
|
||||
|
||||
/// The first `sentences` sentences, cut to `limit` characters.
|
||||
static func shorten(_ text: String, sentences: Int, limit: Int) -> String {
|
||||
var result = ""
|
||||
var count = 0
|
||||
var remainder = Substring(text)
|
||||
while count < sentences, !remainder.isEmpty {
|
||||
if let end = remainder.range(of: ". ") {
|
||||
result += remainder[..<end.lowerBound] + "."
|
||||
remainder = remainder[end.upperBound...]
|
||||
if count + 1 < sentences { result += " " }
|
||||
} else {
|
||||
result += remainder
|
||||
remainder = ""
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
if result.count > limit { result = String(result.prefix(limit - 1)) + "…" }
|
||||
return result
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
@@ -0,0 +1,21 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* OAuth sign-in in the system browser sheet (ASWebAuthenticationSession, which shares
|
||||
* Safari's cookies, so a provider the user is already signed in to needs only a consent
|
||||
* tap) for providers that redirect to a loopback address: a FaceclawLoopbackRedirect
|
||||
* catches the redirect, then the sheet is closed. Presenting needs Faceclaw in the
|
||||
* foreground, so a sign-in started from the glasses waits until the user opens the app.
|
||||
* One sign-in at a time; starting another cancels the first. Call from the main thread.
|
||||
*/
|
||||
@interface FaceclawWebSignIn : NSObject
|
||||
+ (instancetype)shared;
|
||||
/**
|
||||
* completion(redirectUrl, nil) once the provider redirects to http://127.0.0.1:<port><path>;
|
||||
* (nil, "cancelled") when the user closes the sheet or cancel is called; (nil, message)
|
||||
* when the sheet or the receiver cannot start. Called once, on the main queue.
|
||||
*/
|
||||
- (void)signIn:(NSString *)authorizeUrl port:(NSInteger)port path:(NSString *)path
|
||||
completion:(void (^)(NSString *redirectUrl, NSString *error))completion;
|
||||
- (void)cancel;
|
||||
@end
|
||||
@@ -0,0 +1,110 @@
|
||||
#import "FaceclawWebSignIn.h"
|
||||
#import "FaceclawLoopbackRedirect.h"
|
||||
#import <AuthenticationServices/AuthenticationServices.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
|
||||
/** Never reached: the loopback receiver ends the session before any custom-scheme redirect. */
|
||||
static NSString *const kUnusedCallbackScheme = @"faceclaw-signin";
|
||||
|
||||
@interface FaceclawWebSignIn () <ASWebAuthenticationPresentationContextProviding>
|
||||
@property(nonatomic, strong) FaceclawLoopbackRedirect *receiver;
|
||||
@property(nonatomic, strong) ASWebAuthenticationSession *session;
|
||||
@property(nonatomic, copy) void (^completion)(NSString *redirectUrl, NSString *error);
|
||||
/** The sign-in page waiting for Faceclaw to come to the foreground. */
|
||||
@property(nonatomic, copy) NSURL *pendingUrl;
|
||||
@property(nonatomic, strong) id activeObserver;
|
||||
@end
|
||||
|
||||
@implementation FaceclawWebSignIn
|
||||
+ (instancetype)shared {
|
||||
static FaceclawWebSignIn *instance;
|
||||
static dispatch_once_t once;
|
||||
dispatch_once(&once, ^{ instance = [FaceclawWebSignIn new]; });
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (void)signIn:(NSString *)authorizeUrl port:(NSInteger)port path:(NSString *)path
|
||||
completion:(void (^)(NSString *, NSString *))completion {
|
||||
[self finishWithUrl:nil error:@"cancelled"];
|
||||
NSURL *url = [NSURL URLWithString:authorizeUrl];
|
||||
if (!url || ![url.scheme isEqualToString:@"https"]) {
|
||||
if (completion) completion(nil, @"Invalid sign-in address");
|
||||
return;
|
||||
}
|
||||
FaceclawLoopbackRedirect *receiver = [FaceclawLoopbackRedirect new];
|
||||
NSString *error = [receiver startOnPort:port path:path];
|
||||
if (error) {
|
||||
if (completion) completion(nil, error);
|
||||
return;
|
||||
}
|
||||
__weak typeof(self) weakSelf = self;
|
||||
receiver.redirectHandler = ^(NSString *redirectUrl) {
|
||||
[weakSelf finishWithUrl:redirectUrl error:nil];
|
||||
};
|
||||
self.receiver = receiver;
|
||||
self.completion = completion;
|
||||
if (UIApplication.sharedApplication.applicationState == UIApplicationStateActive) {
|
||||
[self present:url];
|
||||
return;
|
||||
}
|
||||
self.pendingUrl = url;
|
||||
self.activeObserver = [NSNotificationCenter.defaultCenter addObserverForName:UIApplicationDidBecomeActiveNotification
|
||||
object:nil queue:NSOperationQueue.mainQueue usingBlock:^(NSNotification *note) {
|
||||
FaceclawWebSignIn *strongSelf = weakSelf;
|
||||
NSURL *pending = strongSelf.pendingUrl;
|
||||
if (!pending) return;
|
||||
[strongSelf removeActiveObserver];
|
||||
[strongSelf present:pending];
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)cancel {
|
||||
[self finishWithUrl:nil error:@"cancelled"];
|
||||
}
|
||||
|
||||
- (void)present:(NSURL *)url {
|
||||
__weak typeof(self) weakSelf = self;
|
||||
ASWebAuthenticationSession *session = [[ASWebAuthenticationSession alloc] initWithURL:url
|
||||
callbackURLScheme:kUnusedCallbackScheme completionHandler:^(NSURL *callbackURL, NSError *error) {
|
||||
// The sheet closed: the user dismissed it, or finishing already cancelled it.
|
||||
BOOL dismissed = !error || ([error.domain isEqualToString:ASWebAuthenticationSessionErrorDomain]
|
||||
&& error.code == ASWebAuthenticationSessionErrorCodeCanceledLogin);
|
||||
[weakSelf finishWithUrl:callbackURL.absoluteString
|
||||
error:callbackURL ? nil : dismissed ? @"cancelled" : error.localizedDescription];
|
||||
}];
|
||||
session.presentationContextProvider = self;
|
||||
session.prefersEphemeralWebBrowserSession = NO;
|
||||
self.session = session;
|
||||
if (![session start]) [self finishWithUrl:nil error:@"Could not open the sign-in page"];
|
||||
}
|
||||
|
||||
/** Report the outcome once; tear down the receiver and close the sheet. */
|
||||
- (void)finishWithUrl:(NSString *)url error:(NSString *)error {
|
||||
void (^completion)(NSString *, NSString *) = self.completion;
|
||||
ASWebAuthenticationSession *session = self.session;
|
||||
self.completion = nil;
|
||||
self.session = nil;
|
||||
self.pendingUrl = nil;
|
||||
[self removeActiveObserver];
|
||||
[self.receiver stop];
|
||||
self.receiver = nil;
|
||||
// Re-enters through the session's completion handler, which then finds nothing to report.
|
||||
[session cancel];
|
||||
if (completion) completion(url, url ? nil : error);
|
||||
}
|
||||
|
||||
- (void)removeActiveObserver {
|
||||
if (self.activeObserver) [NSNotificationCenter.defaultCenter removeObserver:self.activeObserver];
|
||||
self.activeObserver = nil;
|
||||
}
|
||||
|
||||
- (ASPresentationAnchor)presentationAnchorForWebAuthenticationSession:(ASWebAuthenticationSession *)session {
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
if (![scene isKindOfClass:UIWindowScene.class]) continue;
|
||||
NSArray<UIWindow *> *windows = ((UIWindowScene *)scene).windows;
|
||||
for (UIWindow *window in windows) if (window.isKeyWindow) return window;
|
||||
if (windows.firstObject) return windows.firstObject;
|
||||
}
|
||||
return [UIWindow new];
|
||||
}
|
||||
@end
|
||||
@@ -11,5 +11,8 @@ module FaceclawGraphics {
|
||||
header "FaceclawEvenHubWebView.h"
|
||||
header "FaceclawCrypto.h"
|
||||
header "FaceclawQrScanner.h"
|
||||
header "FaceclawLoopbackRedirect.h"
|
||||
header "FaceclawWebSignIn.h"
|
||||
header "FaceclawResourceUsage.h"
|
||||
export *
|
||||
}
|
||||
@@ -1,17 +1,36 @@
|
||||
# Changelog
|
||||
|
||||
## 0.8.5 - Future
|
||||
## 0.8.6 - Future
|
||||
|
||||
* Add a distance unit option
|
||||
* Make BLE benchmark work on iOS
|
||||
* Ensure connection is re-set to 2M on reconnect
|
||||
* Add support for embedded icons and custom layouts in Android notifications
|
||||
* Port the Developer app's resource-usage monitor and various test pages to iOS
|
||||
* iOS: Support Apple Intelligence as an on-device model for voice assistant/etc
|
||||
* iOS: Notifications are selectable via the task switcher
|
||||
* Android: Optionally sign in to Spotify's Web API for library browsing and other features
|
||||
* Mark stock firmware 2.4.0 as validated for installing custom firmware over
|
||||
|
||||
## 0.8.5 - 2026-10-25
|
||||
|
||||
### Features
|
||||
|
||||
* Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover
|
||||
* Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view
|
||||
* Add an Apple Music integration on iOS
|
||||
* Add a Spotify integration on iOS by voice (needs your own Spotify developer client ID and Spotify Premium)
|
||||
* Add a Translate app
|
||||
|
||||
### Bug fixes
|
||||
|
||||
* Inverting Android notification-icons to indicate selectionis now gamma-correct
|
||||
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
|
||||
* Fix missing R1 ring battery indicator on iOS
|
||||
* Fix a bug on iOS that would cause corrupted installs of EvenHub apps
|
||||
* EvenHub apps can use the glasses microphone on iOS (such as Chess's voice moves, which said "Mic unavailable")
|
||||
* EvenHub apps on iOS should now be able to use the mic
|
||||
* Fix a bug that caused fade-in on wake to be shown at the wrong depth on iOS
|
||||
|
||||
## 0.8.4 - 2026-10-17
|
||||
|
||||
|
||||
@@ -116,12 +116,30 @@ provider; Faceclaw does not manage accounts or process billing information.
|
||||
|
||||
## AI Assistant Transcripts
|
||||
|
||||
Faceclaw has a builtin AI voice assistant. Using the voice assistant requires
|
||||
providing an API key for a model provider such as Anthropic or OpenAI. Prompts
|
||||
and responses are subject to that model provider's privacy policy:
|
||||
Faceclaw has a builtin AI voice assistant. Using a cloud model requires
|
||||
providing an API key for a model provider such as Anthropic, OpenAI, or Venice,
|
||||
or configuring an OpenAI-compatible endpoint such as a private Open WebUI
|
||||
server. Prompts and responses are subject to the selected provider or server
|
||||
operator's privacy policy:
|
||||
|
||||
OpenAI: https://openai.com/policies/row-privacy-policy/
|
||||
Anthropic: https://www.anthropic.com/legal/privacy
|
||||
Venice: https://venice.ai/privacy
|
||||
|
||||
With an OpenAI-compatible provider, the configured server receives the
|
||||
assistant system prompt, conversation history, available tool definitions, and
|
||||
tool results needed to answer. Its URL, model ID, and optional API key are
|
||||
stored on the phone. Faceclaw does not operate that server and cannot make
|
||||
representations about its logging or retention; for a private Open WebUI
|
||||
server, the server's operator controls those policies. If you choose Discover
|
||||
models, Faceclaw also requests that server's model catalog using the same
|
||||
configured URL and optional API key.
|
||||
|
||||
If you enable **Venice web search** for the official Venice API URL, supported
|
||||
models may generate web-search queries from your assistant conversation. Those
|
||||
queries and their results are processed by Venice's search service and are
|
||||
subject to Venice's privacy policy and any applicable charges. This setting is
|
||||
off by default and is never sent to Open WebUI or other compatible providers.
|
||||
|
||||
AI conversation histories and memory files may be stored on your phone.
|
||||
Faceclaw does not transmit voice-assistant prompts or responses to any party
|
||||
@@ -153,6 +171,20 @@ Apple's on iOS) for the name of the place.
|
||||
Open-Meteo: https://open-meteo.com/en/terms
|
||||
|
||||
|
||||
## Spotify
|
||||
|
||||
On iOS, the Music app, Glanceboard and the voice assistant can show and
|
||||
control Spotify through Spotify's Web API, once the user has created their own
|
||||
Spotify developer app and signed in. Signing in happens on Spotify's website in
|
||||
the system's sign-in sheet; Faceclaw never sees the password, and stores the
|
||||
resulting sign-in tokens on the phone. While Spotify is shown or used,
|
||||
Faceclaw requests the user's playback state, queue, library and search results
|
||||
from Spotify and sends it playback commands, directly from the phone. This is
|
||||
subject to Spotify's privacy policy.
|
||||
|
||||
Spotify: https://www.spotify.com/legal/privacy-policy/
|
||||
|
||||
|
||||
## EvenHub
|
||||
|
||||
The EvenHub app store is subject to Even Realities' privacy policy:
|
||||
|
||||
@@ -32,7 +32,8 @@ walk you through pairing with the glasses and installing custom firmware.
|
||||
|
||||
For the NativeScript iOS port, see [iOS development](notes/ios-development.md).
|
||||
Use `scripts/pull_config_ios.sh` and `scripts/push_config_ios.sh` to transfer iPhone
|
||||
settings; see [iOS config transfer](scripts/ios-config.md) for options and Android XML support.
|
||||
settings. Android and iOS share one JSON config format, so a config pulled from either
|
||||
platform can be pushed to the other; see [config transfer](scripts/config-transfer.md).
|
||||
The instructions below build the Android app.
|
||||
|
||||
If you want to customize faceclaw, the best way to do it is to download the
|
||||
@@ -54,7 +55,7 @@ port into your computer, authorize access on the phone, and run
|
||||
Note that if you have installed a signed release build, you will have to
|
||||
uninstall it before installing a customized version. Use scripts/pull_config.sh
|
||||
to export a copy of your configuration first, and scripts/push_config.sh to
|
||||
import it into the new version.
|
||||
import it into the new version (see [config transfer](scripts/config-transfer.md)).
|
||||
|
||||
|
||||
## Features
|
||||
@@ -64,8 +65,10 @@ import it into the new version.
|
||||
* **A voice assistant** that wakes up when you say "Hey Even", transcribes
|
||||
text with an on-device model, OpenAI Whisper, ElevenLabs, or Soniox API,
|
||||
and responds to queries and commands using an onboard model (Qwen3 4B,
|
||||
slow) or with an Anthropic or OpenAI model (requires an API key), or using
|
||||
your own long-running OpenClaw agent.
|
||||
slow) or with an Anthropic, OpenAI, or OpenAI-compatible Chat Completions
|
||||
model (including Venice and private Open WebUI servers; model ID and
|
||||
usually an API key required), with optional Venice web search for supported
|
||||
models, or using your own long-running OpenClaw agent.
|
||||
* **Multitasking**, with an app-switcher sidebar and app launcher.
|
||||
* **Glanceboard**, a dashboard with widgets that you can quickly view with a
|
||||
* **Mostly-compatible with EvenHub.** Install and runapps from the Even app
|
||||
@@ -95,6 +98,51 @@ import it into the new version.
|
||||
the glasses, and you can use the phone touchscreen as an input device..
|
||||
<!-- END GENERATED: features -->
|
||||
|
||||
## OpenAI-compatible assistant providers (Venice / Open WebUI)
|
||||
|
||||
The assistant can call an OpenAI-compatible **Chat Completions** endpoint directly
|
||||
from the phone. This uses the same conversation history and glasses tool loop as
|
||||
the built-in cloud providers; it does not require different glasses firmware.
|
||||
|
||||
In **Settings > Assistant**:
|
||||
|
||||
1. **Assistant model** — choose **OpenAI-compatible**. Auto never selects it.
|
||||
The settings in steps 2–4 appear only while it is selected.
|
||||
2. **Compatible API URL** — the explicit API root.
|
||||
3. **Compatible API key** — optional; when set, it is sent only to the configured
|
||||
compatible endpoint as a Bearer token, not to OpenAI's Whisper service.
|
||||
4. **Compatible model ID** — the exact model identifier accepted by the server.
|
||||
Or choose **Discover models** and pick one from the server catalog. The
|
||||
picker shows tool capability when advertised and hides offline models by
|
||||
default. Refresh never changes the saved model by itself.
|
||||
|
||||
Provider examples:
|
||||
|
||||
| Provider | Compatible API URL | Notes |
|
||||
| --- | --- | --- |
|
||||
| Venice | `https://api.venice.ai/api/v1` | Use a current Venice model ID. Choose one with function-calling support if you want glasses tools. |
|
||||
| Open WebUI | `https://your-openwebui-host/api` | This produces `/api/chat/completions`. Use the model ID exposed by your server and an Open WebUI API key if required. |
|
||||
|
||||
You may paste the full `/chat/completions` endpoint instead; Faceclaw removes
|
||||
that suffix. It otherwise preserves the path you enter and does not guess `/v1`
|
||||
or `/api`. Discovery requests `{API root}/models` only when you choose it; there
|
||||
is no background discovery or automatic model selection. Manual model entry
|
||||
remains available for servers without a compatible model-list endpoint.
|
||||
|
||||
When **OpenAI-compatible** is the assistant model and its URL is the official
|
||||
Venice URL, **Settings > Assistant** also shows **Venice web search**. It is off
|
||||
by default and hidden otherwise. When enabled, supported Venice models may
|
||||
search the web for current information; the search model must support the
|
||||
feature. Open WebUI and other compatible servers never receive Venice-specific
|
||||
parameters, including after an endpoint switch.
|
||||
|
||||
The same provider can supply Calculator's rewrite fallback and Translate.
|
||||
Actual tool calling depends on the selected model and server. Use HTTPS where
|
||||
possible; mobile cleartext policies may block plain HTTP. For a private server,
|
||||
make it reachable from the phone (for example through your tailnet) without
|
||||
exposing it publicly. Exported settings contain API keys and must be treated as
|
||||
secrets.
|
||||
|
||||
## Watch remote (Wear OS)
|
||||
|
||||
The `wear/` directory holds a companion app for Wear OS 3+ watches that turns
|
||||
|
||||
+54
@@ -154,6 +154,60 @@ Button.-primary {
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
/* Keyboard-input panel: tighter than the setting editor, with the text
|
||||
* box's edges in line with the title and the one-strip buttons (the theme's
|
||||
* 16dp TextView side margins doubled the inset). */
|
||||
.system-name-modal.keyboard-input-panel {
|
||||
margin: 8;
|
||||
padding: 8 16 12 16;
|
||||
}
|
||||
|
||||
.keyboard-input-panel TextView {
|
||||
margin: 4 0 0 0;
|
||||
padding: 6;
|
||||
}
|
||||
|
||||
/* Glasses-only: the field stays focused for the IME but is an invisible
|
||||
* sliver; the hint line takes its place. */
|
||||
.keyboard-input-panel TextView.keyboard-input-field-hidden {
|
||||
opacity: 0;
|
||||
height: 1;
|
||||
min-height: 0;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.system-name-modal Label.keyboard-input-hidden-hint,
|
||||
.ns-dark .system-name-modal Label.keyboard-input-hidden-hint {
|
||||
color: #666666;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.keyboard-input-buttons {
|
||||
margin-top: 8;
|
||||
}
|
||||
|
||||
.keyboard-input-panel Button,
|
||||
.keyboard-input-panel Button.-primary {
|
||||
margin: 0;
|
||||
font-size: 16;
|
||||
}
|
||||
|
||||
.keyboard-input-panel Button.keyboard-input-gap {
|
||||
margin-left: 8;
|
||||
}
|
||||
|
||||
/* Flat icon buttons in the panel's title row (the modal stays light in dark
|
||||
* mode, so one tint). */
|
||||
.panel-icon-button {
|
||||
padding: 6 8;
|
||||
}
|
||||
|
||||
.panel-icon {
|
||||
height: 20;
|
||||
tint-color: #333333;
|
||||
}
|
||||
|
||||
/* Modal behind the action bar's warning triangle: the problem-status warnings
|
||||
* that used to sit inline on the main page, each with its fix-it button.
|
||||
* The width is set from the view model (near-full-width on phones, capped on
|
||||
|
||||
@@ -56,6 +56,9 @@ Page.main-page {
|
||||
color: #222222;
|
||||
padding: 16;
|
||||
}
|
||||
/* The modal stays light in dark mode; outgun the theme's `.ns-dark .h2`
|
||||
* (white) so the title doesn't go white-on-light-gray. */
|
||||
.main-page .system-name-modal .h2 { color: #222222; }
|
||||
.main-page .system-name-modal TextView {
|
||||
margin: 0;
|
||||
padding: 4;
|
||||
@@ -67,3 +70,15 @@ Page.main-page {
|
||||
.main-landscape .system-name-modal { padding: 6 12; }
|
||||
.main-landscape .system-name-modal .h2 { visibility: collapse; }
|
||||
.main-landscape .system-name-modal Button { padding: 6 8; margin-top: 0; margin-bottom: 0; }
|
||||
|
||||
/* Keyboard-input panel: the one-strip buttons and title-row icons (see app.css). */
|
||||
.main-page .system-name-modal.keyboard-input-panel { padding: 8 16 12 16; }
|
||||
.main-landscape .system-name-modal.keyboard-input-panel { padding: 6 12; }
|
||||
.main-page .keyboard-input-panel TextView { margin: 4 0 0 0; padding: 6; }
|
||||
.main-page .keyboard-input-panel TextView.keyboard-input-field-hidden { opacity: 0; height: 1; min-height: 0; margin: 0; padding: 0; }
|
||||
.main-page .keyboard-input-hidden-hint { color: #666666; padding: 8 0; }
|
||||
.main-page .keyboard-input-buttons { margin-top: 8; }
|
||||
.main-page .keyboard-input-panel Button { margin: 0; }
|
||||
.main-page .keyboard-input-panel Button.keyboard-input-gap { margin-left: 8; }
|
||||
.main-page .panel-icon-button { padding: 6 8; }
|
||||
.main-page .panel-icon { height: 20; tint-color: #333333; }
|
||||
@@ -4,7 +4,7 @@ import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { ASSISTANT_MODEL_CHOICES, assistantModelLabel, resolveAssistantModel } from "../../assistant/models";
|
||||
import type { AssistantTranscriptEntry } from "../../assistant/session";
|
||||
import type { AssistantConversations, ReasoningLevel } from "../../assistant/conversations";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { anthropicApiKeySetting, assistantBackendSetting, assistantOpenAiCompatibleKeySetting, assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import { GESTURE_CLICK, GESTURE_DOUBLE_CLICK, GESTURE_LONG_PRESS, gestureHints, type InputEvent } from "../../ui/gestures";
|
||||
import type { Layer, LayerContext } from "../../ui/layers";
|
||||
import type { MenuItem } from "../../ui/menu";
|
||||
@@ -102,7 +102,7 @@ class AiChatLayer implements Layer {
|
||||
}
|
||||
case "long-press":
|
||||
if (!this.conversations.current().session?.isTurnActive()) {
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(global.isIOS ? "Set an OpenAI or Anthropic API key in Settings." : "Set an API key or download the on-phone model in Settings."); return; }
|
||||
if (!shell.isAssistantAvailable()) { shell.showAlert(shell.assistantSetupHint()); return; }
|
||||
void this.draft.start();
|
||||
}
|
||||
return;
|
||||
@@ -133,7 +133,12 @@ class AiChatLayer implements Layer {
|
||||
onSelect: (pickCtx) => { if (this.conversations.configure(model, record.reasoning)) pickCtx.stack.clearToBase(); },
|
||||
}))) },
|
||||
{ label: `Reasoning: ${record.reasoning}`, disabled: () => {
|
||||
const llm = resolveAssistantModel(record.model, { anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get() });
|
||||
const llm = resolveAssistantModel(record.model, {
|
||||
anthropic: anthropicApiKeySetting.get(), openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
return external || busy() || !llm?.effort;
|
||||
}, onSelect: (ctx) => submenu(ctx, "Reasoning", (["default", "low", "medium", "high"] as ReasoningLevel[]).map((reasoning) => ({
|
||||
label: `${reasoning === record.reasoning ? "✓ " : ""}${reasoning === "default" ? "Model default" : reasoning}`,
|
||||
|
||||
@@ -7,13 +7,17 @@
|
||||
* surfacing a transport error.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import {
|
||||
anthropicApiKeySetting,
|
||||
assistantModelSetting,
|
||||
assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting,
|
||||
assistantOpenAiCompatibleUrlSetting,
|
||||
openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { CALCULATOR_REWRITE_SYSTEM_PROMPT, buildCalculatorRewriteUserMessage } from "../../prompts";
|
||||
@@ -22,6 +26,9 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
const resolved = resolveAssistantModel(assistantModelSetting.get(), {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
if (!resolved) return Promise.resolve(null);
|
||||
|
||||
@@ -40,8 +47,11 @@ export function rewriteSpokenProblem(spoken: string): Promise<string | null> {
|
||||
case "openai":
|
||||
streamOpenAiResponse(options);
|
||||
break;
|
||||
case "openai-compatible":
|
||||
streamOpenAiCompatibleChat({ ...options, baseUrl: resolved.baseUrl ?? "" });
|
||||
break;
|
||||
case "local":
|
||||
streamLocalQwen(options);
|
||||
streamLocalModel(options);
|
||||
break;
|
||||
default:
|
||||
streamAnthropicMessage(options);
|
||||
|
||||
@@ -14,6 +14,9 @@ import {
|
||||
const MESSAGE_SIZES = [250, 500, 1000, 2000, 3800];
|
||||
const WINDOW_SIZES = [1, 2, 3, 4, 6];
|
||||
const LINK_MODES = ["Current", "Request HIGH", "Request 2M", "HIGH + 2M"];
|
||||
// CoreBluetooth has no connection-priority or PHY request API (iOS's
|
||||
// prepareBenchmarkLink is a no-op), so iOS always runs on the current link.
|
||||
const LINK_REQUESTS_SUPPORTED = !global.isIOS;
|
||||
const DURATION_MS = 15_000;
|
||||
// After painting the "running" screen, wait for that frame to reach the
|
||||
// glasses before the benchmark blocks normal image sends — otherwise the
|
||||
@@ -40,7 +43,7 @@ type ConfigRow = "size" | "window" | "link" | "start";
|
||||
export class BandwidthBenchmarkLayer implements Layer {
|
||||
private phase: Phase = "config";
|
||||
private readonly menu = new Menu<ConfigRow>({
|
||||
items: ["size", "window", "link", "start"],
|
||||
items: LINK_REQUESTS_SUPPORTED ? ["size", "window", "link", "start"] : ["size", "window", "start"],
|
||||
wrap: false,
|
||||
rowGap: 1,
|
||||
highlight: { radius: 4 },
|
||||
@@ -134,7 +137,8 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
font,
|
||||
LIST_X,
|
||||
HEADER_HEIGHT + 30,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}, ${LINK_MODES[this.linkModeIndex]}`,
|
||||
`${MESSAGE_SIZES[this.sizeIndex]} B, window ${WINDOW_SIZES[this.windowIndex]}`
|
||||
+ (LINK_REQUESTS_SUPPORTED ? `, ${LINK_MODES[this.linkModeIndex]}` : ""),
|
||||
150,
|
||||
);
|
||||
image.drawText(font, LIST_X, HEADER_HEIGHT + 52, "Display updates pause until the run finishes.", 110);
|
||||
@@ -161,8 +165,11 @@ export class BandwidthBenchmarkLayer implements Layer {
|
||||
`Throughput: ${(payloadRate / 1024).toFixed(2)} KB/s payload (${(wireRate / 1024).toFixed(2)} KB/s wire)`,
|
||||
`Messages: ${r.messagesAcked}/${r.messagesSent} acked, ${r.timeouts} timeouts`,
|
||||
`Elapsed: ${seconds.toFixed(1)} s` + (r.aborted ? " (aborted early)" : ""),
|
||||
`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}`,
|
||||
];
|
||||
if (LINK_REQUESTS_SUPPORTED) {
|
||||
const phy = r.leftPhy || r.rightPhy ? `; PHY L ${r.leftPhy ?? "?"}, R ${r.rightPhy ?? "?"}` : "";
|
||||
lines.push(`Link request: ${LINK_MODES[r.linkMode] ?? "Current"}${phy}`);
|
||||
}
|
||||
for (const line of lines) {
|
||||
image.drawText(font, LIST_X, y, line, 190);
|
||||
y += 20;
|
||||
|
||||
@@ -51,24 +51,18 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Buzzer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Glasses sound playback is not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new BuzzerDemoLayer());
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Accelerometer demo",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new AccelerometerDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
},
|
||||
{
|
||||
label: "Light sensor",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new LightSensorDemoLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
@@ -83,8 +77,6 @@ function debugTestsMenu(openInputEvents: OpenPage, openMicrophones: OpenPage): M
|
||||
},
|
||||
{
|
||||
label: "Microphones",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: openMicrophones,
|
||||
},
|
||||
{
|
||||
@@ -131,8 +123,6 @@ export function createDeveloperAppWindow(appContext: AppContext, options: InProc
|
||||
},
|
||||
{
|
||||
label: "Show resource usage",
|
||||
disabled: global.isIOS,
|
||||
description: global.isIOS ? "Not available on iOS yet." : undefined,
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.push(new ResourceUsageLayer(DEVELOPER_WINDOW_ID, ctx.actions.requestRender));
|
||||
},
|
||||
|
||||
@@ -4,25 +4,23 @@ import { clamp } from "../../util/numeric-util";
|
||||
import { GESTURE_DOUBLE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
|
||||
declare const com: any;
|
||||
import {
|
||||
MEMORY_SERIES,
|
||||
formatMb,
|
||||
sampleResourceUsage,
|
||||
type ResourceSample,
|
||||
} from "../../native/resource-usage";
|
||||
|
||||
const SAMPLE_INTERVAL_MS = 1_000;
|
||||
const HISTORY_MS = 60_000;
|
||||
const MAX_SAMPLES = Math.ceil(HISTORY_MS / SAMPLE_INTERVAL_MS) + 2;
|
||||
/** Line brightness per memory series, headline first. */
|
||||
const MEMORY_SHADES = [240, 155, 80];
|
||||
|
||||
type ResourceSample = {
|
||||
atMs: number;
|
||||
rssMb: number;
|
||||
nativeMb: number;
|
||||
javaMb: number;
|
||||
javaCommittedMb: number;
|
||||
cpuPercent: number;
|
||||
threads: number;
|
||||
};
|
||||
type ChartSample = ResourceSample & { cpuPercent: number };
|
||||
|
||||
type Series = {
|
||||
value: (sample: ResourceSample) => number;
|
||||
value: (sample: ChartSample) => number;
|
||||
shade: number;
|
||||
};
|
||||
|
||||
@@ -30,9 +28,10 @@ type Series = {
|
||||
* One-minute process monitor. Sampling deliberately continues while another
|
||||
* window is foreground so switching back reveals the resource cost of that
|
||||
* workload. Repaints remain visibility-gated to avoid creating hidden frames.
|
||||
* Which memory figures are charted is up to the platform (MEMORY_SERIES).
|
||||
*/
|
||||
export class ResourceUsageLayer implements Layer {
|
||||
private readonly samples: ResourceSample[] = [];
|
||||
private readonly samples: ChartSample[] = [];
|
||||
private timer: ReturnType<typeof setInterval> | null = null;
|
||||
private removed = false;
|
||||
private previousElapsedMs = 0;
|
||||
@@ -68,34 +67,34 @@ export class ResourceUsageLayer implements Layer {
|
||||
return image;
|
||||
}
|
||||
|
||||
const cpuText = `CPU ${formatPercent(latest.cpuPercent)} ${latest.threads} threads`;
|
||||
image.drawText(font, width - 16 - font.measureText(cpuText), 5, cpuText, 200);
|
||||
image.drawText(font, 16, 21, this.sampleError || latest.status, this.sampleError ? 170 : 140);
|
||||
|
||||
const memorySeries = MEMORY_SERIES.map((label, index) => ({
|
||||
label,
|
||||
shade: MEMORY_SHADES[index] ?? 80,
|
||||
value: (sample: ChartSample) => sample.memoryMb[index] ?? 0,
|
||||
}));
|
||||
const memoryPeak = Math.max(...this.samples.flatMap((sample) => sample.memoryMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
const legendLeft = drawLegend(
|
||||
image,
|
||||
font,
|
||||
width,
|
||||
48,
|
||||
memorySeries.map((series) => [`${series.label} ${formatMb(series.value(latest))}`, series.shade] as const),
|
||||
);
|
||||
const scaleLabel = `Memory 0–${formatMb(memoryScale)}`;
|
||||
const shortScaleLabel = `0–${formatMb(memoryScale)}`;
|
||||
image.drawText(
|
||||
font,
|
||||
16,
|
||||
21,
|
||||
`RSS ${formatMb(latest.rssMb)} native ${formatMb(latest.nativeMb)} Java ${formatMb(latest.javaMb)}/${formatMb(latest.javaCommittedMb)}`,
|
||||
190,
|
||||
48,
|
||||
16 + font.measureText(scaleLabel) + 8 <= legendLeft ? scaleLabel : shortScaleLabel,
|
||||
140,
|
||||
);
|
||||
image.drawText(
|
||||
font,
|
||||
width - 16 - font.measureText(`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`),
|
||||
47 - font.lineHeight,
|
||||
`CPU ${formatPercent(latest.cpuPercent)} threads ${latest.threads}`,
|
||||
155,
|
||||
);
|
||||
|
||||
const memoryPeak = Math.max(...this.samples.map((sample) => sample.rssMb), 64);
|
||||
const memoryScale = roundUp(memoryPeak * 1.08, 64);
|
||||
image.drawText(font, 16, 48, `Memory 0–${formatMb(memoryScale)}`, 140);
|
||||
drawLegend(image, font, width, 48, [
|
||||
["RSS", 240],
|
||||
["native", 155],
|
||||
["Java", 80],
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, [
|
||||
{ value: (sample) => sample.rssMb, shade: 240 },
|
||||
{ value: (sample) => sample.nativeMb, shade: 155 },
|
||||
{ value: (sample) => sample.javaMb, shade: 80 },
|
||||
]);
|
||||
drawChart(image, this.samples, latest.atMs, { x: 16, y: 64, width: width - 32, height: 70 }, memoryScale, memorySeries);
|
||||
|
||||
const cpuPeak = Math.max(...this.samples.map((sample) => sample.cpuPercent), 100);
|
||||
const cpuScale = roundUp(cpuPeak * 1.08, 25);
|
||||
@@ -119,26 +118,16 @@ export class ResourceUsageLayer implements Layer {
|
||||
private takeSample(): void {
|
||||
if (this.removed) return;
|
||||
try {
|
||||
const raw = com.faceclaw.app.FaceclawResourceUsage.sample();
|
||||
const atMs = Number(raw[0]);
|
||||
const cpuMs = Number(raw[1]);
|
||||
const elapsedDelta = atMs - this.previousElapsedMs;
|
||||
const cpuDelta = cpuMs - this.previousCpuMs;
|
||||
const sample = sampleResourceUsage();
|
||||
const elapsedDelta = sample.atMs - this.previousElapsedMs;
|
||||
const cpuDelta = sample.cpuMs - this.previousCpuMs;
|
||||
const cpuPercent = this.previousElapsedMs > 0 && elapsedDelta > 0
|
||||
? Math.max(0, cpuDelta / elapsedDelta * 100)
|
||||
: 0;
|
||||
this.previousElapsedMs = atMs;
|
||||
this.previousCpuMs = cpuMs;
|
||||
this.samples.push({
|
||||
atMs,
|
||||
cpuPercent,
|
||||
rssMb: kbToMb(Number(raw[2])),
|
||||
nativeMb: kbToMb(Number(raw[3])),
|
||||
javaMb: kbToMb(Number(raw[4])),
|
||||
javaCommittedMb: kbToMb(Number(raw[5])),
|
||||
threads: Number(raw[6]),
|
||||
});
|
||||
const cutoff = atMs - HISTORY_MS;
|
||||
this.previousElapsedMs = sample.atMs;
|
||||
this.previousCpuMs = sample.cpuMs;
|
||||
this.samples.push({ ...sample, cpuPercent });
|
||||
const cutoff = sample.atMs - HISTORY_MS;
|
||||
while (this.samples.length > 1 && this.samples[0]!.atMs < cutoff) this.samples.shift();
|
||||
while (this.samples.length > MAX_SAMPLES) this.samples.shift();
|
||||
this.sampleError = "";
|
||||
@@ -153,7 +142,7 @@ export class ResourceUsageLayer implements Layer {
|
||||
|
||||
function drawChart(
|
||||
image: GrayImage,
|
||||
samples: readonly ResourceSample[],
|
||||
samples: readonly ChartSample[],
|
||||
nowMs: number,
|
||||
bounds: { x: number; y: number; width: number; height: number },
|
||||
scale: number,
|
||||
@@ -163,7 +152,8 @@ function drawChart(
|
||||
const midY = bounds.y + Math.round((bounds.height - 1) / 2);
|
||||
image.drawLine(bounds.x + 1, midY, bounds.x + bounds.width - 2, midY, 24);
|
||||
const startMs = nowMs - HISTORY_MS;
|
||||
for (const item of series) {
|
||||
// Back to front, so the headline series is drawn on top where lines cross.
|
||||
for (const item of [...series].reverse()) {
|
||||
let previous: { x: number; y: number } | null = null;
|
||||
for (const sample of samples) {
|
||||
if (sample.atMs < startMs) continue;
|
||||
@@ -177,36 +167,30 @@ function drawChart(
|
||||
}
|
||||
}
|
||||
|
||||
/** Right-aligned legend of line swatches and labels; returns its left edge. */
|
||||
function drawLegend(
|
||||
image: GrayImage,
|
||||
font: ReturnType<typeof getDefaultSmallFont>,
|
||||
width: number,
|
||||
y: number,
|
||||
entries: ReadonlyArray<readonly [string, number]>,
|
||||
): void {
|
||||
): number {
|
||||
const totalWidth = entries.reduce((sum, [label]) => sum + font.measureText(label) + 14, 0);
|
||||
let x = width - 16 - totalWidth;
|
||||
const left = width - 16 - totalWidth;
|
||||
let x = left;
|
||||
for (const [label, shade] of entries) {
|
||||
image.drawLine(x, y + 6, x + 8, y + 6, shade);
|
||||
x += 12;
|
||||
image.drawText(font, x, y, label, shade);
|
||||
x += font.measureText(label) + 2;
|
||||
}
|
||||
}
|
||||
|
||||
function kbToMb(kb: number): number {
|
||||
return Math.max(0, kb) / 1024;
|
||||
return left;
|
||||
}
|
||||
|
||||
function roundUp(value: number, step: number): number {
|
||||
return Math.max(step, Math.ceil(value / step) * step);
|
||||
}
|
||||
|
||||
function formatMb(value: number): string {
|
||||
if (value >= 1024) return `${(value / 1024).toFixed(1)}G`;
|
||||
return `${Math.round(value)}M`;
|
||||
}
|
||||
|
||||
function formatPercent(value: number): string {
|
||||
return `${value < 10 ? value.toFixed(1) : Math.round(value)}%`;
|
||||
}
|
||||
@@ -176,6 +176,21 @@ export function readEvenHubPackageManifestBytes(bytes: Uint8Array): EvenHubManif
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete an installed EHPK that is all zeros, keeping its registry entry and
|
||||
* icon so the store treats it as missing and offers Reinstall. iOS builds
|
||||
* before the binary-http.ios.ts copy fix could store a large download that
|
||||
* way. Returns whether the package was discarded.
|
||||
*/
|
||||
export function discardZeroedEvenHubPackage(packageId: string): boolean {
|
||||
const path = installedEvenHubPackagePath(packageId);
|
||||
const bytes = readBinaryFile(path);
|
||||
if (!bytes || bytes.length === 0) return false;
|
||||
for (let i = 0; i < bytes.length; i++) if (bytes[i] !== 0) return false;
|
||||
deletePathRecursively(path);
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Remove the stored EHPK, unpacked runtime files, and registry entry. */
|
||||
export function uninstallEvenHubPackage(packageId: string): boolean {
|
||||
const apps = getInstalledEvenHubApps();
|
||||
|
||||
@@ -27,6 +27,7 @@ export type EvenHubMicClient = {
|
||||
deliverAudioPcm(pcm: Uint8Array): void;
|
||||
};
|
||||
|
||||
/** Android's session for the raw tap; iOS binds its own to the bridge (setRawSession). */
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
|
||||
@@ -726,7 +726,6 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
* flows (foreground + screen on + no assistant modal).
|
||||
*/
|
||||
private audioControl(data: Record<string, unknown>): boolean {
|
||||
if (global.isIOS) return false; // EvenHub PCM routing is not ported yet.
|
||||
if (!permissionsIncludeMicrophone(this.manifest.permissions)) {
|
||||
this.log("evenhub: audioControl denied (app declares no microphone permission)");
|
||||
return false;
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
import { type GrayImage, type UiFont } from "../../../graphics/image";
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../../native/location-permissions";
|
||||
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||
import { NOTIFICATION_ICON_SIZE } from "../../../native/notification-types";
|
||||
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
||||
import { weatherBridge } from "../../../native/weather";
|
||||
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
||||
@@ -37,7 +38,6 @@ import { glanceFont } from "../glance-font";
|
||||
import { type GlanceWidget } from "../widget";
|
||||
|
||||
const PAD = 8;
|
||||
const NOTIFICATION_ICON_SIZE = 24;
|
||||
const NOTIFICATION_ICON_GAP = 4;
|
||||
const BATTERY_ITEM_GAP = 12;
|
||||
const BATTERY_LABEL_VALUE = 150;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
||||
|
||||
@@ -245,7 +245,7 @@ export function askConversations(
|
||||
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
return streamLocalQwen({
|
||||
return streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { useMicControlSetting } from "../../ui/dashboard-settings";
|
||||
import { toJavaBytes } from "../../native/cloud-stt";
|
||||
import { toUint8Array } from "../../util/array-util";
|
||||
import { addMicStatusListener, micControlSession, sendMicControlRecord } from "../../native/mic-control-link";
|
||||
import {
|
||||
MIC_LEASE_MS,
|
||||
decodeMicStatus,
|
||||
@@ -14,8 +13,6 @@ import {
|
||||
type MicStatus,
|
||||
} from "./mic-protocol";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
/**
|
||||
* Host-side controller for the CFW mic_control array: sends the same config
|
||||
* to both temples (each temple is its own endpoint and its own 2-mic array),
|
||||
@@ -30,15 +27,6 @@ const LEASE_RENEW_MS = Math.floor(MIC_LEASE_MS / 3);
|
||||
|
||||
export type MicArrayStatuses = { left: MicStatus | null; right: MicStatus | null };
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
try {
|
||||
return com.faceclaw.app.FaceclawBleCommunicator.getActive();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* True when the "Use microphone control" developer setting is on. With the
|
||||
* setting off, Faceclaw behaves exactly as it would on firmware without the
|
||||
@@ -71,9 +59,9 @@ export function saveMicConfig(config: MicConfig): void {
|
||||
class MicArrayController {
|
||||
private readonly statusListeners = new Set<(statuses: MicArrayStatuses) => void>();
|
||||
private statuses: MicArrayStatuses = { left: null, right: null };
|
||||
// The Java listener proxy must stay referenced or it gets GC'd.
|
||||
private statusProxy: any = null;
|
||||
private proxyCommunicator: any = null;
|
||||
// Holds the native listener proxy, which must stay referenced or it gets GC'd.
|
||||
private unsubscribeStatus: (() => void) | null = null;
|
||||
private statusSession: unknown = null;
|
||||
private leaseTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private armed = false;
|
||||
|
||||
@@ -158,39 +146,28 @@ class MicArrayController {
|
||||
rightTemple = true,
|
||||
leftTemple = true,
|
||||
): boolean {
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return false;
|
||||
const session = micControlSession();
|
||||
if (!session) return false;
|
||||
try {
|
||||
this.ensureStatusProxy(communicator);
|
||||
const record = encodeMicControl(op, config);
|
||||
// False when the session wasn't ready and nothing was queued.
|
||||
return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
this.ensureStatusListener(session);
|
||||
return sendMicControlRecord(session, encodeMicControl(op, config), label, rightTemple, leftTemple);
|
||||
} catch (error) {
|
||||
console.warn(`mic control ${label} failed: ${error}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private ensureStatusProxy(communicator: any): void {
|
||||
if (this.statusProxy && this.proxyCommunicator === communicator) return;
|
||||
if (this.statusProxy && this.proxyCommunicator) {
|
||||
try {
|
||||
this.proxyCommunicator.removeMicStatusListener(this.statusProxy);
|
||||
} catch {
|
||||
// The old communicator instance may be gone after a reconnect.
|
||||
}
|
||||
}
|
||||
this.statusProxy = new com.faceclaw.app.FaceclawMicStatusListener({
|
||||
onMicStatus: (body: any, arm: string) => {
|
||||
const status = decodeMicStatus(toUint8Array(body));
|
||||
if (!status) return;
|
||||
const side: MicSide = String(arm) === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
},
|
||||
private ensureStatusListener(session: unknown): void {
|
||||
if (this.unsubscribeStatus && this.statusSession === session) return;
|
||||
this.unsubscribeStatus?.();
|
||||
this.statusSession = session;
|
||||
this.unsubscribeStatus = addMicStatusListener(session, (body, arm) => {
|
||||
const status = decodeMicStatus(body);
|
||||
if (!status) return;
|
||||
const side: MicSide = arm === "L" ? "left" : "right";
|
||||
this.statuses = { ...this.statuses, [side]: { ...status, side } };
|
||||
this.statusListeners.forEach((listener) => listener(this.statuses));
|
||||
});
|
||||
this.proxyCommunicator = communicator;
|
||||
communicator.addMicStatusListener(this.statusProxy);
|
||||
}
|
||||
|
||||
private syncLeaseTimer(): void {
|
||||
|
||||
@@ -64,6 +64,7 @@ import { analyzeLine, searchableMetadata } from "./sentiment";
|
||||
import { generateSessionInsights } from "./speaker-insights";
|
||||
import { speakerRegistry, type SpeakerMatch } from "./speakers";
|
||||
import { postAmbientCard } from "../../ui/shell/ambient-cards";
|
||||
import { pcmLevel, rmsToLevel } from "../../util/audio-level";
|
||||
import { formatRelativeTime } from "../../util/date-util";
|
||||
import { deviceLanguage, identifyLanguage, translateText } from "./translate";
|
||||
|
||||
@@ -1669,31 +1670,12 @@ function wireConfig(config: MicConfig): MicConfig {
|
||||
return { ...config, channelMask: 0b11 };
|
||||
}
|
||||
|
||||
function pcmLevel(pcm: Uint8Array): number {
|
||||
let sumSquares = 0;
|
||||
const count = pcm.length / 2;
|
||||
if (count === 0) return 0;
|
||||
for (let i = 0; i < count; i++) {
|
||||
let value = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8);
|
||||
if (value >= 0x8000) value -= 0x10000;
|
||||
sumSquares += value * value;
|
||||
}
|
||||
return rmsToLevel(Math.sqrt(sumSquares / count) / 32768);
|
||||
}
|
||||
|
||||
function floatLevel(samples: Float32Array): number {
|
||||
let sumSquares = 0;
|
||||
for (let i = 0; i < samples.length; i++) sumSquares += samples[i]! * samples[i]!;
|
||||
return rmsToLevel(Math.sqrt(sumSquares / Math.max(1, samples.length)));
|
||||
}
|
||||
|
||||
/** Map RMS in [0,1] to a meter position with a log-ish curve (mic peaks ~0.1 FS). */
|
||||
function rmsToLevel(rms: number): number {
|
||||
if (rms <= 0) return 0;
|
||||
const db = 20 * Math.log10(rms);
|
||||
return Math.max(0, Math.min(1, (db + 60) / 60));
|
||||
}
|
||||
|
||||
function floatToPcm16(samples: Float32Array): Uint8Array {
|
||||
const out = new Uint8Array(samples.length * 2);
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import type { NameInference } from "./name-guess";
|
||||
|
||||
/**
|
||||
@@ -38,7 +38,7 @@ export function verifyIntroducedName(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), VERDICT_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
|
||||
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
||||
import { isLocalModelReady, streamLocalModel } from "../../native/local-llm";
|
||||
import { speakerRegistry } from "./speakers";
|
||||
|
||||
declare const com: any;
|
||||
@@ -124,7 +124,7 @@ function generateForSpeaker(
|
||||
};
|
||||
const timer = setTimeout(() => finish(null), GENERATION_TIMEOUT_MS);
|
||||
try {
|
||||
streamLocalQwen({
|
||||
streamLocalModel({
|
||||
apiKey: "",
|
||||
model: "local",
|
||||
system:
|
||||
|
||||
+53
-14
@@ -8,8 +8,10 @@ import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { mediaControllerBridge, type MediaControllerState, type MediaQueueItem } from "../../native/media-controller";
|
||||
import { mediaBrowserBridge, type MediaBrowserApp } from "../../native/media-browser";
|
||||
import { spotifyService } from "../../native/spotify";
|
||||
import { MediaBrowseLayer } from "./media-browse";
|
||||
import { MusicSettingsLayer } from "./music-settings";
|
||||
import { spotifyMenuItem } from "./spotify-menu";
|
||||
import { Layer, type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { createInProcessWindow, YieldAtRootLayer, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
|
||||
@@ -33,6 +35,8 @@ const QUEUE_X = 204;
|
||||
/** Horizontal inset of row text from its selection box. */
|
||||
const LIST_TEXT_INSET = 6;
|
||||
const PLAYLIST_ACTION_INDEX = 1;
|
||||
/** Minimum room for the dim artist after a queue row's title; less and it is left off. */
|
||||
const QUEUE_SUBTITLE_MIN_WIDTH = 40;
|
||||
|
||||
type MusicAction =
|
||||
| { kind: "play-pause"; label: string; enabled: boolean }
|
||||
@@ -40,14 +44,19 @@ type MusicAction =
|
||||
| { kind: "next"; label: string; enabled: boolean }
|
||||
| { kind: "volume"; label: string; enabled: boolean }
|
||||
| { kind: "playlist"; label: string; enabled: boolean }
|
||||
| { kind: "browse"; label: string; enabled: boolean };
|
||||
| { kind: "browse"; label: string; enabled: boolean }
|
||||
/** A player-specific action from the state's customActions (Spotify's shuffle / repeat / like). */
|
||||
| { kind: "custom"; label: string; enabled: boolean; action: string };
|
||||
|
||||
type FocusColumn = "actions" | "playlist";
|
||||
|
||||
/**
|
||||
* Music controller app: metadata + album art + transport controls for the
|
||||
* active Android media session (any player that publishes one) or, on iOS,
|
||||
* the Apple Music system player, plus the player's queue when it exposes one
|
||||
* the Apple Music system player; on both, Spotify through its Web API once
|
||||
* signed in from the window menu (optional on Android, where it adds
|
||||
* library browsing, shuffle / repeat / like and Spotify on other devices),
|
||||
* plus the player's queue when it exposes one
|
||||
* (scroll to a track, click to jump). With no session, the front page lists
|
||||
* the apps whose library can be browsed (click one to open its
|
||||
* MediaBrowseLayer).
|
||||
@@ -110,7 +119,7 @@ class MusicAppLayer implements Layer {
|
||||
const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh);
|
||||
this.libraryAppsFresh = true;
|
||||
this.libraryMenu.setItems(apps);
|
||||
image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180);
|
||||
image.drawText(font, 24, 16, global.isIOS ? media.status || "Nothing is playing." : "No active media session.", 180);
|
||||
const prompt = global.isIOS
|
||||
? "Browse your library, or start playback on the phone."
|
||||
: apps.length
|
||||
@@ -121,6 +130,11 @@ class MusicAppLayer implements Layer {
|
||||
image.drawText(font, 24, y, line, 150);
|
||||
y += lineStep(font);
|
||||
}
|
||||
// Why the last command failed (e.g. playing from the Spotify library with no device open).
|
||||
for (const line of media.notice ? wrapText(font, media.notice, width - 48) : []) {
|
||||
image.drawText(font, 24, y, line, 200);
|
||||
y += lineStep(font);
|
||||
}
|
||||
if (apps.length) {
|
||||
const listTop = y + 6;
|
||||
const listHeight = height - listTop - LIST_BOTTOM_MARGIN;
|
||||
@@ -156,12 +170,13 @@ class MusicAppLayer implements Layer {
|
||||
if (media.album) {
|
||||
image.drawText(font, META_X, metaTop + 4 + (titleSlots + 1) * metaStep, truncateText(font, media.album, metaWidth), 150);
|
||||
}
|
||||
// The player line, or for a few seconds why a command failed.
|
||||
image.drawText(
|
||||
font,
|
||||
META_X,
|
||||
metaTop + 4 + (titleSlots + 2) * metaStep,
|
||||
truncateText(font, media.appName || media.packageName, metaWidth),
|
||||
110,
|
||||
truncateText(font, media.notice || media.appName || media.packageName, metaWidth),
|
||||
media.notice ? 200 : 110,
|
||||
);
|
||||
this.drawProgress(image, media, metaWidth);
|
||||
|
||||
@@ -212,7 +227,16 @@ class MusicAppLayer implements Layer {
|
||||
private drawQueueItem({ image, item, x, y, width, height, selected }: MenuDrawArgs<MediaQueueItem>): void {
|
||||
const font = getDefaultSmallFont();
|
||||
const label = `${item.active ? "> " : " "}${item.title || "(untitled)"}`;
|
||||
image.drawText(font, x + LIST_TEXT_INSET, centeredTextY(font, y, height), truncateText(font, label, width - 2 * LIST_TEXT_INSET), selected ? 255 : 200);
|
||||
const textWidth = width - 2 * LIST_TEXT_INSET;
|
||||
const textY = centeredTextY(font, y, height);
|
||||
const title = truncateText(font, label, textWidth);
|
||||
image.drawText(font, x + LIST_TEXT_INSET, textY, title, selected ? 255 : 200);
|
||||
// The artist, dimmer, in whatever room the title leaves.
|
||||
const subtitleX = font.measureText(`${label} `);
|
||||
if (item.subtitle && title === label && textWidth - subtitleX >= QUEUE_SUBTITLE_MIN_WIDTH) {
|
||||
image.drawText(font, x + LIST_TEXT_INSET + subtitleX, textY,
|
||||
truncateText(font, item.subtitle, textWidth - subtitleX), selected ? 150 : 110);
|
||||
}
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
@@ -278,6 +302,7 @@ class MusicAppLayer implements Layer {
|
||||
else if (action.kind === "previous") await mediaControllerBridge.skipPrevious();
|
||||
else if (action.kind === "next") await mediaControllerBridge.skipNext();
|
||||
else if (action.kind === "browse") this.openBrowse(ctx);
|
||||
else if (action.kind === "custom") await mediaControllerBridge.sendCustomAction?.(action.action);
|
||||
else if (action.kind === "volume") {
|
||||
ctx.stack.push(new VolumeModalLayer(mediaControllerBridge.getMediaVolumePercent()));
|
||||
} else if (action.kind === "playlist") {
|
||||
@@ -342,6 +367,12 @@ class MusicAppLayer implements Layer {
|
||||
{ kind: "volume", label: volume >= 0 ? `Volume (${volume})` : "Volume", enabled: volume >= 0 },
|
||||
{ kind: "next", label: "Next track", enabled: media.canSkipNext },
|
||||
{ kind: "previous", label: "Previous track", enabled: media.canSkipPrevious },
|
||||
...(mediaControllerBridge.sendCustomAction ? media.customActions ?? [] : []).map((custom): MusicAction => ({
|
||||
kind: "custom",
|
||||
label: custom.name,
|
||||
enabled: true,
|
||||
action: custom.action,
|
||||
})),
|
||||
];
|
||||
}
|
||||
|
||||
@@ -496,14 +527,22 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin
|
||||
iconLetter: "M",
|
||||
icon: "music",
|
||||
closeable: true,
|
||||
// Music settings choose among Android media apps; iOS has only Apple Music.
|
||||
menuItems: () => global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}],
|
||||
// Music settings choose among Android media apps (iOS has Apple Music
|
||||
// alone); Spotify's sign-in is optional on Android, where its media
|
||||
// session works without it.
|
||||
menuItems: () => {
|
||||
const spotify = spotifyService();
|
||||
return [
|
||||
...(global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx: LayerContext) => {
|
||||
ctx.stack.pop();
|
||||
ctx.stack.push(new MusicSettingsLayer());
|
||||
},
|
||||
}]),
|
||||
...(spotify ? [spotifyMenuItem(spotify)] : []),
|
||||
];
|
||||
},
|
||||
actions: options.actions,
|
||||
baseLayer: new MusicRootLayer(musicLayer),
|
||||
submitFrame: options.submitFrame,
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import type { GrayImage } from "../../graphics/image";
|
||||
import { isValidSpotifyClientId, SPOTIFY_DASHBOARD_URL, SPOTIFY_REDIRECT_URI } from "../../native/spotify-auth";
|
||||
import type { SpotifyService } from "../../native/spotify-service";
|
||||
import { spotifyClientIdSetting, textSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type LayerContext, type PaintBelow } from "../../ui/layers";
|
||||
import { submenuItem, TextPageLayer, type MenuItem } from "../../ui/menu";
|
||||
import { WindowMenuLayer } from "../../ui/window-menu";
|
||||
import { TextViewerLayer } from "../files/text-viewer";
|
||||
|
||||
const SETUP_INTRO = global.isIOS
|
||||
? "Faceclaw controls Spotify through Spotify's Web API, which needs a Spotify developer app of your own. Creating one is free; Spotify requires its owner to have Premium, and controlling playback needs Premium too."
|
||||
: "Faceclaw already shows and controls Spotify on this phone, like any music app. Signing in to Spotify's Web API is optional and adds browsing your Spotify library, playing from Spotify by voice, shuffle / repeat / like, and Spotify playing on your other devices. It needs a Spotify developer app of your own: creating one is free, Spotify requires its owner to have Premium, and controlling playback through it needs Premium too.";
|
||||
|
||||
const SETUP_HELP = [
|
||||
SETUP_INTRO,
|
||||
"",
|
||||
`1. On a computer, open ${SPOTIFY_DASHBOARD_URL.replace("https://", "")} and sign in with your Spotify account.`,
|
||||
"2. Create an app. Any name and description will do.",
|
||||
`3. Add this Redirect URI exactly: ${SPOTIFY_REDIRECT_URI}`,
|
||||
"4. Under \"Which API/SDKs are you planning to use?\", tick Web API, then save.",
|
||||
"5. Open the app's settings, copy its Client ID, and enter it as Client ID here.",
|
||||
"6. Choose Sign in, then finish on your phone: Faceclaw opens Spotify's sign-in page there (once Faceclaw is open on the phone, if it was in the background).",
|
||||
"",
|
||||
"Spotify plays on whichever of your devices is open (this phone, a computer, a speaker). To start music when nothing is playing, Spotify must be open on one of them.",
|
||||
"",
|
||||
"Others can sign in with your client ID once you add their Spotify email under User Management in the dashboard (up to five people).",
|
||||
].join("\n");
|
||||
|
||||
/** The Music app menu's Spotify row: sign in, sign out, client ID, setup help. */
|
||||
export function spotifyMenuItem(service: SpotifyService): MenuItem {
|
||||
return submenuItem("Spotify", (ctx) => ctx.stack.push(new SpotifyMenuLayer(service)), {
|
||||
description: global.isIOS
|
||||
? "Sign in to Spotify to show and control it from the glasses."
|
||||
: "Optional: sign in to Spotify to browse its library and play from it by voice.",
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows rebuilt every paint and input from the account and sign-in progress,
|
||||
* which change asynchronously. A WindowMenuLayer, so the shell's system menu
|
||||
* closes it like the window menu it opens from.
|
||||
*/
|
||||
class SpotifyMenuLayer extends WindowMenuLayer {
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
|
||||
constructor(private readonly service: SpotifyService) {
|
||||
super("Spotify", []);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext, paintBelow: PaintBelow): GrayImage {
|
||||
this.unsubscribe ??= this.service.onChange(() => ctx.actions.requestRender());
|
||||
this.setItems(this.rows());
|
||||
return super.paint(ctx, paintBelow);
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent, ctx: LayerContext): Promise<void> {
|
||||
this.setItems(this.rows());
|
||||
await super.handleInput(event, ctx);
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
}
|
||||
|
||||
private rows(): MenuItem[] {
|
||||
const service = this.service;
|
||||
const account = service.account();
|
||||
const status = service.signInStatus();
|
||||
const help: MenuItem = {
|
||||
label: "Setup help",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextViewerLayer(SETUP_HELP, "Spotify setup")),
|
||||
};
|
||||
if (account) {
|
||||
return [
|
||||
info(`Signed in as ${account.displayName || account.userId}`),
|
||||
...(account.product && account.product !== "premium" ? [info("Playback control needs Premium")] : []),
|
||||
{ label: "Sign out", onSelect: () => service.signOut() },
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
];
|
||||
}
|
||||
if (status.phase === "waiting") {
|
||||
return [
|
||||
info("Finish signing in on the phone"),
|
||||
{ label: "Cancel sign-in", onSelect: () => service.cancelSignIn() },
|
||||
help,
|
||||
];
|
||||
}
|
||||
const failure: MenuItem[] = status.error
|
||||
? [{
|
||||
label: "Sign-in failed (details)",
|
||||
onSelect: (ctx) => ctx.stack.push(new TextPageLayer("Spotify sign-in failed", status.error)),
|
||||
}]
|
||||
: [];
|
||||
return [
|
||||
{
|
||||
label: "Sign in",
|
||||
description: "Opens Spotify's sign-in page on the phone.",
|
||||
disabled: () => !isValidSpotifyClientId(spotifyClientIdSetting.get()),
|
||||
onSelect: () => void service.signIn(),
|
||||
},
|
||||
textSettingMenuItem(spotifyClientIdSetting),
|
||||
help,
|
||||
...failure,
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
function info(label: string): MenuItem {
|
||||
return { label, disabled: true, onSelect: () => {} };
|
||||
}
|
||||
@@ -28,6 +28,7 @@ import { WindowMenu, WindowMenuLayer } from "../../ui/window-menu";
|
||||
import type { LayerContext } from "../../ui/layers";
|
||||
import { truncateText, wrapText } from "../../graphics/textwrap";
|
||||
import { onSettingsStoreChanged } from "../../native/settings-store";
|
||||
import { formatClockTime } from "../../ui/clock-format";
|
||||
import { openUrlOnPhone } from "../../native/open-url";
|
||||
import {
|
||||
navigateDestinationAddressDraftSetting,
|
||||
@@ -35,6 +36,9 @@ import {
|
||||
navigateRememberRecentSetting,
|
||||
navigateDisplayLayoutSetting,
|
||||
mapboxApiKeySetting,
|
||||
distanceUnitSetting,
|
||||
DISTANCE_UNIT_SETTING_KEY,
|
||||
TIME_FORMAT_SETTING_KEY,
|
||||
enumSettingMenuItem,
|
||||
type ConfigSettingString,
|
||||
} from "../../ui/dashboard-settings";
|
||||
@@ -266,8 +270,15 @@ function post(message: WorkerAppReply): void {
|
||||
// the edit screen live.
|
||||
onSettingsStoreChanged((key) => {
|
||||
// The token key repaints too: the phone editor writes it live during
|
||||
// Edit token, and the idle page changes shape once one is set.
|
||||
if (key.startsWith("navigate.") || key === MAPBOX_TOKEN_SETTING_KEY) render();
|
||||
// Edit token, and the idle page changes shape once one is set. So do the
|
||||
// distance unit and time format (Settings > Customization), for the
|
||||
// guidance panel.
|
||||
if (
|
||||
key.startsWith("navigate.") ||
|
||||
key === MAPBOX_TOKEN_SETTING_KEY ||
|
||||
key === DISTANCE_UNIT_SETTING_KEY ||
|
||||
key === TIME_FORMAT_SETTING_KEY
|
||||
) render();
|
||||
});
|
||||
|
||||
// The host queues messages until this arrives: posts to a worker whose bundle
|
||||
@@ -1569,15 +1580,33 @@ function drawFooter(image: GrayImage, text: string, x = 16): void {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Formatting (imperial; matches the rest of the UI)
|
||||
// Formatting (distances and clock times follow Settings > Customization)
|
||||
|
||||
function metricDistances(): boolean {
|
||||
return distanceUnitSetting.get() === "kilometers";
|
||||
}
|
||||
|
||||
/** Under a kilometer: 10 m steps up to 100 m, then 50 m steps. */
|
||||
function formatMeters(meters: number): string {
|
||||
const step = meters < 100 ? 10 : 50;
|
||||
return `${Math.max(0, Math.round(meters / step) * step)} m`;
|
||||
}
|
||||
|
||||
function formatDistanceShort(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
return meters < 950 ? formatMeters(meters) : `${(meters / 1000).toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 900) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
return `${(feet / 5280).toFixed(1)} mi`;
|
||||
}
|
||||
|
||||
function formatDistance(meters: number): string {
|
||||
if (metricDistances()) {
|
||||
if (meters < 950) return formatMeters(meters);
|
||||
const km = meters / 1000;
|
||||
return km >= 10 ? `${Math.round(km)} km` : `${km.toFixed(1)} km`;
|
||||
}
|
||||
const feet = meters * 3.28084;
|
||||
if (feet < 1000) return `${Math.max(0, Math.round(feet / 50) * 50)} ft`;
|
||||
const miles = feet / 5280;
|
||||
@@ -1597,14 +1626,9 @@ function formatDurationLong(seconds: number): string {
|
||||
return `${hours}h ${minutes % 60}m`;
|
||||
}
|
||||
|
||||
/** "14:05" / "2:05 PM", as the top-bar clock shows it. */
|
||||
function formatClock(timestampMs: number): string {
|
||||
const date = new Date(timestampMs);
|
||||
let hours = date.getHours();
|
||||
const minutes = String(date.getMinutes()).padStart(2, "0");
|
||||
const meridiem = hours >= 12 ? "PM" : "AM";
|
||||
hours = hours % 12;
|
||||
if (hours === 0) hours = 12;
|
||||
return `${hours}:${minutes} ${meridiem}`;
|
||||
return formatClockTime(new Date(timestampMs));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
@@ -78,7 +78,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSCRIBE_WINDOW_ID));
|
||||
options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
return app;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -18,6 +19,7 @@ export class TranscribeLayer implements Layer {
|
||||
private detail = "";
|
||||
private readonly transcript = new TranscriptModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private readonly meter = new MicLevelMeter();
|
||||
private saveNotice = "";
|
||||
private unsubscribeTranscript: (() => void) | null = null;
|
||||
private unsubscribePause: (() => void) | null = null;
|
||||
@@ -26,7 +28,9 @@ export class TranscribeLayer implements Layer {
|
||||
/** `stamp` names the saved transcript, matching the session's audio recording. */
|
||||
constructor(private readonly stamp: string = transcriptTimestamp()) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.transcript.accept(event);
|
||||
requestRender();
|
||||
@@ -48,11 +52,8 @@ export class TranscribeLayer implements Layer {
|
||||
const text = this.transcript.text || (this.listening ? "Listening..." : this.detail);
|
||||
const wrapped = wrapTranscribeText((line) => font.measureText(line), text, width - 64);
|
||||
|
||||
image.drawText(font, 24, 20, "Transcribe", 200);
|
||||
image.drawText(font, 24, 40, this.saveNotice || this.status, 110);
|
||||
|
||||
const bodyTop = paintListeningHeader(image, font, "Transcribe", this.saveNotice || this.status, this.listening ? this.meter : null);
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(wrapped.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(wrapped.length, firstLine + bodyLines); index++) {
|
||||
@@ -82,6 +83,7 @@ export class TranscribeLayer implements Layer {
|
||||
}
|
||||
|
||||
onRemoved(): void {
|
||||
this.meter.stop();
|
||||
this.unsubscribePause?.();
|
||||
this.unsubscribePause = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
|
||||
@@ -1,6 +1,12 @@
|
||||
import { TranslateLayer } from "./translate";
|
||||
import { resolveTranslationModel } from "./translate-llm";
|
||||
import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings";
|
||||
import {
|
||||
resolveTranslateLanguage,
|
||||
resolveTranslateProvider,
|
||||
translateLanguageSetting,
|
||||
translateProviderSetting,
|
||||
translateShowOriginalSetting,
|
||||
} from "./translate-settings";
|
||||
import { MIC_ICON } from "../transcribe/transcribe-app";
|
||||
import { transcriptTimestamp } from "../transcribe/transcribe";
|
||||
import { type ContinuousCaptureOptions } from "../../ui/layers";
|
||||
@@ -22,8 +28,8 @@ const TRAY_ICON_ID = "translate";
|
||||
const NO_PROVIDER_PROBLEM =
|
||||
"Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice).";
|
||||
const NO_MODEL_PROBLEM = global.isIOS
|
||||
? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)."
|
||||
: "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant).";
|
||||
? "Translate needs a language model: configure a cloud or OpenAI-compatible model in Settings > Assistant."
|
||||
: "Translate needs a language model: configure one in Settings > Assistant, or download the on-phone model.";
|
||||
|
||||
export type TranslateAppOptions = InProcessAppOptions & {
|
||||
/** Begin (or retarget) continuous mic capture, kept on while the window is open. */
|
||||
@@ -38,8 +44,9 @@ export type TranslateAppOptions = InProcessAppOptions & {
|
||||
* provider (Translate settings), which a shared capture switches to.
|
||||
*/
|
||||
export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow {
|
||||
// Checks the setup and (re)starts capture with the chosen provider: at
|
||||
// launch, when the provider setting changes, and on a tap after a problem.
|
||||
// Checks the setup and (re)starts capture with the chosen provider and
|
||||
// language: at launch, when either setting changes, and on a tap after a
|
||||
// problem.
|
||||
const startCapture = () => {
|
||||
const provider = resolveTranslateProvider();
|
||||
const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null;
|
||||
@@ -49,7 +56,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
shell.setTrayIcon(TRAY_ICON_ID, null);
|
||||
return;
|
||||
}
|
||||
options.startContinuousVoiceCapture({ multilingualProvider: provider });
|
||||
options.startContinuousVoiceCapture({ transcription: { provider, language: resolveTranslateLanguage() } });
|
||||
shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON);
|
||||
};
|
||||
const layer = new TranslateLayer(transcriptTimestamp(), startCapture);
|
||||
@@ -75,6 +82,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
openSettingsSubMenu(ctx, "Translate settings", [
|
||||
enumSettingMenuItem(translateLanguageSetting, { onChange: startCapture }),
|
||||
enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }),
|
||||
toggleSettingMenuItem(translateShowOriginalSetting),
|
||||
]);
|
||||
@@ -93,7 +101,7 @@ export function createTranslateAppWindow(options: TranslateAppOptions): InProces
|
||||
});
|
||||
// Wire async state changes to this window's renderer, rather than the
|
||||
// controller action from which the window-specific actions were derived.
|
||||
layer.start(app.requestRender);
|
||||
layer.start(app.requestRender, () => shell.isWindowVisible(TRANSLATE_WINDOW_ID));
|
||||
startCapture();
|
||||
return app;
|
||||
}
|
||||
@@ -4,11 +4,15 @@
|
||||
* TranslationModel for how stretches are batched and given context.
|
||||
*/
|
||||
import { streamAnthropicMessage } from "../../native/anthropic";
|
||||
import { streamLocalQwen } from "../../native/llama";
|
||||
import { streamLocalModel } from "../../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../../native/openai-chat";
|
||||
import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models";
|
||||
import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol";
|
||||
import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings";
|
||||
import {
|
||||
anthropicApiKeySetting, assistantModelSetting, assistantOpenAiCompatibleKeySetting,
|
||||
assistantOpenAiCompatibleModelSetting, assistantOpenAiCompatibleUrlSetting, openAiApiKeySetting,
|
||||
} from "../../ui/dashboard-settings";
|
||||
import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts";
|
||||
import { type TranslationTurn } from "./translation-model";
|
||||
|
||||
@@ -23,12 +27,17 @@ export function resolveTranslationModel(): ResolvedAssistantModel | null {
|
||||
return resolveAssistantModel(selection === "hermes" ? "auto" : selection, {
|
||||
anthropic: anthropicApiKeySetting.get(),
|
||||
openai: openAiApiKeySetting.get(),
|
||||
openAiCompatibleUrl: assistantOpenAiCompatibleUrlSetting.get(),
|
||||
openAiCompatibleKey: assistantOpenAiCompatibleKeySetting.get(),
|
||||
openAiCompatibleModel: assistantOpenAiCompatibleModelSetting.get(),
|
||||
});
|
||||
}
|
||||
|
||||
export type TranslationRequest = {
|
||||
speech: string;
|
||||
context: TranslationTurn[];
|
||||
/** English name of the spoken language when the user set one, else "". */
|
||||
language: string;
|
||||
onProgress: (textSoFar: string) => void;
|
||||
onDone: (translation: string) => void;
|
||||
onError: (message: string) => void;
|
||||
@@ -41,7 +50,7 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
system: TRANSLATE_SYSTEM_PROMPT,
|
||||
effort: model.effort,
|
||||
maxTokens: 1024,
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }],
|
||||
messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context, request.language) }],
|
||||
onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)),
|
||||
onDone: (result) => {
|
||||
if (result.stopReason === "refusal") request.onError("The model declined to translate this.");
|
||||
@@ -52,8 +61,10 @@ export function streamTranslation(model: ResolvedAssistantModel, request: Transl
|
||||
switch (model.provider) {
|
||||
case "openai":
|
||||
return streamOpenAiResponse(options);
|
||||
case "openai-compatible":
|
||||
return streamOpenAiCompatibleChat({ ...options, baseUrl: model.baseUrl ?? "" });
|
||||
case "local":
|
||||
return streamLocalQwen(options);
|
||||
return streamLocalModel(options);
|
||||
default:
|
||||
return streamAnthropicMessage(options);
|
||||
}
|
||||
|
||||
@@ -47,6 +47,58 @@ export const translateProviderSetting = new ConfigSettingEnum<TranslateProvider>
|
||||
"Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Spoken languages Translate can be told to expect, by ISO 639-1 code (the
|
||||
* form every multilingual provider accepts), in picker order.
|
||||
*/
|
||||
const SPOKEN_LANGUAGES = {
|
||||
ar: "Arabic",
|
||||
zh: "Chinese",
|
||||
nl: "Dutch",
|
||||
fr: "French",
|
||||
de: "German",
|
||||
el: "Greek",
|
||||
he: "Hebrew",
|
||||
hi: "Hindi",
|
||||
id: "Indonesian",
|
||||
it: "Italian",
|
||||
ja: "Japanese",
|
||||
ko: "Korean",
|
||||
pl: "Polish",
|
||||
pt: "Portuguese",
|
||||
ru: "Russian",
|
||||
es: "Spanish",
|
||||
sv: "Swedish",
|
||||
th: "Thai",
|
||||
tr: "Turkish",
|
||||
uk: "Ukrainian",
|
||||
vi: "Vietnamese",
|
||||
} as const;
|
||||
|
||||
export type SpokenLanguage = keyof typeof SPOKEN_LANGUAGES;
|
||||
|
||||
export const translateLanguageSetting = new ConfigSettingEnum<"auto" | SpokenLanguage>({
|
||||
id: "translate-language",
|
||||
label: "Spoken language",
|
||||
storageKey: "translate.language",
|
||||
defaultValue: "auto",
|
||||
values: ["auto", ...(Object.keys(SPOKEN_LANGUAGES) as SpokenLanguage[])],
|
||||
formatValue: (value) => (value === "auto" ? "Auto-detect" : SPOKEN_LANGUAGES[value]),
|
||||
description:
|
||||
"The language being spoken. Auto-detect can mistake it, especially in short or noisy speech, and then transcribes nonsense in the wrong language; naming it makes transcription much more reliable. ElevenLabs then transcribes only that language.",
|
||||
});
|
||||
|
||||
/** The language to tell the transcription provider (ISO 639-1), or "" to let it detect one. */
|
||||
export function resolveTranslateLanguage(): SpokenLanguage | "" {
|
||||
const value = translateLanguageSetting.get();
|
||||
return value === "auto" ? "" : value;
|
||||
}
|
||||
|
||||
/** The English name of a spoken language, for the translation prompt. */
|
||||
export function spokenLanguageName(language: SpokenLanguage): string {
|
||||
return SPOKEN_LANGUAGES[language];
|
||||
}
|
||||
|
||||
export const translateShowOriginalSetting = new ConfigSettingBoolean({
|
||||
id: "translate-show-original",
|
||||
label: "Show original",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import { MicLevelMeter, paintListeningHeader } from "../../ui/listening-header";
|
||||
import { writeTextToDownloads } from "../../native/file-access";
|
||||
import { voiceControlBridge } from "../../native/voice-control";
|
||||
import { Layer, type LayerContext } from "../../ui/layers";
|
||||
@@ -9,15 +10,16 @@ import { type LlmStreamHandle } from "../../assistant/llm-protocol";
|
||||
import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model";
|
||||
import { resolveTranslationModel, streamTranslation } from "./translate-llm";
|
||||
import { translateShowOriginalSetting } from "./translate-settings";
|
||||
import { resolveTranslateLanguage, spokenLanguageName, translateShowOriginalSetting } from "./translate-settings";
|
||||
|
||||
const NO_MODEL_NOTICE = "No language model is set up.";
|
||||
|
||||
/**
|
||||
* Live translation view. Continuous mic capture is owned by the app wrapper
|
||||
* (createTranslateAppWindow); this layer feeds finalized speech to the
|
||||
* assistant's model one stretch at a time, shows the English as it streams
|
||||
* in, and saves the session to Downloads from the app menu.
|
||||
* (createTranslateAppWindow); this layer feeds speech to the assistant's
|
||||
* model one block at a time (provisionally while a block is still being
|
||||
* spoken), shows the English as it streams in, and saves the session to
|
||||
* Downloads from the app menu.
|
||||
*/
|
||||
export class TranslateLayer implements Layer {
|
||||
private status = "Starting microphone...";
|
||||
@@ -30,7 +32,8 @@ export class TranslateLayer implements Layer {
|
||||
private problem: string | null = null;
|
||||
private readonly model = new TranslationModel();
|
||||
private readonly scroll = new TranscriptScroll();
|
||||
private activeJob: TranslationJob | null = null;
|
||||
private readonly meter = new MicLevelMeter();
|
||||
// The request in flight (the model tracks which job it is).
|
||||
private activeHandle: LlmStreamHandle | null = null;
|
||||
private requestRender: () => void = () => {};
|
||||
private removed = false;
|
||||
@@ -40,10 +43,16 @@ export class TranslateLayer implements Layer {
|
||||
/** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */
|
||||
constructor(private readonly stamp: string, private readonly retry: () => void) {}
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
/** `isVisible` says whether the window is on screen (the level meter repaints only then). */
|
||||
start(requestRender: () => void, isVisible: () => boolean): void {
|
||||
this.requestRender = requestRender;
|
||||
this.meter.start(requestRender, isVisible);
|
||||
this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => {
|
||||
this.model.accept(event);
|
||||
if (this.model.accept(event)) {
|
||||
// The request in flight translates text the transcript has replaced.
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
}
|
||||
this.pump();
|
||||
requestRender();
|
||||
});
|
||||
@@ -68,9 +77,9 @@ export class TranslateLayer implements Layer {
|
||||
const measure = (line: string) => font.measureText(line);
|
||||
const maxWidth = width - 64;
|
||||
|
||||
image.drawText(font, 24, 20, "Translate", 200);
|
||||
const status = this.problem ? "Setup needed" : this.translationError || this.status;
|
||||
image.drawText(font, 24, 40, this.saveNotice || status, 110);
|
||||
const meter = this.listening && !this.problem ? this.meter : null;
|
||||
const bodyTop = paintListeningHeader(image, font, "Translate", this.saveNotice || status, meter);
|
||||
|
||||
let lines: TranslationLine[];
|
||||
if (this.problem) {
|
||||
@@ -84,7 +93,6 @@ export class TranslateLayer implements Layer {
|
||||
}
|
||||
|
||||
const step = lineStep(font) + 2;
|
||||
const bodyTop = 62;
|
||||
const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1);
|
||||
const firstLine = this.scroll.layout(lines.length, bodyLines);
|
||||
for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) {
|
||||
@@ -119,7 +127,7 @@ export class TranslateLayer implements Layer {
|
||||
|
||||
onRemoved(): void {
|
||||
this.removed = true;
|
||||
this.activeJob = null;
|
||||
this.meter.stop();
|
||||
this.activeHandle?.cancel();
|
||||
this.activeHandle = null;
|
||||
this.unsubscribeTranscript?.();
|
||||
@@ -139,7 +147,10 @@ export class TranslateLayer implements Layer {
|
||||
this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access).";
|
||||
}
|
||||
|
||||
/** Start translating the next waiting stretch of speech, unless one is in flight. */
|
||||
/**
|
||||
* Start the next translation request (a final block, else the live block's
|
||||
* provisional one), unless one is in flight.
|
||||
*/
|
||||
private pump(): void {
|
||||
if (this.removed) return;
|
||||
const job = this.model.nextJob();
|
||||
@@ -147,36 +158,34 @@ export class TranslateLayer implements Layer {
|
||||
const llm = resolveTranslationModel();
|
||||
if (!llm) {
|
||||
// The model went away mid-session (a key was cleared): keep the original.
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = NO_MODEL_NOTICE;
|
||||
this.pump();
|
||||
return;
|
||||
}
|
||||
this.activeJob = job;
|
||||
const language = resolveTranslateLanguage();
|
||||
const handle = streamTranslation(llm, {
|
||||
speech: job.segment.source,
|
||||
speech: job.source,
|
||||
context: job.context,
|
||||
language: language ? spokenLanguageName(language) : "",
|
||||
onProgress: (textSoFar) => {
|
||||
if (this.activeJob !== job) return;
|
||||
this.model.progress(job.segment, textSoFar);
|
||||
this.requestRender();
|
||||
if (this.model.progress(job, textSoFar)) this.requestRender();
|
||||
},
|
||||
onDone: (translation) => this.settle(job, () => {
|
||||
this.model.complete(job.segment, translation);
|
||||
this.model.complete(job, translation);
|
||||
this.translationError = "";
|
||||
}),
|
||||
onError: (message) => this.settle(job, () => {
|
||||
this.model.fail(job.segment);
|
||||
this.model.fail(job);
|
||||
this.translationError = `Translation failed: ${message}`;
|
||||
}),
|
||||
});
|
||||
// A provider that fails synchronously has already settled (and moved on).
|
||||
if (this.activeJob === job) this.activeHandle = handle;
|
||||
if (this.model.isActive(job)) this.activeHandle = handle;
|
||||
}
|
||||
|
||||
private settle(job: TranslationJob, apply: () => void): void {
|
||||
if (this.activeJob !== job) return;
|
||||
this.activeJob = null;
|
||||
if (!this.model.isActive(job)) return;
|
||||
this.activeHandle = null;
|
||||
apply();
|
||||
this.pump();
|
||||
|
||||
@@ -2,21 +2,32 @@ import { wrapTranscribeText } from "../transcribe/transcript-model";
|
||||
|
||||
type TranscriptEvent = { text: string; isFinal: boolean };
|
||||
|
||||
export type TranslationState = "pending" | "translating" | "done" | "failed";
|
||||
|
||||
/** One stretch of finalized speech and its English translation. */
|
||||
/**
|
||||
* One block of speech (up to a pause) and its English translation. The last
|
||||
* block may still be live, its transcript revising; it is translated
|
||||
* provisionally as it grows, then again once final if its text changed.
|
||||
*/
|
||||
export type TranslationSegment = {
|
||||
source: string;
|
||||
/** The translation so far (streamed while translating). */
|
||||
/** Committed by the transcriber; a live block's source still revises. */
|
||||
final: boolean;
|
||||
/** The latest finished translation, of `translatedSource`. */
|
||||
translation: string;
|
||||
state: TranslationState;
|
||||
/** What `translation` translates: null until one finishes, behind `source` while provisional. */
|
||||
translatedSource: string | null;
|
||||
/** The text streamed so far by the request in flight for this block, else null. */
|
||||
streaming: string | null;
|
||||
/** A source whose translation failed; not retried until the source changes. */
|
||||
failedSource: string | null;
|
||||
};
|
||||
|
||||
export type TranslationTurn = { source: string; translation: string };
|
||||
|
||||
/** A segment handed to the model, with the finished turns before it. */
|
||||
/** A request: a block's source as sent, with the finished turns before it. */
|
||||
export type TranslationJob = {
|
||||
segment: TranslationSegment;
|
||||
/** The source sent (a live block's moves on while the request is in flight). */
|
||||
source: string;
|
||||
/** Earlier translated turns, oldest first. */
|
||||
context: TranslationTurn[];
|
||||
};
|
||||
@@ -27,59 +38,123 @@ export type TranslationLine = { text: string; gray: number };
|
||||
// over between requests; bounded so a long session's requests stay small.
|
||||
const CONTEXT_TURNS = 4;
|
||||
const CONTEXT_CHARS = 800;
|
||||
// A live block is translated again once its text has changed length by this
|
||||
// much since the last request, so a trickle of revisions (punctuation, one
|
||||
// corrected word) doesn't spend a request each. A final is always redone.
|
||||
const PROVISIONAL_MIN_CHANGE = 3;
|
||||
|
||||
const TRANSLATION_GRAY = 230;
|
||||
const PROVISIONAL_GRAY = 170;
|
||||
const ORIGINAL_GRAY = 120;
|
||||
const LIVE_GRAY = 90;
|
||||
|
||||
function newSegment(source: string, final: boolean): TranslationSegment {
|
||||
return { source, final, translation: "", translatedSource: null, streaming: null, failedSource: null };
|
||||
}
|
||||
|
||||
/** Whether a block's current source still needs a translation request. */
|
||||
function wantsTranslation(segment: TranslationSegment): boolean {
|
||||
const { source, translatedSource } = segment;
|
||||
if (!source || source === translatedSource || source === segment.failedSource) return false;
|
||||
return segment.final || translatedSource === null || Math.abs(source.length - translatedSource.length) >= PROVISIONAL_MIN_CHANGE;
|
||||
}
|
||||
|
||||
/**
|
||||
* The Translate app's conversation: finalized speech segments with their
|
||||
* translations, plus the live partial of the utterance in progress.
|
||||
* The Translate app's conversation: blocks of speech with their translations.
|
||||
* Translation is serial (one request at a time, in order, so each can carry
|
||||
* the turns before it as context); finals that arrive while a request is in
|
||||
* flight merge into one pending segment, so a slow model batches the backlog
|
||||
* instead of falling further behind one request per pause.
|
||||
* the turns before it as context): final blocks first, then the live one.
|
||||
* Finals that arrive while a request is in flight merge into one waiting
|
||||
* block, so a slow model batches the backlog instead of falling further
|
||||
* behind one request per pause.
|
||||
*/
|
||||
export class TranslationModel {
|
||||
readonly segments: TranslationSegment[] = [];
|
||||
/** The utterance in progress (replace semantics), not yet final. */
|
||||
live = "";
|
||||
private active: TranslationJob | null = null;
|
||||
|
||||
accept(event: TranscriptEvent): void {
|
||||
/**
|
||||
* Take a transcript event (replace semantics within the live block).
|
||||
* Returns true when it made the request in flight moot (its block was
|
||||
* finalized as other text, or dropped); that request is abandoned here, and
|
||||
* the caller should cancel it.
|
||||
*/
|
||||
accept(event: TranscriptEvent): boolean {
|
||||
const text = event.text.trim();
|
||||
if (!event.isFinal) {
|
||||
this.live = text;
|
||||
return;
|
||||
}
|
||||
const finalText = text || this.live;
|
||||
this.live = "";
|
||||
if (!finalText) return;
|
||||
const last = this.segments[this.segments.length - 1];
|
||||
if (last?.state === "pending") last.source = `${last.source} ${finalText}`;
|
||||
else this.segments.push({ source: finalText, translation: "", state: "pending" });
|
||||
const live = last && !last.final ? last : null;
|
||||
if (!event.isFinal) {
|
||||
if (live) live.source = text;
|
||||
else if (text) this.segments.push(newSegment(text, false));
|
||||
return false;
|
||||
}
|
||||
const finalText = text || live?.source || "";
|
||||
if (!live) {
|
||||
if (finalText && !this.mergeIntoWaiting(finalText)) this.segments.push(newSegment(finalText, true));
|
||||
return false;
|
||||
}
|
||||
const moot = this.active?.segment === live && this.active.source !== finalText;
|
||||
if (moot) this.abandonActive();
|
||||
if (!finalText || (this.active?.segment !== live && live.translatedSource === null && this.mergeIntoWaiting(finalText, live))) {
|
||||
this.segments.pop();
|
||||
return moot;
|
||||
}
|
||||
live.source = finalText;
|
||||
live.final = true;
|
||||
return moot;
|
||||
}
|
||||
|
||||
/** Take the next segment to translate, or null while one is in flight or none waits. */
|
||||
/** Take the next request to make, or null while one is in flight or nothing waits. */
|
||||
nextJob(): TranslationJob | null {
|
||||
if (this.segments.some((segment) => segment.state === "translating")) return null;
|
||||
const segment = this.segments.find((candidate) => candidate.state === "pending");
|
||||
if (this.active) return null;
|
||||
const segment = this.segments.find((candidate) => wantsTranslation(candidate));
|
||||
if (!segment) return null;
|
||||
segment.state = "translating";
|
||||
return { segment, context: this.contextBefore(segment) };
|
||||
segment.streaming = "";
|
||||
this.active = { segment, source: segment.source, context: this.contextBefore(segment) };
|
||||
return this.active;
|
||||
}
|
||||
|
||||
progress(segment: TranslationSegment, textSoFar: string): void {
|
||||
if (segment.state === "translating") segment.translation = textSoFar;
|
||||
isActive(job: TranslationJob): boolean {
|
||||
return this.active === job;
|
||||
}
|
||||
|
||||
complete(segment: TranslationSegment, translation: string): void {
|
||||
segment.translation = translation;
|
||||
segment.state = "done";
|
||||
/** Streamed text for the request in flight; false when `job` isn't it any more. */
|
||||
progress(job: TranslationJob, textSoFar: string): boolean {
|
||||
if (this.active !== job) return false;
|
||||
job.segment.streaming = textSoFar;
|
||||
return true;
|
||||
}
|
||||
|
||||
fail(segment: TranslationSegment): void {
|
||||
segment.translation = "";
|
||||
segment.state = "failed";
|
||||
complete(job: TranslationJob, translation: string): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.translation = translation;
|
||||
job.segment.translatedSource = job.source;
|
||||
}
|
||||
|
||||
fail(job: TranslationJob): void {
|
||||
if (this.active !== job) return;
|
||||
this.active = null;
|
||||
job.segment.streaming = null;
|
||||
job.segment.failedSource = job.source;
|
||||
}
|
||||
|
||||
private abandonActive(): void {
|
||||
if (this.active) this.active.segment.streaming = null;
|
||||
this.active = null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Append a final to the block before `before` (default: the end) when that
|
||||
* block is final but still waiting for its first request. Returns whether
|
||||
* it did.
|
||||
*/
|
||||
private mergeIntoWaiting(finalText: string, before?: TranslationSegment): boolean {
|
||||
const index = (before ? this.segments.indexOf(before) : this.segments.length) - 1;
|
||||
const waiting = this.segments[index];
|
||||
if (!waiting?.final || waiting.translatedSource !== null || waiting.failedSource !== null) return false;
|
||||
if (this.active?.segment === waiting) return false;
|
||||
waiting.source = `${waiting.source} ${finalText}`;
|
||||
return true;
|
||||
}
|
||||
|
||||
private contextBefore(segment: TranslationSegment): TranslationTurn[] {
|
||||
@@ -87,7 +162,7 @@ export class TranslationModel {
|
||||
let chars = 0;
|
||||
for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) {
|
||||
const earlier = this.segments[index]!;
|
||||
if (earlier.state !== "done" || !earlier.translation) continue;
|
||||
if (!earlier.final || earlier.translatedSource !== earlier.source || !earlier.translation) continue;
|
||||
chars += earlier.source.length + earlier.translation.length;
|
||||
if (turns.length && chars > CONTEXT_CHARS) break;
|
||||
turns.unshift({ source: earlier.source, translation: earlier.translation });
|
||||
@@ -96,9 +171,11 @@ export class TranslationModel {
|
||||
}
|
||||
|
||||
/**
|
||||
* Display lines: each segment's translation, bright; its original, dim,
|
||||
* above it when `showOriginal` is on and in its place until a translation
|
||||
* arrives (or when it failed); then the live partial, dimmer.
|
||||
* Display lines: each block's translation, bright once settled (final, and
|
||||
* of its final text) and dimmer while provisional; its original, dim, above
|
||||
* it when `showOriginal` is on and in its place until a translation
|
||||
* arrives. A redo streams in over the translation it replaces only once it
|
||||
* is as long, so the text doesn't shrink and regrow.
|
||||
*/
|
||||
lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] {
|
||||
const lines: TranslationLine[] = [];
|
||||
@@ -106,21 +183,25 @@ export class TranslationModel {
|
||||
for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray });
|
||||
};
|
||||
for (const segment of this.segments) {
|
||||
const translation = segment.translation.trim();
|
||||
// A finished, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY);
|
||||
if (translation) push(translation, TRANSLATION_GRAY);
|
||||
if (!segment.source) continue;
|
||||
const settled = segment.final && segment.translatedSource === segment.source;
|
||||
const finished = segment.translation.trim();
|
||||
const streaming = segment.streaming?.trim() ?? "";
|
||||
const translation = streaming.length >= finished.length ? streaming : finished;
|
||||
// A settled, empty translation is noise the model chose to drop.
|
||||
if (showOriginal || (!translation && !settled)) push(segment.source, segment.final ? ORIGINAL_GRAY : LIVE_GRAY);
|
||||
if (translation) push(translation, settled ? TRANSLATION_GRAY : PROVISIONAL_GRAY);
|
||||
}
|
||||
if (this.live) push(this.live, LIVE_GRAY);
|
||||
return lines;
|
||||
}
|
||||
|
||||
/** Plain-text transcript for saving: each original, then its translation. */
|
||||
/** Plain-text transcript for saving: each final original, then its translation. */
|
||||
savedText(): string {
|
||||
return this.segments
|
||||
.filter((segment) => segment.final)
|
||||
.map((segment) => {
|
||||
const translation = segment.translation.trim();
|
||||
if (segment.state === "failed") return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation && segment.failedSource === segment.source) return `${segment.source}\n→ (not translated)`;
|
||||
if (!translation || translation === segment.source.trim()) return segment.source;
|
||||
return `${segment.source}\n→ ${translation}`;
|
||||
})
|
||||
|
||||
@@ -6,6 +6,7 @@ import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { distanceUnitSetting } from "../../ui/dashboard-settings";
|
||||
import { renderWeatherIcon } from "./weather-icons";
|
||||
import { formatTemperature, widestTemperatureSample } from "./temperature-format";
|
||||
|
||||
@@ -142,7 +143,7 @@ export class WeatherLayer implements Layer {
|
||||
const details: string[] = [];
|
||||
if (current.humidityPercent !== null) details.push(`Humidity ${Math.round(current.humidityPercent)}%`);
|
||||
if (current.windSpeedMph !== null) {
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${Math.round(current.windSpeedMph)} mph`);
|
||||
details.push(`Wind ${current.windDirection ? `${current.windDirection} ` : ""}${formatWindSpeed(current.windSpeedMph)}`);
|
||||
}
|
||||
image.drawText(small, textX, cy, details.join(" ") || "Current forecast", 170);
|
||||
cy += lineStep(small);
|
||||
@@ -247,6 +248,11 @@ function forecastRowHeight(font: UiFont): number {
|
||||
return Math.max(FORECAST_MIN_ROW_HEIGHT, tightRowHeight(font) + 3);
|
||||
}
|
||||
|
||||
/** A wind speed (weather data is kept in mph) in the Distance unit setting's speed unit: "12 mph" / "19 km/h". */
|
||||
function formatWindSpeed(mph: number): string {
|
||||
return distanceUnitSetting.get() === "kilometers" ? `${Math.round(mph * 1.609344)} km/h` : `${Math.round(mph)} mph`;
|
||||
}
|
||||
|
||||
function formatAge(ageMs: number): string {
|
||||
const minutes = Math.max(0, Math.round(ageMs / 60_000));
|
||||
if (minutes < 1) return "just now";
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
import { streamAnthropicMessage } from "../native/anthropic";
|
||||
import { streamHermesChat } from "../native/hermes";
|
||||
import { streamLocalQwen } from "../native/llama";
|
||||
import { streamLocalModel } from "../native/local-llm";
|
||||
import { streamOpenAiResponse } from "../native/openai";
|
||||
import { streamOpenAiCompatibleChat } from "../native/openai-chat";
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
@@ -40,6 +41,8 @@ export type DirectTurnOptions = {
|
||||
effort?: "low" | "medium" | "high";
|
||||
/** Hermes: the API root, and the server-side session this conversation maps to. */
|
||||
baseUrl?: string;
|
||||
/** Venice-only web search for this interactive turn. */
|
||||
webSearch?: boolean;
|
||||
sessionId?: string;
|
||||
system: string;
|
||||
/** Owned by the session; this loop appends assistant + tool_result messages. */
|
||||
@@ -85,11 +88,12 @@ export class DirectAssistantBackend {
|
||||
return;
|
||||
}
|
||||
|
||||
const streamOptions: LlmStreamOptions = {
|
||||
const streamOptions: LlmStreamOptions & { webSearch?: boolean } = {
|
||||
apiKey: options.apiKey,
|
||||
model: options.model,
|
||||
system: options.system,
|
||||
effort: options.effort,
|
||||
webSearch: options.webSearch,
|
||||
messages: options.messages,
|
||||
// Hermes runs its own tools; Faceclaw's are not offered to it.
|
||||
tools: options.provider === "hermes" ? [] : options.buildTools(),
|
||||
@@ -143,10 +147,12 @@ export class DirectAssistantBackend {
|
||||
streamHandle =
|
||||
options.provider === "hermes"
|
||||
? streamHermesChat({ ...streamOptions, baseUrl: options.baseUrl ?? "", sessionId: options.sessionId ?? "" })
|
||||
: options.provider === "openai-compatible"
|
||||
? streamOpenAiCompatibleChat({ ...streamOptions, baseUrl: options.baseUrl ?? "" })
|
||||
: options.provider === "openai"
|
||||
? streamOpenAiResponse(streamOptions)
|
||||
: options.provider === "local"
|
||||
? streamLocalQwen(streamOptions)
|
||||
? streamLocalModel(streamOptions)
|
||||
: streamAnthropicMessage(streamOptions);
|
||||
};
|
||||
|
||||
|
||||
@@ -15,7 +15,7 @@ export type LlmContentBlock =
|
||||
}
|
||||
| { type: "tool_result"; tool_use_id: string; content: string; is_error?: boolean }
|
||||
/** Opaque state required to continue a provider's stateless conversation. */
|
||||
| { type: "provider_item"; provider: "openai"; item: any };
|
||||
| { type: "provider_item"; provider: "openai" | "openai-compatible"; item: any };
|
||||
|
||||
export type LlmMessage = {
|
||||
role: "user" | "assistant";
|
||||
|
||||
+38
-9
@@ -1,4 +1,5 @@
|
||||
import { isLocalModelReady, LOCAL_MODEL } from "../native/llama";
|
||||
import { isLocalModelReady, isLocalModelSupported, LOCAL_MODEL } from "../native/local-llm";
|
||||
import { isVeniceApiBaseUrl } from "../native/openai-chat";
|
||||
|
||||
export const ASSISTANT_MODEL_VALUES = [
|
||||
"auto",
|
||||
@@ -11,17 +12,23 @@ export const ASSISTANT_MODEL_VALUES = [
|
||||
"sol",
|
||||
"qwen",
|
||||
"hermes",
|
||||
"openai-compatible",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* "qwen" is the on-phone model, whatever it is on this platform: Qwen3-4B on
|
||||
* Android, the Apple Intelligence system model on iOS (the ID predates iOS
|
||||
* support and stays for saved settings and conversations).
|
||||
*/
|
||||
export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number];
|
||||
/** Keep saved model IDs portable, while offering only supported choices. */
|
||||
export const ASSISTANT_MODEL_CHOICES: readonly AssistantModel[] =
|
||||
global.isIOS ? ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen") : ASSISTANT_MODEL_VALUES;
|
||||
isLocalModelSupported() ? ASSISTANT_MODEL_VALUES : ASSISTANT_MODEL_VALUES.filter(value => value !== "qwen");
|
||||
export function supportedAssistantModel(value: AssistantModel): AssistantModel {
|
||||
return global.isIOS && value === "qwen" ? "auto" : value;
|
||||
return value === "qwen" && !isLocalModelSupported() ? "auto" : value;
|
||||
}
|
||||
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes";
|
||||
export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes" | "openai-compatible";
|
||||
|
||||
export type AssistantApiKeys = {
|
||||
anthropic: string;
|
||||
@@ -29,6 +36,11 @@ export type AssistantApiKeys = {
|
||||
/** A Hermes Agent API server; only the voice assistant passes these. */
|
||||
hermesUrl?: string;
|
||||
hermesKey?: string;
|
||||
openAiCompatibleUrl?: string;
|
||||
openAiCompatibleKey?: string;
|
||||
openAiCompatibleModel?: string;
|
||||
/** Venice web search, applied only when the resolved root is Venice. */
|
||||
veniceWebSearch?: boolean;
|
||||
};
|
||||
|
||||
export type ResolvedAssistantModel = {
|
||||
@@ -37,8 +49,10 @@ export type ResolvedAssistantModel = {
|
||||
model: string;
|
||||
apiKey: string;
|
||||
effort?: "low" | "medium" | "high";
|
||||
/** API root for providers that are not fixed services (Hermes). */
|
||||
/** API root for Hermes or an OpenAI-compatible provider. */
|
||||
baseUrl?: string;
|
||||
/** Whether Venice web search is enabled for this interactive assistant turn. */
|
||||
webSearch?: boolean;
|
||||
};
|
||||
|
||||
type ModelDefinition = {
|
||||
@@ -58,11 +72,13 @@ const MODEL_DEFINITIONS: Record<Exclude<AssistantModel, "auto">, ModelDefinition
|
||||
luna: { label: "Luna", provider: "openai", model: "gpt-5.6-luna", effort: "low" },
|
||||
terra: { label: "Terra", provider: "openai", model: "gpt-5.6-terra", effort: "low" },
|
||||
sol: { label: "Sol", provider: "openai", model: "gpt-5.6-sol", effort: "low" },
|
||||
// Runs on the phone itself (llama.cpp); needs the downloaded model file,
|
||||
// not an API key. The default for users who haven't set any key.
|
||||
// Runs on the phone itself (Android: llama.cpp with the downloaded model
|
||||
// file; iOS: Apple Intelligence), not an API key. The default for users who
|
||||
// haven't set any key.
|
||||
qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id },
|
||||
// Your own Hermes Agent (its API server), with its own tools and memory.
|
||||
hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" },
|
||||
"openai-compatible": { label: "OpenAI-compatible", provider: "openai-compatible", model: "custom" },
|
||||
};
|
||||
|
||||
export function assistantModelLabel(value: AssistantModel): string {
|
||||
@@ -76,8 +92,8 @@ export function assistantModelProvider(value: AssistantModel): AssistantProvider
|
||||
/**
|
||||
* Resolve Auto and attach the credential needed by the chosen provider.
|
||||
* Auto intentionally prefers Terra over Sonnet when both keys are available,
|
||||
* and falls back to the on-phone model when no key is set but the model file
|
||||
* has been downloaded.
|
||||
* and falls back to the on-phone model when no key is set but that model is
|
||||
* ready (downloaded on Android; Apple Intelligence on and ready on iOS).
|
||||
*/
|
||||
export function resolveAssistantModel(
|
||||
selection: AssistantModel,
|
||||
@@ -106,6 +122,19 @@ export function resolveAssistantModel(
|
||||
if (!baseUrl) return null;
|
||||
return { selection, provider: "hermes", model: definition.model, apiKey: keys.hermesKey?.trim() ?? "", baseUrl };
|
||||
}
|
||||
if (definition.provider === "openai-compatible") {
|
||||
const baseUrl = keys.openAiCompatibleUrl?.trim() ?? "";
|
||||
const model = keys.openAiCompatibleModel?.trim() ?? "";
|
||||
if (!baseUrl || !model) return null;
|
||||
return {
|
||||
selection,
|
||||
provider: definition.provider,
|
||||
model,
|
||||
apiKey: keys.openAiCompatibleKey?.trim() ?? "",
|
||||
baseUrl,
|
||||
webSearch: isVeniceApiBaseUrl(baseUrl) && keys.veniceWebSearch === true,
|
||||
};
|
||||
}
|
||||
if (definition.provider === "local") {
|
||||
if (!isLocalModelReady()) return null;
|
||||
return {
|
||||
|
||||
@@ -95,14 +95,20 @@ export class AssistantSession {
|
||||
// original model. Preserve visible conversation when switching engines.
|
||||
if (this.config.kind !== config.kind ||
|
||||
(this.config.kind === "direct" && config.kind === "direct" &&
|
||||
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider))) {
|
||||
(this.config.llm.model !== config.llm.model || this.config.llm.provider !== config.llm.provider ||
|
||||
(this.config.llm.baseUrl ?? "") !== (config.llm.baseUrl ?? "")))) {
|
||||
this.rebuildTextHistory();
|
||||
}
|
||||
this.config = config;
|
||||
}
|
||||
|
||||
private engine(): string {
|
||||
return this.config.kind === "direct" ? `${this.config.llm.provider}:${this.config.llm.model}` : "external";
|
||||
if (this.config.kind !== "direct") return "external";
|
||||
const llm = this.config.llm;
|
||||
// Keep the legacy engine ID for built-in providers so upgrading does not
|
||||
// discard their opaque history. Custom endpoints need endpoint isolation.
|
||||
const endpoint = llm.provider === "openai-compatible" ? `:${llm.baseUrl ?? ""}` : "";
|
||||
return `${llm.provider}:${llm.model}${endpoint}`;
|
||||
}
|
||||
|
||||
private rebuildTextHistory(): void {
|
||||
@@ -192,6 +198,7 @@ export class AssistantSession {
|
||||
model: llm.model,
|
||||
effort: llm.effort,
|
||||
baseUrl: llm.baseUrl,
|
||||
webSearch: llm.webSearch,
|
||||
sessionId: this.hermesSessionId,
|
||||
system: llm.provider === "hermes" ? HERMES_SYSTEM_PROMPT
|
||||
: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx),
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
import { readUpcomingEventsAsync, type CalendarEvent } from "../native/calendar";
|
||||
import { hasCalendarPermission } from "../native/calendar-permissions";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
import { spotifyService } from "../native/spotify";
|
||||
import { dismissNotification, readActiveNotifications } from "../native/notification-icons";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { toolRegistry, type ToolRegistry, type ToolResult } from "./tool-registry";
|
||||
@@ -93,7 +94,11 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
proactive: true,
|
||||
},
|
||||
() => ok(describeNowPlaying()),
|
||||
async () => {
|
||||
// Spotify's state is polled only while something shows it.
|
||||
await mediaControllerBridge.refresh?.();
|
||||
return ok(describeNowPlaying());
|
||||
},
|
||||
);
|
||||
|
||||
registry.registerSystemTool(
|
||||
@@ -103,6 +108,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
await mediaControllerBridge.refresh?.();
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canPlayPause) return err("No media session can be controlled right now.");
|
||||
await mediaControllerBridge.playPause();
|
||||
@@ -117,6 +123,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
inputSchema: { type: "object", properties: {}, additionalProperties: false },
|
||||
},
|
||||
async () => {
|
||||
await mediaControllerBridge.refresh?.();
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.canSkipNext) return err("The current media session can't skip to the next track.");
|
||||
await mediaControllerBridge.skipNext();
|
||||
@@ -124,6 +131,45 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
},
|
||||
);
|
||||
|
||||
const spotify = spotifyService();
|
||||
if (spotify) {
|
||||
registry.register({
|
||||
spec: {
|
||||
name: "media.spotify_play",
|
||||
description:
|
||||
"Search Spotify and start playing the best match on the user's Spotify (this phone or another of their Spotify devices). Use for requests like \"play Daft Punk\" (artist), \"play the album Discovery\" (album) or \"play Harder Better Faster Stronger\" (track).",
|
||||
inputSchema: {
|
||||
type: "object",
|
||||
properties: {
|
||||
query: { type: "string", description: "What to search for, e.g. an artist, album, song or playlist name." },
|
||||
kind: {
|
||||
type: "string",
|
||||
enum: ["track", "album", "artist", "playlist"],
|
||||
description: "What kind of thing to play (default track).",
|
||||
},
|
||||
},
|
||||
required: ["query"],
|
||||
additionalProperties: false,
|
||||
},
|
||||
availability: "always",
|
||||
// Search, then play (with a device lookup when none is active).
|
||||
timeoutMs: 20_000,
|
||||
},
|
||||
handler: async (args) => {
|
||||
const query = String(args?.query ?? "").trim();
|
||||
if (!query) return err("media.spotify_play requires a query");
|
||||
const kind = ["track", "album", "artist", "playlist"].includes(args?.kind) ? args.kind : "track";
|
||||
try {
|
||||
return ok(`Playing ${await spotify.playSearch(query, kind)} on Spotify.`);
|
||||
} catch (error) {
|
||||
return err(String((error as Error)?.message ?? error));
|
||||
}
|
||||
},
|
||||
// Offered once signed in from the Music app menu.
|
||||
isAvailable: () => spotify.account() !== null,
|
||||
});
|
||||
}
|
||||
|
||||
// Android notification tools are still platform-specific.
|
||||
if (global.isIOS) return;
|
||||
|
||||
@@ -198,8 +244,8 @@ function formatEvent(event: CalendarEvent): string {
|
||||
return `- ${event.title || "(untitled)"} — ${when}${location}`;
|
||||
}
|
||||
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string }): string {
|
||||
const title = notification.title || "(no title)";
|
||||
function formatNotification(notification: { key: string; appName: string; title: string; text: string; customText?: string[] }): string {
|
||||
const title = notification.title || notification.customText?.join(" ") || "(no title)";
|
||||
const body = notification.text ? `: ${notification.text}` : "";
|
||||
return `- [${notification.appName}] ${title}${body} (key: ${notification.key})`;
|
||||
}
|
||||
|
||||
@@ -76,9 +76,9 @@ import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEdi
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence";
|
||||
import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers";
|
||||
import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle, type TranscriptionChoice } from "../ui/layers";
|
||||
import { type KeyboardInputSession } from "../ui/shell/keyboard-input";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings";
|
||||
import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings";
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import {
|
||||
getInstalledEvenHubAppById,
|
||||
@@ -126,12 +126,10 @@ export type DashboardSnapshot = {
|
||||
activeTextEditorToggleValue: boolean;
|
||||
activeTextEditorToggleVisible: boolean;
|
||||
/**
|
||||
* The keyboard dialog is up on the glasses (opened by the phone's keyboard
|
||||
* button): the phone shows its typing panel, with a send button per
|
||||
* destination in the glasses menu's row order.
|
||||
* The keyboard dialog is open (opened by the phone's keyboard button): the
|
||||
* phone shows its typing panel, driven by keyboardInputSession.
|
||||
*/
|
||||
keyboardInputActive: boolean;
|
||||
keyboardInputTargets: ReadonlyArray<{ id: string; label: string }>;
|
||||
evenAppConflictMessage: string;
|
||||
evenAppConflictWarningVisible: boolean;
|
||||
firmwareWarningMessage: string;
|
||||
@@ -1148,7 +1146,6 @@ class DashboardController {
|
||||
activeTextEditorToggleValue: this.activeTextEditorToggle?.setting.get() ?? false,
|
||||
activeTextEditorToggleVisible: this.activeTextEditorToggle !== null,
|
||||
keyboardInputActive: this.keyboardInput !== null,
|
||||
keyboardInputTargets: this.keyboardInput?.targets ?? [],
|
||||
evenAppConflictMessage: this.evenAppConflictMessage,
|
||||
evenAppConflictWarningVisible: this.evenAppConflictMessage.length > 0,
|
||||
firmwareWarningMessage: this.firmwareWarningMessage,
|
||||
@@ -1347,23 +1344,9 @@ class DashboardController {
|
||||
shell.startKeyboardInput();
|
||||
}
|
||||
|
||||
/** The phone's typing panel changed: mirror the text on the glasses. */
|
||||
setKeyboardInputText(text: string): void {
|
||||
shell.noteUserActivity();
|
||||
this.keyboardInput?.setText(text);
|
||||
}
|
||||
|
||||
/** Send to one destination, or (no id: the IME's send key) the one highlighted on the glasses. */
|
||||
sendKeyboardInput(targetId?: string): void {
|
||||
if (targetId) {
|
||||
this.keyboardInput?.sendTo(targetId);
|
||||
} else {
|
||||
this.keyboardInput?.send();
|
||||
}
|
||||
}
|
||||
|
||||
discardKeyboardInput(): void {
|
||||
this.keyboardInput?.discard();
|
||||
/** The open keyboard dialog, for the phone's typing panel (KeyboardInputViewModel). */
|
||||
get keyboardInputSession(): KeyboardInputSession | null {
|
||||
return this.keyboardInput;
|
||||
}
|
||||
|
||||
setActiveTextEditorToggleValue(value: boolean): void {
|
||||
@@ -2150,10 +2133,10 @@ class DashboardController {
|
||||
voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator));
|
||||
}
|
||||
|
||||
/** The provider a continuous-capture window needs (Translate's multilingual one), if any. */
|
||||
private multilingualProvider(): VoiceProvider | undefined {
|
||||
/** The provider and language a continuous-capture window needs (Translate's), if any. */
|
||||
private transcriptionChoice(): TranscriptionChoice | undefined {
|
||||
for (const options of this.continuousClients.values()) {
|
||||
if (options.multilingualProvider) return options.multilingualProvider;
|
||||
if (options.transcription) return options.transcription;
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
@@ -2178,20 +2161,21 @@ class DashboardController {
|
||||
* stands in for the G2 mic.
|
||||
*/
|
||||
private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) {
|
||||
// A window that needs a multilingual provider sets it for every capture
|
||||
// while it is open, so push-to-talk still shares its stream.
|
||||
const multilingual = this.multilingualProvider();
|
||||
// A window that needs its own provider and language sets them for every
|
||||
// capture while it is open, so push-to-talk still shares its stream.
|
||||
// Otherwise only Whisper server is told a language (its own setting).
|
||||
const choice = this.transcriptionChoice();
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get();
|
||||
return {
|
||||
communicator: communicator?.getNativeCommunicator() ?? null,
|
||||
usePhoneMic: communicator === null,
|
||||
provider: multilingual ?? voiceProviderSetting.get(),
|
||||
provider,
|
||||
language: choice ? choice.language : provider === "whisper-server" ? whisperServerLanguageSetting.get() : "",
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(),
|
||||
sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(),
|
||||
whisperServerModel: whisperServerModelSetting.get(),
|
||||
// Whisper detects the language itself when the speech may be in any.
|
||||
whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(),
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get(),
|
||||
saveRecording: saveVoiceRecordingsSetting.get(),
|
||||
// "My voice only" (Microphones app): verify command utterances
|
||||
|
||||
@@ -26,7 +26,7 @@ export type FirmwareInfo = {
|
||||
};
|
||||
|
||||
/** The Faceclaw firmware revision this build of the app needs. */
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 37;
|
||||
export const REQUIRED_FACECLAW_FIRMWARE_VERSION = 38;
|
||||
|
||||
const FACECLAW_PREFIX = "Faceclaw/";
|
||||
const LEGACY_PREFIX = "EVENCFW";
|
||||
@@ -36,7 +36,7 @@ const LEGACY_PREFIX = "EVENCFW";
|
||||
// unrecognized (its layout may differ from what our patch set targets).
|
||||
export const BASE_STOCK_VERSION = [2, 3, 0, 24];
|
||||
export const BASE_STOCK_VERSION_TEXT = BASE_STOCK_VERSION.join(".");
|
||||
export const VALIDATED_STOCK_VERSION = [2, 3, 2, 14];
|
||||
export const VALIDATED_STOCK_VERSION = [2, 4, 0, 24];
|
||||
export const VALIDATED_STOCK_VERSION_TEXT = VALIDATED_STOCK_VERSION.join(".");
|
||||
|
||||
export type FirmwareExtension =
|
||||
|
||||
@@ -19,12 +19,12 @@ export type FirmwarePatchSet = {
|
||||
export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"base": "g2_2.3.0.24.bin",
|
||||
"baseSha256": "187ccf2bcc5c17a212106e8a376745511e8289c4232b634a7ea94b9bf25a0979",
|
||||
"outputSha256": "90b932552e987fc554e6f748e50989503aaff0a93a3915702eacd96e3bd99e9b",
|
||||
"outputSha256": "4aac023f49ec6965186d9106e328a516f20a2915ab400d561f662f984023a432",
|
||||
"patches": [
|
||||
{
|
||||
"offset": 1045797,
|
||||
"old": "38b584b0",
|
||||
"new": "5cf377bb",
|
||||
"new": "5cf371bb",
|
||||
"desc": "ring receiver entry: timestamped unfiltered reports under framebuffer lease"
|
||||
},
|
||||
{
|
||||
@@ -36,43 +36,43 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 1437923,
|
||||
"old": "fef788ff",
|
||||
"new": "fdf292f8",
|
||||
"new": "fdf2a2f8",
|
||||
"desc": "cfw_ancs_event"
|
||||
},
|
||||
{
|
||||
"offset": 1437951,
|
||||
"old": "fef7e8fb",
|
||||
"new": "fcf2c0fd",
|
||||
"new": "fcf2d0fd",
|
||||
"desc": "cfw_ancs_open"
|
||||
},
|
||||
{
|
||||
"offset": 1437969,
|
||||
"old": "fef7e7fb",
|
||||
"new": "fcf2dbfd",
|
||||
"new": "fcf2ebfd",
|
||||
"desc": "cfw_ancs_close"
|
||||
},
|
||||
{
|
||||
"offset": 1388387,
|
||||
"old": "39694d00",
|
||||
"new": "05597d00",
|
||||
"new": "25597d00",
|
||||
"desc": "connection-bound ANCS EUS write callback"
|
||||
},
|
||||
{
|
||||
"offset": 971497,
|
||||
"old": "fff717f8",
|
||||
"new": "6ef3a7fc",
|
||||
"new": "6ef3a1fc",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 971845,
|
||||
"old": "fef769ff",
|
||||
"new": "6ef3f9fb",
|
||||
"new": "6ef3f3fb",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
"offset": 979579,
|
||||
"old": "fdf74ef8",
|
||||
"new": "6cf3defc",
|
||||
"new": "6cf3d8fc",
|
||||
"desc": "bl faceclaw_input_bridge_received (ring relay and ordered private transport before worker pool)"
|
||||
},
|
||||
{
|
||||
@@ -99,6 +99,12 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
"new": "a325",
|
||||
"desc": "_connectParamReq_impl: force requested mode to fast (0xa3); disables idle slow requests"
|
||||
},
|
||||
{
|
||||
"offset": 1058287,
|
||||
"old": "fff754fb",
|
||||
"new": "59f342fb",
|
||||
"desc": "bl ble_classify_link (only a <= 11.25 ms / latency-0 link is 'already fast')"
|
||||
},
|
||||
{
|
||||
"offset": 1130227,
|
||||
"old": "5ff43432",
|
||||
@@ -132,31 +138,31 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 975103,
|
||||
"old": "07f085fb",
|
||||
"new": "6df3acfc",
|
||||
"desc": "bl headup_gate (head-up sensor event -> EvenHub sys event 12 under lease)"
|
||||
"new": "6df3a6fc",
|
||||
"desc": "bl headup_gate (idle input gate: tap/long/release -> field-102 events, stock result kept)"
|
||||
},
|
||||
{
|
||||
"offset": 830549,
|
||||
"old": "19f03efb",
|
||||
"new": "90f35bfe",
|
||||
"desc": "bl gesture_ring_press_mode (observe ring touch-down, preserve stock dispatch)"
|
||||
"new": "90f37bfe",
|
||||
"desc": "bl gesture_ring_press_mode (ring touch-down 14, head up 12 / down 15; stock dispatch preserved)"
|
||||
},
|
||||
{
|
||||
"offset": 830795,
|
||||
"old": "1ff071ff",
|
||||
"new": "90f314fe",
|
||||
"new": "90f334fe",
|
||||
"desc": "bl gesture_press (owned event 9 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
"offset": 830887,
|
||||
"old": "1ff043ff",
|
||||
"new": "90f30efe",
|
||||
"new": "90f32efe",
|
||||
"desc": "bl gesture_short_long (Faceclaw lease gates event 11 vs stock Menu path)"
|
||||
},
|
||||
{
|
||||
"offset": 831763,
|
||||
"old": "1ff08dfd",
|
||||
"new": "90f344fc",
|
||||
"new": "90f364fc",
|
||||
"desc": "bl gesture_release (owned event 10 before stock menu gate)"
|
||||
},
|
||||
{
|
||||
@@ -210,43 +216,43 @@ export const CFW_PATCH_SET: FirmwarePatchSet = {
|
||||
{
|
||||
"offset": 4537963,
|
||||
"old": "",
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f084fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f22c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
|
||||
"new": "b0b581ea00039b0701d0034637e0c2b110f0030315d000f1010ca2f1010e0023cc5c9e45c45403f1010404d0634413f003032346f4d121440319121b102a14d302e00346102a10d3d1f800c0d1f804e0cc688d68103adc609d60c3f804e043f810cb0f2a01f11001eed8042a07d300bf51f8045b043a032a43f8045bf8d8002a08bfb0bd0139013b11f8015f013a03f8015ff9d1b0bd00bf884218bf002a00d1704710b50b1a38bf431a934207d288420ad281ea00039b0726d0034669e0fff79fffbde81040704781ea000300eb020c9b0701eb020e42d15fea8c732bd0a0f1010c00bf8b1813f801ec8418531e04f801ec04d0624412f003021a46f2d101eb030e00eb030c17e0830736d0a2f1010c00f1010e002300bfcc5c9c45c45403f1010404d0734413f003032346f4d1121b2144031922e01346042b0ed3aef10402acf1040152f80449043b032b41f80449f8d802f1040e01f1040c1a4653b1aef10101acf1010300bf11f80149013a03f80149f9d1bde8104070470346042a06d351f8044b043a032a43f8044bf8d8002af0d00139013b00bf11f8014f013a03f8014ff9d1e6e700bf80b5002a18bf10f0030301d186460ce000f1010c864600bf013a0ef8011b04d01cf003030cf1010cf6d1102a11d35ffa81fc4ff001330cfb03fc00bf103acef80cc0cef808c0cef804c04ef810cb0f2af4d8042a0bd35ffa81fc4ff001330cfb03fc00bf043a032a4ef804cbfad8002a08bf80bdaef10103013a03f8011ffbd180bd00bf2de9f04fc7b04ff29c5bc2f2290b4ff6ff754ff47d7400274ff0010847f67f7a7f261191cde90f3007e000bf48f2a351c0f24501bff32f8f8847dbf80090bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14068b0f57d7f00f096804ff47d7006f004fd002800f0cb814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160446080f80e80a0f8c4a180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001f49457ff452afbff35f8f4be8000121b15be8001f4945f8d047e74ff47d7800274ff0010a47f67f797f2607e000bf48f2a351c0f24501bff32f8f8847dbf80040bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f098804ff47d7006f042fc002800f009814ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80ea0a0f8c49180f8c361058445848584c584058545858585c585058645868586c586058745878587c5875be8001fa1427ff451afbff35f8f00bf4be8000121b15be8001fa142f8d045e7d0f8948242f6f76ac0f2440ab8f1000fbff35f8f4ad00af19a0240464ff0ff314ff0ff379047002840f090811198002861d010980778a7f1170002285bd811980c2840f07d81109c6078a178084300f07781e0780138022800f27281a279e379667a42ea0322237a207942ea034242ea0663a27ae67a6179192f42ea062200f0de81172f40f0f481002a40f05981fe2800f0b086812800f0a386002840f0b68651ea030040f04c8100f000bf04460020d047e8b104f5257151e8008fb8f1000f0cd1bff35f8f00bf41e80002002a00f0e58051e8008fb8f1000ff5d00021bff32f8f0029bff35f8f00f0d480804692e74ff0ff372ae147f6df40c0f24f00cdf83880804700274ff47d784ff0010947f67f744ff07f0a1397adf8507007e000bf48f2a351c0f24501bff32f8f8847dbf80060bff35f8fdbf800000321cff680710140b1f1005fbff35f8f0bd101684ef66c62ccf2ff02914204d14168b1f57d7f00f09f804ff47d7006f0f8fa002800f097804ff60c41cff6ff7100bf42182e3182f8f47382f8f57382f8f67382f8f77382f8f87382f8f97382f8fa7382f8fb7382f8fc7382f8fd7382f8fe7382f8ff7382f8007482f8017482f8027482f8037482f8047482f8057482f8067482f8077482f8087482f8097482f80a7482f80b7482f80c7482f80d7482f80e7482f80f7482f8107482f8117482f8127482f8137482f8147482f8157482f8167482f8177482f8187482f8197482f81a7482f81b7482f81c7482f81d7482f81e7482f81f7482f8207482f82174a0d14ef66c61ccf2ff010160c0f8048080f80e90a0f8c44180f8c3a1058445848584c584058545858585c585058645868586c586058745878587c5875be8001fb1427ff451afbff35f8f00bf4be8000159b15be8001fb142f8d045e70af5a9718847bce6012120e70020109f42f6f76addf83880c0f2440a9fb1119989b1397801f07f02b2f10b0418bf0124b2f11c0618bf01268a2924dc0b2926d01c2924d03ce00f9012a803f045ff4ff0ff3701249df85100002812a818bf4ff0ff3704f05ff8054644b99df8500028b94af21f30c0f247004c3080470f98002818bf056114e09c2901d08b2918d168b14af21f31c0f247010746884797f8681121b997f824133846109f49b14ff0ff370af5807140468847384647b0bde8f08f0f9012a8344003f007ff387800f07f061e2e05d11199384607f094fe0746b9e71d2e00f07e0018bf1a2819d1dbf800100322cff680720a40b2f1005fbff35f8f12d10a684ef66c63ccf2ff039a420bd14a68b2f57d7f04bf002281f8a12303e0a6f115010129e1d90f2e2cdd142e40f31082152e00f05d83162e00f069831f2e40f02c841198012840f0fc8438781f2840f0f88445f64d55c0f24605a847012841f0828140f24c71c0f2010105f5e2777944e02029220023b8470746002818bf4ff0ff3766e7092e00f3fe80052e00f03e82072e40f0028400274ff0010947f67f764ff07f080d943ae000294ff0ff3708bf002b7ff47caf1328bff479af102a4ff0000138bf01211228b2fa82f218bf510901297ff46caf0Line truncated
|
||||
"desc": "append injected blobs to main-app payload"
|
||||
},
|
||||
{
|
||||
"offset": 779123,
|
||||
"old": "805a3900",
|
||||
"new": "48703a00",
|
||||
"new": "68703a00",
|
||||
"desc": "main-app subheader payload size (ps)"
|
||||
},
|
||||
{
|
||||
"offset": 152,
|
||||
"old": "005b3900",
|
||||
"new": "c8703a00",
|
||||
"new": "e8703a00",
|
||||
"desc": "main-app TOC entry size (ps + 128)"
|
||||
},
|
||||
{
|
||||
"offset": 779243,
|
||||
"old": "805a3904",
|
||||
"new": "48703a04",
|
||||
"new": "68703a04",
|
||||
"desc": "main-app preamble length (low 24 bits)"
|
||||
},
|
||||
{
|
||||
"offset": 779247,
|
||||
"old": "effdbcea",
|
||||
"new": "6a8cfe55",
|
||||
"new": "0b0c48b0",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin preamble crc32"
|
||||
},
|
||||
{
|
||||
"offset": 156,
|
||||
"old": "c0c6322a",
|
||||
"new": "7f295d98",
|
||||
"new": "52937d21",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (TOC)"
|
||||
},
|
||||
{
|
||||
"offset": 779127,
|
||||
"old": "c0c6322a",
|
||||
"new": "7f295d98",
|
||||
"new": "52937d21",
|
||||
"desc": "[5] ota/s200_firmware_ota.bin component crc32c (subheader)"
|
||||
}
|
||||
]
|
||||
|
||||
@@ -4,7 +4,7 @@ import { acceptInput, resetRingInputFilter } from "../ui/input-monitor";
|
||||
import { bindIosNotifications, iosNotificationsChanged, onIosNotificationPopup, readActiveNotifications } from '../native/notification-icons.ios'
|
||||
import { shouldShowNotificationOnGlasses } from '../native/notification-sources'
|
||||
import { launcherEntries } from '../apps/launcher'
|
||||
import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
||||
import { discardZeroedEvenHubPackage, getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps'
|
||||
import { isInstalledPackagePresent } from '../apps/evenhub/updates'
|
||||
import { openEvenHubStoreForPackage } from '../apps/evenhub'
|
||||
import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager'
|
||||
@@ -14,6 +14,7 @@ import { registerNavigateTools } from '../assistant/navigate-tools'
|
||||
import { registerRoamTools } from '../assistant/roam-tools'
|
||||
import { IosNavigationSensors } from '../native/ios-navigation-sensors'
|
||||
import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios'
|
||||
import { bindBandwidthBenchmarkSession } from '../native/bandwidth-benchmark.ios'
|
||||
import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core'
|
||||
import { iosVoiceInput } from '../native/ios-voice-input'
|
||||
import { voiceActivity } from '../ui/shell/voice-activity'
|
||||
@@ -31,6 +32,7 @@ import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmwar
|
||||
import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy'
|
||||
import { deviceAddressError } from './ios-peripheral-identity'
|
||||
import { createLauncherWindow, LAUNCHER_SURFACE_ID } from '../apps/launcher/launcher-app'
|
||||
import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from '../apps/notifications/notifications-app'
|
||||
import { ALL_APPS } from '../apps/all-apps'
|
||||
import type { AppContext, AppDefinition, AppLaunchParams } from '../apps/app-definition'
|
||||
import { WorkerAppHost } from '../ui/shell/worker-window'
|
||||
@@ -65,7 +67,7 @@ const FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS = 2000
|
||||
|
||||
export type SessionPhase = 'disconnected' | 'connecting' | 'connected' | 'retrying' | 'disconnecting' | 'error'
|
||||
export type SessionState = { phase: SessionPhase; status: string; battery: number | null; charging: boolean | null;
|
||||
ring: boolean; frames: number; capabilities: string; leftVersion: string; rightVersion: string;
|
||||
ring: boolean; ringBattery: number | null; ringCharging: boolean | null; frames: number; capabilities: string; leftVersion: string; rightVersion: string;
|
||||
/** Why the connected firmware can't run Faceclaw (firmware-compat.ts), or '' when it can or is unknown. */
|
||||
firmwareWarning: string }
|
||||
|
||||
@@ -84,7 +86,7 @@ export class IosPreviewController {
|
||||
private communicator: FaceclawCommunicatorBridge | null = null
|
||||
private readonly startedCommunicators = new WeakSet<FaceclawCommunicatorBridge>()
|
||||
state: SessionState = { phase: 'disconnected', status: 'Preview only', battery: null, charging: null, ring: false,
|
||||
frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' }
|
||||
ringBattery: null, ringCharging: null, frames: 0, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '' }
|
||||
private notifications: AncsClient | null = null
|
||||
private readonly sessionOffs: (() => void)[] = []
|
||||
private stopping: Promise<void> | null = null
|
||||
@@ -131,6 +133,8 @@ export class IosPreviewController {
|
||||
private clockMinute = -1
|
||||
private shellRenderInProgress = false
|
||||
private shellRenderQueued = false
|
||||
/** The shell woke; the next shell render unblanks the compositor after submitting. */
|
||||
private unblankAfterShellRender = false
|
||||
private nextFrameId = 1
|
||||
private nextWorkerFrame = 0
|
||||
private inputQueue: Promise<void> = Promise.resolve()
|
||||
@@ -149,11 +153,12 @@ export class IosPreviewController {
|
||||
private logLines: string[] = []
|
||||
private logTimer: ReturnType<typeof setTimeout> | null = null
|
||||
// Windows that want continuous capture, and what each needs of it (a
|
||||
// recording, a multilingual provider). They share one capture, which runs
|
||||
// recording, a provider and language). They share one capture, which runs
|
||||
// while any of them wants it.
|
||||
private readonly continuousClients = new Map<string, ContinuousCaptureOptions>()
|
||||
// transcriptionKey of the running continuous capture's provider; one that no
|
||||
// longer matches what its windows need is drained and restarted.
|
||||
// transcriptionKey of the running continuous capture (provider and
|
||||
// language); one that no longer matches what its windows need is drained
|
||||
// and restarted.
|
||||
private continuousTranscription = ''
|
||||
// Continuous capture should (re)start once audio is available: a window
|
||||
// asked, the glasses connected or disconnected, or the glasses lock or the
|
||||
@@ -198,11 +203,16 @@ export class IosPreviewController {
|
||||
send: () => { if (this.active && !this.glassesLocked) session.send() },
|
||||
sendTo: id => { if (this.active && !this.glassesLocked) session.sendTo(id) },
|
||||
discard: () => session.discard(),
|
||||
get visibility() { return session.visibility },
|
||||
setVisibility: visibility => { if (this.active && !this.glassesLocked) session.setVisibility(visibility) },
|
||||
get history() { return session.history },
|
||||
} : null),
|
||||
prepareVoiceCapture: () => this.prepareVoiceCapture(),
|
||||
getScreenTimeoutMs: () => null,
|
||||
requestShellRender: () => this.requestShellRender(),
|
||||
onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() },
|
||||
openNotificationsWindow: () =>
|
||||
this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow),
|
||||
onSwitcherBarHeightChanged: () => {
|
||||
this.relayout()
|
||||
shell.foregroundWindow()?.requestRender()
|
||||
@@ -210,7 +220,12 @@ export class IosPreviewController {
|
||||
},
|
||||
onScreenStateChanged: on => {
|
||||
if (on) this.glance.dismiss()
|
||||
void this.display?.setScreenBlanked(!on).catch(error => this.fail(error))
|
||||
// A wake unblanks only once the next shell render has staged an awake
|
||||
// scene: unblanking composites the retained one, painted while asleep
|
||||
// with no chrome and depth 0, and that first frame is the one the
|
||||
// brightness fade-in starts on.
|
||||
this.unblankAfterShellRender = on
|
||||
if (!on) void this.display?.setScreenBlanked(true).catch(error => this.fail(error))
|
||||
this.requestShellRender()
|
||||
},
|
||||
})
|
||||
@@ -442,6 +457,7 @@ export class IosPreviewController {
|
||||
iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.',
|
||||
'Glasses locked. Transcription resumes when you unlock your phone.')
|
||||
} else this.syncContinuousCapture()
|
||||
this.syncRawMicSession()
|
||||
void (async () => {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
@@ -519,10 +535,19 @@ export class IosPreviewController {
|
||||
await this.displayReady
|
||||
const display = this.display
|
||||
if (!display) return
|
||||
const startedAt = Date.now()
|
||||
const bytes = shell.paintScene()
|
||||
// Taken with the paint, so a render that painted before the wake can't unblank.
|
||||
const unblank = this.unblankAfterShellRender
|
||||
this.unblankAfterShellRender = false
|
||||
const frameId = this.allocateFrameId()
|
||||
await display.submitShellScene(bytes, Date.now() - startedAt, frameId)
|
||||
try {
|
||||
const startedAt = Date.now()
|
||||
const bytes = shell.paintScene()
|
||||
await display.submitShellScene(bytes, Date.now() - startedAt, frameId)
|
||||
} finally {
|
||||
// Even if the render failed, so the screen doesn't stay dark; not if it
|
||||
// went back to sleep meanwhile (that sleep re-blanked).
|
||||
if (unblank && shell.isScreenOn()) await display.setScreenBlanked(false)
|
||||
}
|
||||
// Backpressure: the next shell render waits for this one to reach the
|
||||
// glasses (the preview target resolves immediately; nothing transmits).
|
||||
await display.waitForFrameFinished(frameId, FRAME_TRANSMIT_BACKPRESSURE_TIMEOUT_MS)
|
||||
@@ -601,6 +626,7 @@ export class IosPreviewController {
|
||||
const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy)
|
||||
const x = Math.max(0, Math.min(639, Math.floor(nx * 640)))
|
||||
const y = Math.max(0, Math.min(479, Math.floor(ny * 480)))
|
||||
if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) { this.requestShellRender(); return }
|
||||
const window = shell.foregroundWindow()
|
||||
if (!shell.hasOverlay() && sidebarStripVisible(shell.getFocus(), window?.appId)
|
||||
&& isOnSwitcherEdge(x, y, window?.heightMode ?? 'min', window?.appId)) {
|
||||
@@ -624,9 +650,15 @@ export class IosPreviewController {
|
||||
const host = ALL_APPS.find(candidate => candidate.appId === 'evenhub')
|
||||
if (installed && host) {
|
||||
try {
|
||||
if (isInstalledPackagePresent(installed.packageId))
|
||||
await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed)
|
||||
else await openEvenHubStoreForPackage(this.buildAppContext(host), installed)
|
||||
if (isInstalledPackagePresent(installed.packageId)) {
|
||||
try { await launchInstalledPackage(this.buildAppContext({ ...host, appId: id }), installed); return }
|
||||
catch (error) {
|
||||
// Checked only after a failure: it reads the whole package again.
|
||||
if (!discardZeroedEvenHubPackage(installed.packageId)) throw error
|
||||
console.log(`[ios-preview] Discarded zeroed package for ${installed.packageId}; opening store page`)
|
||||
}
|
||||
}
|
||||
await openEvenHubStoreForPackage(this.buildAppContext(host), installed)
|
||||
}
|
||||
catch (error) { this.fail(error) }
|
||||
}
|
||||
@@ -656,6 +688,7 @@ export class IosPreviewController {
|
||||
appId: app.appId, apps: ALL_APPS, actions: this.actions,
|
||||
launchApp: (id, params) => this.launchApp(id, params), uninstallApp: id => this.uninstallApp(id),
|
||||
launchInProcessApp: (id, surface, create) => this.launchInProcessApp(id, surface, create),
|
||||
openInProcessAppInBackground: (id, surface, create) => this.openInProcessAppInBackground(id, surface, create),
|
||||
ensureWorkerHost: create => this.ensureWorkerHost(app.appId, create),
|
||||
submitWindowFrame: (id, planes) => this.submit(id, planes),
|
||||
setWindowSurfaceVisible: (id, visible) => this.setWindowSurfaceVisible(id, visible),
|
||||
@@ -667,14 +700,44 @@ export class IosPreviewController {
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): Promise<void> {
|
||||
const existing = this.inProcessApps.get(windowId)
|
||||
if (existing) { shell.focusWindow(windowId); existing.requestRender(); this.requestShellRender(); return }
|
||||
// Apps can request a render while their factory runs (Nightscout's tray
|
||||
// subscription does). The surface only exists once the factory returns.
|
||||
// The explicit render below supplies a fresh frame after configuration.
|
||||
let surfaceReady = false
|
||||
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
||||
await this.configureWindow(app.window)
|
||||
surface.ready = true
|
||||
shell.registerWindow(app.window); shell.focusWindow(windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
}
|
||||
/**
|
||||
* An in-process singleton app's window, opened in the background (neither
|
||||
* foregrounded nor focused) unless it is open already, for the shell to
|
||||
* bring forward itself: the Notifications window under a notification
|
||||
* selected in the switcher, or an app widget's window under the widget.
|
||||
* Display calls run in order and frames wait on displayReady before
|
||||
* queueing, so the surface configure queued here lands ahead of any frame
|
||||
* and of the shell foregrounding the window, which it may do at once.
|
||||
*/
|
||||
private openInProcessAppInBackground(windowId: string, surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): { window: InProcessWindow; opened: boolean } {
|
||||
const existing = this.inProcessApps.get(windowId)
|
||||
if (existing) return { window: existing, opened: false }
|
||||
const { app, surface } = this.createInProcessApp(windowId, surfaceId, create)
|
||||
void this.configureWindow(app.window).catch(error => this.fail(error))
|
||||
surface.ready = true
|
||||
shell.registerWindow(app.window)
|
||||
return { window: app, opened: true }
|
||||
}
|
||||
/**
|
||||
* Create an in-process singleton app's window (not yet registered with the
|
||||
* shell). Its frames reach the compositor only while `surface.ready`: apps
|
||||
* can request a render while their factory runs (Nightscout's tray
|
||||
* subscription does), before the caller has configured the surface.
|
||||
*/
|
||||
private createInProcessApp(windowId: string, surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow): { app: InProcessWindow; surface: { ready: boolean } } {
|
||||
const surface = { ready: false }
|
||||
const app = create({
|
||||
actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(),
|
||||
actions: this.actions, submitFrame: planes => surface.ready ? this.submit(surfaceId, planes, () => surface.ready) : Promise.resolve(),
|
||||
setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible),
|
||||
removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) },
|
||||
removeSurface: () => { surface.ready = false; this.removeWindowSurface(surfaceId) },
|
||||
reconfigureSurface: () => {
|
||||
const window = shell.getWindows().find(w => w.windowId === windowId)
|
||||
if (window) void this.configureWindow(window).catch(error => this.fail(error))
|
||||
@@ -683,10 +746,7 @@ export class IosPreviewController {
|
||||
onClosed: () => { this.inProcessApps.delete(windowId) },
|
||||
})
|
||||
this.inProcessApps.set(windowId, app)
|
||||
await this.configureWindow(app.window)
|
||||
surfaceReady = true
|
||||
shell.registerWindow(app.window); shell.focusWindow(windowId)
|
||||
app.requestRender(); this.requestShellRender()
|
||||
return { app, surface }
|
||||
}
|
||||
private ensureWorkerHost(appId: string, create: () => Worker): WorkerAppHost {
|
||||
const existing = this.appHosts.get(appId)
|
||||
@@ -738,14 +798,22 @@ export class IosPreviewController {
|
||||
this.pushBrightness()
|
||||
this.maybeLaunchOnboarding(this.state)
|
||||
if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null
|
||||
// Unbound first, so a speech capture ending below doesn't hand the mic back to the dead session.
|
||||
this.syncRawMicSession()
|
||||
if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded()
|
||||
this.syncContinuousSession()
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||
this.pushBatteryLevels()
|
||||
this.syncPresence()
|
||||
this.syncRuntime()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
}
|
||||
/** The ring level arrives through the glasses, so like Android it is shown only while connected. */
|
||||
private pushBatteryLevels(): void {
|
||||
const connected = this.state.phase === 'connected'
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging,
|
||||
ring: connected ? this.state.ringBattery : null, ringCharging: connected ? this.state.ringCharging : null })
|
||||
}
|
||||
async connect(): Promise<void> {
|
||||
if (!this.active || this.connecting || this.stopping) return
|
||||
if (this.communicator && ['connecting', 'connected', 'retrying', 'disconnecting'].includes(this.state.phase)) return
|
||||
@@ -754,7 +822,8 @@ export class IosPreviewController {
|
||||
this.connecting = true
|
||||
this.sessionCreated = true
|
||||
try {
|
||||
this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false }
|
||||
this.state = { ...this.state, capabilities: '', leftVersion: '', rightVersion: '', firmwareWarning: '', battery: null, charging: null, ring: false,
|
||||
ringBattery: null, ringCharging: null }
|
||||
this.firmwareCompatible = false
|
||||
this.update('connecting', 'Finding configured devices…')
|
||||
let identifiers
|
||||
@@ -796,16 +865,21 @@ export class IosPreviewController {
|
||||
this.notifications = notifications
|
||||
bindIosNotifications(notifications)
|
||||
bindCompassSession(communicator)
|
||||
bindBandwidthBenchmarkSession(communicator)
|
||||
communicator.setRequiresAncs(true)
|
||||
this.sessionOffs.push(
|
||||
communicator.onStateChange(state => this.handleSessionState(communicator, state)),
|
||||
communicator.onRingEvent(input => this.handleRingEvent(input)),
|
||||
communicator.onBatteryState(state => {
|
||||
if (this.communicator !== communicator) return
|
||||
// Unlike the headset level, an unavailable ring reading (-1: ring away or old
|
||||
// firmware) clears the indicator, as on Android.
|
||||
const ringBattery = state.ringBattery ?? -1, hasRingBattery = Number.isInteger(ringBattery) && ringBattery >= 0 && ringBattery <= 100
|
||||
this.state = { ...this.state, battery: state.battery >= 0 && state.battery <= 100 ? state.battery : this.state.battery,
|
||||
charging: state.chargingStatus >= 0 ? state.chargingStatus > 0 : this.state.charging,
|
||||
ring: (state.ringBattery ?? -1) >= 0 || this.state.ring }
|
||||
shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging })
|
||||
ring: hasRingBattery || this.state.ring,
|
||||
ringBattery: hasRingBattery ? ringBattery : null, ringCharging: hasRingBattery ? state.ringChargingStatus === 1 : null }
|
||||
this.pushBatteryLevels()
|
||||
if (this.active) this.onConnectionState({ ...this.state })
|
||||
this.requestShellRender()
|
||||
}),
|
||||
@@ -934,6 +1008,9 @@ export class IosPreviewController {
|
||||
for (const off of this.sessionOffs.splice(0)) off()
|
||||
this.notifications = null
|
||||
this.communicator = null
|
||||
bindBandwidthBenchmarkSession(null)
|
||||
// Releases an EvenHub app's mic before the session closes.
|
||||
this.syncRawMicSession()
|
||||
this.glance.reset()
|
||||
await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`))
|
||||
}
|
||||
@@ -977,7 +1054,7 @@ export class IosPreviewController {
|
||||
this.continuousClients.set(client, options)
|
||||
if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) {
|
||||
this.continuousClients.delete(client)
|
||||
this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
this.onError('On-device transcription for Transcribe needs iOS 26 or later; before that, choose a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).')
|
||||
return
|
||||
}
|
||||
this.syncContinuousRecording()
|
||||
@@ -1047,6 +1124,15 @@ export class IosPreviewController {
|
||||
}
|
||||
if (current()) this.syncContinuousClaim()
|
||||
}
|
||||
/**
|
||||
* EvenHub apps' mic (the bridge's raw tap) listens through a connected,
|
||||
* unlocked glasses session, as speech capture does; the EvenHub mic router
|
||||
* decides which app hears it and when.
|
||||
*/
|
||||
private syncRawMicSession(): void {
|
||||
const ready = this.state.phase === 'connected' && !this.glassesLocked
|
||||
iosVoiceInput.setRawSession(ready ? this.communicator : null, message => this.logBluetooth(message))
|
||||
}
|
||||
/** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */
|
||||
private syncContinuousClaim(): void {
|
||||
voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER)
|
||||
@@ -1058,15 +1144,17 @@ export class IosPreviewController {
|
||||
}
|
||||
/**
|
||||
* The transcription provider setting and the keys the cloud ones need. A
|
||||
* window that needs a multilingual provider (Translate) sets it for every
|
||||
* capture while it is open, with Whisper left to detect the language.
|
||||
* window that needs its own provider and language (Translate) sets them for
|
||||
* every capture while it is open; otherwise only Whisper server is told a
|
||||
* language (its own setting).
|
||||
*/
|
||||
private voiceProvider(): VoiceProviderSelection {
|
||||
const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider
|
||||
return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(),
|
||||
openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
const choice = [...this.continuousClients.values()].find(options => options.transcription)?.transcription
|
||||
const provider = choice?.provider ?? voiceProviderSetting.get()
|
||||
return { provider, language: choice ? choice.language : provider === 'whisper-server' ? whisperServerLanguageSetting.get() : '',
|
||||
elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(),
|
||||
whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(),
|
||||
whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
whisperServerApiKey: whisperServerApiKeySetting.get() }
|
||||
}
|
||||
private logBluetooth(message: string): void {
|
||||
const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}`
|
||||
|
||||
@@ -25,6 +25,7 @@ import { EVENHUB_RUNTIME_FONT_FILENAME, isEvenHubFontAsset } from "../g2/firmwar
|
||||
import { lvglMetrics, lvglGlyph } from "../native/lvgl-font";
|
||||
import { GrayImage, grayToNibble, type FwTextFont, type PlacedFwText, type PlacedFwTextGlyph } from "./image";
|
||||
import { TtfFont } from "./ttf-font";
|
||||
import { decodeBase64 } from "../util/base64";
|
||||
|
||||
const CJK_ASSET_PATH = "fonts/source-han-sans/SourceHanSansSC-Light-20.lvgl.bin";
|
||||
|
||||
@@ -55,25 +56,6 @@ type CjkGlyph = {
|
||||
bitmap: Uint8Array;
|
||||
};
|
||||
|
||||
const BASE64_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
|
||||
|
||||
function base64Decode(text: string): Uint8Array {
|
||||
const clean = text.replace(/[^A-Za-z0-9+/]/g, "");
|
||||
const out = new Uint8Array(Math.floor((clean.length * 3) / 4));
|
||||
let bits = 0;
|
||||
let bitCount = 0;
|
||||
let outIndex = 0;
|
||||
for (let i = 0; i < clean.length; i++) {
|
||||
bits = (bits << 6) | BASE64_ALPHABET.indexOf(clean[i]!);
|
||||
bitCount += 6;
|
||||
if (bitCount >= 8) {
|
||||
bitCount -= 8;
|
||||
out[outIndex++] = (bits >> bitCount) & 0xff;
|
||||
}
|
||||
}
|
||||
return out.subarray(0, outIndex);
|
||||
}
|
||||
|
||||
/**
|
||||
* Codepoint ranges whose glyphs the font asset merged from the flash CJK
|
||||
* font with a baseline-shifted ofsY (see scripts/build-evenhub-font.mjs).
|
||||
@@ -144,7 +126,7 @@ export class EvenHubFont implements FwTextFont {
|
||||
this.useTtfFallback = useTtfFallback;
|
||||
this.lineHeight = asset.lineHeight;
|
||||
this.baseline = asset.baseline;
|
||||
this.bitmap = base64Decode(asset.bitmapBase64);
|
||||
this.bitmap = decodeBase64(asset.bitmapBase64);
|
||||
this.glyphs = new Map();
|
||||
for (const key of Object.keys(asset.glyphs)) {
|
||||
this.glyphs.set(Number(key), asset.glyphs[key]!);
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Generated by scripts/generate-pebble-icons.cjs; do not edit by hand.
|
||||
//
|
||||
// Pebble notification icons from https://github.com/pebble-dev/iconography
|
||||
// (commit 16a9ea42ea7ac079127af2ecc54f453dfbc314c8): the original Pebble Technology icons and
|
||||
// Rebble community additions, licensed under the Apache License 2.0 (see
|
||||
// app/graphics/pebble-icons.LICENSE). Modified: rasterized at 25x25 and
|
||||
// inverted for a black display (Pebble draws them black on white).
|
||||
//
|
||||
// Each icon is SIZE x SIZE pixels at 2 bits each (0, 1, 2 = coverage 0,
|
||||
// 128, 255), four pixels to a byte with the first in the high bits,
|
||||
// row-major, base64-encoded.
|
||||
|
||||
export const PEBBLE_ICON_SIZE = 25;
|
||||
|
||||
export const PEBBLE_ICON_DATA = {
|
||||
generic: "AAACgAAAAAAAoAAAAAAAKAAAAAACqqgAAAACqqqAAAACoACoAAACoAAKgAACoAAAqAAAoAAACgAAKAAAAoAACgAAAKAAAoAAACgAAKAAAAoAACgAAAKAACoAAACoACoAAAAKgCqqqAAAqAqqqgAACgKAAAAAAoCgAAAAAKAqqqqqqqgKqqqqqqoAAAAABkAAAAAAAKAAAAAAACgAAA==", // pebbleos/Pebble_25x25_Generic_notification.svg
|
||||
activity: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAFAAAACkAGkAAAAqkKqQAAAKaqhqAAACgagqpAAAoAQaakAAKAABGqkACgAAClqkAoAAAUZqQKqpAAKBqSlWqqqgBooAFVWkAKKAAAAZACilVVValVpqqqqqqqqQAAAAUAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Activity.svg
|
||||
alarmClock: "AAAAAAAAAAAAAAAAAAoAAACgAAqCqqgqAAqCqqqCoAqCoACoKgKCoAAKgoACoAoAqAACoAKACoACoACgAKgAoAAoAAoAKAAKAAKACgAKgACgAoAKgAAoAKAKgAAKACgKgAACgAqCgAACoACoAAACoAAKgAACoAACqAACqAABmoACpkAAZKqqoZAAKAqqoCgACgAAAAoAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Alarm_clock.svg
|
||||
appStore: "AAAAAAAAAAACqoAAAAACqqgAAAACooqAAAACoKCoAAAAoCgKAAAAKAoCgAAACgKgoAAAAoAqKAAAAKgCqgAAAAqAKgAAAACqqgAAAAAKqgAABAAAAAAARpQAAAAFpGqqqqqqpAGqqqqqkAAoAAAAoAAKAAAAKAACgAAACgAAoAAAAoAAKAAAAKAACqqqqqgAAqqqqqoAAAAAAAAAAA==", // icons/25px_Appstore.svg
|
||||
calendar: "AAKAAKAAAACgACgAAKqqqqqqgCqqqqqqoAoCgACgKAKAoAAoCgCiqoCqooAoqqAqqKAKAAAAACgCgAAAAAoAoAAAAAKAKAAACgCgCgAABoAoAoAABoAKAKAKBoACgCgCkpAAoAoAKpAAKAKABpAACgCgAKAAAoAoAAAAAKAKAAAAACgCgAAAAAoAqqqqqqqAKqqqqqqgAAAAAAAAAA==", // pebbleos/Pebble_25x25_Calendar.svg
|
||||
car: "AAAAAAAAAAqqqqpAAAKqqqqgAAGQAAAoAACgAAAGQAAoAAAAoAAZAAAAKQAKAAAABkCqgAAAAKqqqqqqqqqgqqqqqqqAKAKACgCgCgCgAoAoAqqoAKqqAKqqACqqgCgAKqAAoAoACqgAKAKAAAAACgCqqqqqqoAqqqqqqqAAoKACgoAAKCgAoKAACgoAKCgCqqqqqqqqqqqqqqqqgA==", // pebbleos/Pebble_25x25_Car_rental.svg
|
||||
contacts: "AAAAAAAAACqqqqqqAAqqqqqqgAKAAAAAoAqqAAAAKAKqgAAACgAKAAAAAoACgAAAAKAAoAApACgAKAAaQAoACgAGkAKAKqgAAACgCqoAGAAoACgAGoAKAAoAGqgCgAKACqpAoACgAAAAKAAoAAAACgCqoAAAAoAqqAAAAKAAoAAAACgAKAAAAAoACqqqqqqAAqqqqqqgAAAAAAAAAA==", // icons/25px_Contacts_book.svg
|
||||
dinner: "AAAAAAAAAAAqqqAAAAAqqqoAAAKqAAKqAAKqAAAqoAKigAAKKgKgoAACgqCgKgACoCgoAqACoAoKACgAoAKCoAAAAAKgKgAAAAKgAqoAACqgACqAAAqgAACgAAKAAAAooCigAAAKKAooAAACigKKAAAAqqqqgAAAKqqqoAAACgAAKAAAAoAACgAAAKqqqoAAACqqqqAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Dinner_reservation.svg
|
||||
email: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqqoqqqqqqqqKqAAAACqiioAAACoooKgAACoKKAqAACoCigCoACoAooAKgCoAKKAAqCoACigACqoAAooAoKoKAKKAqAoCoCigqAAAKgooqAAAAqKKqAAAACqiqAAAAAKoqqqqqqqqKqqqqqqqgAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Generic_e-mail.svg
|
||||
flight: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaAAAAAABqoAAAAAWpagqoABqkCqKqQGqQCqpkpalAAoKZGqQAAACmQpAAAAAooAAAqoAKKAAAKqAChkAUAGQCoZAKACgCoGQCgAoCoAqqoAaqoAKqqAKqoAAACgCgAAAAAoBkAAAAAKqoAAAAACqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Scheduled_flight.svg
|
||||
glucose: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAACgCgAAAAooAoAAAAqKAKAAAAqCgCgAoAqAoAoAqgqAKAKAqqqACgCgqCqAAoAoqAKAAKAKKAAAACgCgAAAAAoAoAAAAAKAKKqoAACgCiqqAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Glucose_monitor.svg
|
||||
heart: "AAAAAAAAAAKAACgAAAKoACqAAAKqgCqoAAKgqCoKgAKgCqoAqAKgAKoACoCgAAoAAKAoAAAAACgKAAAAAAoCgAAAAAKAoAAAAACgKAAAAAAoCoAAAAAqAKgAAAAqAAqAAAAqAACoAAAqAAAKgAAqAAAAqAAqAAAACoAqAAAAAKgqAAAAAAqqAAAAAACqAAAAAAAKAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Heart.svg
|
||||
incomingCall: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqgAACgAAAoAAAoqqqKAAAKKqqigAACigAooAAAooAKKCgAKKACigqACigAooCoAoqqqKAKgKKqqigAqCgAAAoACgoqiqKAAAKKoqiqqgCgAAAqqoAoqiqKAAAKKoqigAKCgAAAoAKgoAAAKAKgKAAACgKgCgAAAoKgAqqqqqCgAKqqqqgA==", // pebbleos/Pebble_25x25_Incoming_call.svg
|
||||
location: "AAACqqqoAAAAqqqqAAAAKAAqAAAACgAqAAAAAoAqAAAAAKAKgAAAACgAqAAAAAoACoAAAAKqqqgAAACqqqoAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAACqqqqAAACqqqqoAACoAoAKgACoAKAAqACoACgACoCoAAAAAKiqqqqqqqqqqqqqqqqgA==", // pebbleos/Pebble_25x25_Location.svg
|
||||
microphone: "AAAqqoAAAAAqqqgAAAAqCgqAAAAKBpSgAAACqqqoAAAAqmmqAAAAKAoCgAAACqqqoAAAAqqqqAAAAKAoCgAAACgKAoAAAAqmmqAAAAKqqqgAAACgaQoAAAAqCgqAAAACqqqAAACgKqqAoAAqAAAAqAACoAAAqAAAKqqqqAAAAqqqqAAAAAAoAAAAAAAKAAAAAAAqqAAAAAAKqgAAAA==", // icons/25px_Microphone.svg
|
||||
missedCall: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqiqoCgAAAoqqAoqqqKKoAKKqqiiqgCigAoooqAooAKKKCoKKACigAKiigAooACooqqqKACoKKqqigCoCgAAAoCoAoqiqKCoAKKoqiioACgAAAqoAAoqiqKoAAKKoqigAACgAAAoAAAoqiqKAAAKKoqigAACgAAAoAAAqqqqqAAAKqqqqgA==", // pebbleos/Pebble_25x25_Missed_call.svg
|
||||
movie: "AAAAAAAAAAAAAAAAAKqqqqqqACqqqqqqgAoqACoAoAKCoAKgKACgKgAqCgAoAqACooAKqqqqqqACqqqqqqgAoAqACooAKAqACoKACgqACoCgAoqACoAoAKqqqqqqACqqqqqqgAoAAAAAoAKAAAAAKACgAAAACgAoAAAAAoAKqqqqqqACqqqqqqgAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Movie_event.svg
|
||||
music: "AAKQAAAAAACqkAAAAAApqpAAAAAKAaqQAAACgAGqkAAApAABqoAAKqQAAaAACmqkACgAAoBqpAoKqqAAaqaCqqgAAGqgoAoAAABoKAKAAAAKCgCgAAACgoAoAAAAoKAKAAKqqCqqgACqqgqqoAAoAoAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACqqgAAAAAqqoAAAAAAAAAA==", // icons/25px_Music_note.svg
|
||||
news: "AAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqoAqqqqqqqAKAAAAACgCgAAAAAoAoKqqqoKAKCqqqqCgCgAAAAAoAoKqqqgKAKCqqqoCgCgAAAAAoAoKqqqAKAKCqqqgCgCgAAAAAoAoKqqAAKAKCqqgACgCgAAAAAoAoAAAAAKAKqqqqqqgCqqqqqqoAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_News_event.svg
|
||||
payBill: "AAAAAAAAAAAAAAAAAAAAAAAAApQAAAAAWqqlAABaqplqqAKqpZZAWgClAGSgAZBkACgoACgoAAoKqqqpAAZCqqqqQAGQoAACgAGgKAACkAaoCgACpBqqAoACqmqSgKACoqpAoCgAqCoAKAoACqoACgKAAKoAAoCgAAoAAKAoAAAAACgKqqqqqqoCqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Pay_bill.svg
|
||||
phone: "AAAACgAAAAAAAoAAAAAAqqqqqAAAKqqqqgAACgAAAoAAAoqqqKAAAKKqqigAACigAooAAAooAKKCgAKKACigqACigAooCoAoqqqKAKgKKqqigAqCgAAAoACgoqiqKAAAKKoqiqqgCgAAAqqoAoqiqKAAAKKoqigAKCgAAAoAKgoAAAKAKgKAAACgKgCgAAAoKgAqqqqqCgAKqqqqgA==", // pebbleos/Pebble_25x25_During_phone_call.svg
|
||||
podcast: "ACgAAAAAAAoAAAAAAAZAAAAAAAGQAAAAAACgAAAAAAAoAAAAAAAZAAAAAAAGQAAAAAAqqqqqqqoKqqqqqqqCgAAAAACgoAAAAAAoKAqoAqqKCgqqgKqigoqAqAAAoKKACgAAKCigAoAACgooAKAAAoKKgKgAAKCgqqgAACgoCqgAAAoKAAAAAAKCgAAAAACgqqqqqqqoKqqqqqqqAA==", // pebbleos/Pebble_25x25_Radio_show.svg
|
||||
reminder: "AAAACqoAAAAAAqqCgAAAAKCioAAAACgqoAAAAAoKoAAAAAKqoAAAAACqqAAqqqqoKoAKqqqqCqgCgoKCgoqAoKCgoKCgKCgoKCgACgoKqqqAAoKCqqqgAKqqoAAoACqqqAAKAAqACqoGgACoAqqGgAAKgAACkAAAoAACkAAAKAAAoAAACgAAKAAAAoAACgAAAKqqqoAAACqqqqAAAA==", // pebbleos/Pebble_25x25_Generic_reminder.svg
|
||||
settings: "AAAKqqAAAAACqqgAAAAAoAoAAqqqqAKqqqqqqgCqqoAAAoAoAAAAAKqqAAAAACqqgAAAAAAAAAAAqqoAAAAAKqqAAAAACgCgAAAAqoAqqqqqqqAKqqqqoCgCgAAAAAqqoAAAAAKqqAAAAAAAAAAKqqAAAAACqqgAAAAAoAqqqqqqqAKqqqqqqgCgAAAAAoAoAAAAAKqqAAAAACqqgA==", // pebbleos/Pebble_25x25_Settings.svg
|
||||
shopping: "AAAAaAAAAAABqoAAAAACqaAAAAAApCgAAAAAKAoAAAAACgKAAACqqqqqqoAqqqqqqqAGgAAAAGQApAAAACgAGgAAAAoAApAAAAZAAGgAAAKAAAoAAACgAAZAAABkAAKAAABoAACgAAGqQABkAAKpoAAoAACkKQAKAAAABoAGQAAAAKQCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // icons/25px_Shopping_bag.svg
|
||||
sms: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACqqqqqqqAqqqqqqqoCoAAAAAKgKgAAAAAoAoAAAAAKAKCqqqqCgCgqqqqgoAoAAAAAKAKAAAAACgCgqqgAAoAoKqoAAKAKAAAAACgCgAAAAAoAqAAAAAqACqqqqqqAAKqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // icons/25px_SMS.svg
|
||||
stocks: "AAAAAAAACgAAAAAAAoAAAAAAAKAAAAAAACgAAAAAAAoAAAAoAAKAAAAqigCgAAAqqqAoAAAqCqoKAoAqCoKigqgqAqCooqqqACqoKqCqAAKoCqAKAAAoAqAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACgCgAAAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Stocks_event.svg
|
||||
tv: "AAAAAAoAAAAAAAqAAAAKAAqAAAACoAqAAAAAKgqAAAAAAqqAAAKqqqqqqqCqqqqqqqgoAAAAAAoKKqqqqqKCiqqqqqigooAAAAooKKAAAAKKCigAAACigooAAAAooKKAAAAKKCiqqqqqigoqqqqqooKAAAAAAKCgACqgACgoAAqoAAoKAAAAAAKCqqqqqqqgqqqqqqqoAAAAAAAAAA==", // pebbleos/Pebble_25x25_TV_show.svg
|
||||
voicemail: "AABAUAAAAABkaAAAAAAZKQAAAAAKKQAAAAKCigEAAACoABaQAABqgCqQAAApqApAAAAKCoAAAAAGQKgAAAACgAqAAAAAoBqgAAAAZCqQAAAAKQqAAAAACoBpAKAAAKgGkGoAAAaQaSqgAABpBqoqAAAGkCpCoAAAaQKAKgAABpAABqAAAGkAGqgAAAKhqpAAAAAqqQAAAAACkAAAAA==", // icons/25px_Phone.svg
|
||||
warning: "AAAAAAAAAAAAoAAAAAAAaQAAAAAAKoAAAAAAGqQAAAAACgoAAAAABkGQAAAAAoooAAAAAZKGQAAAAaCgpAAAAKQoGgAAAGgKApAAACkCgGgAABoAoApAAApAKAGgAAZAAAAZAAKAAoACgAGQAKAAZACgACgACgCkAAoAAaBoAAAAACkqqqqqqqqKqqqqqqqgAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Generic_warning.svg
|
||||
weather: "AAAAAAKAAAAAAACgAAAACgAoAAAAAqAKAAAAACoAAAAAAAKKqgAAAAAKqqAAAAAKgCoAAKqCgAKAAKqooACgAKgKqAAoKqgAqqqqKqoACqqqqgKgAAACqgAoAAAAKoAAAAAACqAAAAAAAqgAAAAAAKqAAAAAAKiqqqqqqqgKqqqqqqgAAAAAAAAAKCgoAAAACgoKAAAAAoKCgAAAAA==", // pebbleos/Pebble_25x25_Generic_weather.svg
|
||||
airmail: "AAAAAAAAAAAAAAAAAAAqqqoAAAAqqqqgAAAqAAAqAAAqAAACoAAqAAAAKgAqAAAAAqAKkAAAAagCqkAABqoApqkAGqaAKBqkapCgCgBqqkAoAoABqQAKAKAAAAACgCgAAAAAoAoAAAAAKAKgAAAAKgAaQAAAKgABpAAAKgAAGkAAKgAAAKqqqgAAAAqqqgAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Airmail_25px.svg
|
||||
amazon: "AACqqgAAAACqqqAAAACoACoAAACoAAKgAAAqqoAoAAAKqqgKAAAAAAqCgAAAAACgoAAAAqqoKAAAAqqqCgAAAqAAAoAAAqAAAKAAAKAqqCgAACgqqgoAAAoKAoKAAAKCgKCoAACgqqgKgAAoCqgAqAAKgAAoKgKAqAAqqgqoCqqqqgqKgKqqCgqAqAAAAAqACqqqqqqAAKqqqqqAAA==", // pebbleos/Pebble_25x25_Amazon.svg
|
||||
appleShortcuts: "AAAKqAAAAAAKqoAAAAAKgKgAAAAKgAqAAAAKgACoAAAKgAAKgAAKgAAAqAAKgCqgCoACgCqqAKAAoCoCoCgAKgoAKioAAqAAAqoAACoAACoAACqgACqgACoqACoqAAoCoCoCgAKAKqoAoACoAqoAqAAKgAAAqAAAqAAAqAAACoAAqAAAAKgAqAAAAAqAqAAAAACqqAAAAAAKqAAAAA==", // brand-logos/Apple_Shortcuts_25px.svg
|
||||
bbcNews: "qqqqqqqqoAAAAAAAKCqgqoCqCgqqKqiqooKCigooKKCqoqqKACgqqKqigAoKCigooKKCqoqqKqigqoKqAqgoAAAAAAAKAAAKqqACgAACqqgAoAAAoAoAKACqqAKACgAqqgCgAoAKAoAoAKACgKqqACgAoCqqgAoAKqoKAAKACqqCgACgAACqoAAoAAAqqAAKAAAAAAACqqqqqqqqgA==", // brand-logos/BBC_News_25px.svg
|
||||
beeper: "AAqqqqAAAAqqqqoAAAqAAAKgAAqAAAAqAAqAAAACoAKAAAAAKACgAAAACgAoAAAAAoAKAAAAAKACoAAAAKgAKgAAAKgAAqAAAKgAAKqqoKgAACqqqKoAAAoAGaqgAAKACqgqAACgAqgCoAAoAagAKAAKAKgACgACgCgAAoAAoAAAAqAAKAAAAqAACgAAAqAAAqqqqqAAAKqqqqAAAA==", // brand-logos/Beeper_25px.svg
|
||||
bluesky: "AAAAAAAAAGgAAAApAqqAAAAqqKWoAAAqWigKgAAqAooAqAAqAKKACoAqACigAKgqAAooAAqqAAKKAACqAACigAAKAAAopAAAAAAaGgAAAAAKQpAAAAAGgGqoAAKqkAqqAACqoAKpAAAGqACkAAAAGgAoAAAAAoAKAAKAAKACgAGkACgAqACqACoACoBqkCoAAKqpaqoAAAqqCqoAAA==", // brand-logos/Bluesky_25px.svg
|
||||
discord: "AAAAAAAAAAAAAAAAAAAAAAAAAABoAACkAAKqqqqqoAClqqqpaABkAAAABkAoAAAAAKAKAAAAACgGQAAAAAZCgAAAAACgoCqgKqAoZAqoCqgGaAKKAooAqgCqgKqAKoAqoCqgCqAAAAAAAqkAAAAAAaqkCqqoBqhqmqqqmqQBqoAAqpAABoAACkAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Discord_25px.svg
|
||||
duolingo: "AAAAAAAAAAAAAAAAAAoAAAAoAKagAAAqaCqqAAAqqhqmoAAqappoKgAqCmqAAqAqAAUKqCgKCqgKqoAACqqKgKgACoCqgAoAAoAKoKKAAKKCqCigACigqgooAAooKoKKAAKKCqCiiqiigqoCqqqqAqKqqoCqqqAqoqqqKqAAAKqqgAAAAAoCgAAAAAKqoAAAAAAqoAAAAAAAAAAAAA==", // brand-logos/Duolingo_25px.svg
|
||||
ebay: "AAAqqgAAAAAqqqAAAAAqACoAAAAqAAKgAAAKAAAoAAACgAAKAAKqqqqqqqCqqqqqqqgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKACgAoAKCgAoAKACgoAKACgAoKgCgAoAqAqAoAKAqACqqqqqqAAKqqqqqAAA==", // brand-logos/eBay_25px.svg
|
||||
element: "AACqqoAAAAAqqqgAAAAKAAqAAAACgACoAAAAqqgKAACqqqqCgACqqqCgoACoACgoKACoAAoKCqqoCqqCgqqqCqqgqqgqgoAAKqoKoKAAAAKCqCqqAACgqgqqgqqoKqqgoKqoCqqoKCgACoAKCgoACoACgoKqqoAAoKqqqoAAKAqqgAAACoAAoAAAAKgAKAAAAAqqqgAAAACqqoAAAA==", // brand-logos/Element_25px.svg
|
||||
facebook: "AAAAAAAAAAAAAAAAAAAAAAAAAAqqqqqqgAKqqqqqoACgAAAAKAAoAAAACgAKAAAKgoACgAAKoKAAoAAKgCgAKAACgAoACgAAoAKAAoACqqCgAKAAqqgoACgAAoAKAAoAAKACgAKAACgAoACgAAoAKAAoAAKACgAKAACgAoACqqqqqqAAqqqqqqgAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Facebook_notification.svg
|
||||
facetime: "AAAoAAAAAAAKAAAAAoACgAKAAKgAoAKgAAqAKAKgAACoCgKgAAAKgoKgAAAAoACgAAAAAAAAAAAAAAAAAAAAKqqqoACgKqqqqgCoKgAAAqCqKgCqgCqqigCqqAKoooCoCoCoKKAoAKAoCigKACgKAooCgAoCgKKAqAqAqCigCqqAKooqAKqAKqqCoAAAKgqgKqqqqgCoAqqqqgAKAA==", // pebbleos/Pebble_25x25_Facetime_incoming.svg
|
||||
gmail: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqqqqiqqqqqqqooAqAAqAKKACoAqACigAKgqAAooKAqqAoKKCoCqAqCigqgKAqgooKqAAqoKKCioAqKCigoKgqCgooKCqqgoKKCiqqqKCigqoKCqgooKoAAKoKKCoAAAqCiqqqqqqqoqqqqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Gmail_notification.svg
|
||||
googleChat: "AAAAAAAAAAAAAAAAAKqqqqoAACqqqqqgAAoAAAAqAAKAAAACoACgAAAAKgAoCqqqgqAKAqqqoCgCgKAAKAoAoCgACgKAKAoAAoCgCgKAAKAoAoCqqqgKAKAqqqoCgCgKgAAAoAoCgAAAKAKAoAAACgCqqAqqqoAqqgqqqqAAAoqAAAAAAKqAAAAAACqAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Google_Chat_25px.svg
|
||||
googleMaps: "AAAAAKqgAAAAAKqqAKqqqqgCoKqqqqgAKqgAACgAAqgKoAoAAKoKqAKAACqKgACgAAqigqgqAAqooKoCoAqKKgqAKgqCgqqAKoKgoCqAKmWoKAAAKgqqCgAAKgKqgoAAKgBpoKAAKqAKKCgAKioCigoAKgKgAoKAKgAqAKCgKgACoCgoKgAAKgoKqgAAAqqAqqqqqqqACqqqqqqAAA==", // pebbleos/Pebble_25x25_Google_Maps.svg
|
||||
googlePhotos: "AAAAoAAAAAAAKgAAAAAACqAAAAAAAqoAAAAAAKKgAAAAqqgqAAAAqqoCoAAAqAKAKAAAqACgCgAAqACqAoAAqACqoKAAqqqoKqqqKqqqCqqqgAKCqoAKgACgKoAKgAAoAoAKgAAKAKAKgAACoCqqgAAAKgqqgAAAAqKAAAAAACqgAAAAAAKoAAAAAAAqAAAAAAACgAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Google_Photos.svg
|
||||
googleTasks: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAqqqAqAAAqqqoqAAAqAAKqAAAqAAAqAAAqAAAqoAAKAAAqKAACgAAqCgAAooAqAoAAKKgqAKAACgqqACgAAoCqAAoAAKAKAAKAACoAAAKgAAKgAAKgAAAqAAKgAAACqqqgAAAAKqqgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Google_Tasks_25px.svg
|
||||
homeAssistant: "AAAAAAAAAAAAKAAAAAAAKoAAAAAAKqgAAAAAKgqAAAAAKgCoAAAAKgAKgAAAKgAAqAAAKgqqCoAAKgKqgKgAKgCgoAqAKgAoKACoKgAKqgAKigACqoAAooAACgqqKKAAAoKqiiiqoKCgoooqqCgoKKKKCgoKqiiigoKKqoooqqCqgAKKKqoqgACigAKqgAAoqqqqqqqqKqqqqqqqgA==", // brand-logos/Home_Assistant_25px.svg
|
||||
instagram: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAqqgCgAAAKqoAoAKqoAKAKAKqqgCgCgKgAqAoAqqgACqqAKqgAAKqgCgoAACgoAoKAAAoKAKCgAAKCgCgoAACgoAoKgACoKAKAqACoCgCgCqqoAoAoAKqoAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Instagram.svg
|
||||
iosPhotos: "AAAKqAAAAAAKqoAAAAqqgKqoAAqqgAqqgAqBoAKQqAKAKACgCgCgBkBkAoApAKAoAaAKpBkZBqgKqpaKWqqKgGqqqkCqgAGqqQAKoAAKqAACqAAaqpAAqoBqqqpAqKqpaKWqqAqkGRkGqAKQCgKAGgCgBkBkAoAoAoAKAKAKgaACkKgAqqgAqqgACqqAqqgAAACqqAAAAAAKqAAAAA==", // pebbleos/Pebble_25x25_iOS_Photos.svg
|
||||
kakaoTalk: "AAAAAAAAAAAAAAAAAAKqqqqAAAKqqqqoAAKgAAAKgAKgAAAAqAKgAAAACoCgCqqqAKAoAqqqgCgKAACgAAoCgAAoAAKAoAAKAACgKAACgAAoCgAAoAAKAoAAKAACgKAACgAAoCoAAAAAqAKgAAAAqAAqAAAAqAACgKqqqAAAoaqqqAAAKaQAAAAACqQAAAAAAqQAAAAAAKAAAAAAAA==", // pebbleos/Pebble_25x25_KakaoTalk.svg
|
||||
kik: "AoAAAAAAAqgAAAAAAqqAAAAAAKCgAAAAACgoAAAAAAoKAKqAAAKCgKqgAACgoKgoAAAoKKgKAAAKCqgKgCgCgqgKgGqQoKgKgKlqKCgKgCgCigAKgAoAooAKgAKAKKACoACpaigAKgAGqQoKAqAAKAKCoCoAAACgqgKgAAAoKqAqAAAKCioCgAACqoKgoAAAKoAqqAAAAoACqgAAAA==", // pebbleos/Pebble_25x25_Kik.svg
|
||||
line: "AKqqqqqAAKqqqqqoAKgAAAAKgKgAAAAAqKgAAAAACqgAAAAAAKoKAoqoqiqCgKKqKoqgoCiiigKoKAoooqiqCgKKKKoqgoCiiigKoKooooqiqCqKKKKoqoAAAAAAKKgAAAAAKgqAAAAAKgCoAAAAKgAKqqqAKgAAqqqgKgAAAAAZKgAAAAAGagAAAAABqgAAAAAAKgAAAAAACgAAAA==", // pebbleos/Pebble_25x25_Line.svg
|
||||
linkedIn: "AAAAAAAAAAAAAAAAAKqqqqqqgCqqqqqqoAoAAAAAKAKAAAAACgCgKAAAAoAoCgAAAKAKAAAAACgCgKAAAAoAoCgqqAKAKAoKqoCgCgKCgKAoAoCgoCgKAKAoKAoCgCgKCgKAoAoCgoCgKAKAoKAoCgCgAAAAAoAoAAAAAKAKAAAAACgCqqqqqqoAqqqqqqqAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_LinkedIn.svg
|
||||
messenger: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAoAAAAAABqAACgAACqAABqQAGqQAApqAKqgAAZCpamkAAaAGqRoAAKQAKAoAAGgAAApAACgKAAaAACkapAKQABpqWoGQAAqqAKmgAAapAAakAAKoAAAoAAKkAAAAAACgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Facebook_messenger_notification.svg
|
||||
omniFocus: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKgAAAAAAKqAAAAAAKioAAAAAKgKAAAAAKgKgAAAAKgKgACoAKgKgACqgKgKgACoqKgKgAAoCqgKgAAKgKgKgAAAqAQKgAAACoAKgAAAAKgKgAAAAAqKgAAAAACqgAAAAAAKgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/OmniFocus_25px.svg
|
||||
outlook: "AAAAAAAAAAAAFAAAAAAAapAAAAABqqpAAAAGqQapAAAapAAapABqkAAAapCqQAAAAaopAAAAAAaKAAAAAACikAAAAABqqqqoAAGqqqqqAAaqqAACgBqkqiqooGqQKoqqKqpACqKCiqkAAqigopqkAKoqqKBqgCqKqigBoAqgAAoAKAKqqqqAKgKqqqqgKgKgCqqqqqqgAKqqqqqgAA==", // brand-logos/Outlook_25px.svg
|
||||
reddit: "AAAAAAAAAAAAAAAAAAAAAAqgAAAAAAqqAAAAAAKCgAAAAACgoAAAAAKqqAAAAACmqAAAAABkAAAAAAAoAAAAqkaqpGqAqqqqqqqoqBpAAGkKqAAAAAAAqgAKACgAKqECgAoBKiqQoAKBqgKgAAAAKgAoAqqgCgAKQKqoBoACqQqoGqAAGqQAapAAAGqqqkAAAAGqqQAAAAAAAAAAAA==", // brand-logos/Reddit_25px.svg
|
||||
signal: "AAqqCqgAAAqqgqqAAAKAAACoAAACqqoKAAoCqqqgAAqCoAAqAoqCoAACoKqCoAAAKgqioAAAAqKooAAAACigKAAAAAooCgAAAAKKooAAAACgKKAAAAAoCigAAAAKKooAAAACiqKAAAACoqigAAACoKBkAAACoKgoAAACoKgKBqqqoCgGqqqqoAACqkAAAKAApAqoKqgAAAKqCqgAAA==", // brand-logos/Signal_25px.svg
|
||||
simplex: "AAAAAAAAAAAoAoAAAAAqgqgAAAAqqqqAAAAqCqCoAAAKgKAqAAAoqAAqKAAqipQqKoAqqqpKqqgqCqGQqgqKgKAoCgKgqAAqgAKgCoAqqAKgAqAKqgCoAqAAqgAKgqAoCgKAqKgqhkKoKgqqqpKqqgCqKlCoqgAKKgAKigAAKgKAqAAACoKoKgAAAKqqqgAAAAqgqgAAAACgCgAAAA==", // brand-logos/SimpleX_25px.svg
|
||||
slack: "AAAAGgAAAAAAqoAAAAAAKWWgAAGgBkqoAAqoAaKWQAKWWqgBoABkqpAKqAAaKQGikACqgBqoZAApAKqWSgAGWikAooAAqoZAKGWgKWSgBkqoAAooAaKWQAKGWqgBoABkqpAKqAAaKQGikACqgBqoZAApAKqWWgAGWikAqoAAqoZAKQAAKWWgAAAAAAqoAAAAAAKQAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Slack.svg
|
||||
snapchat: "AACqqgAAAACqqqAAAACoACoAAACoAAKgAAAoKCgoAAAKCgoKAAKqgoKCqoCqoKCgqqAoACgoACgKQAAAABoCqQAAAGqAGqAAAKpAAGgAACkAAAoAAAoAAAqAAAKgAAqAAAAqAAqAAAACoAqAAAAAKgqAAAAAAqKAAAAAACikAAAAABoqlAAAAWqAapAABqkAAWqqqpQAAAGqqkAAAA==", // pebbleos/Pebble_25x25_Snapchat.svg
|
||||
steam: "AAAAAAAAAAAAAAAAAAAAACqqAAAAACqqoAAAACoAKgAAACoqoqAAAAoqqigAAAKKAooAAACigKKAAAAooCigAAAaKqooAAAKAqoqAAKqQAAqKQKqgCqqCqqgAGqqABqpAKkAAAAqqCgAAAAKGgoAAAACgAKAAAAAqAKgAAAACqqgAAAAAKqgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Steam_25px.svg
|
||||
substack: "AAAAAAAAAAAAAAAAACqqqqqqAAqqqqqqgAKAAAAAoACgAAAAKAAoAAAACgAKqqqqqoACqqqqqqAAoAAAACgAKAAAAAoACqqqqqqAAqqqqqqgAKAAAAAoACgAAAAKAAoAAAACgAKAAAAAoACgAAAAKAAoAGpACgAKAaqpAoAChqkapKAAqqQAaqgAKpAAAaoACkAAAAaAAAAAAAAAAA==", // brand-logos/Substack_25px.svg
|
||||
swarm: "AAAAAKQAAAAAAKqAACkAAKmgACqpACgoACpaqAoKACoAGkKGgApAAaCmkAGgACkqoAApAAaKkAAGqqqgAAAAqqpABqkAAAAAaqqkAAACqqBqgAABpCgBoAAAqQoAKAAAakKACgAAKpBkAoAAGmgKAKAACkoCoCgABoKQKioAApApAqoAAKpGgGoAAAaqqqoAAAAaqqQAAAAAAAAAAA==", // brand-logos/Swarm_25px.svg
|
||||
tapo: "AAABkAAAAAACqgAAAAACoqAAAAACoCoAAAACoCqgAAACoCoqAAACoAoCoAACoAKAKgACoACgAqACoAAoACoCoAAqAAKhoAAqoAApoAAqKgACqAAqAqAAqgAqACoAKoAKAAKACqACgACgAqgAoAAoAKoAKAAKACqACgACgAqgAoAAoAKoAKAAKACqACgACgAqqqoAAqqqqqqAAKqqgA==", // brand-logos/Tapo_25px.svg
|
||||
teams: "AAKoAAAAAAKqgAAAAAKgqAAAAACgCgAAAAAoAoAqgAAKgqAqqAAAqqAKCgAACqACgoACqqqgqqACqqqqCqACoAACqqoCqqqoKqqgqqqqCgAqKAACgoACiiqooKAAooqqKCgAKKAoCgoACigKAoKAAooCgKCgAKKAoCgoACigAAoKAAoqqqqCoAKKqqqgKgKgCqqqqqqgAKqqqqqgAA==", // brand-logos/Teams_25px.svg
|
||||
telegram: "AAAAAAAAAAAAAAAAAAAAAAAAaAAAAAABqgAAAAAWqkAAAABqkpAAAAGqQKAAAAapAGQAAGqQABkAAapAAAoABqkAGgKAWpQBaoGQqkAGqoCgKqRappAoAGqqUaAZAABpAKAGQAAGQKQCgAABkGqRoAAAKCmqpAAACikGqAAAAaoAGgAAAGoAAAAAAApAAAAAAAKAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Telegram_app.svg
|
||||
threads: "AAKqqqgAAAKqqqqAAAKgAACoAAKgAAAKgAKgKqqAqAKgKqqoCoKgKgAKgqCgCoAAoqAoAKqoKqAKAKqqCqACgKgAACoAoKgAAAKgKCgqqAAoCgoKqgoKAoKCgoKCgKCgqqCgoCgoKqgoKAoKgAAqCgKAqAAqAoCoCqqqAqAKgKqqAqAAqAAAAqAACoAAAqAAAKqqqqAAAAqqqqAAAA==", // brand-logos/Threads_25px.svg
|
||||
tiktok: "AAAAAAAAAAAAKqgAAAAACqpAAAAAAoGgAAAAAKAqAAAAACgCpQAAAAoAKoAAAAKAAKAACqigoCgACqooKkoACoKKCqqACoCigpqgCoGooKAAAoGqKCgAAKCkCgoAACgoAoKAAAoKAKCgAAKCkGgoAACgaqkKAAAqBqkKgAACoAAKgAAAKgAKgAAAAqqqgAAAACqqgAAAAAAAAAAAAA==", // brand-logos/TikTok_25px.svg
|
||||
twitch: "AAqqqqqqAAqqqqqqgAqgAAAAoAqoAAAAKAqKAAAACgKCgCgKAoCgoAoCgKAoKAKAoCgKCgCgKAoCgoAoCgKAoKAKAoCgKCgAAAAoCgoAAAAqAoKAAAAqAKCgAAAqACgqqqqqAAoKqqqqAAKAAqAqAACgAKAqAAAqqgqqAAAKqoqqAAAAAKqAAAAAACqAAAAAAAqAAAAAAAKAAAAAAA==", // brand-logos/Twitch_25px.svg
|
||||
twitter: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAWgAKQAACqqFqqkAApSqqKqpAZAKaRpaqWQAGgKQGqgACkApABoAAoAGgAAAAKAAaAAAACgACkAAAAoAAaAAAAKAABoAAAGgABqAAAGgACpAAAGkAGoAAACkAKoAAAWkACqpQWqoAABaqqpQAAAAFpQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_Twitter_notification.svg
|
||||
unifiProtect: "AAAAAAAAAAAaqkAAAAAqqqgAAAAqQBqAAAAaCqCkAAAKSqoaAAACgoKCgAAAoKCgoAAAKCqoKAAACgKoCgAAAoAAAoAAAKAAAKAAACgCgCgAAAoAoAoAAAKAKAKAAACgAACgAAAoAAAoAAAKAAAKAAACgAACgAAAoAAAoAAAKAAAKAAACoAAKgAAAKqqqgAAAAqqqgAAAAAAAAAAAA==", // brand-logos/Unifi_Protect_25px.svg
|
||||
viber: "AAAAAAAAAAACqqgAAAAAqqqAAAAAAACoAAKAAAAKgAKoAKqAqAKqgCqoCoKgqAAKgKigCoAAqAooAKgoCoKKACoKgKCigCoAqCgooCoACgoKKAqAAAKCigCoAAAAoqAKgCgAACoAqCqAAAKgCqqoAAAqAKoKgAACoAoAqAAAKgAACoAAAqAAAqAAACoAAqAAAAKqqqAAAAAqqqAAAA==", // pebbleos/Pebble_25x25_Viber.svg
|
||||
waze: "AAAAAAAAAAACqqoAAAACqqqgAAACoAAqAAACoAACoAACoAAAKgACoAFBQqAAoACgoCgAKAAoKAoACgAKCgKAAoABQUCgAKAAAAAoqqgCgCgKKqoAqqoCgqAACqoAoCoAAAAAqAKgAAAAqAAqqgAAqAACqqAAqAAAoCqqqAAAKAqqqgAACgKKAoAAAqqiqqAAACqgKqAAAAAAAAAAAA==", // brand-logos/Waze_25px.svg
|
||||
weChat: "AAAAAAAAACqqqAAAACqqqoAAACoAAKgAACoAAAqAACoAAACoAAoKoKoKAAKCqCqCgACgAAAAoAAoAAAKqqgKAAAKqqqCgAAKgACoqAAKgAAKioACgAAAoKAAoqiqKCqqqKoqigqqqgAAAoKgAoAAAKCgAKgAAKgAAAqAACgAAACqqqoAAAAKqqqAAAAAAAKgAAAAAAAoAAAAAAAAAA==", // pebbleos/Pebble_25x25_WeChat.svg
|
||||
whatsApp: "AAAAAAAAAAAAAAAAAAAKqqoAAAAKqqqgAAAKgAAqAAAKgAACoAAKigAAKgAKiqAAAqAKiqoAACoCgoKgAAKAoKCoAACgKCioAAAoCgoqAAAKAoKCoKACgKCoKqoAoCgKgqqgKAoAqCgqCgKgCoAKioBoAKqqioAZAAqqioAKAAAACoAGlqAACoACqqqqqoAAqVKqqoAAAAAAAAAAAA==", // pebbleos/Pebble_25x25_WhatsApp_notification.svg
|
||||
whatsAppBusiness: "AAAAAKqgAAAAACqoAAAqqqoKAAAqqqqCqgAqACqgqoAqAAoAAKAqKAKAACgqKoCqgqoqKqgqoKqKCgqAKCoCgoKgCqqAoKKgAqqgKCioAAAoCgoKgoAKAoKgqqgCgKAqCqqAoCgCoKCoKAqAKgAqKgGgAqqqKgBkACqqKgAoAAAAKgAaWoAAKgAKqqqqqgACpUqqqgAAAAAAAAAAAA==", // brand-logos/WhatsApp_Business_25px.svg
|
||||
yahooMail: "CqqqqqqgAqqqqqqoAKAAAAAKACgqoAACgAoKqKqAoAKAKiqgKACgAqqACgAoACqAAoAKAAKAAKACgAqqACgAoAKqgAoCqAAAAAKoqgACgACqKoACqAAqiqgCqoAqooqCoKgqKKCqoAqqCigKoACqAooCoAAKgKKCoAAAqCiioAAACooqoAAAAKqKqqqqqqqiqqqqqqqoAAAAAAAAAA==", // pebbleos/Pebble_25x25_Yahoo_mail.svg
|
||||
youtube: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqqoAKqqqqqqgKgAAAAAqCgAAAAACgoACkAAAoKAAqkAAKCgAKqkACgoAChqgAoKAAoBoAKCgAKGqACgoACqpAAoKAAqkAAKCgAKQAACgoAAAAAAoKgAAAAAqAqqqqqqqACqqqqqqAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Youtube_25px.svg
|
||||
zoom: "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAKqqqqoABqqqqqqgGqoAAAAqaqqAAAACqkqgAAAAKQKoAAAAAACqAAAAAAAqgAAAAAAKoAAAAAACqAAAAAAAqgAAAAAAKoAAAAAACqgAAAApAoqAAAAKpKCoAAACqqgKqqqqoaoAqqqqqAaAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA==", // brand-logos/Zoom_25px.svg
|
||||
} as const;
|
||||
|
||||
export type PebbleIconName = keyof typeof PEBBLE_ICON_DATA;
|
||||
@@ -0,0 +1,201 @@
|
||||
Apache License
|
||||
Version 2.0, January 2004
|
||||
http://www.apache.org/licenses/
|
||||
|
||||
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
|
||||
|
||||
1. Definitions.
|
||||
|
||||
"License" shall mean the terms and conditions for use, reproduction,
|
||||
and distribution as defined by Sections 1 through 9 of this document.
|
||||
|
||||
"Licensor" shall mean the copyright owner or entity authorized by
|
||||
the copyright owner that is granting the License.
|
||||
|
||||
"Legal Entity" shall mean the union of the acting entity and all
|
||||
other entities that control, are controlled by, or are under common
|
||||
control with that entity. For the purposes of this definition,
|
||||
"control" means (i) the power, direct or indirect, to cause the
|
||||
direction or management of such entity, whether by contract or
|
||||
otherwise, or (ii) ownership of fifty percent (50%) or more of the
|
||||
outstanding shares, or (iii) beneficial ownership of such entity.
|
||||
|
||||
"You" (or "Your") shall mean an individual or Legal Entity
|
||||
exercising permissions granted by this License.
|
||||
|
||||
"Source" form shall mean the preferred form for making modifications,
|
||||
including but not limited to software source code, documentation
|
||||
source, and configuration files.
|
||||
|
||||
"Object" form shall mean any form resulting from mechanical
|
||||
transformation or translation of a Source form, including but
|
||||
not limited to compiled object code, generated documentation,
|
||||
and conversions to other media types.
|
||||
|
||||
"Work" shall mean the work of authorship, whether in Source or
|
||||
Object form, made available under the License, as indicated by a
|
||||
copyright notice that is included in or attached to the work
|
||||
(an example is provided in the Appendix below).
|
||||
|
||||
"Derivative Works" shall mean any work, whether in Source or Object
|
||||
form, that is based on (or derived from) the Work and for which the
|
||||
editorial revisions, annotations, elaborations, or other modifications
|
||||
represent, as a whole, an original work of authorship. For the purposes
|
||||
of this License, Derivative Works shall not include works that remain
|
||||
separable from, or merely link (or bind by name) to the interfaces of,
|
||||
the Work and Derivative Works thereof.
|
||||
|
||||
"Contribution" shall mean any work of authorship, including
|
||||
the original version of the Work and any modifications or additions
|
||||
to that Work or Derivative Works thereof, that is intentionally
|
||||
submitted to Licensor for inclusion in the Work by the copyright owner
|
||||
or by an individual or Legal Entity authorized to submit on behalf of
|
||||
the copyright owner. For the purposes of this definition, "submitted"
|
||||
means any form of electronic, verbal, or written communication sent
|
||||
to the Licensor or its representatives, including but not limited to
|
||||
communication on electronic mailing lists, source code control systems,
|
||||
and issue tracking systems that are managed by, or on behalf of, the
|
||||
Licensor for the purpose of discussing and improving the Work, but
|
||||
excluding communication that is conspicuously marked or otherwise
|
||||
designated in writing by the copyright owner as "Not a Contribution."
|
||||
|
||||
"Contributor" shall mean Licensor and any individual or Legal Entity
|
||||
on behalf of whom a Contribution has been received by Licensor and
|
||||
subsequently incorporated within the Work.
|
||||
|
||||
2. Grant of Copyright License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
copyright license to reproduce, prepare Derivative Works of,
|
||||
publicly display, publicly perform, sublicense, and distribute the
|
||||
Work and such Derivative Works in Source or Object form.
|
||||
|
||||
3. Grant of Patent License. Subject to the terms and conditions of
|
||||
this License, each Contributor hereby grants to You a perpetual,
|
||||
worldwide, non-exclusive, no-charge, royalty-free, irrevocable
|
||||
(except as stated in this section) patent license to make, have made,
|
||||
use, offer to sell, sell, import, and otherwise transfer the Work,
|
||||
where such license applies only to those patent claims licensable
|
||||
by such Contributor that are necessarily infringed by their
|
||||
Contribution(s) alone or by combination of their Contribution(s)
|
||||
with the Work to which such Contribution(s) was submitted. If You
|
||||
institute patent litigation against any entity (including a
|
||||
cross-claim or counterclaim in a lawsuit) alleging that the Work
|
||||
or a Contribution incorporated within the Work constitutes direct
|
||||
or contributory patent infringement, then any patent licenses
|
||||
granted to You under this License for that Work shall terminate
|
||||
as of the date such litigation is filed.
|
||||
|
||||
4. Redistribution. You may reproduce and distribute copies of the
|
||||
Work or Derivative Works thereof in any medium, with or without
|
||||
modifications, and in Source or Object form, provided that You
|
||||
meet the following conditions:
|
||||
|
||||
(a) You must give any other recipients of the Work or
|
||||
Derivative Works a copy of this License; and
|
||||
|
||||
(b) You must cause any modified files to carry prominent notices
|
||||
stating that You changed the files; and
|
||||
|
||||
(c) You must retain, in the Source form of any Derivative Works
|
||||
that You distribute, all copyright, patent, trademark, and
|
||||
attribution notices from the Source form of the Work,
|
||||
excluding those notices that do not pertain to any part of
|
||||
the Derivative Works; and
|
||||
|
||||
(d) If the Work includes a "NOTICE" text file as part of its
|
||||
distribution, then any Derivative Works that You distribute must
|
||||
include a readable copy of the attribution notices contained
|
||||
within such NOTICE file, excluding those notices that do not
|
||||
pertain to any part of the Derivative Works, in at least one
|
||||
of the following places: within a NOTICE text file distributed
|
||||
as part of the Derivative Works; within the Source form or
|
||||
documentation, if provided along with the Derivative Works; or,
|
||||
within a display generated by the Derivative Works, if and
|
||||
wherever such third-party notices normally appear. The contents
|
||||
of the NOTICE file are for informational purposes only and
|
||||
do not modify the License. You may add Your own attribution
|
||||
notices within Derivative Works that You distribute, alongside
|
||||
or as an addendum to the NOTICE text from the Work, provided
|
||||
that such additional attribution notices cannot be construed
|
||||
as modifying the License.
|
||||
|
||||
You may add Your own copyright statement to Your modifications and
|
||||
may provide additional or different license terms and conditions
|
||||
for use, reproduction, or distribution of Your modifications, or
|
||||
for any such Derivative Works as a whole, provided Your use,
|
||||
reproduction, and distribution of the Work otherwise complies with
|
||||
the conditions stated in this License.
|
||||
|
||||
5. Submission of Contributions. Unless You explicitly state otherwise,
|
||||
any Contribution intentionally submitted for inclusion in the Work
|
||||
by You to the Licensor shall be under the terms and conditions of
|
||||
this License, without any additional terms or conditions.
|
||||
Notwithstanding the above, nothing herein shall supersede or modify
|
||||
the terms of any separate license agreement you may have executed
|
||||
with Licensor regarding such Contributions.
|
||||
|
||||
6. Trademarks. This License does not grant permission to use the trade
|
||||
names, trademarks, service marks, or product names of the Licensor,
|
||||
except as required for reasonable and customary use in describing the
|
||||
origin of the Work and reproducing the content of the NOTICE file.
|
||||
|
||||
7. Disclaimer of Warranty. Unless required by applicable law or
|
||||
agreed to in writing, Licensor provides the Work (and each
|
||||
Contributor provides its Contributions) on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
implied, including, without limitation, any warranties or conditions
|
||||
of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
|
||||
PARTICULAR PURPOSE. You are solely responsible for determining the
|
||||
appropriateness of using or redistributing the Work and assume any
|
||||
risks associated with Your exercise of permissions under this License.
|
||||
|
||||
8. Limitation of Liability. In no event and under no legal theory,
|
||||
whether in tort (including negligence), contract, or otherwise,
|
||||
unless required by applicable law (such as deliberate and grossly
|
||||
negligent acts) or agreed to in writing, shall any Contributor be
|
||||
liable to You for damages, including any direct, indirect, special,
|
||||
incidental, or consequential damages of any character arising as a
|
||||
result of this License or out of the use or inability to use the
|
||||
Work (including but not limited to damages for loss of goodwill,
|
||||
work stoppage, computer failure or malfunction, or any and all
|
||||
other commercial damages or losses), even if such Contributor
|
||||
has been advised of the possibility of such damages.
|
||||
|
||||
9. Accepting Warranty or Additional Liability. While redistributing
|
||||
the Work or Derivative Works thereof, You may choose to offer,
|
||||
and charge a fee for, acceptance of support, warranty, indemnity,
|
||||
or other liability obligations and/or rights consistent with this
|
||||
License. However, in accepting such obligations, You may act only
|
||||
on Your own behalf and on Your sole responsibility, not on behalf
|
||||
of any other Contributor, and only if You agree to indemnify,
|
||||
defend, and hold each Contributor harmless for any liability
|
||||
incurred by, or claims asserted against, such Contributor by reason
|
||||
of your accepting any such warranty or additional liability.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
APPENDIX: How to apply the Apache License to your work.
|
||||
|
||||
To apply the Apache License to your work, attach the following
|
||||
boilerplate notice, with the fields enclosed by brackets "[]"
|
||||
replaced with your own identifying information. (Don't include
|
||||
the brackets!) The text should be enclosed in the appropriate
|
||||
comment syntax for the file format. We also recommend that a
|
||||
file or class name and description of purpose be included on the
|
||||
same "printed page" as the copyright notice for easier
|
||||
identification within third-party archives.
|
||||
|
||||
Copyright [yyyy] [name of copyright owner]
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
@@ -0,0 +1,28 @@
|
||||
import { GrayImage } from "./image";
|
||||
import { decodeBase64 } from "../util/base64";
|
||||
import { PEBBLE_ICON_DATA, PEBBLE_ICON_SIZE, type PebbleIconName } from "./pebble-icons-data";
|
||||
|
||||
export { PEBBLE_ICON_SIZE, type PebbleIconName };
|
||||
|
||||
// 2-bit codes in pebble-icons-data.ts -> coverage.
|
||||
const LEVELS = [0, 128, 255, 255];
|
||||
|
||||
const cache = new Map<PebbleIconName, GrayImage>();
|
||||
|
||||
/**
|
||||
* One of the bundled Pebble notification icons (see pebble-icons-data.ts and
|
||||
* scripts/generate-pebble-icons.cjs), as a PEBBLE_ICON_SIZE square coverage
|
||||
* bitmap. Decoded once and shared: clone it before drawing on it.
|
||||
*/
|
||||
export function pebbleIcon(name: PebbleIconName): GrayImage {
|
||||
let icon = cache.get(name);
|
||||
if (!icon) {
|
||||
const bytes = decodeBase64(PEBBLE_ICON_DATA[name]);
|
||||
icon = new GrayImage(PEBBLE_ICON_SIZE, PEBBLE_ICON_SIZE, 0);
|
||||
for (let i = 0; i < icon.pixels.length; i++) {
|
||||
icon.pixels[i] = LEVELS[(bytes[i >> 2]! >> (6 - 2 * (i & 3))) & 3]!;
|
||||
}
|
||||
cache.set(name, icon);
|
||||
}
|
||||
return icon;
|
||||
}
|
||||
@@ -11,6 +11,11 @@ export interface WrapFont {
|
||||
|
||||
export type WrapTextOptions = {
|
||||
preserveLeadingWhitespace?: boolean;
|
||||
/**
|
||||
* A word too wide for a line of its own starts on the current line and is
|
||||
* split at the width, rather than first being pushed to a line of its own.
|
||||
* Words that fit on a line still wrap whole.
|
||||
*/
|
||||
breakLongWords?: boolean;
|
||||
};
|
||||
|
||||
@@ -89,17 +94,20 @@ function wrapParagraph(font: WrapFont, paragraph: string, targetWidth: number, o
|
||||
|
||||
while (start < paragraph.length) {
|
||||
let best = -1;
|
||||
let overflow = -1;
|
||||
for (const candidate of wrapOffsets) {
|
||||
if (candidate <= start) continue;
|
||||
if (measureSpan(widths, candidate) - measureSpan(widths, start) <= targetWidth) {
|
||||
best = candidate;
|
||||
continue;
|
||||
}
|
||||
overflow = candidate;
|
||||
break;
|
||||
}
|
||||
|
||||
const splitOffset = furthestFittingOffset(widths, start, targetWidth);
|
||||
if (opts.breakLongWords && splitOffset > start && (best < 0 || splitOffset > trimEndOffset(paragraph, best))) {
|
||||
if (opts.breakLongWords && splitOffset > start && splitOffset > best
|
||||
&& (best < 0 || (overflow >= 0 && wordWidth(paragraph, widths, best, overflow) > targetWidth))) {
|
||||
best = splitOffset;
|
||||
}
|
||||
if (best < 0) {
|
||||
@@ -157,6 +165,11 @@ function measureSpan(widths: Map<number, number>, offset: number): number {
|
||||
return widths.get(offset) ?? 0;
|
||||
}
|
||||
|
||||
/** Width of the word between two wrap offsets, without surrounding whitespace. */
|
||||
function wordWidth(text: string, widths: Map<number, number>, from: number, to: number): number {
|
||||
return measureSpan(widths, trimEndOffset(text, to)) - measureSpan(widths, skipLeadingWhitespace(text, from));
|
||||
}
|
||||
|
||||
function skipLeadingWhitespace(text: string, offset: number): number {
|
||||
let cursor = offset;
|
||||
while (cursor < text.length) {
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
/**
|
||||
* Platform-neutral ambient-light types and the CFW report decoder (settings
|
||||
* field 105), shared by ambient-light.ts and ambient-light.ios.ts.
|
||||
*/
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
|
||||
/** The options with their defaults filled in. */
|
||||
export function resolvePollingOptions(options: AmbientLightPollingOptions): Required<AmbientLightPollingOptions> {
|
||||
return {
|
||||
intervalMs: options.intervalMs ?? 500,
|
||||
minDelta: options.minDelta ?? 1,
|
||||
heartbeatMs: options.heartbeatMs ?? 5000,
|
||||
bindToLease: options.bindToLease ?? true,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
/**
|
||||
* Ambient light sensor access on iOS: the same API as ambient-light.ts (see
|
||||
* there for the CFW query and passive-polling semantics) over the running
|
||||
* shared-Kotlin session. No-ops while no session is running.
|
||||
*/
|
||||
|
||||
/** Ask the firmware for one ambient-light report (delivered to listeners). */
|
||||
export function queryAmbientLight(): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.queryAmbientLight();
|
||||
} catch (error) {
|
||||
console.warn(`queryAmbientLight failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Start or stop CFW passive light-sensor polling. */
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to decoded ambient-light reports. Delivered on the main thread. */
|
||||
export function addAmbientLightListener(listener: (report: AmbientLightReport) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addAmbientLightListener((body) => {
|
||||
const report = decodeAmbientLightReport(body);
|
||||
if (report) listener(report);
|
||||
});
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -16,88 +16,17 @@
|
||||
* Reports are 24-byte ['A','L',version,...] records on settings field 105.
|
||||
*/
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import {
|
||||
decodeAmbientLightReport,
|
||||
resolvePollingOptions,
|
||||
type AmbientLightPollingOptions,
|
||||
type AmbientLightReport,
|
||||
} from "./ambient-light-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const global: any;
|
||||
|
||||
export const ALS_REPORT_VERSION = 1;
|
||||
|
||||
export type AmbientLightReason = "query" | "started" | "poll" | "stopped";
|
||||
|
||||
export type AmbientLightReport = {
|
||||
reason: AmbientLightReason;
|
||||
/** The sensor driver is open (readings are live). */
|
||||
opened: boolean;
|
||||
/** CFW passive polling is installed (stock auto-brightness is not stepping). */
|
||||
passive: boolean;
|
||||
/** The user's stock auto-brightness setting is on. */
|
||||
autoBrightness: boolean;
|
||||
/** The last passive read succeeded (only meaningful while passive). */
|
||||
readOk: boolean;
|
||||
/** Stock driver status: 1 start-read, 2 adjusting, 3 polling. */
|
||||
status: number;
|
||||
/** Ambient light in the stock driver's units (roughly tenths of a lux). */
|
||||
alsValue: number;
|
||||
/** Max of the last five readings (what the stock curve is applied to). */
|
||||
peak: number;
|
||||
/** Brightness level (0..100) the stock curve would choose for `peak`. */
|
||||
stockTarget: number;
|
||||
/** Current brightness setting (0..100). */
|
||||
brightness: number;
|
||||
/** Row of the stock brightness curve (0..5) for `peak`. */
|
||||
gear: number;
|
||||
/** Stock learned scale factor, Q10 (1024 = 1.0). */
|
||||
scaleQ10: number;
|
||||
/** Firmware millisecond tick when the report was built. */
|
||||
tickMs: number;
|
||||
};
|
||||
|
||||
const REASONS: AmbientLightReason[] = ["query", "started", "poll", "stopped"];
|
||||
|
||||
function rd16(b: Uint8Array, i: number): number {
|
||||
return b[i] | (b[i + 1] << 8);
|
||||
}
|
||||
|
||||
function rd32(b: Uint8Array, i: number): number {
|
||||
return (b[i] | (b[i + 1] << 8) | (b[i + 2] << 16) | (b[i + 3] << 24)) >>> 0;
|
||||
}
|
||||
|
||||
/** Decode a field-105 record body, or null when it isn't one. */
|
||||
export function decodeAmbientLightReport(body: Uint8Array): AmbientLightReport | null {
|
||||
if (body.length < 24 || body[0] !== 0x41 || body[1] !== 0x4c || body[2] !== ALS_REPORT_VERSION) {
|
||||
return null;
|
||||
}
|
||||
const flags = body[4];
|
||||
return {
|
||||
reason: REASONS[body[3]] ?? "poll",
|
||||
opened: (flags & 0x01) !== 0,
|
||||
passive: (flags & 0x02) !== 0,
|
||||
autoBrightness: (flags & 0x04) !== 0,
|
||||
readOk: (flags & 0x08) !== 0,
|
||||
status: body[5],
|
||||
alsValue: rd32(body, 6),
|
||||
peak: rd32(body, 10),
|
||||
stockTarget: rd16(body, 14),
|
||||
brightness: body[16],
|
||||
gear: body[17],
|
||||
scaleQ10: rd16(body, 18),
|
||||
tickMs: rd32(body, 20),
|
||||
};
|
||||
}
|
||||
|
||||
export type AmbientLightPollingOptions = {
|
||||
/** Poll period in ms; the firmware clamps to 100..5000. Default 500. */
|
||||
intervalMs?: number;
|
||||
/** Report when the reading moved by at least this much. Default 1. */
|
||||
minDelta?: number;
|
||||
/** Also report after this many ms without one (0 = never). Default 5000. */
|
||||
heartbeatMs?: number;
|
||||
/**
|
||||
* Stop automatically when the Faceclaw framebuffer lease is released or
|
||||
* lapses (the phone went away). Default true.
|
||||
*/
|
||||
bindToLease?: boolean;
|
||||
};
|
||||
export * from "./ambient-light-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -123,14 +52,9 @@ export function queryAmbientLight(): void {
|
||||
* on every session start.
|
||||
*/
|
||||
export function setAmbientLightPolling(enabled: boolean, options: AmbientLightPollingOptions = {}): void {
|
||||
const { intervalMs, minDelta, heartbeatMs, bindToLease } = resolvePollingOptions(options);
|
||||
try {
|
||||
activeCommunicator()?.setAmbientLightPolling(
|
||||
enabled,
|
||||
options.intervalMs ?? 500,
|
||||
options.minDelta ?? 1,
|
||||
options.heartbeatMs ?? 5000,
|
||||
options.bindToLease ?? true,
|
||||
);
|
||||
activeCommunicator()?.setAmbientLightPolling(enabled, intervalMs, minDelta, heartbeatMs, bindToLease);
|
||||
} catch (error) {
|
||||
console.warn(`setAmbientLightPolling failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
|
||||
/**
|
||||
* Passive view of the glasses' audio traffic on iOS: the same API as
|
||||
* audio-monitor.ts over the running shared-Kotlin session. Nothing here sends
|
||||
* anything to the glasses.
|
||||
*/
|
||||
|
||||
export type AudioArm = "L" | "R" | "?";
|
||||
|
||||
/** The session listeners attach to; changes across reconnects. */
|
||||
export function audioMonitorCommunicator(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.getActive();
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio to one of its apps. */
|
||||
export function isFaceclawAudioCaptureActive(): boolean {
|
||||
try {
|
||||
return Boolean(FaceclawCommunicatorBridge.getActive()?.isAudioCaptureActive());
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Subscribe `communicator` to every audio-characteristic packet and every
|
||||
* field-104 mic status record body (undecoded). Delivered on the main thread.
|
||||
* Returns an unsubscribe function.
|
||||
*/
|
||||
export function addAudioMonitorListeners(
|
||||
communicator: FaceclawCommunicatorBridge | null,
|
||||
onPacket: (data: Uint8Array, arm: AudioArm, arrivalMs: number) => void,
|
||||
onStatus: (body: Uint8Array, arm: AudioArm) => void,
|
||||
): () => void {
|
||||
if (!communicator) return () => {};
|
||||
const offPackets = communicator.addAudioMonitorListener((data, arm, arrivalMs) => onPacket(data, toArm(arm), arrivalMs));
|
||||
const offStatus = communicator.addMicStatusListener((body, arm) => onStatus(body, toArm(arm)));
|
||||
return () => {
|
||||
try {
|
||||
offPackets();
|
||||
offStatus();
|
||||
} catch {
|
||||
// The session may have been closed during a disconnect.
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function toArm(arm: string): AudioArm {
|
||||
return arm === "L" || arm === "R" ? arm : "?";
|
||||
}
|
||||
@@ -0,0 +1,49 @@
|
||||
import type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
export type { BandwidthBenchmarkStatus } from './bandwidth-benchmark'
|
||||
|
||||
// The iOS twin of the Android benchmark bridge. iOS has no static active
|
||||
// communicator, so the preview controller binds its session here on connect
|
||||
// and unbinds it on close; the run itself is the same shared-Kotlin
|
||||
// GlassesSessionCore stream as on Android.
|
||||
type BenchmarkSession = {
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean
|
||||
cancelBandwidthBenchmark(): void
|
||||
getBandwidthBenchmarkStatus(): string
|
||||
}
|
||||
let session: BenchmarkSession | null = null
|
||||
|
||||
export function bindBandwidthBenchmarkSession(next: BenchmarkSession | null): void {
|
||||
session = next
|
||||
}
|
||||
|
||||
/** Start a run. False if not connected or a run is already active. */
|
||||
export function startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode = 0): boolean {
|
||||
if (!session) return false
|
||||
try {
|
||||
return session.startBandwidthBenchmark(
|
||||
Math.round(messageSize), Math.round(windowSize), Math.round(durationMs), Math.round(linkMode))
|
||||
} catch (error) {
|
||||
console.warn(`startBandwidthBenchmark failed: ${error}`)
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
/** Cancel an in-progress run. Safe to call when idle. */
|
||||
export function cancelBandwidthBenchmark(): void {
|
||||
try {
|
||||
session?.cancelBandwidthBenchmark()
|
||||
} catch (error) {
|
||||
console.warn(`cancelBandwidthBenchmark failed: ${error}`)
|
||||
}
|
||||
}
|
||||
|
||||
/** Status of the current or most recent run; null when not connected. */
|
||||
export function getBandwidthBenchmarkStatus(): BandwidthBenchmarkStatus | null {
|
||||
if (!session) return null
|
||||
try {
|
||||
return JSON.parse(session.getBandwidthBenchmarkStatus()) as BandwidthBenchmarkStatus
|
||||
} catch (error) {
|
||||
console.warn(`getBandwidthBenchmarkStatus failed: ${error}`)
|
||||
return null
|
||||
}
|
||||
}
|
||||
@@ -22,6 +22,9 @@ export type BandwidthBenchmarkStatus = {
|
||||
payloadBytesAcked: number;
|
||||
wireBytesAcked: number;
|
||||
aborted: boolean;
|
||||
/** Each arm's current PHY ("2M", "1M", ...); null when the platform doesn't report it. */
|
||||
leftPhy?: string | null;
|
||||
rightPhy?: string | null;
|
||||
};
|
||||
|
||||
function activeCommunicator(): any {
|
||||
|
||||
@@ -2,11 +2,18 @@ import { Http } from '@nativescript/core';
|
||||
import { withUserAgent } from '../util/http';
|
||||
import type { BinaryPostResponse, BinaryResponse } from './binary-http';
|
||||
|
||||
/** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */
|
||||
/**
|
||||
* Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies.
|
||||
*
|
||||
* toArrayBuffer() is interop.bufferFromData, which in @nativescript/ios 9.0.3
|
||||
* views the NSData's bytes without retaining it: once the response is
|
||||
* collected, the buffer is dangling (a store EHPK was saved as 41 MB of
|
||||
* zeros). Copy while the response is still alive.
|
||||
*/
|
||||
export async function downloadBinary(url: string, timeoutMs: number): Promise<BinaryResponse> {
|
||||
const response = await Http.request({ url, method: 'GET', headers: withUserAgent(), timeout: timeoutMs });
|
||||
const length = Object.entries(response.headers).find(([name]) => name.toLowerCase() === 'content-length')?.[1];
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) };
|
||||
return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()).slice(), contentLength: Number(length ?? 0) };
|
||||
}
|
||||
|
||||
/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */
|
||||
|
||||
@@ -12,13 +12,17 @@ import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServ
|
||||
*/
|
||||
export type VoiceProviderSelection = {
|
||||
provider: VoiceProviderKind;
|
||||
/**
|
||||
* Spoken language the provider should expect (ISO 639-1); empty lets it
|
||||
* detect the language. Cloud providers only.
|
||||
*/
|
||||
language?: string;
|
||||
elevenLabsApiKey: string;
|
||||
openAiApiKey: string;
|
||||
sonioxApiKey: string;
|
||||
/** Self-hosted Whisper ("whisper-server"); the key is optional there. */
|
||||
whisperServerUrl?: string;
|
||||
whisperServerModel?: string;
|
||||
whisperServerLanguage?: string;
|
||||
whisperServerApiKey?: string;
|
||||
};
|
||||
|
||||
@@ -58,7 +62,7 @@ const CLOUD_PROVIDERS: Partial<Record<VoiceProviderKind, CloudProvider>> = {
|
||||
create: (options, selection) => new WhisperServerSttClient(options, {
|
||||
baseUrl: selection.whisperServerUrl ?? "",
|
||||
model: selection.whisperServerModel ?? "",
|
||||
language: selection.whisperServerLanguage ?? "",
|
||||
language: selection.language ?? "",
|
||||
}),
|
||||
finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS,
|
||||
},
|
||||
@@ -69,14 +73,11 @@ function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection
|
||||
}
|
||||
|
||||
/**
|
||||
* What a capture transcribes with: the provider, and for Whisper server the
|
||||
* language it is told to expect. A running capture is shared only by holders
|
||||
* asking for the same.
|
||||
* What a capture transcribes with: the provider and the language it is told
|
||||
* to expect. A running capture is shared only by holders asking for the same.
|
||||
*/
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "whisperServerLanguage">): string {
|
||||
return selection.provider === "whisper-server"
|
||||
? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}`
|
||||
: selection.provider;
|
||||
export function transcriptionKey(selection: Pick<VoiceProviderSelection, "provider" | "language">): string {
|
||||
return `${selection.provider}:${(selection.language ?? "").trim().toLowerCase()}`;
|
||||
}
|
||||
|
||||
/** Whether createCloudSttClient would return a cloud client for this selection. */
|
||||
@@ -105,7 +106,7 @@ export function createCloudSttClient(
|
||||
const apiKey = provider.apiKey(selection).trim();
|
||||
return new ReconnectingSttClient(
|
||||
(clientOptions) => provider.create(clientOptions, selection),
|
||||
{ ...options, apiKey },
|
||||
{ ...options, apiKey, language: (selection.language ?? "").trim() },
|
||||
continuous,
|
||||
provider.finalTimeoutMs,
|
||||
);
|
||||
|
||||
@@ -17,6 +17,8 @@ export type CloudSttTranscriptEvent = {
|
||||
|
||||
export type CloudSttOptions = {
|
||||
apiKey: string;
|
||||
/** Spoken language to expect (ISO 639-1); empty or absent lets the provider detect it. */
|
||||
language?: string;
|
||||
onTranscript: (event: CloudSttTranscriptEvent) => void;
|
||||
onStatus: (status: string) => void;
|
||||
onError: (message: string) => void;
|
||||
|
||||
@@ -33,7 +33,10 @@ export class ElevenLabsSttClient {
|
||||
|
||||
start(): void {
|
||||
if (this.closed || this.ws) return;
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`;
|
||||
// Without language_code the model detects the language, which can go
|
||||
// badly wrong on short or noisy segments (Japanese heard as Russian).
|
||||
const language = this.options.language ? `&language_code=${encodeURIComponent(this.options.language)}` : "";
|
||||
const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual${language}`;
|
||||
const listener = {
|
||||
onOpen: () => {
|
||||
if (this.closed) return;
|
||||
|
||||
@@ -2,6 +2,7 @@ import type { RingInput } from "../g2/ring-input";
|
||||
import { File, knownFolders, path, type ImageSource } from "@nativescript/core";
|
||||
import { REQUIRED_FACECLAW_FIRMWARE_VERSION, type FirmwareInfo } from "../g2/firmware-compat";
|
||||
import { type CompassEvent } from "./compass-types";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
import { fromData, toData } from "./kotlin-data";
|
||||
import { previewPixels } from "./ios-graphics";
|
||||
import { iosBluetooth } from "./ios-bluetooth";
|
||||
@@ -14,6 +15,9 @@ declare const FaceclawKitFaceclawBleCommunicatorListener: any;
|
||||
declare const FaceclawKitIosAncsListener: any;
|
||||
declare const FaceclawKitIosAudioPacketListener: any;
|
||||
declare const FaceclawKitFaceclawCompassListener: any;
|
||||
declare const FaceclawKitFaceclawImuListener: any;
|
||||
declare const FaceclawKitIosAmbientLightListener: any;
|
||||
declare const FaceclawKitIosMicStatusListener: any;
|
||||
|
||||
|
||||
export type CommunicatorPhase =
|
||||
@@ -164,6 +168,29 @@ class CompassListener extends NSObject {
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class ImuListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitFaceclawImuListener];
|
||||
handler!: (reading: ImuReading) => void;
|
||||
onImuDataXYZEventSource(x: number, y: number, z: number, eventSource: number): void {
|
||||
this.handler({ x: Number(x), y: Number(y), z: Number(z), source: Number(eventSource) });
|
||||
}
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class AmbientLightListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosAmbientLightListener];
|
||||
handler!: (body: Uint8Array) => void;
|
||||
onNsDataAmbientLightBody(body: NSData): void { this.handler(fromData(body)); }
|
||||
}
|
||||
|
||||
@NativeClass()
|
||||
class MicStatusListener extends NSObject {
|
||||
static ObjCProtocols = [FaceclawKitIosMicStatusListener];
|
||||
handler!: (body: Uint8Array, arm: string) => void;
|
||||
onNsDataMicStatusBodyArm(body: NSData, arm: string): void { this.handler(fromData(body), String(arm ?? "")); }
|
||||
}
|
||||
|
||||
/**
|
||||
* The iOS twin of the Android communicator bridge: the same public API over
|
||||
* the shared Kotlin session (FaceclawKitIosGlassesSession, CoreBluetooth
|
||||
@@ -173,6 +200,8 @@ class CompassListener extends NSObject {
|
||||
* Android so callers never re-enter the bridge from a Kotlin frame.
|
||||
*/
|
||||
export class FaceclawCommunicatorBridge {
|
||||
// The started, not yet closed bridge (see getActive).
|
||||
private static active: FaceclawCommunicatorBridge | null = null;
|
||||
private readonly communicator: any;
|
||||
private readonly listenerProxy: SessionListener;
|
||||
private ancsProxy: AncsListener | null = null;
|
||||
@@ -200,6 +229,17 @@ export class FaceclawCommunicatorBridge {
|
||||
private audioListener: AudioPacketListener | null = null;
|
||||
private microphoneWork: Promise<void> = Promise.resolve();
|
||||
private microphoneToken = 0;
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* The bridge whose session is running, for modules that reach the glasses
|
||||
* without the bridge threaded through (imu, ambient-light, audio-monitor):
|
||||
* the iOS stand-in for Android's FaceclawBleCommunicator.getActive().
|
||||
* Set by start(), cleared by close().
|
||||
*/
|
||||
static getActive(): FaceclawCommunicatorBridge | null {
|
||||
return FaceclawCommunicatorBridge.active;
|
||||
}
|
||||
|
||||
constructor(addresses: { right: string; left: string; ring?: string; ringRole?: string }) {
|
||||
this.communicator = FaceclawKitIosGlassesSession.alloc().initWithRightAddressLeftAddressRingAddress(
|
||||
@@ -454,7 +494,9 @@ export class FaceclawCommunicatorBridge {
|
||||
// ----- session ---------------------------------------------------------------------
|
||||
|
||||
async start(): Promise<void> {
|
||||
await this.enqueueNativeCall(() => { this.communicator.start(); });
|
||||
await this.enqueueNativeCall(() => {
|
||||
if (this.communicator.start() && !this.closed) FaceclawCommunicatorBridge.active = this;
|
||||
});
|
||||
}
|
||||
|
||||
async setG2ScreenOn(screenOn: boolean): Promise<void> {
|
||||
@@ -644,6 +686,8 @@ export class FaceclawCommunicatorBridge {
|
||||
}
|
||||
|
||||
async close(): Promise<void> {
|
||||
this.closed = true;
|
||||
if (FaceclawCommunicatorBridge.active === this) FaceclawCommunicatorBridge.active = null;
|
||||
this.setAncsListeners(false);
|
||||
this.previewAnimationListeners.clear();
|
||||
await this.enqueueNativeCall(() => {
|
||||
@@ -667,6 +711,85 @@ export class FaceclawCommunicatorBridge {
|
||||
return () => { this.communicator.removeCompassListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Developer > Debug tests > BLE bandwidth; see app/native/bandwidth-benchmark.ios.ts. */
|
||||
startBandwidthBenchmark(messageSize: number, windowSize: number, durationMs: number, linkMode: number): boolean {
|
||||
return Boolean(this.communicator.startBandwidthBenchmarkWithLinkModeMessageSizeWindowSizeDurationMsLinkMode(
|
||||
messageSize, windowSize, durationMs, linkMode));
|
||||
}
|
||||
|
||||
cancelBandwidthBenchmark(): void {
|
||||
this.communicator.cancelBandwidthBenchmark();
|
||||
}
|
||||
|
||||
/** Status/results of the current or most recent run, as JSON. */
|
||||
getBandwidthBenchmarkStatus(): string {
|
||||
return String(this.communicator.getBandwidthBenchmarkStatus());
|
||||
}
|
||||
|
||||
// ----- sensors and passive audio, for the developer demos -----------------------------
|
||||
// Fire-and-forget, so they skip the native-call queue like the compass controls do.
|
||||
|
||||
/** Enable/disable the accelerometer report stream (an ImuReportPace code, 100..1000). */
|
||||
setImuReportEnabled(enable: boolean, reportFrq: number): void {
|
||||
this.communicator.setImuReportEnabledEnableReportFrq(Boolean(enable), Math.max(0, Math.round(reportFrq)));
|
||||
}
|
||||
|
||||
/** Accelerometer samples, delivered on the main queue. */
|
||||
addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const proxy = ImuListener.new() as ImuListener;
|
||||
proxy.handler = listener;
|
||||
this.communicator.addImuListenerListener(proxy);
|
||||
return () => { this.communicator.removeImuListenerListener(proxy); };
|
||||
}
|
||||
|
||||
/** Ask the CFW for one ambient-light report (delivered to the ambient-light listeners). */
|
||||
queryAmbientLight(): void {
|
||||
this.communicator.queryAmbientLight();
|
||||
}
|
||||
|
||||
setAmbientLightPolling(enable: boolean, intervalMs: number, minDelta: number, heartbeatMs: number, bindToLease: boolean): void {
|
||||
this.communicator.setAmbientLightPollingEnableIntervalMsMinDeltaHeartbeatMsBindToLease(Boolean(enable),
|
||||
Math.round(intervalMs), Math.round(minDelta), Math.round(heartbeatMs), Boolean(bindToLease));
|
||||
}
|
||||
|
||||
/** Raw field-105 ambient-light records, delivered on the main queue. */
|
||||
addAmbientLightListener(listener: (body: Uint8Array) => void): () => void {
|
||||
const proxy = AmbientLightListener.new() as AmbientLightListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAmbientLightListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAmbientLightListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Raw field-104 mic_control status records and their temple ("L", "R" or "?"), on the main queue. */
|
||||
addMicStatusListener(listener: (body: Uint8Array, arm: string) => void): () => void {
|
||||
const proxy = MicStatusListener.new() as MicStatusListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addMicStatusListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeMicStatusListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** Queue a CFW mic_control record for the chosen temples; false when nothing was queued. */
|
||||
sendFaceclawMicControl(record: Uint8Array, label: string, rightTemple: boolean, leftTemple: boolean): boolean {
|
||||
return Boolean(this.communicator.sendFaceclawMicControlRecordLabelRightTempleLeftTemple(
|
||||
toData(new Uint8Array(record)), label, Boolean(rightTemple), Boolean(leftTemple)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Every audio-characteristic packet, whatever enabled the mics, on the
|
||||
* main queue. Passive: sends nothing and doesn't affect capture.
|
||||
*/
|
||||
addAudioMonitorListener(listener: (packet: Uint8Array, arm: string, arrivalMs: number) => void): () => void {
|
||||
const proxy = AudioPacketListener.new() as AudioPacketListener;
|
||||
proxy.handler = listener;
|
||||
const handle = this.communicator.addAudioMonitorListenerNsDataListener(proxy);
|
||||
return () => { this.communicator.removeAudioMonitorListenerListener(handle); };
|
||||
}
|
||||
|
||||
/** True while Faceclaw itself is forwarding glasses audio (speech capture or an app's mic). */
|
||||
isAudioCaptureActive(): boolean {
|
||||
return Boolean(this.communicator.isAudioCaptureActive());
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialized glasses microphone enable/disable (the voice bridge's
|
||||
* IosMicrophoneSession contract). A release while an enable awaits its ACK
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
/** Platform-neutral IMU types, shared by imu.ts and imu.ios.ts. */
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
import { FaceclawCommunicatorBridge } from "./faceclaw-communicator.ios";
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
/**
|
||||
* IMU (accelerometer) access on iOS: the same API as imu.ts over the running
|
||||
* shared-Kotlin session. As there, the stream is experimental, the calls are
|
||||
* no-ops while no session is running, and a listener stays with the session
|
||||
* it subscribed to.
|
||||
*/
|
||||
|
||||
/** Enable/disable the accelerometer report stream. No-op if not connected. */
|
||||
export function setImuReportEnabled(enable: boolean, reportFreq: number): void {
|
||||
try {
|
||||
FaceclawCommunicatorBridge.getActive()?.setImuReportEnabled(enable, reportFreq);
|
||||
} catch (error) {
|
||||
console.warn(`setImuReportEnabled failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Subscribe to IMU readings; returns an unsubscribe function. */
|
||||
export function addImuListener(listener: (reading: ImuReading) => void): () => void {
|
||||
const active = FaceclawCommunicatorBridge.getActive();
|
||||
if (!active) return () => {};
|
||||
const unsubscribe = active.addImuListener(listener);
|
||||
return () => {
|
||||
try {
|
||||
unsubscribe();
|
||||
} catch {
|
||||
// The session may be gone after a disconnect; nothing to clean up.
|
||||
}
|
||||
};
|
||||
}
|
||||
+3
-15
@@ -12,7 +12,9 @@ declare const com: any;
|
||||
* static, so it works from any isolate without threading the bridge through.
|
||||
*/
|
||||
|
||||
export type ImuReading = { x: number; y: number; z: number; source: number };
|
||||
import { type ImuReading } from "./imu-types";
|
||||
|
||||
export * from "./imu-types";
|
||||
|
||||
function activeCommunicator(): any {
|
||||
if (!global.isAndroid) return null;
|
||||
@@ -52,17 +54,3 @@ export function addImuListener(listener: (reading: ImuReading) => void): () => v
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/** Map an IMU sample's EventSource to a short label. */
|
||||
export function imuSourceLabel(source: number): string {
|
||||
switch (source) {
|
||||
case 1:
|
||||
return "R arm";
|
||||
case 2:
|
||||
return "ring";
|
||||
case 3:
|
||||
return "L arm";
|
||||
default:
|
||||
return "device";
|
||||
}
|
||||
}
|
||||
+4
-6
@@ -6,12 +6,14 @@ import type {
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import { buildQwenPrompt } from "../assistant/qwen-prompt";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
|
||||
/**
|
||||
* On-phone LLM backend: model download management plus a streaming client
|
||||
* Android's on-phone LLM backend (reached through local-llm.ts; iOS uses
|
||||
* local-llm.ios.ts instead): model download management plus a streaming client
|
||||
* with the same LlmStreamOptions surface as the Anthropic/OpenAI clients, so
|
||||
* the direct assistant loop can treat "local" as just another provider.
|
||||
* Inference is llama.cpp on the CPU (FaceclawLlamaRunner over JNI).
|
||||
@@ -38,11 +40,7 @@ const TOP_K = 20;
|
||||
/** Free the ~3GB of model + KV this long after the last generation. */
|
||||
const IDLE_UNLOAD_MS = 2 * 60 * 1000;
|
||||
|
||||
export type LocalModelState = {
|
||||
status: "absent" | "downloading" | "ready";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
};
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
let runner: any = null;
|
||||
let downloader: any = null;
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
/** Shared shapes for the on-phone LLM (local-llm.ts on Android, local-llm.ios.ts on iOS). */
|
||||
|
||||
export type LocalModelState = {
|
||||
/**
|
||||
* Android downloads its model: absent → downloading → ready. iOS uses the
|
||||
* system's Apple Intelligence model: ready, or unavailable (see detail).
|
||||
*/
|
||||
status: "absent" | "downloading" | "ready" | "unavailable";
|
||||
bytesDownloaded: number;
|
||||
totalBytes: number;
|
||||
/** Why the model can't be used, when status is "unavailable" (short, for a settings row). */
|
||||
detail?: string;
|
||||
};
|
||||
@@ -0,0 +1,383 @@
|
||||
import type {
|
||||
LlmContentBlock,
|
||||
LlmMessage,
|
||||
LlmStreamHandle,
|
||||
LlmStreamOptions,
|
||||
} from "../assistant/llm-protocol";
|
||||
import type { LocalModelState } from "./local-llm-types";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
declare const FaceclawSystemLanguageModel: any;
|
||||
|
||||
/**
|
||||
* The on-phone LLM on iOS: Apple's Foundation Models system model (Apple
|
||||
* Intelligence, iOS 26+), through FaceclawSystemLanguageModel.swift. Same
|
||||
* surface as Android's local-llm.ts, so the direct assistant loop, Translate,
|
||||
* Calculator and the Microphones helpers treat it as the "local" provider.
|
||||
*
|
||||
* Why not Android's Qwen3-4B on llama.cpp: iOS forbids GPU work in the
|
||||
* background (where Faceclaw usually is, driving the glasses with the phone
|
||||
* locked), CPU-only inference of a 4B model is far too slow on a phone, and a
|
||||
* ~3 GB in-process model makes the app the first thing jetsam kills. The
|
||||
* system model runs out of process on the Neural Engine, needs no download,
|
||||
* and decodes tool arguments under a schema constraint (like our GBNF grammar
|
||||
* on Android). Its costs: a 4096-token context (the Swift side fits history
|
||||
* and tool definitions into it), Apple's guardrails, a background request
|
||||
* budget, and Apple Intelligence-capable hardware.
|
||||
*
|
||||
* Tool calls: the framework runs tools itself, mid-response. The Swift side
|
||||
* parks each call and reports it; here that ends the loop iteration with
|
||||
* stopReason "tool_use", the direct backend executes the tools as for any
|
||||
* provider, and its next request (ending in the matching tool_result blocks)
|
||||
* resumes the parked generation instead of starting a new one.
|
||||
*/
|
||||
|
||||
export const LOCAL_MODEL = {
|
||||
id: "apple-foundation-model",
|
||||
label: "Apple Intelligence",
|
||||
sizeBytes: 0,
|
||||
};
|
||||
|
||||
const DEFAULT_MAX_TOKENS = 512;
|
||||
/** A generation parked in a tool call is abandoned if the tool loop never returns. */
|
||||
const PARKED_TIMEOUT_MS = 3 * 60 * 1000;
|
||||
/**
|
||||
* Only used when tool results arrive with no parked generation to resume (it
|
||||
* timed out, or the app restarted mid-turn): the framework has no way to
|
||||
* continue after a tool output without a new prompt.
|
||||
*/
|
||||
export const CONTINUE_PROMPT = "Use the tool results above to finish answering my last request.";
|
||||
|
||||
type Availability =
|
||||
| "available"
|
||||
| "unsupportedOS"
|
||||
| "deviceNotEligible"
|
||||
| "appleIntelligenceNotEnabled"
|
||||
| "modelNotReady"
|
||||
| "unavailable";
|
||||
|
||||
const UNAVAILABLE_DETAIL: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "needs iOS 26",
|
||||
deviceNotEligible: "not on this iPhone",
|
||||
appleIntelligenceNotEnabled: "Apple Intelligence off",
|
||||
modelNotReady: "downloading",
|
||||
unavailable: "unavailable",
|
||||
};
|
||||
|
||||
const UNAVAILABLE_MESSAGE: Record<Exclude<Availability, "available">, string> = {
|
||||
unsupportedOS: "The on-phone model needs iOS 26 or later with Apple Intelligence.",
|
||||
deviceNotEligible: "This iPhone doesn't support Apple Intelligence, which the on-phone model uses.",
|
||||
appleIntelligenceNotEnabled:
|
||||
"Turn on Apple Intelligence (iOS Settings > Apple Intelligence & Siri) to use the on-phone model.",
|
||||
modelNotReady: "Apple Intelligence is still downloading its model. Try again later.",
|
||||
unavailable: "Apple Intelligence's on-device model is unavailable right now.",
|
||||
};
|
||||
|
||||
export function localModelAvailability(): Availability {
|
||||
try {
|
||||
if (typeof FaceclawSystemLanguageModel === "undefined") return "unsupportedOS";
|
||||
const value = String(FaceclawSystemLanguageModel.availability());
|
||||
return value in UNAVAILABLE_MESSAGE || value === "available" ? (value as Availability) : "unavailable";
|
||||
} catch {
|
||||
return "unsupportedOS";
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether this iPhone can ever run the model (Apple Intelligence may still be off or downloading). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
const availability = localModelAvailability();
|
||||
return availability !== "unsupportedOS" && availability !== "deviceNotEligible";
|
||||
}
|
||||
|
||||
export function isLocalModelReady(): boolean {
|
||||
return localModelAvailability() === "available";
|
||||
}
|
||||
|
||||
export function localModelState(): LocalModelState {
|
||||
const availability = localModelAvailability();
|
||||
if (availability === "available") return { status: "ready", bytesDownloaded: 0, totalBytes: 0 };
|
||||
return { status: "unavailable", bytesDownloaded: 0, totalBytes: 0, detail: UNAVAILABLE_DETAIL[availability] };
|
||||
}
|
||||
|
||||
/** The user-facing reason the model can't run, or "" when it can. */
|
||||
export function localModelUnavailableMessage(): string {
|
||||
const availability = localModelAvailability();
|
||||
return availability === "available" ? "" : UNAVAILABLE_MESSAGE[availability];
|
||||
}
|
||||
|
||||
// iOS manages the system model itself: there is nothing to download, cancel,
|
||||
// delete or unload, and no progress to watch.
|
||||
export function onLocalModelStateChanged(_listener: (state: LocalModelState) => void): () => void {
|
||||
return () => {};
|
||||
}
|
||||
export function startLocalModelDownload(): void {}
|
||||
export function cancelLocalModelDownload(): void {}
|
||||
export function deleteLocalModel(): void {}
|
||||
export function unloadLocalModel(): void {}
|
||||
|
||||
type HistoryEntry =
|
||||
| { type: "prompt" | "response"; text: string }
|
||||
| { type: "toolCalls"; calls: Array<{ id: string; name: string; arguments: string }> }
|
||||
| { type: "toolOutput"; id: string; name: string; text: string };
|
||||
|
||||
export type SystemModelRequest = {
|
||||
system: string;
|
||||
tools: Array<{ name: string; description: string; parameters: object }>;
|
||||
history: HistoryEntry[];
|
||||
prompt: string;
|
||||
maxTokens: number;
|
||||
permissive: boolean;
|
||||
};
|
||||
|
||||
type ToolResultBlock = Extract<LlmContentBlock, { type: "tool_result" }>;
|
||||
|
||||
/**
|
||||
* Translate provider-neutral history into the Swift request: everything but
|
||||
* the newest user prompt becomes Transcript entries, and that prompt is what
|
||||
* the model answers.
|
||||
*/
|
||||
export function buildSystemModelRequest(
|
||||
options: Pick<LlmStreamOptions, "system" | "messages" | "tools" | "maxTokens">,
|
||||
): SystemModelRequest {
|
||||
const messages = options.messages;
|
||||
const last = messages[messages.length - 1];
|
||||
const endsWithPrompt = last?.role === "user" && !toolResultsOf(last).length;
|
||||
const history: HistoryEntry[] = [];
|
||||
const toolNames = new Map<string, string>();
|
||||
for (const message of endsWithPrompt ? messages.slice(0, -1) : messages) {
|
||||
appendHistory(history, message, toolNames);
|
||||
}
|
||||
const tools = (options.tools ?? []).map((tool) => ({
|
||||
name: tool.name,
|
||||
description: tool.description,
|
||||
parameters: tool.input_schema,
|
||||
}));
|
||||
return {
|
||||
system: options.system?.trim() || "You are a helpful assistant.",
|
||||
tools,
|
||||
history,
|
||||
prompt: endsWithPrompt ? textOf(last.content) : CONTINUE_PROMPT,
|
||||
maxTokens: options.maxTokens ?? DEFAULT_MAX_TOKENS,
|
||||
// Tool-free requests transform user text (translate, summarize, Q&A over transcripts).
|
||||
permissive: tools.length === 0,
|
||||
};
|
||||
}
|
||||
|
||||
function appendHistory(history: HistoryEntry[], message: LlmMessage, toolNames: Map<string, string>): void {
|
||||
if (message.role === "user") {
|
||||
for (const result of toolResultsOf(message)) {
|
||||
const body = result.is_error ? `Error: ${result.content}` : result.content || "(no output)";
|
||||
history.push({ type: "toolOutput", id: result.tool_use_id, name: toolNames.get(result.tool_use_id) ?? "tool", text: body });
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "prompt", text });
|
||||
return;
|
||||
}
|
||||
const text = textOf(message.content);
|
||||
if (text) history.push({ type: "response", text });
|
||||
if (typeof message.content === "string") return;
|
||||
const calls: Array<{ id: string; name: string; arguments: string }> = [];
|
||||
for (const block of message.content) {
|
||||
if (block.type !== "tool_use") continue;
|
||||
toolNames.set(block.id, block.name);
|
||||
calls.push({ id: block.id, name: block.name, arguments: JSON.stringify(block.input ?? {}) });
|
||||
}
|
||||
if (calls.length) history.push({ type: "toolCalls", calls });
|
||||
}
|
||||
|
||||
function textOf(content: string | LlmContentBlock[]): string {
|
||||
if (typeof content === "string") return content.trim();
|
||||
return content
|
||||
.filter((block): block is Extract<LlmContentBlock, { type: "text" }> => block.type === "text")
|
||||
.map((block) => block.text.trim())
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
function toolResultsOf(message: LlmMessage | undefined): ToolResultBlock[] {
|
||||
if (!message || message.role !== "user" || typeof message.content === "string") return [];
|
||||
return message.content.filter((block): block is ToolResultBlock => block.type === "tool_result");
|
||||
}
|
||||
|
||||
function parseArguments(json: string): any {
|
||||
try {
|
||||
const value = JSON.parse(json);
|
||||
return value && typeof value === "object" ? value : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
type NativeEvent = {
|
||||
kind: string;
|
||||
text?: string;
|
||||
calls?: Array<{ id: string; name: string; arguments: string }>;
|
||||
code?: string;
|
||||
message?: string;
|
||||
};
|
||||
|
||||
/** Generations parked in tool calls, by call id. */
|
||||
const parked = new Map<string, Generation>();
|
||||
/** Every unfinished generation, so its native object outlives the caller's handle. */
|
||||
const live = new Set<Generation>();
|
||||
|
||||
/**
|
||||
* One native generation, possibly spanning several loop iterations: each
|
||||
* iteration ends at a batch of tool calls (or the end of the reply), and the
|
||||
* next one resumes it with the results.
|
||||
*/
|
||||
class Generation {
|
||||
private readonly native: any;
|
||||
private options: LlmStreamOptions | null;
|
||||
private iteration = 0;
|
||||
private text = "";
|
||||
private callIds: string[] = [];
|
||||
private expiry: ReturnType<typeof setTimeout> | null = null;
|
||||
private closed = false;
|
||||
|
||||
constructor(options: LlmStreamOptions) {
|
||||
this.options = options;
|
||||
this.native = FaceclawSystemLanguageModel.new();
|
||||
this.native.eventHandler = (json: string) => this.onEvent(String(json));
|
||||
live.add(this);
|
||||
}
|
||||
|
||||
start(request: SystemModelRequest): LlmStreamHandle {
|
||||
this.native.start(JSON.stringify(request));
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
/** Whether these results answer this generation's parked calls. */
|
||||
awaits(results: ToolResultBlock[]): boolean {
|
||||
return !this.closed && results.some((result) => this.callIds.includes(result.tool_use_id));
|
||||
}
|
||||
|
||||
resume(options: LlmStreamOptions, results: ToolResultBlock[]): LlmStreamHandle {
|
||||
this.unpark();
|
||||
this.options = options;
|
||||
this.iteration++;
|
||||
this.text = "";
|
||||
this.native.resolveToolCalls(
|
||||
JSON.stringify(results.map((result) => ({ id: result.tool_use_id, content: result.content, isError: Boolean(result.is_error) }))),
|
||||
);
|
||||
return this.handle();
|
||||
}
|
||||
|
||||
cancel(): void {
|
||||
if (this.closed) return;
|
||||
this.close();
|
||||
try {
|
||||
this.native.cancel();
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
private handle(): LlmStreamHandle {
|
||||
const iteration = this.iteration;
|
||||
// A stale handle (an earlier iteration's) still cancels a generation parked
|
||||
// since then: its owner gave up on the turn.
|
||||
return { cancel: () => { if (this.iteration === iteration || this.options === null) this.cancel(); } };
|
||||
}
|
||||
|
||||
private close(): void {
|
||||
this.closed = true;
|
||||
this.unpark();
|
||||
this.options = null;
|
||||
this.native.eventHandler = null;
|
||||
live.delete(this);
|
||||
}
|
||||
|
||||
private park(ids: string[]): void {
|
||||
this.callIds = ids;
|
||||
for (const id of ids) parked.set(id, this);
|
||||
this.expiry = setTimeout(() => {
|
||||
console.warn("[local-llm] abandoning an on-device generation whose tool results never came back");
|
||||
this.cancel();
|
||||
}, PARKED_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private unpark(): void {
|
||||
for (const id of this.callIds) {
|
||||
if (parked.get(id) === this) parked.delete(id);
|
||||
}
|
||||
this.callIds = [];
|
||||
if (this.expiry !== null) clearTimeout(this.expiry);
|
||||
this.expiry = null;
|
||||
}
|
||||
|
||||
private onEvent(json: string): void {
|
||||
if (this.closed) return;
|
||||
let event: NativeEvent;
|
||||
try {
|
||||
event = JSON.parse(json);
|
||||
} catch {
|
||||
return;
|
||||
}
|
||||
if (event.kind === "log") {
|
||||
console.log(`[local-llm] ${event.message ?? ""}`);
|
||||
return;
|
||||
}
|
||||
const options = this.options;
|
||||
if (!options) return;
|
||||
switch (event.kind) {
|
||||
case "delta": {
|
||||
const delta = event.text ?? "";
|
||||
this.text += delta;
|
||||
options.onTextDelta?.(delta, this.text);
|
||||
return;
|
||||
}
|
||||
case "toolCalls": {
|
||||
const calls = event.calls ?? [];
|
||||
const text = this.text.trim();
|
||||
const content: LlmContentBlock[] = text ? [{ type: "text", text }] : [];
|
||||
for (const call of calls) {
|
||||
content.push({ type: "tool_use", id: call.id, name: call.name, input: parseArguments(call.arguments) });
|
||||
}
|
||||
this.options = null;
|
||||
this.park(calls.map((call) => call.id));
|
||||
options.onDone({ text, content, stopReason: "tool_use" });
|
||||
return;
|
||||
}
|
||||
case "done": {
|
||||
this.close();
|
||||
const text = this.text.trim();
|
||||
options.onDone({ text, content: text ? [{ type: "text", text }] : [], stopReason: "end_turn" });
|
||||
return;
|
||||
}
|
||||
case "error": {
|
||||
this.close();
|
||||
if (event.code === "refusal") {
|
||||
options.onDone({ text: this.text.trim(), content: [], stopReason: "refusal" });
|
||||
} else {
|
||||
options.onError(event.message || "The on-device model failed");
|
||||
}
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream a reply from the system model. A request whose history ends in tool
|
||||
* results for a parked generation resumes it; anything else starts afresh.
|
||||
*/
|
||||
export function streamLocalModel(options: LlmStreamOptions): LlmStreamHandle {
|
||||
const availability = localModelAvailability();
|
||||
if (availability !== "available") {
|
||||
setTimeout(() => options.onError(UNAVAILABLE_MESSAGE[availability]), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
const results = toolResultsOf(options.messages[options.messages.length - 1]);
|
||||
const waiting = results.map((result) => parked.get(result.tool_use_id)).find(Boolean);
|
||||
if (waiting?.awaits(results)) return waiting.resume(options, results);
|
||||
let generation: Generation | null = null;
|
||||
try {
|
||||
generation = new Generation(options);
|
||||
return generation.start(buildSystemModelRequest(options));
|
||||
} catch (error) {
|
||||
generation?.cancel();
|
||||
setTimeout(() => options.onError(`On-device model failed: ${String((error as Error)?.message ?? error)}`), 0);
|
||||
return { cancel: () => {} };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The on-phone LLM: one entry point for every feature that can run on the
|
||||
* phone without an API key (the direct assistant's "local" provider,
|
||||
* Translate, Calculator's rewrite fallback, the Microphones helpers). Each
|
||||
* platform supplies a streaming client with the cloud clients' LlmStreamOptions
|
||||
* contract, plus model status for Settings.
|
||||
*
|
||||
* - Android (this file): Qwen3-4B on llama.cpp, downloaded on demand (llama.ts).
|
||||
* - iOS (local-llm.ios.ts): Apple's Foundation Models system model; nothing
|
||||
* to download, available on Apple Intelligence devices running iOS 26+.
|
||||
*/
|
||||
export {
|
||||
LOCAL_MODEL,
|
||||
isLocalModelReady,
|
||||
localModelState,
|
||||
onLocalModelStateChanged,
|
||||
startLocalModelDownload,
|
||||
cancelLocalModelDownload,
|
||||
deleteLocalModel,
|
||||
unloadLocalModel,
|
||||
streamLocalQwen as streamLocalModel,
|
||||
} from "./llama";
|
||||
export type { LocalModelState } from "./local-llm-types";
|
||||
|
||||
/** Whether this phone can offer an on-phone model at all (Android: always, by download). */
|
||||
export function isLocalModelSupported(): boolean {
|
||||
return true;
|
||||
}
|
||||
Loaded 100 of 218 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user