Convert most Android-specific kotlin to shared or shareable

This commit is contained in:
James Babcock committed 2026-09-22 18:43:55 -07:00
1 parent f9724ddcab
commit 587364e5e6
92 files changed
+14347 -9193

No files matched your search

@@ -0,0 +1,41 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
/** [SessionLink] over the Android GATT boundary ([FaceclawBleManager]). */
class AndroidSessionLink(private val bleManager: FaceclawBleManager) : SessionLink {
override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs)
override fun requestHighPriority(address: String) {
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
}
override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean =
bleManager.requestMtu(address, mtu, timeoutMs)
override fun negotiatedMtu(address: String): Int = bleManager.getNegotiatedMtu(address)
override fun discoverServices(address: String, timeoutMs: Int): Boolean =
bleManager.discoverServices(address, timeoutMs)
override fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean =
bleManager.enableNotifications(address, characteristicUuid, enable, timeoutMs)
override fun writeFrames(
address: String,
characteristicUuid: String,
frames: List<ByteArray>,
mode: GattWriteMode,
timeoutMs: Int,
): Boolean = bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs)
override fun disconnect(address: String) = bleManager.disconnect(address)
override fun close() = bleManager.close()
override fun isBonded(address: String): Boolean = bleManager.isBonded(address)
override fun prepareBenchmarkLink(address: String, mode: Int) = bleManager.prepareBenchmarkLink(address, mode)
override fun recordDisplayFrameSent() = FaceclawBleManager.recordDisplayFrameSent()
}
@@ -0,0 +1,147 @@
package com.faceclaw.app
import android.util.Log
import com.k2fsa.sherpa.onnx.FeatureConfig
import com.k2fsa.sherpa.onnx.OfflineModelConfig
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizer
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult
import com.k2fsa.sherpa.onnx.OfflineStream
import com.k2fsa.sherpa.onnx.OfflineWhisperModelConfig
import java.io.File
/**
* Android implementations of the shared voice ports: the sherpa-onnx offline recognizer as an
* [OfflineTranscriber], the phone microphone as a [PcmAudioSource], and the JNI LC3 decoder as
* an [AudioPacketDecoder]. The pipelines themselves live in the shared VoiceCaptureSession and
* CaptionSession.
*/
internal object AndroidSpeechEngines {
private const val TAG = "FaceclawVoice"
private const val SAMPLE_RATE = 16000
private const val FEATURE_DIM = 80
/** Sherpa-onnx configuration for the on-device model files in [modelDir]. */
fun recognizerConfig(modelDir: File, kind: VoiceModelKind): OfflineRecognizerConfig {
val modelConfig = OfflineModelConfig.builder()
.setNumThreads(1)
if (kind == VoiceModelKind.WHISPER) {
modelConfig
.setWhisper(OfflineWhisperModelConfig.builder()
.setEncoder(File(modelDir, "base.en-encoder.int8.onnx").absolutePath)
.setDecoder(File(modelDir, "base.en-decoder.int8.onnx").absolutePath)
.setLanguage("en")
.setTask("transcribe")
.build())
.setTokens(File(modelDir, "base.en-tokens.txt").absolutePath)
} else {
modelConfig
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(File(modelDir, "encoder_model.ort").absolutePath)
.setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath)
.build())
.setTokens(File(modelDir, "tokens.txt").absolutePath)
}
return OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(modelConfig.build())
.build()
}
class SherpaTranscriber(config: OfflineRecognizerConfig) : OfflineTranscriber {
private var recognizer: OfflineRecognizer? = OfflineRecognizer(config)
override fun recognize(samples: FloatArray, count: Int): String {
val currentRecognizer = recognizer ?: return ""
val segment = if (count == samples.size) samples else samples.copyOf(count)
val offlineStream: OfflineStream = currentRecognizer.createStream()
try {
offlineStream.acceptWaveform(segment, SAMPLE_RATE)
currentRecognizer.decode(offlineStream)
val result: OfflineRecognizerResult? = currentRecognizer.getResult(offlineStream)
val raw: String? = if (result == null) "" else result.text
return raw?.trim() ?: ""
} finally {
offlineStream.release()
}
}
override fun release() {
recognizer?.release()
recognizer = null
}
}
/** The JNI LC3 decoder behind the shared packet-decoder port. */
class Lc3DecoderAdapter : AudioPacketDecoder {
private val decoder = FaceclawLc3Decoder()
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int = decoder.decodePacket(packet, pcmOut)
override val lastAngleDegrees: Int get() = decoder.getLastAngleDegrees()
override val lastSsr: Int get() = decoder.getLastSsr()
override val realPackets: Long get() = decoder.getRealPackets()
override val duplicatePackets: Long get() = decoder.getDuplicatePackets()
override val missingPackets: Long get() = decoder.getMissingPackets()
override val decodeErrors: Long get() = decoder.getDecodeErrors()
override fun close() = decoder.close()
}
/**
* Open the phone's own microphone at the pipeline's native format (16 kHz
* mono PCM16), or null when it cannot start — the permission is missing
* (SecurityException) or the device refuses the configuration.
*/
fun openPhoneMic(chunkSamples: Int): PcmAudioSource? {
var record: android.media.AudioRecord? = null
try {
val minBytes = android.media.AudioRecord.getMinBufferSize(
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT)
val bufferBytes = Math.max(minBytes, chunkSamples * 2 * 4)
record = android.media.AudioRecord(
android.media.MediaRecorder.AudioSource.VOICE_RECOGNITION,
SAMPLE_RATE,
android.media.AudioFormat.CHANNEL_IN_MONO,
android.media.AudioFormat.ENCODING_PCM_16BIT,
bufferBytes)
if (record.state != android.media.AudioRecord.STATE_INITIALIZED) {
record.release()
return null
}
record.startRecording()
if (record.recordingState != android.media.AudioRecord.RECORDSTATE_RECORDING) {
record.release()
return null
}
return AudioRecordSource(record)
} catch (t: Throwable) {
Log.w(TAG, "phone mic open failed", t)
if (record != null) {
record.release()
}
return null
}
}
private class AudioRecordSource(private val record: android.media.AudioRecord) : PcmAudioSource {
override fun read(pcm: ShortArray): Int = record.read(pcm, 0, pcm.size)
override fun close() {
try {
record.stop()
} catch (ignored: Throwable) {
// Already stopped or never recording; release below either way.
}
record.release()
}
}
}
@@ -0,0 +1,64 @@
package com.faceclaw.app
import android.bluetooth.BluetoothDevice
import android.bluetooth.BluetoothGatt
import android.content.Context
/**
* Android [StockLink]: a 1:1 wrapper over FaceclawBleManager for the stock-firmware flows
* (device-info probe, flash prompt, OTA flasher). Owns its own GATT manager, exactly as each
* flow did before the shared cores existed.
*/
class AndroidStockLink(context: Context) : StockLink, FaceclawBleListener {
private val bleManager = FaceclawBleManager(context.applicationContext)
@Volatile
private var listener: StockLinkListener? = null
init {
bleManager.setListener(this)
}
override fun setListener(listener: StockLinkListener?) {
this.listener = listener
}
override fun connect(address: String, timeoutMs: Int): Boolean = bleManager.connect(address, timeoutMs)
override fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) {
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, desiredMtu, timeoutMs)
}
override fun discoverServices(address: String, timeoutMs: Int): Boolean = bleManager.discoverServices(address, timeoutMs)
override fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean =
bleManager.enableNotifications(address, characteristicUuid, true, timeoutMs)
override fun writeFrames(address: String, characteristicUuid: String, frames: List<ByteArray>, mode: GattWriteMode, timeoutMs: Int): Boolean =
bleManager.writeFrames(address, characteristicUuid, frames, AndroidProtocolPlatform.writeType(mode), timeoutMs)
override fun isConnected(address: String): Boolean = bleManager.isConnected(address)
override fun bondState(address: String): BondState =
when (bleManager.getBondState(address)) {
BluetoothDevice.BOND_NONE -> BondState.NONE
BluetoothDevice.BOND_BONDING -> BondState.BONDING
BluetoothDevice.BOND_BONDED -> BondState.BONDED
else -> BondState.UNKNOWN
}
override fun disconnect(address: String) = bleManager.disconnect(address)
override fun close() = bleManager.close()
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (address == null || characteristicUuid == null || data == null) return
listener?.onNotification(address, characteristicUuid, data)
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (address == null) return
listener?.onConnectionStateChange(address, connected)
}
}
@@ -22,9 +22,6 @@ import android.os.PowerManager
import android.os.VibrationEffect
import android.os.Vibrator
import java.util.ArrayList
import java.util.Collections
import java.util.LinkedHashMap
/**
* The ringing foreground service. One instance carries every item currently
@@ -37,22 +34,14 @@ import java.util.LinkedHashMap
* to total silence) Do Not Disturb do not silence it.
*/
class FaceclawAlarmService : Service() {
/** The shared item model; kept as a nested subclass so the lock-screen activity's references hold. */
internal class Ringing(
@JvmField val id: Long,
id: Long,
title: String?,
text: String?,
kind: String?,
snoozeMinutes: Int
) {
@JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title
@JvmField val text: String = text ?: ""
@JvmField val kind: String = kind ?: FaceclawAlarms.KIND_ALARM
@JvmField val snoozeMinutes: Int = Math.max(1, snoozeMinutes)
@JvmField val ringAtMs: Long = System.currentTimeMillis()
@JvmField var deliveredAtMs: Long = 0
@JvmField var escalated: Boolean = false
@JvmField var silenced: Boolean = false
}
) : RingingItem(id, title, text, kind, snoozeMinutes, System.currentTimeMillis())
companion object {
const val CHANNEL_ID = "faceclaw-alarms"
@@ -72,17 +61,11 @@ class FaceclawAlarmService : Service() {
/** Alarm-stream level (fraction of max) used when the wearer has it muted. */
internal const val MUTED_VOLUME_FRACTION = 0.6f
/** Ringing items, oldest first; static so the activity can read them. */
private val ringing: MutableMap<Long, Ringing> = Collections.synchronizedMap(LinkedHashMap())
/**
* Signals that arrived before the ring intent was processed (the JS
* engine fires, launches its window and reports delivery on the same
* main thread the service's onStartCommand queues behind), keyed by id
* with their arrival time. Consumed by startRinging.
* The escalation policy is process-wide (the activity reads it, the static entry points
* consult it before the service exists); the live service attaches its timers/effects.
*/
private val deliveredEarly: MutableMap<Long, Long> = Collections.synchronizedMap(LinkedHashMap())
private val stoppedEarly: MutableMap<Long, Long> = Collections.synchronizedMap(LinkedHashMap())
private const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L
private val policy = AlarmRingingPolicy(FaceclawAlarms.glassesGate)
// ------------------------------------------------------------------
// Static entry points
@@ -102,8 +85,7 @@ class FaceclawAlarmService : Service() {
@JvmStatic
fun delivered(context: Context, id: Long) {
if (!ringing.containsKey(id)) {
deliveredEarly[id] = System.currentTimeMillis()
if (!policy.deliveredOrDefer(id)) {
return
}
val appContext = context.applicationContext
@@ -114,8 +96,7 @@ class FaceclawAlarmService : Service() {
@JvmStatic
fun stopItem(context: Context, id: Long) {
if (!ringing.containsKey(id)) {
stoppedEarly[id] = System.currentTimeMillis()
if (!policy.stopOrDefer(id)) {
return
}
val appContext = context.applicationContext
@@ -125,21 +106,13 @@ class FaceclawAlarmService : Service() {
}
@JvmStatic
fun isRinging(id: Long): Boolean {
return ringing.containsKey(id)
}
fun isRinging(id: Long): Boolean = policy.isRinging(id)
@JvmStatic
fun isAnythingRinging(): Boolean {
return !ringing.isEmpty()
}
fun isAnythingRinging(): Boolean = policy.isAnythingRinging()
@JvmStatic
internal fun snapshot(): List<Ringing> {
synchronized(ringing) {
return ArrayList(ringing.values)
}
}
internal fun snapshot(): List<Ringing> = policy.snapshot().map { it as Ringing }
@JvmStatic
internal fun actionIntent(appContext: Context, action: String, id: Long): PendingIntent {
@@ -157,16 +130,6 @@ class FaceclawAlarmService : Service() {
return PendingIntent.getService(appContext, code, intent, flags)
}
/** Consume an early signal for the id; stale entries are dropped. */
private fun takeEarlySignal(signals: MutableMap<Long, Long>, id: Long): Boolean {
val at = signals.remove(id)
val now = System.currentTimeMillis()
synchronized(signals) {
signals.values.removeIf { stamp -> now - stamp > EARLY_SIGNAL_MAX_AGE_MS }
}
return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS
}
private fun notificationId(id: Long): Int {
return 0x41000000 or (FaceclawAlarms.requestCode(id) and 0x00ffffff)
}
@@ -188,6 +151,11 @@ class FaceclawAlarmService : Service() {
return null
}
override fun onCreate() {
super.onCreate()
policy.attach(timers, effects)
}
override fun onStartCommand(intent: Intent?, flags: Int, startId: Int): Int {
val action = intent?.action
val id = if (intent == null) 0L else intent.getLongExtra(FaceclawAlarms.EXTRA_ID, 0)
@@ -206,13 +174,13 @@ class FaceclawAlarmService : Service() {
intent.getIntExtra(FaceclawAlarms.EXTRA_SNOOZE_MINUTES, 10)
))
ACTION_DELIVERED ->
markDelivered(id)
policy.markDelivered(id)
ACTION_DISMISS ->
dismiss(id)
policy.dismiss(id)
ACTION_SNOOZE ->
snooze(id)
policy.snooze(id)
ACTION_STOP_ITEM ->
stopItemQuietly(id, "acknowledged")
policy.stopQuietly(id, "acknowledged")
else -> {}
}
finishIfIdle()
@@ -220,6 +188,7 @@ class FaceclawAlarmService : Service() {
}
override fun onDestroy() {
policy.detach()
handler.removeCallbacksAndMessages(null)
stopSound()
releaseWakeLock()
@@ -234,120 +203,69 @@ class FaceclawAlarmService : Service() {
return
}
ensureChannel()
val existing = ringing[item.id]
if (existing != null) {
// The engine's JS timeout and the AlarmManager alarm both fired: one ring.
showNotification(existing)
return
}
if (takeEarlySignal(stoppedEarly, item.id)) {
// Acknowledged or cancelled before this intent was processed.
FaceclawAlarms.log(this, "ring " + item.id + " already acknowledged")
return
}
ringing[item.id] = item
FaceclawAlarms.log(this, "ring " + item.kind + " " + item.id + " (" + FaceclawAlarms.glassesStatusDescription() + ")")
acquireWakeLock()
showNotification(item)
if (!FaceclawAlarms.glassesCanCarryAlarm()) {
escalate(item, FaceclawAlarms.glassesStatusDescription())
return
}
if (takeEarlySignal(deliveredEarly, item.id)) {
markDelivered(item.id)
return
}
// The glasses could carry it: give the JS side a moment to confirm
// they are actually showing it, then hold for the acknowledgement.
handler.postDelayed({
val current = ringing[item.id]
if (current != null && !current.escalated && current.deliveredAtMs == 0L) {
escalate(current, "not delivered to the glasses")
policy.ring(item)
}
/** Escalation timers on the service's main-thread Handler. */
private val timers = object : AlarmScheduler {
private val pending = HashMap<Any, Runnable>()
override fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) {
pending.remove(token)?.let { handler.removeCallbacks(it) }
val runnable = Runnable {
pending.remove(token)
action()
}
}, DELIVERY_WAIT_MS)
pending[token] = runnable
handler.postDelayed(runnable, delayMs)
}
override fun cancel(token: Any) {
pending.remove(token)?.let { handler.removeCallbacks(it) }
}
override fun cancelAll() {
pending.clear()
handler.removeCallbacksAndMessages(null)
}
}
private fun markDelivered(id: Long) {
val item = ringing[id]
if (item == null || item.deliveredAtMs != 0L) {
return
}
item.deliveredAtMs = System.currentTimeMillis()
FaceclawAlarms.log(this, "delivered $id to glasses")
handler.postDelayed({
val current = ringing[id]
if (current != null && !current.escalated) {
escalate(current, "not acknowledged on the glasses")
/** The phone-side effects of the shared policy: notifications, sound, schedule, journal. */
private val effects = object : RingingSink {
override fun showNotification(item: RingingItem) = this@FaceclawAlarmService.showNotification(item as Ringing)
override fun itemRemoved(item: RingingItem) {
val manager = getSystemService(NotificationManager::class.java)
manager?.cancel(notificationId(item.id))
if (item.id == foregroundId) {
promoteAnotherToForeground()
}
}, ACK_WAIT_MS)
}
private fun escalate(item: Ringing, reason: String) {
if (item.escalated) {
return
}
item.escalated = true
FaceclawAlarms.log(this, "phone sound for " + item.id + ": " + reason)
showNotification(item)
startSound()
handler.postDelayed({
val current = ringing[item.id]
if (current != null && current.escalated && !current.silenced) {
current.silenced = true
FaceclawAlarms.log(this, "auto-silenced " + current.id)
showNotification(current)
syncSound()
}
}, AUTO_SILENCE_MS)
}
private fun dismiss(id: Long) {
val item = ringing[id] ?: return
FaceclawAlarms.recordPhoneAction(this, id, "dismiss", 0)
removeItem(item)
// Also drop the schedule entry (a one-off is over; the engine re-arms repeats).
FaceclawAlarms.cancel(this, id)
}
override fun ringingStarted() = acquireWakeLock()
private fun snooze(id: Long) {
val item = ringing[id] ?: return
val minutes = item.snoozeMinutes
FaceclawAlarms.recordPhoneAction(this, id, "snooze", minutes)
removeItem(item)
// Re-arm on the phone directly so the snooze holds even with the JS
// side gone; the engine replays the journal and lands on the same id.
FaceclawAlarms.schedule(
this,
id,
System.currentTimeMillis() + minutes * 60_000L,
item.title,
item.text,
item.kind,
minutes
)
}
override fun startSound() = this@FaceclawAlarmService.startSound()
private fun stopItemQuietly(id: Long, reason: String) {
val item = ringing[id] ?: return
FaceclawAlarms.log(this, "$reason $id")
removeItem(item)
}
override fun stopSound() = this@FaceclawAlarmService.stopSound()
private fun removeItem(item: Ringing) {
ringing.remove(item.id)
val manager = getSystemService(NotificationManager::class.java)
manager?.cancel(notificationId(item.id))
syncSound()
if (item.id == foregroundId) {
promoteAnotherToForeground()
override fun reschedule(item: RingingItem, atMs: Long, minutes: Int) {
// The engine replays the journal and lands on the same id.
FaceclawAlarms.schedule(this@FaceclawAlarmService, item.id, atMs, item.title, item.text, item.kind, minutes)
}
override fun cancelSchedule(id: Long) = FaceclawAlarms.cancel(this@FaceclawAlarmService, id)
override fun recordPhoneAction(id: Long, action: String, minutes: Int) =
FaceclawAlarms.recordPhoneAction(this@FaceclawAlarmService, id, action, minutes)
override fun log(line: String) = FaceclawAlarms.log(this@FaceclawAlarmService, line)
}
private fun finishIfIdle() {
if (!ringing.isEmpty()) {
if (policy.isAnythingRinging()) {
return
}
handler.removeCallbacksAndMessages(null)
policy.cancelTimers()
stopSound()
releaseWakeLock()
stopForeground(STOP_FOREGROUND_REMOVE)
@@ -359,7 +277,7 @@ class FaceclawAlarmService : Service() {
private fun showNotification(item: Ringing) {
val notification = buildNotification(item)
if (foregroundId == 0L || foregroundId == item.id || !ringing.containsKey(foregroundId)) {
if (foregroundId == 0L || foregroundId == item.id || !policy.isRinging(foregroundId)) {
foregroundId = item.id
startForegroundCompat(notificationId(item.id), notification)
} else {
@@ -456,21 +374,6 @@ class FaceclawAlarmService : Service() {
// ------------------------------------------------------------------
// Sound and vibration (only while some item is escalated and not yet silenced)
private fun syncSound() {
var wanted = false
for (item in snapshot()) {
if (item.escalated && !item.silenced) {
wanted = true
break
}
}
if (wanted) {
startSound()
} else {
stopSound()
}
}
private fun startSound() {
if (player != null) {
return
@@ -47,32 +47,19 @@ class FaceclawAlarms private constructor() {
internal const val EXTRA_KIND = "kind"
internal const val EXTRA_SNOOZE_MINUTES = "snoozeMinutes"
const val KIND_TIMER = "timer"
const val KIND_ALARM = "alarm"
const val KIND_TIMER = AlarmSchedule.KIND_TIMER
const val KIND_ALARM = AlarmSchedule.KIND_ALARM
private const val PREFS_NAME = "faceclaw-alarms"
private const val KEY_SCHEDULED = "scheduled"
private const val KEY_JOURNAL = "journal"
private const val KEY_LOG = "log"
private const val LOG_LINES = 60
/**
* A schedule entry found already past due (after a reboot, or when the
* process was asleep) rings if it is at most this late; older ones are
* dropped as missed. Matches the JS engine's grace window.
*/
internal const val LATE_GRACE_MS = 5 * 60_000L
/** A glasses status report older than this is not trusted (the JS side is probably gone). */
internal const val GLASSES_STATUS_MAX_AGE_MS = 3 * 60_000L
/** Grace window for past-due entries found on replay (shared policy). */
internal const val LATE_GRACE_MS = AlarmSchedule.LATE_GRACE_MS
/** A glasses status report older than this is not trusted (shared policy). */
internal const val GLASSES_STATUS_MAX_AGE_MS = GlassesStatusGate.MAX_AGE_MS
@Volatile
private var glassesConnected = false
@Volatile
private var glassesWorn = false
@Volatile
private var glassesCharging = false
@Volatile
private var glassesStatusAtMs = 0L
/** The last glasses status pushed by the JS side; read by the ringing service. */
@JvmField
internal val glassesGate = GlassesStatusGate()
@Volatile
private var listener: FaceclawAlarmListener? = null
@@ -95,66 +82,30 @@ class FaceclawAlarms private constructor() {
return base.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
}
// The Java statics were `synchronized`, i.e. locked on the class object;
// the explicit blocks below keep that single monitor.
private fun readArray(context: Context, key: String): JSONArray {
synchronized(FaceclawAlarms::class.java) {
val raw = prefs(context).getString(key, "[]")
return try {
JSONArray(raw)
} catch (error: JSONException) {
JSONArray()
}
}
}
@Volatile
private var scheduleInstance: AlarmSchedule? = null
private fun writeArray(context: Context, key: String, array: JSONArray) {
/** The shared schedule/journal/log core over the device-protected prefs (one per process). */
@JvmStatic
internal fun scheduleCore(context: Context): AlarmSchedule {
scheduleInstance?.let { return it }
synchronized(FaceclawAlarms::class.java) {
prefs(context).edit().putString(key, array.toString()).apply()
scheduleInstance?.let { return it }
val prefs = prefs(context)
val created = AlarmSchedule(object : AlarmStore {
override fun read(key: String): String? = prefs.getString(key, null)
override fun write(key: String, value: String?) {
prefs.edit().putString(key, value).apply()
}
})
scheduleInstance = created
return created
}
}
@JvmStatic
internal fun findScheduled(context: Context, id: Long): JSONObject? {
val scheduled = readArray(context, KEY_SCHEDULED)
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index)
if (entry != null && entry.optLong("id") == id) {
return entry
}
}
return null
}
private fun putScheduled(context: Context, entry: JSONObject) {
synchronized(FaceclawAlarms::class.java) {
val scheduled = readArray(context, KEY_SCHEDULED)
val next = JSONArray()
val id = entry.optLong("id")
for (index in 0 until scheduled.length()) {
val existing = scheduled.optJSONObject(index)
if (existing != null && existing.optLong("id") != id) {
next.put(existing)
}
}
next.put(entry)
writeArray(context, KEY_SCHEDULED, next)
}
}
private fun removeScheduled(context: Context, id: Long) {
synchronized(FaceclawAlarms::class.java) {
val scheduled = readArray(context, KEY_SCHEDULED)
val next = JSONArray()
for (index in 0 until scheduled.length()) {
val existing = scheduled.optJSONObject(index)
if (existing != null && existing.optLong("id") != id) {
next.put(existing)
}
}
writeArray(context, KEY_SCHEDULED, next)
}
}
internal fun findScheduled(context: Context, id: Long): AlarmEntry? = scheduleCore(context).find(id)
// ------------------------------------------------------------------
// Scheduling
@@ -177,26 +128,16 @@ class FaceclawAlarms private constructor() {
snoozeMinutes: Int
) {
val appContext = context.applicationContext
val entry = JSONObject()
try {
entry.put("id", id)
entry.put("at", triggerAtMs)
entry.put("title", title ?: "")
entry.put("text", text ?: "")
entry.put("kind", kind ?: KIND_ALARM)
entry.put("snoozeMinutes", snoozeMinutes)
} catch (ignored: JSONException) {
return
}
putScheduled(appContext, entry)
val entry = AlarmEntry(id, triggerAtMs, title, text, kind ?: KIND_ALARM, snoozeMinutes)
scheduleCore(appContext).put(entry)
armAlarm(appContext, entry)
}
private fun armAlarm(appContext: Context, entry: JSONObject) {
private fun armAlarm(appContext: Context, entry: AlarmEntry) {
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
?: return
val id = entry.optLong("id")
val triggerAt = Math.max(System.currentTimeMillis(), entry.optLong("at"))
val id = entry.id
val triggerAt = Math.max(System.currentTimeMillis(), entry.at)
val fire = expiryPendingIntent(appContext, entry)
try {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
@@ -219,15 +160,9 @@ class FaceclawAlarms private constructor() {
val appContext = context.applicationContext
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
if (manager != null) {
val entry = JSONObject()
try {
entry.put("id", id)
} catch (ignored: JSONException) {
// unreachable for a numeric put
}
manager.cancel(expiryPendingIntent(appContext, entry))
manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10)))
}
removeScheduled(appContext, id)
scheduleCore(appContext).remove(id)
FaceclawAlarmService.stopItem(appContext, id)
}
@@ -239,25 +174,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun rescheduleAll(context: Context, reason: String?) {
val appContext = context.applicationContext
val scheduled = readArray(appContext, KEY_SCHEDULED)
val now = System.currentTimeMillis()
var armed = 0
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index) ?: continue
val at = entry.optLong("at")
if (at <= now) {
if (now - at <= LATE_GRACE_MS) {
ringEntry(appContext, entry)
} else {
log(appContext, "missed " + entry.optString("kind") + " " + entry.optLong("id") + " (" + reason + ")")
removeScheduled(appContext, entry.optLong("id"))
}
continue
}
armAlarm(appContext, entry)
armed++
}
log(appContext, "rescheduled $armed ($reason)")
scheduleCore(appContext).replay(reason, { ringEntry(appContext, it) }, { armAlarm(appContext, it) })
}
/**
@@ -271,24 +188,13 @@ class FaceclawAlarms private constructor() {
return false
}
val appContext = context.applicationContext
val scheduled = readArray(appContext, KEY_SCHEDULED)
val now = System.currentTimeMillis()
var soonest = Long.MAX_VALUE
for (index in 0 until scheduled.length()) {
val entry = scheduled.optJSONObject(index)
if (entry != null && entry.optLong("at") > now) {
soonest = Math.min(soonest, entry.optLong("at"))
}
}
if (soonest == Long.MAX_VALUE) {
val core = scheduleCore(appContext)
if (core.soonestPendingAt() == null) {
return false
}
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
?: return false
val next = manager.nextAlarmClock
// Another app's alarm may legitimately be sooner; only a later (or
// absent) system alarm proves ours is missing.
return next == null || next.triggerTime > soonest + 1000
return core.looksStale(manager.nextAlarmClock?.triggerTime)
}
// ------------------------------------------------------------------
@@ -296,15 +202,8 @@ class FaceclawAlarms private constructor() {
/** An item came due: hand it to the ringing service (idempotent per id). */
@JvmStatic
internal fun ringEntry(appContext: Context, entry: JSONObject) {
FaceclawAlarmService.ring(
appContext,
entry.optLong("id"),
entry.optString("title"),
entry.optString("text"),
entry.optString("kind", KIND_ALARM),
entry.optInt("snoozeMinutes", 10)
)
internal fun ringEntry(appContext: Context, entry: AlarmEntry) {
FaceclawAlarmService.ring(appContext, entry.id, entry.title, entry.text, entry.kind, entry.snoozeMinutes)
}
/** The JS engine's own expiry path; the service deduplicates against the alarm's. */
@@ -313,13 +212,7 @@ class FaceclawAlarms private constructor() {
val appContext = context.applicationContext
val manager = appContext.getSystemService(Context.ALARM_SERVICE) as AlarmManager?
if (manager != null) {
val probe = JSONObject()
try {
probe.put("id", id)
} catch (ignored: JSONException) {
// unreachable for a numeric put
}
manager.cancel(expiryPendingIntent(appContext, probe))
manager.cancel(expiryPendingIntent(appContext, AlarmEntry(id, 0L, null, null, null, 10)))
}
FaceclawAlarmService.ring(appContext, id, title, text, kind, snoozeMinutes)
}
@@ -334,7 +227,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun acknowledge(context: Context, id: Long) {
val appContext = context.applicationContext
removeScheduled(appContext, id)
scheduleCore(appContext).remove(id)
FaceclawAlarmService.stopItem(appContext, id)
}
@@ -343,42 +236,15 @@ class FaceclawAlarms private constructor() {
@JvmStatic
fun setGlassesStatus(connected: Boolean, worn: Boolean, charging: Boolean) {
glassesConnected = connected
glassesWorn = worn
glassesCharging = charging
glassesStatusAtMs = System.currentTimeMillis()
glassesGate.set(connected, worn, charging)
}
/** True only when a fresh report says the glasses are connected, on a head, and not charging. */
@JvmStatic
internal fun glassesCanCarryAlarm(): Boolean {
val age = System.currentTimeMillis() - glassesStatusAtMs
if (glassesStatusAtMs == 0L || age > GLASSES_STATUS_MAX_AGE_MS) {
return false
}
return glassesConnected && glassesWorn && !glassesCharging
}
internal fun glassesCanCarryAlarm(): Boolean = glassesGate.canCarryAlarm()
@JvmStatic
internal fun glassesStatusDescription(): String {
if (glassesStatusAtMs == 0L) {
return "no glasses status"
}
val age = System.currentTimeMillis() - glassesStatusAtMs
if (age > GLASSES_STATUS_MAX_AGE_MS) {
return "glasses status stale"
}
if (!glassesConnected) {
return "glasses not connected"
}
if (glassesCharging) {
return "glasses charging"
}
if (!glassesWorn) {
return "glasses not worn"
}
return "glasses worn"
}
internal fun glassesStatusDescription(): String = glassesGate.description()
// ------------------------------------------------------------------
// Phone actions -> JS
@@ -396,21 +262,7 @@ class FaceclawAlarms private constructor() {
@JvmStatic
internal fun recordPhoneAction(context: Context, id: Long, action: String, minutes: Int) {
val appContext = context.applicationContext
val event = JSONObject()
try {
event.put("id", id)
event.put("action", action)
event.put("minutes", minutes)
event.put("at", System.currentTimeMillis())
} catch (ignored: JSONException) {
return
}
synchronized(FaceclawAlarms::class.java) {
val journal = readArray(appContext, KEY_JOURNAL)
journal.put(event)
writeArray(appContext, KEY_JOURNAL, journal)
}
log(appContext, "phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else ""))
scheduleCore(appContext).appendJournal(id, action, minutes)
val current = listener
if (current != null) {
mainHandler.post {
@@ -425,14 +277,7 @@ class FaceclawAlarms private constructor() {
/** Hand the journal to the JS side (as a JSON array) and clear it. */
@JvmStatic
fun drainJournal(context: Context): String {
synchronized(FaceclawAlarms::class.java) {
val appContext = context.applicationContext
val journal = readArray(appContext, KEY_JOURNAL)
writeArray(appContext, KEY_JOURNAL, JSONArray())
return journal.toString()
}
}
fun drainJournal(context: Context): String = scheduleCore(context.applicationContext).drainJournal()
// ------------------------------------------------------------------
// Reliability self-check
@@ -577,45 +422,25 @@ class FaceclawAlarms private constructor() {
@JvmStatic
internal fun log(context: Context, line: String) {
synchronized(FaceclawAlarms::class.java) {
val appContext = context.applicationContext
val log = readArray(appContext, KEY_LOG)
val next = JSONArray()
val start = Math.max(0, log.length() - (LOG_LINES - 1))
for (index in start until log.length()) {
next.put(log.optString(index))
}
next.put(System.currentTimeMillis().toString() + " " + line)
writeArray(appContext, KEY_LOG, next)
}
scheduleCore(context.applicationContext).log(line)
}
/** The ring / miss / action log, newest last, one line per entry. */
@JvmStatic
fun readLog(context: Context): String {
val log = readArray(context.applicationContext, KEY_LOG)
val out = StringBuilder()
for (index in 0 until log.length()) {
if (index > 0) {
out.append('\n')
}
out.append(log.optString(index))
}
return out.toString()
}
fun readLog(context: Context): String = scheduleCore(context.applicationContext).readLog()
// ------------------------------------------------------------------
// Intents
private fun expiryPendingIntent(appContext: Context, entry: JSONObject): PendingIntent {
private fun expiryPendingIntent(appContext: Context, entry: AlarmEntry): PendingIntent {
val intent = Intent(appContext, FaceclawAlarmReceiver::class.java)
intent.action = ACTION_EXPIRE
val id = entry.optLong("id")
val id = entry.id
intent.putExtra(EXTRA_ID, id)
intent.putExtra(EXTRA_TITLE, entry.optString("title"))
intent.putExtra(EXTRA_TEXT, entry.optString("text"))
intent.putExtra(EXTRA_KIND, entry.optString("kind", KIND_ALARM))
intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.optInt("snoozeMinutes", 10))
intent.putExtra(EXTRA_TITLE, entry.title)
intent.putExtra(EXTRA_TEXT, entry.text)
intent.putExtra(EXTRA_KIND, entry.kind)
intent.putExtra(EXTRA_SNOOZE_MINUTES, entry.snoozeMinutes)
var flags = PendingIntent.FLAG_UPDATE_CURRENT
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
flags = flags or PendingIntent.FLAG_IMMUTABLE
@@ -635,9 +460,7 @@ class FaceclawAlarms private constructor() {
}
@JvmStatic
internal fun requestCode(id: Long): Int {
return (id xor (id ushr 32)).toInt() and 0x7fffffff
}
internal fun requestCode(id: Long): Int = AlarmSchedule.requestCode(id)
@JvmStatic
internal fun startServiceCompat(appContext: Context, intent: Intent) {
@@ -9,8 +9,6 @@ import android.util.Log
import java.io.File
import java.io.FileInputStream
import java.io.IOException
import java.nio.ByteBuffer
import java.nio.ByteOrder
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
@@ -28,69 +26,25 @@ class FaceclawAudioTranscoder(
private const val TAG = "FaceclawTranscode"
private const val CODEC_TIMEOUT_US: Long = 10_000
/** Minimal RIFF walk: locates fmt and data chunks, 16-bit PCM only. */
/** Shared RIFF walk over the file's first bytes: locates fmt and data chunks, 16-bit PCM only. */
@JvmStatic
@Throws(IOException::class)
private fun readWavHeader(path: String?): WavInfo {
FileInputStream(path).use { `in` ->
val head = ByteArray(12)
if (`in`.read(head) != 12 || head[0] != 'R'.code.toByte() || head[1] != 'I'.code.toByte() || head[8] != 'W'.code.toByte()) {
throw IOException("not a WAV file")
private fun readWavHeader(path: String?): WavPcmReader.WavInfo {
val file = File(path)
val fileLength = file.length()
FileInputStream(file).use { `in` ->
val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt())
var filled = 0
while (filled < prefix.size) {
val read = `in`.read(prefix, filled, prefix.size - filled)
if (read <= 0) break
filled += read
}
val info = WavInfo()
var offset: Long = 12
val chunkHead = ByteArray(8)
while (`in`.read(chunkHead) == 8) {
val b = ByteBuffer.wrap(chunkHead).order(ByteOrder.LITTLE_ENDIAN)
val id = b.getInt()
val size = b.getInt().toLong() and 0xffffffffL
offset += 8
if (id == 0x20746d66) { // "fmt "
val fmt = ByteArray(Math.min(size, 16L).toInt())
if (`in`.read(fmt) != fmt.size) {
throw IOException("truncated fmt chunk")
}
val f = ByteBuffer.wrap(fmt).order(ByteOrder.LITTLE_ENDIAN)
val audioFormat = f.getShort().toInt()
info.channels = f.getShort().toInt()
info.sampleRate = f.getInt()
f.getInt()
f.getShort()
val bits = f.getShort().toInt()
if (audioFormat != 1 || bits != 16) {
throw IOException("only 16-bit PCM WAV is supported")
}
var skip = size - fmt.size
while (skip > 0) {
val step = `in`.skip(skip)
if (step <= 0) {
break
}
skip -= step
}
offset += size
} else if (id == 0x61746164) { // "data"
info.dataOffset = offset
// A zero size means the header was never patched (crash
// mid-recording); recover the length from the file size.
info.dataBytes = if (size > 0) size else File(path).length() - offset
break
} else {
var skip = size
while (skip > 0) {
val step = `in`.skip(skip)
if (step <= 0) {
break
}
skip -= step
}
offset += size
}
try {
return WavPcmReader.parseHeader(if (filled == prefix.size) prefix else prefix.copyOf(filled), fileLength)
} catch (e: IllegalArgumentException) {
throw IOException(e.message ?: "not a WAV file")
}
if (info.sampleRate == 0 || info.dataOffset == 0L) {
throw IOException("missing fmt or data chunk")
}
return info
}
}
}
@@ -226,11 +180,4 @@ class FaceclawAudioTranscoder(
}
}
}
private class WavInfo {
var sampleRate = 0
var channels = 0
var dataOffset: Long = 0
var dataBytes: Long = 0
}
}
File diff suppressed because it is too large. Load diff
@@ -2,455 +2,74 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import android.util.Log
import com.k2fsa.sherpa.onnx.FeatureConfig
import com.k2fsa.sherpa.onnx.OfflineModelConfig
import com.k2fsa.sherpa.onnx.OfflineMoonshineModelConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizer
import com.k2fsa.sherpa.onnx.OfflineRecognizerConfig
import com.k2fsa.sherpa.onnx.OfflineRecognizerResult
import com.k2fsa.sherpa.onnx.OfflineStream
import java.io.File
import java.util.ArrayDeque
import java.util.Arrays
/**
* Continuous captioning over the decoded mic PCM: segments speech into
* utterances with an adaptive energy gate, transcribes each utterance with
* the on-device Moonshine model, and attaches a speaker voice-print per
* utterance. PCM is pushed in from the TS side (which owns mic arbitration
* and any beam-direction gating), so this engine has no BLE dependencies.
* Continuous captioning over the decoded mic PCM: the shared [CaptionSession] segments speech
* into utterances with an adaptive energy gate, transcribes each with the on-device Moonshine
* model and attaches a speaker voice-print. This Android class only supplies the sherpa-onnx
* recognizer and speaker model and delivers callbacks on the main thread. PCM is pushed in
* from the TS side (which owns mic arbitration and any beam-direction gating), so the engine
* has no BLE dependencies.
*
* Utterance boundaries are measured on the sample clock, not the wall clock:
* BLE delivers mic packets in bursts, so elapsed real time overestimates the
* audio heard. The TS side converts startMs/endMs to wall-clock times using
* the engine start timestamp.
* Utterance boundaries are measured on the sample clock, not the wall clock; the TS side
* converts startMs/endMs to wall-clock times using the engine start timestamp.
*/
class FaceclawCaptionEngine {
companion object {
private const val TAG = "FaceclawCaptions"
private const val SAMPLE_RATE = 16000
private const val FEATURE_DIM = 80
private const val MAX_QUEUE_PACKETS = 200
private val mainHandler = Handler(Looper.getMainLooper())
// Segmentation: RMS thresholds relative to a rolling noise floor, in the
// same spirit as FaceclawVoiceController.EndpointDetector but recurring.
private const val ONSET_FACTOR = 3.0
private const val RELEASE_FACTOR = 1.8
private const val MIN_RMS = 220.0
private const val NOISE_EMA_ALPHA = 0.05
private const val PRE_ROLL_MS = 400
private const val DEFAULT_SILENCE_MS = 800
private const val MIN_UTTERANCE_MS = 350
private const val MAX_UTTERANCE_MS = 15000
// Moonshine v2 fails past ~9.1 s of input; decode long utterances in
// segments cut at the quietest window (mirrors FaceclawVoiceController).
private const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8
private const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2
private const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000
private const val NORMALIZE_TARGET_PEAK = 0.9f
private const val NORMALIZE_MAX_GAIN = 30f
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
private const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10
private val host = object : CaptionHost {
override val dispatcher = CallbackDispatcher { action -> mainHandler.post { action() } }
override fun loadTranscriber(modelDir: String): OfflineTranscriber? {
if (!File(modelDir, "tokens.txt").exists()) {
return null
}
return AndroidSpeechEngines.SherpaTranscriber(
AndroidSpeechEngines.recognizerConfig(File(modelDir), VoiceModelKind.MOONSHINE))
}
override fun loadEmbedder(modelPath: String): SpeakerEmbedder? {
if (!File(modelPath).exists()) {
return null
}
val speakerId = FaceclawSpeakerId(modelPath)
speakerId.ensureLoaded()
return speakerId
}
}
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val queueLock = Any()
private val queue = ArrayDeque<ByteArray>()
@Volatile
private var listener: FaceclawCaptionEngineListener? = null
@Volatile
private var asrModelDir: String? = null
@Volatile
private var speakerModelPath: String? = null
@Volatile
private var silenceMs = DEFAULT_SILENCE_MS
private var workerThread: Thread? = null
@Volatile
private var started = false
private var recognizer: OfflineRecognizer? = null
private var speakerId: FaceclawSpeakerId? = null
// Segmentation state (worker thread only).
private var totalSamples: Long = 0
private var noiseFloor = 0.0
private var inUtterance = false
private var utteranceStartSample: Long = 0
private var silenceRunSamples: Long = 0
private var utterancePeakRms = 0.0
private var utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000))
private var utteranceLength = 0
private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000)
private var preRollLength = 0
private val session = CaptionSession(host)
fun setListener(listener: FaceclawCaptionEngineListener?) {
this.listener = listener
session.setListener(listener)
}
/** Directory holding the Moonshine model files, or null to disable ASR. */
fun setAsrModelDir(dir: String?) {
this.asrModelDir = dir
session.setAsrModelDir(dir)
}
/** Speaker-embedding ONNX model path, or null to disable voice-prints. */
fun setSpeakerModelPath(path: String?) {
this.speakerModelPath = path
session.setSpeakerModelPath(path)
}
fun setSilenceMs(ms: Int) {
this.silenceMs = Math.max(200, Math.min(3000, ms))
session.setSilenceMs(ms)
}
fun start() {
synchronized(lock) {
if (started) {
return
}
started = true
workerThread = Thread(Runnable { runLoop() }, "FaceclawCaptionEngine")
workerThread!!.start()
}
session.start()
}
fun stop() {
val threadToJoin: Thread?
synchronized(lock) {
if (!started) {
return
}
started = false
threadToJoin = workerThread
workerThread = null
}
synchronized(queueLock) {
(queueLock as java.lang.Object).notifyAll()
}
if (threadToJoin != null) {
threadToJoin.interrupt()
if (Thread.currentThread() !== threadToJoin) {
try {
threadToJoin.join(2000)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
}
}
}
session.stop()
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
fun acceptPcm(pcm16le: ByteArray?) {
if (!started || pcm16le == null || pcm16le.size < 2) {
return
}
synchronized(queueLock) {
if (queue.size >= MAX_QUEUE_PACKETS) {
queue.removeFirst()
}
queue.addLast(pcm16le)
(queueLock as java.lang.Object).notifyAll()
}
}
private fun runLoop() {
try {
loadModels()
resetSegmentation()
while (started && !Thread.currentThread().isInterrupted) {
val chunk = takeChunk()
if (chunk == null) {
continue
}
processChunk(chunk)
}
// Flush a trailing utterance so its text isn't lost on stop.
if (inUtterance && utteranceLength > 0) {
finalizeUtterance()
}
} catch (t: Throwable) {
Log.e(TAG, "caption engine failed", t)
emitStatus("Captions failed: " + t.message)
} finally {
releaseModels()
}
}
private fun loadModels() {
val modelDir = asrModelDir
if (modelDir != null && File(modelDir, "tokens.txt").exists()) {
emitStatus("Loading caption model...")
recognizer = OfflineRecognizer(OfflineRecognizerConfig.builder()
.setFeatureConfig(FeatureConfig.builder()
.setSampleRate(SAMPLE_RATE)
.setFeatureDim(FEATURE_DIM)
.build())
.setModelConfig(OfflineModelConfig.builder()
.setMoonshine(OfflineMoonshineModelConfig.builder()
.setEncoder(File(modelDir, "encoder_model.ort").absolutePath)
.setMergedDecoder(File(modelDir, "decoder_model_merged.ort").absolutePath)
.build())
.setTokens(File(modelDir, "tokens.txt").absolutePath)
.setNumThreads(1)
.build())
.build())
} else {
recognizer = null
}
val embedModel = speakerModelPath
if (embedModel != null && File(embedModel).exists()) {
speakerId = FaceclawSpeakerId(embedModel)
speakerId!!.ensureLoaded()
} else {
speakerId = null
}
emitStatus(if (recognizer != null) "Captions listening..." else "Captions listening (no ASR model)...")
}
private fun releaseModels() {
if (recognizer != null) {
recognizer!!.release()
recognizer = null
}
if (speakerId != null) {
speakerId!!.close()
speakerId = null
}
}
private fun takeChunk(): ByteArray? {
synchronized(queueLock) {
while (started && queue.isEmpty()) {
try {
(queueLock as java.lang.Object).wait(250)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return null
}
}
return queue.pollFirst()
}
}
private fun resetSegmentation() {
totalSamples = 0
noiseFloor = 0.0
inUtterance = false
utteranceLength = 0
preRollLength = 0
silenceRunSamples = 0
utterancePeakRms = 0.0
}
private fun processChunk(chunk: ByteArray) {
val count = chunk.size / 2
val pcm = ShortArray(count)
var sumSquares = 0.0
for (i in 0 until count) {
val s = ((chunk[i * 2].toInt() and 0xff) or (chunk[i * 2 + 1].toInt() shl 8)).toShort()
pcm[i] = s
sumSquares += s.toDouble() * s
}
val rms = Math.sqrt(sumSquares / Math.max(1, count))
totalSamples += count.toLong()
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA
val threshold = Math.max(noiseFloor, MIN_RMS)
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true
utteranceLength = 0
silenceRunSamples = 0
utterancePeakRms = rms
appendUtterance(preRoll, preRollLength)
utteranceStartSample = Math.max(0L, totalSamples - count - preRollLength)
appendUtterance(pcm, count)
emitSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE)
} else {
appendPreRoll(pcm, count)
}
return
}
appendUtterance(pcm, count)
utterancePeakRms = Math.max(utterancePeakRms, rms)
val threshold = Math.max(noiseFloor, MIN_RMS)
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count.toLong()
} else {
silenceRunSamples = 0
}
val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE
val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance()
inUtterance = false
preRollLength = 0
}
}
private fun appendPreRoll(pcm: ShortArray, count: Int) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.size) {
System.arraycopy(pcm, count - preRoll.size, preRoll, 0, preRoll.size)
preRollLength = preRoll.size
return
}
val keep = Math.min(preRollLength, preRoll.size - count)
System.arraycopy(preRoll, preRollLength - keep, preRoll, 0, keep)
System.arraycopy(pcm, 0, preRoll, keep, count)
preRollLength = keep + count
}
private fun appendUtterance(pcm: ShortArray, count: Int) {
val room = utterance.size - utteranceLength
val copied = Math.min(room, count)
if (copied > 0) {
System.arraycopy(pcm, 0, utterance, utteranceLength, copied)
utteranceLength += copied
}
}
private fun finalizeUtterance() {
val length = utteranceLength
val startMs = utteranceStartSample * 1000 / SAMPLE_RATE
val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE
if (endMs - startMs < MIN_UTTERANCE_MS) {
return
}
val text = recognizeUtterance(length)
val embedding = embedUtterance(length)
emitUtterance(text, embedding, startMs, endMs, utterancePeakRms)
}
private fun recognizeUtterance(length: Int): String {
if (recognizer == null || length <= 0) {
return ""
}
val joined = StringBuilder()
var offset = 0
while (offset < length) {
val remaining = length - offset
var segment = Math.min(remaining, DECODE_SEGMENT_MAX_SAMPLES)
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = findQuietCut(offset, segment)
}
val part = recognizeRange(offset, segment)
if (part.length > 0) {
if (joined.length > 0 && ".,!?;:%)]}".indexOf(part[0]) < 0) {
joined.append(' ')
}
joined.append(part)
}
offset += segment
}
return joined.toString().trim()
}
/**
* End a decode segment at the center of the quietest window near its end
* so the cut lands between words (same approach as the PTT controller).
*/
private fun findQuietCut(offset: Int, segment: Int): Int {
val searchStart = Math.max(0, segment - CUT_SEARCH_SAMPLES)
val win = CUT_WINDOW_SAMPLES
if (segment - searchStart <= win) {
return segment
}
var sum = 0.0
for (i in searchStart until searchStart + win) {
val s = utterance[offset + i].toDouble()
sum += s * s
}
var best = sum
var bestStart = searchStart
var start = searchStart + 1
while (start + win <= segment) {
val dropped = utterance[offset + start - 1].toDouble()
val added = utterance[offset + start + win - 1].toDouble()
sum += added * added - dropped * dropped
if (sum < best) {
best = sum
bestStart = start
}
start++
}
return bestStart + win / 2
}
private fun recognizeRange(offset: Int, count: Int): String {
val samples = FloatArray(count)
var peak = 0f
for (i in 0 until count) {
val v = utterance[offset + i] / 32768.0f
samples[i] = v
val a = Math.abs(v)
if (a > peak) {
peak = a
}
}
if (peak > 0f) {
val gain = Math.min(NORMALIZE_TARGET_PEAK / peak, NORMALIZE_MAX_GAIN)
if (gain > 1f) {
for (i in 0 until count) {
samples[i] *= gain
}
}
}
val recognizer = this.recognizer!!
val stream: OfflineStream = recognizer.createStream()
try {
stream.acceptWaveform(samples, SAMPLE_RATE)
recognizer.decode(stream)
val result: OfflineRecognizerResult? = recognizer.getResult(stream)
val raw: String? = if (result == null) "" else result.text
return raw?.trim() ?: ""
} finally {
stream.release()
}
}
private fun embedUtterance(length: Int): FloatArray? {
val currentSpeakerId = speakerId
if (currentSpeakerId == null || length <= 0) {
return null
}
val count = Math.min(length, EMBED_MAX_SAMPLES)
val le = ByteArray(count * 2)
for (i in 0 until count) {
val s = utterance[i].toInt()
le[i * 2] = (s and 0xff).toByte()
le[i * 2 + 1] = ((s shr 8) and 0xff).toByte()
}
return currentSpeakerId.embed(le, SAMPLE_RATE)
}
private fun emitUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) {
val currentListener = listener
if (currentListener == null) {
return
}
val embeddingCopy = if (embedding == null) null else Arrays.copyOf(embedding, embedding.size)
mainHandler.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) }
}
private fun emitSpeechStart(startMs: Long) {
val currentListener = listener
if (currentListener == null) {
return
}
mainHandler.post { currentListener.onSpeechStart(startMs) }
}
private fun emitStatus(status: String?) {
val currentListener = listener
if (currentListener == null) {
return
}
mainHandler.post { currentListener.onStatus(status) }
session.acceptPcm(pcm16le)
}
}
@@ -13,8 +13,6 @@ import org.json.JSONException
import org.json.JSONObject
import java.io.File
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.ArrayList
/**
@@ -47,41 +45,6 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
}
}
// ---- blob helpers ----
private fun blobToFloats(blob: ByteArray?): FloatArray? {
if (blob == null || blob.size < 4 || blob.size % 4 != 0) {
return null
}
val buffer = ByteBuffer.wrap(blob).order(ByteOrder.LITTLE_ENDIAN)
val out = FloatArray(blob.size / 4)
for (i in out.indices) {
out[i] = buffer.getFloat()
}
return out
}
private fun floatsToBlob(values: FloatArray): ByteArray {
val buffer = ByteBuffer.allocate(values.size * 4).order(ByteOrder.LITTLE_ENDIAN)
for (v in values) {
buffer.putFloat(v)
}
return buffer.array()
}
private fun l2NormalizeInPlace(v: FloatArray) {
var sum = 0.0
for (x in v) {
sum += x.toDouble() * x
}
val norm = Math.sqrt(sum)
if (norm <= 0) {
return
}
for (i in v.indices) {
v[i] = (v[i] / norm).toFloat()
}
}
}
override fun onCreate(db: SQLiteDatabase) {
@@ -567,7 +530,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
fun updateSpeakerEmbedding(speakerId: Long, embeddingBase64: String?, maxCount: Int) {
try {
val incoming = Base64.decode(embeddingBase64, Base64.NO_WRAP)
val update = blobToFloats(incoming)
val update = SpeakerEmbeddings.blobToFloats(incoming)
if (update == null) {
return
}
@@ -578,28 +541,16 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
arrayOf(speakerId.toString()))
try {
if (c.moveToFirst()) {
current = if (c.isNull(0)) null else blobToFloats(c.getBlob(0))
current = if (c.isNull(0)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(0))
count = c.getInt(1)
}
} finally {
c.close()
}
val merged: FloatArray
if (current == null || current.size != update.size) {
merged = update
count = 1
} else {
val effective = Math.min(count, Math.max(1, maxCount))
merged = FloatArray(current.size)
for (i in merged.indices) {
merged[i] = (current[i] * effective + update[i]) / (effective + 1)
}
l2NormalizeInPlace(merged)
count = count + 1
}
val centroid = SpeakerEmbeddings.runningMean(current, count, update, maxCount)
val values = ContentValues()
values.put("embedding", floatsToBlob(merged))
values.put("embedding_count", count)
values.put("embedding", SpeakerEmbeddings.floatsToBlob(centroid.embedding))
values.put("embedding_count", centroid.count)
values.put("last_heard_at", System.currentTimeMillis())
writableDatabase.update("speakers", values, "id=?",
arrayOf(speakerId.toString()))
@@ -630,7 +581,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
try {
while (c.moveToNext()) {
val id = c.getLong(0)
val embedding = if (c.isNull(1)) null else blobToFloats(c.getBlob(1))
val embedding = if (c.isNull(1)) null else SpeakerEmbeddings.blobToFloats(c.getBlob(1))
val count = c.getInt(2)
if (id == fromId) {
from = embedding
@@ -644,21 +595,10 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
} finally {
c.close()
}
var merged: FloatArray? = into
if (from != null && into != null && from.size == into.size) {
val blended = FloatArray(into.size)
val total = Math.max(1, fromCount + intoCount)
for (i in blended.indices) {
blended[i] = (into[i] * intoCount + from[i] * fromCount) / total
}
l2NormalizeInPlace(blended)
merged = blended
} else if (into == null) {
merged = from
}
val merged = SpeakerEmbeddings.blend(from, fromCount, into, intoCount)
val values = ContentValues()
if (merged != null) {
values.put("embedding", floatsToBlob(merged))
values.put("embedding", SpeakerEmbeddings.floatsToBlob(merged))
values.put("embedding_count", fromCount + intoCount)
}
if (fromWearer) {
@@ -677,7 +617,7 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
val hasInsights = !c2.isNull(0) && !c2.getString(0).isEmpty()
if (c2.position == 0) {
targetHasInsights = hasInsights
} else if (!targetHasInsights && hasInsights) {
} else if (SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights, hasInsights)) {
values.put("last_recap", c2.getString(0))
values.put("action_items", if (c2.isNull(1)) "" else c2.getString(1))
values.put("facts", if (c2.isNull(2)) "" else c2.getString(2))
@@ -1,99 +1,44 @@
package com.faceclaw.app
import android.bluetooth.BluetoothDevice
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.util.Log
import java.util.HashSet
import java.util.concurrent.CountDownLatch
import java.util.concurrent.TimeUnit
/**
* Connects to the glasses (stock-firmware compatible), reads the device-info /
* settings response, and reports the firmware versions plus the firmware-
* extension string (empty on stock firmware). Used by onboarding to decide whether to
* flash. Reuses FaceclawBleManager + BleProtocol; owns its own connection and
* runs on a single worker thread. Shows nothing on the lens.
* Android host for the shared [DeviceInfoProbeFlow]: owns the GATT link (AndroidStockLink),
* the worker thread the flow runs on, and delivers listener callbacks on the main thread.
* Public API unchanged for the TypeScript wrapper (app/native/device-info-probe.ts).
*/
class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawDeviceInfo"
private const val QUERY_TIMEOUT_MS = 4_000
/** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */
private const val ARM_ATTEMPTS = 3
private const val ARM_RETRY_DELAY_MS = 1_000
/**
* Upper bound on waiting for the auth success while Android reports the arm
* as BOND_BONDING — i.e. an OS pairing dialog may be sitting there waiting
* for the user, which can take a lot longer than the normal 30 s window.
*/
private const val PAIRING_WAIT_CAP_MS = 90_000
/** After a fresh bond, how long the firmware gets to notify success on its own before the request is re-sent. */
private const val POST_BOND_GRACE_MS = 2_000
private const val POLL_MS = 250
private fun bondStateName(state: Int): String {
return when (state) {
BluetoothDevice.BOND_NONE -> "none"
BluetoothDevice.BOND_BONDING -> "bonding"
BluetoothDevice.BOND_BONDED -> "bonded"
else -> "unknown"
}
}
}
private val context: Context
private val rightAddress: String
private val leftAddress: String
private val bleManager: FaceclawBleManager
class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddress: String?) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private var nextSeq = 0x40
private var nextMagic = 100
private var awaitSid = -1
private var awaitMagic = -1
private var awaitPb: ByteArray? = null
private var awaitLatch: CountDownLatch? = null
/**
* Security-auth wait, kept separate from the generic ack wait: only the
* firmware's SUCCESS result (empty result message) may satisfy it. The
* firmware answers the request as soon as it arrives, but with a
* non-success result until the link is encrypted; accepting that first
* reply as "the ack" is what let the probe move on to the other arm while
* Android's pairing dialog for this one was still up.
*/
private val authMagics: MutableSet<Int> = HashSet()
private var authLatch: CountDownLatch? = null
/**
* A sid-0x09 frame that carried firmware versions without matching the
* awaited magic — some firmware pushes the settings snapshot on a magic it
* picked itself instead of (or in addition to) acking the read.
*/
@Volatile
private var unsolicitedSettingsPb: ByteArray? = null
@Volatile
private var listener: FaceclawDeviceInfoProbeListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
private val flow =
DeviceInfoProbeFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
object : FaceclawDeviceInfoProbeListener {
override fun onLog(line: String?) {
android.util.Log.i(DeviceInfoProbeFlow.TAG, line ?: "")
post { it.onLog(line) }
}
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onResult(leftVersion: String?, rightVersion: String?, extension: String?) =
post { it.onResult(leftVersion, rightVersion, extension) }
override fun onError(message: String?) = post { it.onError(message ?: "") }
},
)
fun setListener(listener: FaceclawDeviceInfoProbeListener?) {
this.listener = listener
@@ -104,525 +49,29 @@ class FaceclawDeviceInfoProbe(context: Context, rightAddress: String?, leftAddre
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-device-info")
val w = Thread({ flow.run() }, "faceclaw-device-info")
worker = w
w.start()
}
}
fun cancel() {
cancelled = true
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (rightAddress.trim().isEmpty()) {
emitError("No right-arm address configured.")
return
}
val haveLeft = !leftAddress.trim().isEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true)
// Bring up and authenticate BOTH arms before asking for versions.
// Each arm is its own peripheral with its own bond, so on a fresh
// phone each raises its own pairing prompt; completing both here
// means the flash step (and the app proper) find both bonds in
// place. Strictly one arm at a time: Android pairs with one device
// at a time, and a second arm connected while the first was still
// pairing has been seen to drop within a second and never pair
// (2026-09-11 logcat).
var rightAuthenticated = bringUpArm(rightAddress, "right")
var leftAuthenticated = false
if (haveLeft && !cancelled) {
leftAuthenticated = bringUpArm(leftAddress, "left")
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (!bleManager.isConnected(rightAddress)) {
// The right lens may have dropped while the left one paired;
// it is bonded by now, so this comes back quickly.
emitLog("right lens disconnected while the left lens was set up; reconnecting")
rightAuthenticated = bringUpArm(rightAddress, "right")
if (cancelled) {
emitError("Cancelled.")
return
}
}
var ack: ByteArray? = null
var rightFailure: String? = null
try {
ack = queryArm(rightAddress, "right", rightAuthenticated)
} catch (e: IllegalStateException) {
rightFailure = if (e.message == null) e.toString() else e.message
emitLog("right-lens query failed: $rightFailure")
}
// The right lens is the documented control endpoint, but a silent
// right lens has been observed on stock 2.2.9 even after a
// successful security auth — run the same probe against the left
// lens rather than giving up, and log which lens answered.
if (ack == null && !cancelled && haveLeft) {
emitLog("right lens did not answer the settings query; probing the left lens")
try {
ack = queryArm(leftAddress, "left", leftAuthenticated)
} catch (e: Exception) {
emitLog("left-lens probe failed: " + (if (e.message == null) e.toString() else e.message))
}
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (ack == null) {
throw IllegalStateException(if (rightFailure != null)
rightFailure
else "no response to the device-info query on either lens — check `adb logcat -s "
+ TAG + "` for the frame trace")
}
val info: BleProtocol.FirmwareInfo? = BleProtocol.parseSettingsFirmwareInfo(ack)
val left = if (info == null) "" else info.leftVersion
val right = if (info == null) "" else info.rightVersion
val extension = if (info == null) "" else info.extension
emitLog("device-info: L=$left R=$right ext=[$extension]")
emitResult(left, right, extension)
} catch (e: Exception) {
val message = if (cancelled) "Cancelled." else (if (e.message == null) e.toString() else e.message)
emitError(message)
} finally {
try {
bleManager.close()
} catch (ignored: Exception) {
}
}
}
/**
* Connect to one arm and complete its security-auth exchange, retrying
* when the link drops mid-pairing. Returns whether the auth success was
* seen (false = connected but unconfirmed, tolerated because the custom
* firmware's response to the exchange is not yet hardware-verified);
* throws when the arm cannot be brought up at all.
*/
@Throws(InterruptedException::class)
private fun bringUpArm(address: String, label: String): Boolean {
var lastFailure: String? = null
var attempt = 1
while (attempt <= ARM_ATTEMPTS && !cancelled) {
if (attempt > 1) {
// A link Android dropped while it was still pairing comes back
// once the bond settles; reconnecting sooner just fails again.
waitForBondToSettle(address, label)
if (cancelled) {
break
}
emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure")
Thread.sleep(ARM_RETRY_DELAY_MS.toLong())
}
emitState("connecting", label)
try {
connectArm(address)
} catch (e: IllegalStateException) {
lastFailure = if (e.message == null) e.toString() else e.message
emitLog("$label lens: $lastFailure")
attempt++
continue
}
if (cancelled) {
break
}
// Firmware 2.2.9 answers no queries until the security-auth
// exchange completes over an encrypted link; on a phone with no
// existing bond this is also what triggers SMP pairing (and its OS
// prompt), so it must come before the prelude and query.
emitState("authenticating", label)
val result = authenticate(address, label)
if (result == AuthResult.SUCCESS) {
return true
}
if (result == AuthResult.UNCONFIRMED) {
return false
}
lastFailure = "link dropped during authentication"
attempt++
}
if (cancelled) {
return false
}
throw IllegalStateException("could not connect to the $label lens ($address): "
+ lastFailure + " — if Android showed a Bluetooth pairing request, accept it and try again")
}
private enum class AuthResult { SUCCESS, UNCONFIRMED, LINK_DROPPED }
/**
* Send the sid-0x80 authentication request and wait for the firmware's
* SUCCESS notification, which only arrives once the link is encrypted.
* On an unbonded phone that means waiting through Android's pairing flow
* (BOND_BONDING, possibly with a dialog up), so the wait is extended while
* the OS reports pairing in progress and the request is re-sent once
* after a fresh bond in case the pre-pairing write was dropped. Bond
* state is polled rather than awaited via broadcast; 250 ms granularity
* is plenty here.
*/
@Throws(InterruptedException::class)
private fun authenticate(address: String, label: String): AuthResult {
val start = SystemClock.elapsedRealtime()
var deadline = start + ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS
val hardDeadline = start + PAIRING_WAIT_CAP_MS
val latch = CountDownLatch(1)
synchronized(lock) {
authMagics.clear()
authLatch = latch
}
val initialBond = bleManager.getBondState(address)
emitLog("security auth: $label lens, Android bond state " + bondStateName(initialBond))
try {
var sends = 0
var lastSendFailed = false
var nextSendAt = start
var bondedAt = -1L
var resentAfterBond = false
var loggedBonding = false
while (!cancelled) {
val now = SystemClock.elapsedRealtime()
val bond = bleManager.getBondState(address)
if (bond == BluetoothDevice.BOND_BONDING) {
// OS pairing in progress — the firmware will answer once the
// link is encrypted. Don't write, don't give up, don't move
// on to the other arm.
if (!loggedBonding) {
emitLog("Android is pairing with the $label lens; waiting for the user to accept")
loggedBonding = true
}
if (now >= hardDeadline) {
break
}
} else {
if (bond == BluetoothDevice.BOND_BONDED && initialBond != BluetoothDevice.BOND_BONDED
&& bondedAt < 0) {
bondedAt = now
emitLog("Android bond established with the $label lens; waiting for the auth success")
deadline = Math.max(deadline, now + ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS)
}
val wantSend = sends == 0
|| (lastSendFailed && sends < 3 && now >= nextSendAt)
|| (bondedAt >= 0 && !resentAfterBond && now - bondedAt >= POST_BOND_GRACE_MS)
if (wantSend) {
if (bondedAt >= 0) {
resentAfterBond = true
if (sends > 0) {
emitLog("re-sending the auth request after pairing ($label lens)")
}
}
lastSendFailed = !sendAuthRequest(address)
sends++
nextSendAt = SystemClock.elapsedRealtime() + ARM_RETRY_DELAY_MS
if (lastSendFailed && !bleManager.isConnected(address)) {
return AuthResult.LINK_DROPPED
}
}
if (now >= deadline) {
break
}
}
if (latch.await(POLL_MS.toLong(), TimeUnit.MILLISECONDS)) {
emitLog("security auth complete: $label lens ($address)")
return AuthResult.SUCCESS
}
if (!bleManager.isConnected(address)) {
emitLog("security auth: $label lens disconnected while waiting"
+ (if (bond == BluetoothDevice.BOND_BONDING) " (Android was still pairing)" else ""))
return AuthResult.LINK_DROPPED
}
}
if (cancelled) {
return AuthResult.UNCONFIRMED
}
emitLog("security auth unconfirmed: $label lens ($address), Android bond state "
+ bondStateName(bleManager.getBondState(address)))
return AuthResult.UNCONFIRMED
} finally {
synchronized(lock) {
authLatch = null
authMagics.clear()
}
}
}
private fun sendAuthRequest(address: String): Boolean {
val magic = allocMagic()
val seq: Int
synchronized(lock) {
authMagics.add(magic)
seq = nextSeq++ and 0xff
}
val payload = BleProtocol.buildAuthenticationRequest(magic)
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, magic, seq, payload.size))
val frames = BleProtocol.framePb(payload, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, seq)
var written: Boolean
try {
written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
} catch (e: IllegalStateException) {
// "Not connected" — the link went away underneath us.
written = false
}
if (!written) {
emitLog("tx write FAILED sid=0x80 ($address)")
}
return written
}
/** Block while Android reports this arm as BOND_BONDING (pairing dialog up / SMP in flight). */
@Throws(InterruptedException::class)
private fun waitForBondToSettle(address: String, label: String) {
val cap = SystemClock.elapsedRealtime() + PAIRING_WAIT_CAP_MS
var logged = false
while (!cancelled && bleManager.getBondState(address) == BluetoothDevice.BOND_BONDING
&& SystemClock.elapsedRealtime() < cap) {
if (!logged) {
emitLog("waiting for Android to finish pairing with the $label lens before reconnecting")
logged = true
}
Thread.sleep(POLL_MS.toLong())
}
}
/**
* Prelude + settings read on an already connected/authenticated lens.
* Returns the settings ack protobuf, an unsolicited settings push that
* carried firmware versions, or null when the lens never answered the read.
* Throws on prelude failure; the caller wraps the fallback lens's attempt
* so its failure cannot mask the primary lens's outcome.
*/
@Throws(InterruptedException::class)
private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? {
if (cancelled) {
return null
}
emitState("querying", label)
// Session prelude, then a settings/device-info read (both arms'
// versions and the firmware-extension string ride back in one response).
if (writeAndAwaitAck(address, BleProtocol.PRELUDE_ACK_SID, BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC, BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw IllegalStateException("session prelude not acked ($label lens)"
+ (if (authenticated) "" else "; authentication did not complete —"
+ " if Android shows a Bluetooth pairing request, accept it and try again"))
}
// Two attempts: the first read straight after a fresh pairing has been
// seen to go unanswered while a later one succeeds.
var attempt = 0
while (attempt < 2 && !cancelled) {
val magic = allocMagic()
val ack = writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST,
magic, BleProtocol.buildSettingsQuery(magic), QUERY_TIMEOUT_MS)
if (ack != null) {
return ack
}
// A push with the firmware versions on the device's own magic is
// as good as the ack we asked for.
val pushed = unsolicitedSettingsPb
if (pushed != null) {
emitLog("using unsolicited settings push instead of the read ack ($label lens)")
return pushed
}
emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)")
attempt++
}
return null
}
private fun connectArm(address: String) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("connect failed")
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
throw IllegalStateException("service discovery failed")
}
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw IllegalStateException("could not subscribe to notifications")
}
}
@Throws(InterruptedException::class)
private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? {
val latch = CountDownLatch(1)
synchronized(lock) {
awaitSid = sid
awaitMagic = magic
awaitPb = null
awaitLatch = latch
}
val seq: Int
synchronized(lock) {
seq = nextSeq++ and 0xff
}
emitLog(String.format("tx %s sid=0x%02x flag=0x%02x magic=%d seq=0x%02x len=%d",
address, sid, flag, magic, seq, payload.size))
val frames = BleProtocol.framePb(payload, sid, flag, seq)
val written = bleManager.writeFrames(
address, BleProtocol.WRITE_CHAR_UUID, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
if (!written) {
emitLog("tx write FAILED sid=0x" + Integer.toHexString(sid))
synchronized(lock) {
awaitLatch = null
}
return null
}
val acked = latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)
synchronized(lock) {
awaitLatch = null
return if (acked) awaitPb else null
}
}
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
return
}
// One notification value can carry several envelope frames back to
// back; reading only the first would silently drop the rest.
val frames = BleProtocol.splitFrames(data)
if (frames.size > 1) {
emitLog("rx " + address + " value carries " + frames.size + " frames (raw " + data!!.size + " bytes)")
}
for (buf in frames) {
handleFrame(address, buf, data!!.size)
}
}
private fun handleFrame(address: String?, buf: ByteArray, rawValueLength: Int) {
val frame = BleProtocol.parseFrame(buf)
if (!frame.ok) {
emitLog("rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")"
+ " head=" + FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(buf, Math.min(16, buf.size))))
return
}
// Log every control frame while diagnosing 2.2.9: sid/flag/type/magic
// plus a payload prefix is enough to reconstruct what the lens said.
val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0
val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else ""
emitLog(String.format("rx %s sid=0x%02x flag=0x%02x type=%d magic=%d frag=%d/%d len=%d%s pb=%s",
address, frame.sid, frame.flag, frame.msgType, frame.msgSeq,
if (buf.size > 5) buf[5].toInt() and 0xff else 0, if (buf.size > 4) buf[4].toInt() and 0xff else 0, frame.pb.size, truncated,
FaceclawFirmwareUtil.bytesToHex(java.util.Arrays.copyOf(frame.pb, Math.min(48, frame.pb.size)))))
if (frame.sid == BleProtocol.SID_UI_SETTING
&& BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) {
// Any settings frame carrying firmware versions answers the probe's
// question, whether or not it matches the magic we asked with.
unsolicitedSettingsPb = frame.pb
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
// Async event, not an ack. This includes the lens's own periodic
// sid-0x80 notifies (device-chosen magic, non-empty result).
return
}
if (frame.sid == BleProtocol.SID_SECURITY_AUTH) {
var success = false
var ours = false
synchronized(lock) {
val currentLatch = authLatch
if (currentLatch != null) {
ours = authMagics.contains(frame.msgSeq)
for (magic in authMagics) {
if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) {
success = true
currentLatch.countDown()
break
}
}
}
}
if (ours && !success) {
// Expected before the link is encrypted: the firmware answers
// the request straight away with a non-success result and
// sends the real success once pairing/encryption completes.
emitLog("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting")
}
return
}
synchronized(lock) {
val currentLatch = awaitLatch
if (currentLatch != null && frame.sid == awaitSid && frame.msgSeq == awaitMagic) {
awaitPb = frame.pb
currentLatch.countDown()
}
}
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (!connected) {
Log.i(TAG, "disconnected: $address")
}
}
private fun allocMagic(): Int {
synchronized(lock) {
val magic = nextMagic
nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1
return magic
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawDeviceInfoProbeListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitResult(left: String, right: String, extension: String) {
mainHandler.post {
val current = listener
if (current != null) {
current.onResult(left, right, extension)
}
}
}
private fun emitError(message: String?) {
val safeMessage = message ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onError(safeMessage)
action(current)
}
}
}
@@ -129,17 +129,19 @@ class FaceclawDiarizer(
val file = File(path)
val fileLength = file.length()
DataInputStream(FileInputStream(file)).use { `in` ->
val header = ByteArray(44)
`in`.readFully(header)
if (header[0] != 'R'.code.toByte() || header[1] != 'I'.code.toByte() || header[8] != 'W'.code.toByte()) {
throw IOException("not a WAV file")
// Shared RIFF walk (handles extra chunks and unpatched data sizes); the
// stream is then positioned at the first PCM sample.
val prefix = ByteArray(Math.min(fileLength, WavPcmReader.HEADER_PREFIX_BYTES.toLong()).toInt())
`in`.readFully(prefix)
val info = try {
WavPcmReader.parseHeader(prefix, fileLength)
} catch (e: IllegalArgumentException) {
throw IOException(e.message ?: "not a WAV file")
}
val channels = (header[22].toInt() and 0xff) or (header[23].toInt() shl 8)
val bits = (header[34].toInt() and 0xff) or (header[35].toInt() shl 8)
if (channels != 1 || bits != 16) {
if (info.channels != 1) {
throw IOException("only 16-bit mono WAV is supported")
}
val dataBytes = fileLength - 44
val dataBytes = info.dataBytes
val count = Math.min(dataBytes / 2, Int.MAX_VALUE.toLong()).toInt()
val samples = FloatArray(count)
val chunk = ByteArray(65536)
@@ -147,6 +149,21 @@ class FaceclawDiarizer(
var carried = 0
var carriedByte: Byte = 0
var read = 0
// The prefix already holds the start of the PCM data; consume it first.
val prefixData = prefix.size - info.dataOffset.toInt()
if (prefixData > 0) {
val pairs = prefixData / 2
var i = 0
while (i < pairs && sampleIndex < count) {
val base = info.dataOffset.toInt() + i * 2
samples[sampleIndex++] = AudioSegmentation.pcm16le(prefix[base], prefix[base + 1]) / 32768.0f
i++
}
if ((prefixData and 1) == 1) {
carried = 1
carriedByte = prefix[prefix.size - 1]
}
}
while (sampleIndex < count && `in`.read(chunk).also { read = it } > 0) {
var offset = 0
if (carried == 1) {
@@ -15,7 +15,6 @@ import java.io.FileInputStream
import java.io.IOException
import java.io.InputStream
import java.nio.charset.StandardCharsets
import java.util.Locale
/**
* Serves an unpacked EvenHub app's files to its WebView from a fake per-app
@@ -52,25 +51,25 @@ class FaceclawEvenHubWebViewClient(
if (url == null || host != url.host) {
return null // External hosts: normal network handling.
}
var path = url.path
if (path == null || path.isEmpty() || path == "/") {
path = "/index.html"
}
val path = url.path
try {
// Path rules, MIME table and shim injection are shared (EvenHubAssetServer);
// the canonical-file check below additionally defeats symlinks.
val asset = EvenHubAssetServer.resolve(path)
val root = File(rootDir).canonicalFile
val target = File(root, path.substring(1)).canonicalFile
if (!target.path.startsWith(root.path) || !target.isFile) {
val target = asset?.let { File(root, it.relativePath).canonicalFile }
if (asset == null || target == null || !EvenHubAssetServer.isInsideRoot(root.path, target.path) || !target.isFile) {
Log.w(TAG, "404 $path")
return WebResourceResponse(
"text/plain", "utf-8",
ByteArrayInputStream("not found".toByteArray(StandardCharsets.UTF_8)))
}
val mime = mimeTypeFor(target.name)
if (mime == "text/html") {
return WebResourceResponse(mime, "utf-8", ByteArrayInputStream(injectIntoHtml(target)))
if (asset.isHtml) {
val html = EvenHubAssetServer.injectIntoHtml(String(readAll(target), StandardCharsets.UTF_8), injectScript)
return WebResourceResponse(asset.mime, "utf-8", ByteArrayInputStream(html.toByteArray(StandardCharsets.UTF_8)))
}
val stream: InputStream = FileInputStream(target)
return WebResourceResponse(mime, null, stream)
return WebResourceResponse(asset.mime, null, stream)
} catch (e: IOException) {
Log.e(TAG, "serve failed for $path", e)
return WebResourceResponse(
@@ -94,30 +93,9 @@ class FaceclawEvenHubWebViewClient(
}
}
@Throws(IOException::class)
private fun injectIntoHtml(file: File): ByteArray {
val raw = readAll(file)
val html = String(raw, StandardCharsets.UTF_8)
val tag = "<script>$injectScript</script>"
// After <head...> if present, else before everything.
val headIndex = indexOfIgnoreCase(html, "<head")
if (headIndex >= 0) {
val close = html.indexOf('>', headIndex)
if (close >= 0) {
return (html.substring(0, close + 1) + tag + html.substring(close + 1))
.toByteArray(StandardCharsets.UTF_8)
}
}
return (tag + html).toByteArray(StandardCharsets.UTF_8)
}
companion object {
private const val TAG = "FaceclawEvenHub"
private fun indexOfIgnoreCase(haystack: String, needle: String): Int {
return haystack.lowercase(Locale.ROOT).indexOf(needle)
}
@Throws(IOException::class)
private fun readAll(file: File): ByteArray {
val input = FileInputStream(file)
@@ -134,25 +112,5 @@ class FaceclawEvenHubWebViewClient(
input.close()
}
}
private fun mimeTypeFor(name: String): String {
val lower = name.lowercase(Locale.ROOT)
if (lower.endsWith(".html") || lower.endsWith(".htm")) return "text/html"
if (lower.endsWith(".js") || lower.endsWith(".mjs")) return "application/javascript"
if (lower.endsWith(".css")) return "text/css"
if (lower.endsWith(".json")) return "application/json"
if (lower.endsWith(".png")) return "image/png"
if (lower.endsWith(".jpg") || lower.endsWith(".jpeg")) return "image/jpeg"
if (lower.endsWith(".gif")) return "image/gif"
if (lower.endsWith(".svg")) return "image/svg+xml"
if (lower.endsWith(".webp")) return "image/webp"
if (lower.endsWith(".ico")) return "image/x-icon"
if (lower.endsWith(".wasm")) return "application/wasm"
if (lower.endsWith(".woff")) return "font/woff"
if (lower.endsWith(".woff2")) return "font/woff2"
if (lower.endsWith(".ttf")) return "font/ttf"
if (lower.endsWith(".txt") || lower.endsWith(".map")) return "text/plain"
return "application/octet-stream"
}
}
}
@@ -1,185 +1,54 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.util.Log
import java.io.ByteArrayOutputStream
import java.io.FileInputStream
import java.io.IOException
import java.nio.charset.StandardCharsets
import java.util.ArrayList
import java.util.Arrays
import java.util.concurrent.CountDownLatch
import java.util.concurrent.LinkedBlockingQueue
import java.util.concurrent.TimeUnit
import java.util.concurrent.TimeoutException
/**
* Ports g2flash.py's OTA flash procedure to Android. Streams the EVENOTA
* firmware container to each lens over the firmware-data service using the
* aa21 envelope (BleProtocol.framePb) and the c0/c1 control/data protocol.
*
* Each lens connection completes the sid-0x80 security-auth exchange before
* OTA BEGIN — firmware 2.2.9 closes unauthenticated links ~30 s in, killing
* the transfer mid-component. No control-channel traffic is injected during
* the transfer itself: official OTA captures show none between BEGIN and the
* final END, heartbeats demonstrably do not satisfy the 2.2.9 deadline, and a
* concurrent control writer would interleave with the marker/data
* transactions. (See ../notes/ble-connections-2.2.9.md.)
*
* Lenses are flashed one at a time (left, then right). Finishing either lens
* reboots BOTH lenses, so the second lens is briefly unreachable — the reconnect
* for it uses a generous window rather than failing fast.
*
* Separate from FaceclawBleCommunicator on purpose: this speaks the stock OTA
* protocol and must run before the custom firmware exists. It reuses the GATT
* wrapper (FaceclawBleManager) and framing (BleProtocol).
* Android host for the shared [OtaFlashFlow] (g2flash.py's OTA procedure over the stock
* firmware-data service): owns the GATT link (AndroidStockLink) and the worker thread, and
* delivers listener callbacks on the main thread. Separate from FaceclawBleCommunicator on
* purpose: this speaks the stock OTA protocol and must run before the custom firmware exists.
* Public API unchanged for app/native/firmware-flasher.ts.
*/
class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawFlasher"
// OTA message types (envelope sid byte) and control opcodes.
private const val SID_CTRL = 0xc0
private const val SID_DATA = 0xc1
private const val FLAG_OTA = 0x00
private const val OP_BEGIN = 0x00
private const val OP_FILE_CHECK = 0x01
private const val OP_BLOCK = 0x02
private const val OP_END = 0x03
private const val BLOCK_SIZE = 4096
private const val BLOCK_ACK_TIMEOUT_MS = 4_000
private const val CTRL_ACK_TIMEOUT_MS = 8_000
private const val BLOCK_NAK_RETRIES = 3
private const val COMPONENT_RETRIES = 3
// Reconnect windows. The first lens connects from an idle device; the second
// must ride out the post-first-lens reboot of both lenses.
private const val FIRST_LENS_CONNECT_WINDOW_MS = 30_000
private const val SECOND_LENS_CONNECT_WINDOW_MS = 120_000
private const val REBOOT_SETTLE_MS = 5_000
private const val NOTIFY_SETTLE_MS = 2_500
private const val RETRY_DELAY_MS = 2_500
// END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART.
private val END_OK = intArrayOf(0, 8, 9)
// Firmware layout expectations (mirrors g2flash.py). Firmware 2.2.4 has 5
// components; 2.2.6 has 6.
private const val MIN_SEGMENTS = 5
private const val MAX_SEGMENTS = 6
private const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin"
private const val APP_LOAD_ADDR = 0x00438000L
private const val APP_MAX_END = 0x007F0000L
private const val APP_PREAMBLE = 0x20
// ---- firmware container parsing / validation -----------------------------
private val EMPTY = ByteArray(0)
// ---- helpers -------------------------------------------------------------
private fun isEndOk(status: Int): Boolean {
for (ok in END_OK) {
if (ok == status) {
return true
}
}
return false
}
private fun readFile(path: String): ByteArray {
try {
FileInputStream(path).use { fis ->
val out = ByteArrayOutputStream()
val buf = ByteArray(65536)
var read: Int
while (fis.read(buf).also { read = it } >= 0) {
out.write(buf, 0, read)
}
return out.toByteArray()
}
} catch (e: IOException) {
throw IllegalStateException("could not read firmware: " + e.message, e)
}
}
private fun readU32(buf: ByteArray, offset: Int): Long {
return (buf[offset].toLong() and 0xffL) or
((buf[offset + 1].toLong() and 0xffL) shl 8) or
((buf[offset + 2].toLong() and 0xffL) shl 16) or
((buf[offset + 3].toLong() and 0xffL) shl 24)
}
private fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String {
var end = offset
val limit = Math.min(buf.size, offset + maxLen)
while (end < limit && buf[end].toInt() != 0) {
end++
}
return String(buf, offset, end - offset, StandardCharsets.ISO_8859_1)
}
private val CRC32C_TABLE: IntArray = buildCrc32cTable()
private fun buildCrc32cTable(): IntArray {
val table = IntArray(256)
for (b in 0 until 256) {
var c = b shl 24
for (i in 0 until 8) {
c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1
}
table[b] = c
}
return table
}
/** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */
private fun crc32cMsb(data: ByteArray): Int {
var crc = 0
for (value in data) {
crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (value.toInt() and 0xff)) and 0xff]
}
return crc
}
}
private val context: Context
private val leftAddress: String
private val rightAddress: String
private val firmwarePath: String
private val bleManager: FaceclawBleManager
class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddress: String?, firmwarePath: String?) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val dataAcks = LinkedBlockingQueue<ByteArray>()
@Volatile
private var listener: FaceclawFirmwareFlasherListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
private var nextSeq = 0
private var nextAuthMagic = 0x60
private var pendingAuthMagic = -1
private var authLatch: CountDownLatch? = null
private var authAckPb: ByteArray? = null
private val flow =
OtaFlashFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
firmwarePath ?: "",
object : FaceclawFirmwareFlasherListener {
override fun onLog(line: String?) {
android.util.Log.i(OtaFlashFlow.TAG, line ?: "")
post { it.onLog(line) }
}
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.firmwarePath = firmwarePath ?: ""
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
override fun onProgress(
lens: String?,
componentIndex: Int,
componentCount: Int,
blockIndex: Int,
blockCount: Int,
bytesSent: Long,
bytesTotal: Long,
) = post { it.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal) }
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onComplete(success: Boolean, detail: String?) = post { it.onComplete(success, detail ?: "") }
},
)
fun setListener(listener: FaceclawFirmwareFlasherListener?) {
this.listener = listener
@@ -190,517 +59,29 @@ class FaceclawFirmwareFlasher(context: Context, rightAddress: String?, leftAddre
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-flasher")
val w = Thread({ flow.run() }, "faceclaw-flasher")
worker = w
w.start()
}
}
fun cancel() {
cancelled = true
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (leftAddress.trim().isEmpty() || rightAddress.trim().isEmpty()) {
throw IllegalStateException("Both lens addresses are required to flash.")
}
emitState("validating", "")
val img = readFile(firmwarePath)
val segs = validate(img)
emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes")
flashLens("left", leftAddress, img, segs, FIRST_LENS_CONNECT_WINDOW_MS)
if (cancelled) {
emitState("error", "Cancelled.")
emitComplete(false, "Cancelled after the left lens.")
return
}
emitState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.")
sleepInterruptibly(REBOOT_SETTLE_MS)
flashLens("right", rightAddress, img, segs, SECOND_LENS_CONNECT_WINDOW_MS)
emitState("done", "")
emitComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.")
} catch (e: Exception) {
var message = if (e.message == null) e.toString() else e.message
if (cancelled) {
message = "Cancelled."
}
emitState("error", message)
emitComplete(false, message)
} finally {
teardown()
}
}
@Throws(TimeoutException::class)
private fun flashLens(lens: String, address: String, img: ByteArray, segs: List<Segment>, connectWindowMs: Int) {
emitState("connecting", lens)
connectLensResilient(lens, address, connectWindowMs)
sleepInterruptibly(NOTIFY_SETTLE_MS)
drainAcks()
// Fresh envelope counter for the OTA stream (the auth exchange during
// bring-up used its own); mirrors the stock sequence.
resetSeq()
emitState("flashing", lens)
val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, CTRL_ACK_TIMEOUT_MS)
if (!isEndOk(beginStatus)) {
emitLog("warning: unexpected begin status $beginStatus; continuing")
}
// Progress is reported in bytes across the whole image: the segments
// are one large firmware blob plus several small ones, so a bar
// stepped per segment would sit still through the big one and then
// sprint through the rest.
var totalBytes = 0L
for (seg in segs) {
totalBytes += seg.ps
}
var bytesBefore = 0L
for (i in segs.indices) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes)
bytesBefore += segs[i].ps
}
emitLog("$lens lens: all components verified")
try {
bleManager.disconnect(address)
} catch (ignored: Exception) {
}
}
private fun flashComponentWithRetry(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray,
bytesBefore: Long, totalBytes: Long) {
for (attempt in 0 until COMPONENT_RETRIES) {
if (attempt > 0) {
emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES)
}
var endStatus: Int
try {
endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes)
} catch (e: TimeoutException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
} catch (e: RuntimeException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
}
if (isEndOk(endStatus)) {
emitLog(seg.name + ": END verify OK (status " + endStatus + ")")
return
}
if (endStatus >= 0) {
emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")")
}
drainAcks()
sleepInterruptibly(1_500)
}
throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts")
}
@Throws(TimeoutException::class)
private fun flashComponent(lens: String, address: String, index: Int, count: Int, seg: Segment, img: ByteArray,
bytesBefore: Long, totalBytes: Long): Int {
val sub = Arrays.copyOfRange(img, seg.off, seg.off + 128)
val ps = seg.ps
val payloadStart = seg.off + 128
val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, CTRL_ACK_TIMEOUT_MS)
if (checkStatus != 0) {
throw IllegalStateException("FILE_CHECK rejected status=$checkStatus")
}
val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE
for (b in 0 until blockCount) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
val start = payloadStart + b * BLOCK_SIZE
val end = Math.min(start + BLOCK_SIZE, payloadStart + ps)
val block = Arrays.copyOfRange(img, start, end)
var accepted = false
for (tries in 0 until BLOCK_NAK_RETRIES) {
val status = sendBlock(address, block) // throws TimeoutException -> component re-flash
if (status == 0) {
accepted = true
break
}
emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status
+ " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES)
}
if (!accepted) {
throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times")
}
if (b % 20 == 0 || b == blockCount - 1) {
val bytesSent = bytesBefore + Math.min((b + 1).toLong() * BLOCK_SIZE, ps.toLong())
emitProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes)
}
}
emitLog(seg.name + ": data phase done; sending END")
return sendCtrlAndWait(address, OP_END, EMPTY, CTRL_ACK_TIMEOUT_MS)
}
/** Send one 4 KB block as a marker + data pair sharing one envelope seq. */
@Throws(TimeoutException::class)
private fun sendBlock(address: String, block: ByteArray): Int {
val seq = nextSeq()
drainAcks()
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq)
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_DATA, block, seq)
return waitAck(OP_BLOCK, BLOCK_ACK_TIMEOUT_MS)
}
@Throws(TimeoutException::class)
private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int {
val seq = nextSeq()
drainAcks()
val payload = ByteArray(1 + data.size)
payload[0] = op.toByte()
System.arraycopy(data, 0, payload, 1, data.size)
writeOta(address, BleProtocol.OTA_DATA_WRITE_UUID, SID_CTRL, payload, seq)
return waitAck(op, timeoutMs)
}
private fun writeOta(address: String, writeChar: String, sid: Int, payload: ByteArray, seq: Int): Boolean {
val frames = BleProtocol.framePb(payload, sid, FLAG_OTA, seq)
return bleManager.writeFrames(
address, writeChar, frames, AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE), ConnectionOptions.WRITE_TIMEOUT_MS)
}
@Throws(TimeoutException::class)
private fun waitAck(wantOp: Int, timeoutMs: Int): Int {
val deadline = SystemClock.elapsedRealtime() + timeoutMs
while (true) {
val remaining = deadline - SystemClock.elapsedRealtime()
if (remaining <= 0) {
break
}
val pb: ByteArray?
try {
pb = dataAcks.poll(remaining, TimeUnit.MILLISECONDS)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
throw TimeoutException("interrupted waiting for ack op=$wantOp")
}
if (pb == null) {
break
}
if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) {
return pb[1].toInt() and 0xff
}
// Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting.
}
throw TimeoutException("no ack op=0x" + Integer.toHexString(wantOp) + " within " + timeoutMs + "ms")
}
// ---- connection ----------------------------------------------------------
private fun connectLensResilient(lens: String, address: String, windowMs: Int) {
val deadline = SystemClock.elapsedRealtime() + windowMs
var lastError: String? = "unknown"
var attempt = 0
while (SystemClock.elapsedRealtime() < deadline && !cancelled) {
attempt++
try {
if (bringUpLens(address)) {
emitLog("connected $lens lens (attempt $attempt)")
return
}
lastError = "connect/discover incomplete"
} catch (e: Exception) {
lastError = if (e.message == null) e.toString() else e.message
}
emitLog("$lens connect attempt $attempt failed ($lastError); retrying...")
try {
bleManager.disconnect(address)
} catch (ignored: Exception) {
}
sleepInterruptibly(RETRY_DELAY_MS)
}
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
throw IllegalStateException("could not reach $lens lens: $lastError")
}
private fun bringUpLens(address: String): Boolean {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
return false
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
return false
}
// Control-channel notify carries the auth response; mandatory.
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false
}
if (!bleManager.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
return false
}
if (!authenticate(address)) {
throw IllegalStateException(
"security auth not acknowledged — if Android shows a Bluetooth pairing request, accept it")
}
return true
}
/**
* Complete the sid-0x80 security-auth exchange on this connection. On an
* unbonded phone this triggers SMP pairing (possibly with an OS prompt),
* so the wait is generous. Must succeed before any OTA traffic.
*/
private fun authenticate(address: String): Boolean {
val magic: Int
val latch = CountDownLatch(1)
synchronized(lock) {
magic = nextAuthMagic
nextAuthMagic = if (nextAuthMagic >= 0x7f) 0x60 else nextAuthMagic + 1
pendingAuthMagic = magic
authAckPb = null
authLatch = latch
}
try {
if (!writeOta(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH,
BleProtocol.buildAuthenticationRequest(magic), nextSeq())) {
emitLog("auth write failed: $address")
return false
}
val signalled: Boolean
try {
signalled = latch.await(ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS)
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
return false
}
val pb: ByteArray?
synchronized(lock) {
pb = authAckPb
}
if (!signalled || !BleProtocol.isAuthenticationSuccess(pb, magic)) {
emitLog("security auth not acknowledged (" + address
+ ") — if Android shows a Bluetooth pairing request, accept it")
return false
}
emitLog("security auth complete: $address")
return true
} finally {
synchronized(lock) {
pendingAuthMagic = -1
authLatch = null
}
}
}
// ---- firmware container parsing / validation -----------------------------
private class Segment(
@JvmField val off: Int,
@JvmField val ps: Int,
@JvmField val crc: Long,
@JvmField val name: String
)
private fun parseSegments(img: ByteArray): List<Segment> {
if (img.size < 0x40) {
throw IllegalStateException("file is too small to be a firmware image")
}
val n = readU32(img, 8)
if (n <= 0 || n > 64) {
throw IllegalStateException("implausible component count $n (corrupt image?)")
}
val segs = ArrayList<Segment>()
var i = 0
while (i < n) {
val base = 0x40 + i * 16
val crc = readU32(img, base + 12)
val off = readU32(img, base + 4).toInt()
if (off + 128 > img.size) {
throw IllegalStateException("segment $i subheader runs past end of file")
}
val ps = readU32(img, off + 8).toInt()
val name = readCString(img, off + 48, 80)
segs.add(Segment(off, ps, crc, name))
i++
}
return segs
}
private fun validate(img: ByteArray): List<Segment> {
val segs = parseSegments(img)
if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) {
throw IllegalStateException(
"expected " + MIN_SEGMENTS + "-" + MAX_SEGMENTS + " components, found " + segs.size)
}
var main: Segment? = null
for (s in segs) {
val payload = Arrays.copyOfRange(img, s.off + 128, s.off + 128 + s.ps)
val calc = crc32cMsb(payload).toLong() and 0xffffffffL
val subCrc = readU32(img, s.off + 12)
if (calc != s.crc || calc != subCrc) {
throw IllegalStateException("component " + s.name + " CRC32C is stale (image not checksum-fixed)")
}
if (REQUIRED_SEGMENT == s.name) {
main = s
}
}
if (main == null) {
throw IllegalStateException("required component $REQUIRED_SEGMENT not found")
}
checkMainAppFitsMram(img, main)
return segs
}
private fun checkMainAppFitsMram(img: ByteArray, main: Segment) {
if (main.ps < APP_PREAMBLE) {
throw IllegalStateException("main-app payload is smaller than its preamble")
}
val loadAddr = readU32(img, main.off + 128 + 0x14)
val preLen = readU32(img, main.off + 128) and 0xFFFFFFL
if (loadAddr != APP_LOAD_ADDR) {
throw IllegalStateException(
"main-app preamble load address is 0x" + java.lang.Long.toHexString(loadAddr)
+ ", expected 0x" + java.lang.Long.toHexString(APP_LOAD_ADDR))
}
if (preLen != main.ps.toLong()) {
throw IllegalStateException(
"main-app preamble length (" + preLen + ") != staged payload size (" + main.ps + ")")
}
val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE
if (progEnd > APP_MAX_END) {
val over = progEnd - APP_MAX_END
throw IllegalStateException(
"main-app is too large: programmed region ends at 0x" + java.lang.Long.toHexString(progEnd)
+ ", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)")
}
}
// ---- listener callbacks --------------------------------------------------
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
// Control channel: only the security-auth response is awaited here.
// A value can carry several envelope frames back to back (e.g. the
// response packed with one of the lens's own sid-0x80 notifies).
for (buf in BleProtocol.splitFrames(data)) {
val frame = BleProtocol.parseFrame(buf)
if (frame.ok && frame.sid == BleProtocol.SID_SECURITY_AUTH) {
synchronized(lock) {
val currentLatch = authLatch
if (currentLatch != null && frame.msgSeq == pendingAuthMagic) {
authAckPb = frame.pb
currentLatch.countDown()
}
}
}
}
return
}
if (!BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) {
return // OTA acks arrive on the data-notify char
}
val frame = BleProtocol.parseFrame(data)
if (!frame.ok || frame.pb.size < 2) {
return
}
dataAcks.add(Arrays.copyOf(frame.pb, Math.min(frame.pb.size, 2)))
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (!connected) {
Log.i(TAG, "disconnected: $address")
}
}
// ---- helpers -------------------------------------------------------------
private fun teardown() {
try {
bleManager.close()
} catch (ignored: Exception) {
}
}
private fun resetSeq() {
synchronized(lock) {
nextSeq = 0
}
}
private fun nextSeq(): Int {
synchronized(lock) {
nextSeq = (nextSeq + 1) and 0xff
return nextSeq
}
}
private fun drainAcks() {
dataAcks.clear()
}
private fun sleepInterruptibly(ms: Int) {
try {
Thread.sleep(ms.toLong())
} catch (e: InterruptedException) {
Thread.currentThread().interrupt()
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawFirmwareFlasherListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitProgress(lens: String, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int,
bytesSent: Long, bytesTotal: Long) {
mainHandler.post {
val current = listener
if (current != null) {
current.onProgress(lens, componentIndex, componentCount, blockIndex, blockCount, bytesSent, bytesTotal)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitComplete(success: Boolean, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onComplete(success, safeDetail)
action(current)
}
}
}
@@ -1,110 +1,53 @@
package com.faceclaw.app
import android.bluetooth.BluetoothGatt
import android.content.Context
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.util.concurrent.ConcurrentHashMap
import java.util.concurrent.CountDownLatch
import java.util.concurrent.Executors
import java.util.concurrent.ScheduledExecutorService
import java.util.concurrent.TimeUnit
/**
* A minimal, stock-firmware-compatible BLE communicator used only to show the
* pre-flash confirmation prompt on the glasses and read back the user's Yes/No
* choice. It is deliberately separate from FaceclawBleCommunicator: the main
* app assumes the custom firmware (image containers, compositor, etc.), whereas
* this must work on unmodified glasses, so it speaks only the stock subset —
* session prelude, a text container, a list container, and heartbeats. No
* images.
*
* It reuses the low-level GATT wrapper (FaceclawBleManager) and the framing /
* protobuf helpers (BleProtocol), but owns its own GATT connection and runs its
* whole lifecycle on a single worker thread. It expects the main app to be
* disconnected while it runs (onboarding, before flashing).
*
* Both arms are connected and security-authenticated up front, even though the
* prompt itself only needs the right arm (the lenses relay messages to each
* other, and acks/events only ever come from the right arm): on an unbonded
* phone the auth exchange is what raises the OS pairing prompt, and doing both
* here means both prompts appear at the start rather than one popping up half
* way through flashing the second lens. While connected, each arm's battery
* level is read (they are independent batteries) so the caller can refuse to
* flash on a low charge.
*
* With `skipPrompt` the on-glasses confirmation is not shown: the flow is just
* connect + auth + battery read + result(approved). Used to re-check the
* battery after the user has already confirmed once.
* Android host for the shared [FlashPromptFlow] (the stock-firmware pre-flash Yes/No prompt
* plus battery read): owns the GATT link (AndroidStockLink) and the worker thread, and
* delivers listener callbacks on the main thread. It expects the main app to be disconnected
* while it runs (onboarding, before flashing). Public API unchanged for
* app/native/flash-prompt-communicator.ts.
*/
class FaceclawFlashPromptCommunicator(
context: Context, rightAddress: String?, leftAddress: String?, warningText: String?, skipPrompt: Boolean) : FaceclawBleListener {
companion object {
private const val TAG = "FaceclawFlashPrompt"
private const val TEXT_NAME = "flashwarn"
private const val LIST_NAME = "flashmenu"
private const val TEXT_CONTAINER_ID = 1
private const val LIST_CONTAINER_ID = 2
// Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid.
private val ITEMS = arrayOf("No, cancel", "Yes, flash")
private const val HEARTBEAT_INTERVAL_MS = 4_000
private const val SELECTION_TIMEOUT_MS = 120_000
private const val CREATE_ACK_TIMEOUT_MS = 3_000
private const val BATTERY_ACK_TIMEOUT_MS = 3_000
private fun ackKey(sid: Int, magic: Int): String {
return "$sid:$magic"
}
}
private val context: Context
private val rightAddress: String
private val leftAddress: String
private val warningText: String
private val skipPrompt: Boolean
private val bleManager: FaceclawBleManager
context: Context,
rightAddress: String?,
leftAddress: String?,
warningText: String?,
skipPrompt: Boolean,
) {
private val link = AndroidStockLink(context.applicationContext)
private val mainHandler = Handler(Looper.getMainLooper())
private val lock = Any()
private val pendingAcks = ConcurrentHashMap<String, CountDownLatch>()
private val ackPayloads = ConcurrentHashMap<String, ByteArray>()
private val selectionLatch = CountDownLatch(1)
@Volatile
private var listener: FaceclawFlashPromptListener? = null
@Volatile
private var worker: Thread? = null
@Volatile
private var cancelled = false
@Volatile
private var finished = false
@Volatile
private var rightConnected = false
@Volatile
private var leftConnected = false
@Volatile
private var rightLost = false
@Volatile
private var approved: Boolean? = null
@Volatile
private var heartbeatExecutor: ScheduledExecutorService? = null
private var nextMagic = 100
private var nextSeq = 0x40
private val flow =
FlashPromptFlow(
link,
rightAddress ?: "",
leftAddress ?: "",
warningText ?: "",
skipPrompt,
object : FaceclawFlashPromptListener {
override fun onLog(line: String?) {
android.util.Log.i(FlashPromptFlow.TAG, line ?: "")
post { it.onLog(line) }
}
init {
this.context = context.applicationContext
this.rightAddress = rightAddress ?: ""
this.leftAddress = leftAddress ?: ""
this.warningText = warningText ?: ""
this.skipPrompt = skipPrompt
this.bleManager = FaceclawBleManager(this.context)
this.bleManager.setListener(this)
}
override fun onState(state: String?, detail: String?) = post { it.onState(state, detail ?: "") }
override fun onBattery(rightPercent: Int, leftPercent: Int) = post { it.onBattery(rightPercent, leftPercent) }
override fun onResult(approved: Boolean) = post { it.onResult(approved) }
},
)
fun setListener(listener: FaceclawFlashPromptListener?) {
this.listener = listener
@@ -115,7 +58,7 @@ class FaceclawFlashPromptCommunicator(
if (worker != null) {
return
}
val w = Thread({ run() }, "faceclaw-flash-prompt")
val w = Thread({ flow.run() }, "faceclaw-flash-prompt")
worker = w
w.start()
}
@@ -123,443 +66,22 @@ class FaceclawFlashPromptCommunicator(
/** Abort the prompt (e.g. the user backed out on the phone). */
fun cancel() {
cancelled = true
selectionLatch.countDown()
val w = worker
if (w != null) {
w.interrupt()
}
flow.cancel()
}
fun close() {
cancel()
stopHeartbeat()
try {
bleManager.close()
link.close()
} catch (ignored: Exception) {
}
}
private fun run() {
try {
if (rightAddress.trim().isEmpty()) {
emitState("error", "No right-arm address configured.")
return
}
emitState("connecting", "")
connectArm(rightAddress, "right")
rightConnected = true
if (cancelled) {
teardown()
return
}
if (!leftAddress.trim().isEmpty()) {
connectArm(leftAddress, "left")
leftConnected = true
if (cancelled) {
teardown()
return
}
}
emitState("connected", "")
// Auth both arms now (pairing prompts, if any, happen here) before
// anything else, so both bonds exist by the time flashing starts.
authenticateArm(rightAddress, "right")
if (leftConnected) {
authenticateArm(leftAddress, "left")
}
if (cancelled) {
teardown()
return
}
sendPrelude(rightAddress)
if (!skipPrompt) {
showPrompt(rightAddress)
startHeartbeat()
emitState("prompting", "")
selectionLatch.await(SELECTION_TIMEOUT_MS.toLong(), TimeUnit.MILLISECONDS)
stopHeartbeat()
if (cancelled) {
emitState("cancelled", "")
teardown()
return
}
if (approved == null) {
if (rightLost) {
emitState("disconnected", "Lost connection to the glasses.")
} else {
emitState("timeout", "No response from the glasses.")
}
teardown()
return
}
try {
sendShutdown()
} catch (ignored: Exception) {
}
if (approved != true) {
emitResult(false)
emitState("result", "declined")
teardown()
return
}
}
// Approved (or prompt skipped): read each arm's battery while the
// links are still up, then report the result.
emitState("battery", "")
readBatteries()
if (cancelled) {
emitState("cancelled", "")
teardown()
return
}
if (rightLost) {
emitState("disconnected", "Lost connection to the glasses.")
teardown()
return
}
finished = true
emitResult(true)
emitState("result", "approved")
teardown()
} catch (e: Exception) {
stopHeartbeat()
if (!cancelled) {
val message = if (e.message == null) e.toString() else e.message
emitState("error", message)
}
teardown()
}
}
private fun connectArm(address: String, arm: String) {
if (!bleManager.connect(address, ConnectionOptions.CONNECT_TIMEOUT_MS)) {
throw IllegalStateException("could not connect to the $arm arm ($address)")
}
bleManager.requestConnectionPriority(address, BluetoothGatt.CONNECTION_PRIORITY_HIGH)
bleManager.requestMtu(address, ConnectionOptions.DESIRED_MTU, ConnectionOptions.CONNECT_TIMEOUT_MS)
if (!bleManager.discoverServices(address, ConnectionOptions.SERVICES_TIMEOUT_MS)) {
throw IllegalStateException("discoverServices failed: $arm arm ($address)")
}
if (!bleManager.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, true, ConnectionOptions.DESCRIPTOR_TIMEOUT_MS)) {
throw IllegalStateException("enableNotifications failed: $arm arm ($address)")
}
emitLog("connected $arm arm")
}
/**
* Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a
* session (and closes the link after ~30 s) without it, and on an
* unbonded phone this is what triggers SMP pairing, so it may sit
* waiting on an OS pairing prompt.
*/
@Throws(InterruptedException::class)
private fun authenticateArm(address: String, arm: String) {
val authMagic = allocMagic()
val authAck = writeAndAwaitAck(
address,
BleProtocol.SID_SECURITY_AUTH,
BleProtocol.FLAG_SECURITY_AUTH,
authMagic,
BleProtocol.buildAuthenticationRequest(authMagic),
ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS)
if (authAck == null || !BleProtocol.isAuthenticationSuccess(authAck, authMagic)) {
throw IllegalStateException(
"could not authenticate with the $arm arm ($address"
+ ") — if Android shows a Bluetooth pairing request, accept it and try again")
}
emitLog("security auth complete: $arm arm")
}
/** Mandatory session prelude on sid=0x01 (app-launch). Right arm only; it relays. */
@Throws(InterruptedException::class)
private fun sendPrelude(address: String) {
if (writeAndAwaitAck(
address,
BleProtocol.PRELUDE_ACK_SID,
BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC,
BleProtocol.PRELUDE_F5872_PAYLOAD,
ConnectionOptions.PRELUDE_TIMEOUT_MS) == null) {
throw IllegalStateException("session prelude not acked: $address")
}
}
@Throws(InterruptedException::class)
private fun showPrompt(address: String) {
// Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0.
// Two attempts: on 2.2.9 the first request sent right after the
// prelude can be dropped while the lens emits its own sid-0x80
// notifications (observed with the device-info settings read).
var pageAck: ByteArray? = null
var attempt = 0
while (attempt < 2 && pageAck == null && !cancelled) {
val magic = allocMagic()
val page = BleProtocol.buildCreatePromptPage(
magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS)
pageAck = writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, CREATE_ACK_TIMEOUT_MS)
if (pageAck == null) {
emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address)
}
attempt++
}
if (pageAck == null) {
throw IllegalStateException("prompt page not acked: $address")
}
emitLog("prompt page shown on $address")
}
/**
* Read each arm's battery via a sid-0x09 settings read on that arm's own
* link (the arms have independent batteries and each answers with its
* own). An arm that does not answer reports -1; the caller decides what
* to do with a partial reading.
*/
@Throws(InterruptedException::class)
private fun readBatteries() {
val right = if (rightConnected) readBattery(rightAddress, "right") else -1
val left = if (leftConnected) readBattery(leftAddress, "left") else -1
emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%"))
emitBattery(right, left)
}
@Throws(InterruptedException::class)
private fun readBattery(address: String, arm: String): Int {
var attempt = 0
while (attempt < 2 && !cancelled) {
val magic = allocMagic()
val ack = writeAndAwaitAck(
address,
BleProtocol.SID_UI_SETTING,
BleProtocol.FLAG_REQUEST,
magic,
BleProtocol.buildSettingsQuery(magic),
BATTERY_ACK_TIMEOUT_MS)
if (ack == null) {
emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm")
attempt++
continue
}
val snapshot: BleProtocol.BatterySnapshot? = BleProtocol.parseSettingsBattery(ack)
if (snapshot != null) {
return snapshot.battery
}
emitLog("battery read ack had no battery field: $arm arm")
attempt++
}
return -1
}
/** Write and wait for the matching ack; returns the ack's protobuf (may be empty), or null on timeout/write failure. */
@Throws(InterruptedException::class)
private fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? {
val latch = CountDownLatch(1)
val key = ackKey(sid, magic)
pendingAcks[key] = latch
try {
if (!writeFrame(address, sid, flag, payload)) {
return null
}
if (!latch.await(timeoutMs.toLong(), TimeUnit.MILLISECONDS)) {
return null
}
val pb = ackPayloads[key]
return pb ?: ByteArray(0)
} finally {
pendingAcks.remove(key, latch)
ackPayloads.remove(key)
}
}
private fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean {
val seq: Int
synchronized(lock) {
seq = nextSeq++ and 0xff
}
val frames = BleProtocol.framePb(payload, sid, flag, seq)
return bleManager.writeFrames(
address,
BleProtocol.WRITE_CHAR_UUID,
frames,
AndroidProtocolPlatform.writeType(ConnectionOptions.WRITE_MODE),
ConnectionOptions.WRITE_TIMEOUT_MS)
}
private fun startHeartbeat() {
stopHeartbeat()
val executor = Executors.newSingleThreadScheduledExecutor { runnable ->
val t = Thread(runnable, "faceclaw-flash-hb")
t.isDaemon = true
t
}
heartbeatExecutor = executor
executor.scheduleWithFixedDelay(
{ sendHeartbeat() }, HEARTBEAT_INTERVAL_MS.toLong(), HEARTBEAT_INTERVAL_MS.toLong(), TimeUnit.MILLISECONDS)
}
private fun sendHeartbeat() {
if (finished || cancelled) {
return
}
try {
val heartbeat = BleProtocol.buildHeartbeat(allocMagic())
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat)
}
} catch (e: Exception) {
Log.w(TAG, "heartbeat failed: " + e.message)
}
}
private fun stopHeartbeat() {
val executor = heartbeatExecutor
heartbeatExecutor = null
if (executor != null) {
executor.shutdownNow()
}
}
private fun sendShutdown() {
val shutdown = BleProtocol.buildShutdown(allocMagic(), 0)
if (rightConnected) {
writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown)
}
}
private fun teardown() {
finished = true
stopHeartbeat()
try {
bleManager.close()
} catch (ignored: Exception) {
}
rightConnected = false
leftConnected = false
}
override fun onNotification(address: String?, characteristicUuid: String?, data: ByteArray?) {
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
return
}
// A value can carry several envelope frames back to back.
for (buf in BleProtocol.splitFrames(data)) {
handleFrame(address, buf)
}
}
private fun handleFrame(address: String?, buf: ByteArray) {
val frame = BleProtocol.parseFrame(buf)
if (!frame.ok) {
return
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
handleEvent(frame)
return
}
if (frame.msgSeq >= 0) {
val key = ackKey(frame.sid, frame.msgSeq)
val latch = pendingAcks[key]
if (latch != null) {
ackPayloads[key] = frame.pb
latch.countDown()
}
}
}
private fun handleEvent(frame: BleProtocol.ParsedFrame) {
val selection: BleProtocol.ListSelection? = BleProtocol.parseListSelection(frame)
if (selection == null || LIST_NAME != selection.containerName) {
return
}
// Only a confirmed click is a decision; scroll/highlight changes are ignored.
if (selection.eventType != BleProtocol.EVENT_CLICK) {
return
}
val itemName = selection.itemName
val yes = selection.itemIndex == 1
|| (itemName != null && itemName.lowercase().startsWith("yes"))
synchronized(lock) {
if (finished) {
return
}
finished = true
approved = yes
}
emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")")
selectionLatch.countDown()
}
override fun onConnectionStateChange(address: String?, connected: Boolean) {
if (connected) {
return
}
if (address!!.equals(leftAddress, ignoreCase = true)) {
// The prompt itself only needs the right arm; a dropped left link
// just means no left battery reading.
leftConnected = false
emitLog("left arm disconnected")
return
}
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = false
synchronized(lock) {
if (!finished && !cancelled) {
rightLost = true
finished = true
selectionLatch.countDown()
}
}
}
}
private fun allocMagic(): Int {
synchronized(lock) {
val magic = nextMagic
nextMagic = if (nextMagic >= 255) 100 else nextMagic + 1
return magic
}
}
private fun emitLog(line: String) {
Log.i(TAG, line)
private fun post(action: (FaceclawFlashPromptListener) -> Unit) {
mainHandler.post {
val current = listener
if (current != null) {
current.onLog(line)
}
}
}
private fun emitState(state: String, detail: String?) {
val safeDetail = detail ?: ""
mainHandler.post {
val current = listener
if (current != null) {
current.onState(state, safeDetail)
}
}
}
private fun emitBattery(rightPercent: Int, leftPercent: Int) {
mainHandler.post {
val current = listener
if (current != null) {
current.onBattery(rightPercent, leftPercent)
}
}
}
private fun emitResult(approvedResult: Boolean) {
mainHandler.post {
val current = listener
if (current != null) {
current.onResult(approvedResult)
action(current)
}
}
}
@@ -7,27 +7,18 @@ import okhttp3.Request
import okhttp3.Response
/**
* Shared HTTP identity for the Java networking helpers. The version lives in
* Android okhttp face of the shared HTTP identity. The version lives in
* app/version.ts, so the TypeScript side pushes the real string in at startup
* (see installNativeUserAgent in app/util/http.ts); the default below is only
* a fallback for requests that somehow beat that call.
* (see installNativeUserAgent in app/util/http.ts); the value itself is held
* by the shared HttpIdentity so every platform's networking reads one string.
*/
class FaceclawHttp private constructor() {
companion object {
@Volatile
private var userAgent = "Faceclaw"
@JvmStatic
fun setUserAgent(value: String?) = HttpIdentity.setUserAgent(value)
@JvmStatic
fun setUserAgent(value: String?) {
if (value != null && !value.trim().isEmpty()) {
userAgent = value.trim()
}
}
@JvmStatic
fun getUserAgent(): String {
return userAgent
}
fun getUserAgent(): String = HttpIdentity.getUserAgent()
/** Stamps our User-Agent on any request that doesn't already carry one. */
@JvmStatic
@@ -39,7 +30,7 @@ class FaceclawHttp private constructor() {
if (request.header("User-Agent") != null) {
return chain.proceed(request)
}
return chain.proceed(request.newBuilder().header("User-Agent", userAgent).build())
return chain.proceed(request.newBuilder().header("User-Agent", HttpIdentity.getUserAgent()).build())
}
}
}
@@ -2,23 +2,23 @@ package com.faceclaw.app
import android.util.Log
/**
* Android LC3 decoder for G2 microphone packets: the liblc3 JNI codec behind the shared
* [Lc3PacketFramer], which owns the counter/duplicate/trailer logic and statistics.
*/
class FaceclawLc3Decoder {
companion object {
private const val TAG = "FaceclawLc3"
const val SAMPLE_RATE = 16000
const val FRAME_US = 10000
const val FRAME_BYTES = 40
const val FRAMES_PER_PACKET = 5
const val PACKET_BYTES = 205
const val COUNTER_OFFSET = 204
const val SAMPLES_PER_FRAME = 160
const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
const val SSR_OFFSET = 200
const val ANGLE_OFFSET = 202
const val SAMPLE_RATE = Lc3PacketFramer.SAMPLE_RATE
const val FRAME_US = Lc3PacketFramer.FRAME_US
const val FRAME_BYTES = Lc3PacketFramer.FRAME_BYTES
const val FRAMES_PER_PACKET = Lc3PacketFramer.FRAMES_PER_PACKET
const val PACKET_BYTES = Lc3PacketFramer.PACKET_BYTES
const val COUNTER_OFFSET = Lc3PacketFramer.COUNTER_OFFSET
const val SAMPLES_PER_FRAME = Lc3PacketFramer.SAMPLES_PER_FRAME
const val SAMPLES_PER_PACKET = Lc3PacketFramer.SAMPLES_PER_PACKET
const val SSR_OFFSET = Lc3PacketFramer.SSR_OFFSET
const val ANGLE_OFFSET = Lc3PacketFramer.ANGLE_OFFSET
init {
System.loadLibrary("faceclaw_lc3")
@@ -32,17 +32,15 @@ class FaceclawLc3Decoder {
private external fun nativeDestroy(handle: Long)
}
private var nativeHandle: Long
private var lastCounter = -1
private var realPackets: Long = 0
private var duplicatePackets: Long = 0
private var missingPackets: Long = 0
private var decodeErrors: Long = 0
private var lastSsr = 0
private var lastAngleDegrees = 0
private var nativeHandle: Long = nativeCreate(FRAME_US, SAMPLE_RATE)
private val framer = Lc3PacketFramer(object : Lc3Codec {
override fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int {
// The JNI entry decodes the five 40-byte frames at the start of the packet.
return nativeDecodePacket(nativeHandle, packet, pcmOut)
}
})
init {
nativeHandle = nativeCreate(FRAME_US, SAMPLE_RATE)
if (nativeHandle == 0L) {
throw IllegalStateException("Could not create LC3 decoder")
}
@@ -54,70 +52,30 @@ class FaceclawLc3Decoder {
throw IllegalStateException("LC3 decoder is closed")
}
if (packet == null || packet.size != PACKET_BYTES) {
decodeErrors++
Log.w(TAG, "unexpected G2 audio packet length=" + (packet?.size ?: -1))
return 0
}
if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) {
throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples")
}
val counter = packet[COUNTER_OFFSET].toInt() and 0xff
if (lastCounter >= 0) {
val gap = (counter - lastCounter) and 0xff
if (gap == 0) {
duplicatePackets++
return 0
}
val missing = gap - 1
if (missing > 0) {
missingPackets += missing.toLong()
}
}
val decoded = nativeDecodePacket(nativeHandle, packet, pcmOut)
if (decoded <= 0) {
decodeErrors++
return 0
}
realPackets++
lastCounter = counter
lastSsr = ((packet[SSR_OFFSET].toInt() and 0xff) or (packet[SSR_OFFSET + 1].toInt() shl 8)).toShort().toInt()
lastAngleDegrees = ((packet[ANGLE_OFFSET].toInt() and 0xff) or (packet[ANGLE_OFFSET + 1].toInt() shl 8)).toShort().toInt()
return decoded
return framer.decodePacket(packet, pcmOut)
}
/** Firmware signal-strength ratio from the most recent decoded packet. */
@Synchronized
fun getLastSsr(): Int {
return lastSsr
}
fun getLastSsr(): Int = framer.lastSsr
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
@Synchronized
fun getLastAngleDegrees(): Int {
return lastAngleDegrees
}
fun getLastAngleDegrees(): Int = framer.lastAngleDegrees
@Synchronized
fun getRealPackets(): Long {
return realPackets
}
fun getRealPackets(): Long = framer.realPackets
@Synchronized
fun getDuplicatePackets(): Long {
return duplicatePackets
}
fun getDuplicatePackets(): Long = framer.duplicatePackets
@Synchronized
fun getMissingPackets(): Long {
return missingPackets
}
fun getMissingPackets(): Long = framer.missingPackets
@Synchronized
fun getDecodeErrors(): Long {
return decodeErrors
}
fun getDecodeErrors(): Long = framer.decodeErrors
@Synchronized
fun close() {
@@ -2,13 +2,11 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.io.File
import java.io.FileInputStream
import java.io.FileOutputStream
import java.io.IOException
import java.security.MessageDigest
import java.io.InputStream
import java.util.concurrent.TimeUnit
import okhttp3.Call
@@ -17,10 +15,10 @@ import okhttp3.Request
/**
* Large-file downloader for on-phone model weights (multi-GB, from Hugging
* Face). Downloads to "<dest>.part" with HTTP Range resume across app
* restarts, verifies a pinned sha256 (hashed incrementally, including the
* resumed prefix), then renames into place. Listener callbacks are posted to
* the constructing thread's Looper, like the other Faceclaw bridges.
* Face). The resume/hash/verify/rename policy is the shared ResumableDownload;
* this class supplies okhttp, java.io storage, the worker thread, and posts
* listener callbacks to the constructing thread's Looper, like the other
* Faceclaw bridges.
*/
class FaceclawModelDownloader(
url: String,
@@ -30,35 +28,31 @@ class FaceclawModelDownloader(
listener: FaceclawModelDownloaderListener?
) {
private val callbackHandler: Handler
private val url: String
private val destFile: File
private val partFile: File
private val expectedSha256: String
private val expectedTotalBytes: Long
private val listener: FaceclawModelDownloaderListener
@Volatile
private var cancelled = false
@Volatile
private var call: Call? = null
private val transport = OkHttpTransport()
private val download: ResumableDownload
private var worker: Thread? = null
init {
if (listener == null) throw IllegalArgumentException("listener is required")
val looper = Looper.myLooper()
this.callbackHandler = Handler(looper ?: Looper.getMainLooper())
this.url = url
this.destFile = File(destPath)
this.partFile = File("$destPath.part")
this.expectedSha256 = expectedSha256?.lowercase() ?: ""
this.expectedTotalBytes = expectedTotalBytes
this.listener = listener
val posted = object : FaceclawModelDownloaderListener {
override fun onProgress(bytesDownloaded: Long, totalBytes: Long) {
callbackHandler.post { listener.onProgress(bytesDownloaded, totalBytes) }
}
override fun onDone(path: String?) {
callbackHandler.post { listener.onDone(path) }
}
override fun onError(message: String?) {
callbackHandler.post { listener.onError(message) }
}
}
download = ResumableDownload(url, File(destPath).absolutePath, expectedSha256, expectedTotalBytes, transport, FileStorage, posted)
}
companion object {
private const val TAG = "FaceclawModelDl"
private const val PROGRESS_INTERVAL_MS = 500L
@Volatile
private var sharedClient: OkHttpClient? = null
@@ -79,120 +73,76 @@ class FaceclawModelDownloader(
}
fun start() {
val thread = Thread({ run() }, "FaceclawModelDl")
val thread = Thread({ download.run() }, "FaceclawModelDl")
worker = thread
thread.start()
}
/** Stops the download but keeps the .part file so a later start() resumes. */
fun cancel() {
cancelled = true
val current = call
current?.cancel()
download.cancel()
transport.call?.cancel()
}
private fun run() {
try {
if (destFile.exists() && destFile.length() > 0) {
post { listener.onDone(destFile.absolutePath) }
return
}
destFile.parentFile.mkdirs()
var digest = MessageDigest.getInstance("SHA-256")
var offset = 0L
if (partFile.exists()) {
offset = hashExistingPrefix(digest)
if (cancelled) return
}
/** One call at a time per downloader, so cancel() can abort a connect in progress. */
private class OkHttpTransport : DownloadTransport {
@Volatile
var call: Call? = null
override fun open(url: String, rangeFrom: Long): DownloadResponse {
val builder = Request.Builder().url(url)
if (offset > 0) builder.header("Range", "bytes=$offset-")
val newCall = getClient().newCall(builder.build())
call = newCall
if (rangeFrom > 0) builder.header("Range", "bytes=$rangeFrom-")
val call: Call = getClient().newCall(builder.build())
this.call = call
val response = call.execute()
val body = response.body
val input: InputStream? = body?.byteStream()
return object : DownloadResponse {
override val code: Int = response.code
override val contentLength: Long = body?.contentLength() ?: -1L
newCall.execute().use { response ->
val body = response.body
if (!response.isSuccessful || body == null) {
postError("Model download failed: HTTP " + response.code)
return
}
val resumed = response.code == 206
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0L
digest = MessageDigest.getInstance("SHA-256")
}
val total = if (expectedTotalBytes > 0) expectedTotalBytes
else (if (body.contentLength() > 0) offset + body.contentLength() else -1L)
override fun read(buffer: ByteArray): Int = input?.read(buffer) ?: -1
FileOutputStream(partFile, resumed).use { out ->
body.byteStream().use { input ->
val buffer = ByteArray(1 shl 16)
var done = offset
var lastProgressAt = 0L
var read: Int
while (input.read(buffer).also { read = it } != -1) {
if (cancelled) return
out.write(buffer, 0, read)
digest.update(buffer, 0, read)
done += read
val now = System.currentTimeMillis()
if (now - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = now
val doneFinal = done
val totalFinal = total
post { listener.onProgress(doneFinal, totalFinal) }
}
}
}
override fun close() {
call.cancel()
response.close()
}
}
if (expectedTotalBytes > 0 && partFile.length() != expectedTotalBytes) {
postError("Model download ended early (" + partFile.length() + " of "
+ expectedTotalBytes + " bytes); try again to resume")
return
}
val actualSha = FaceclawFirmwareUtil.bytesToHex(digest.digest())
if (!expectedSha256.isEmpty() && actualSha != expectedSha256) {
partFile.delete()
postError("Model download was corrupted (checksum mismatch); download it again")
return
}
if (!partFile.renameTo(destFile)) {
postError("Could not move the downloaded model into place")
return
}
post { listener.onDone(destFile.absolutePath) }
} catch (e: Exception) {
if (cancelled) return
Log.w(TAG, "download failed", e)
postError("Model download failed: $e")
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
@Throws(IOException::class)
private fun hashExistingPrefix(digest: MessageDigest): Long {
var hashed = 0L
FileInputStream(partFile).use { input ->
val buffer = ByteArray(1 shl 16)
var read: Int
while (input.read(buffer).also { read = it } != -1) {
if (cancelled) return hashed
digest.update(buffer, 0, read)
hashed += read
private object FileStorage : DownloadStorage {
override fun length(path: String): Long = File(path).let { if (it.exists()) it.length() else -1L }
override fun prepareParent(path: String) {
File(path).parentFile?.mkdirs()
}
override fun openWrite(path: String, append: Boolean): FileSink {
val out = FileOutputStream(path, append)
return object : FileSink {
override fun write(bytes: ByteArray, offset: Int, length: Int) = out.write(bytes, offset, length)
override fun close() = out.close()
}
}
return hashed
}
private fun post(runnable: Runnable) {
callbackHandler.post(runnable)
}
override fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long {
var consumed = 0L
FileInputStream(path).use { input ->
val buffer = ByteArray(1 shl 16)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (!consumer(buffer, read)) break
consumed += read
}
}
return consumed
}
private fun postError(message: String) {
post { listener.onError(message) }
override fun rename(from: String, to: String): Boolean = File(from).renameTo(File(to))
override fun delete(path: String): Boolean = File(path).delete()
}
}
@@ -2,20 +2,19 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import java.io.ByteArrayOutputStream
import java.io.IOException
import java.net.InetAddress
import java.net.NetworkInterface
import java.util.Collections
import org.json.JSONArray
import java.net.ServerSocket
import java.net.Socket
import java.nio.charset.StandardCharsets
import java.nio.ByteBuffer
import java.security.MessageDigest
import java.security.SecureRandom
import org.json.JSONObject
import java.util.Collections
/** Explicit-address, bounded input port. Complete requests notify JS on the main thread. */
/**
* Explicit-address, bounded input port. The request state machine lives in the shared
* [RemoteInputSession]; this adapter supplies java.net sockets, SecureRandom and main-thread
* delivery, and keeps the JVM API TypeScript calls (app/native/remote-input.ts).
*/
class FaceclawRemoteInput {
companion object {
private val INSTANCE = FaceclawRemoteInput()
@@ -23,130 +22,91 @@ class FaceclawRemoteInput {
@JvmStatic
fun interfaces(): String {
val result = JSONArray()
val all = ArrayList<InterfaceAddress>()
try {
for (device in Collections.list(NetworkInterface.getNetworkInterfaces())) {
if (!device.isUp) continue
for (address in Collections.list(device.inetAddresses)) {
if (address.isLoopbackAddress || address.isLinkLocalAddress || address.isAnyLocalAddress) continue
result.put(JSONObject().put("name", device.name).put("address", address.hostAddress)
.put("pointToPoint", device.isPointToPoint))
all.add(InterfaceAddress(
interfaceName = device.name,
address = address.hostAddress ?: continue,
up = device.isUp,
pointToPoint = device.isPointToPoint,
loopback = address.isLoopbackAddress,
linkLocal = address.isLinkLocalAddress,
anyLocal = address.isAnyLocalAddress,
))
}
}
} catch (ignored: Exception) {}
return result.toString()
}
private fun hex(bytes: ByteArray): String {
val result = StringBuilder(bytes.size * 2)
for (b in bytes) result.append(String.format(java.util.Locale.ROOT, "%02x", b.toInt() and 255))
return result.toString()
return RemoteInputSession.interfacesJson(all)
}
}
private var server: ServerSocket? = null
private var client: Socket? = null
private var pending: String? = null
private var reply: String? = null
private var nextId = 0L
private var requestId = 0L
private var deadline = 0L
private val mainHandler = Handler(Looper.getMainLooper())
private var requestListener: Runnable? = null
private val session = RemoteInputSession(
port = AndroidLocalServerPort,
random = object : RandomSource {
private val secureRandom = SecureRandom()
override fun nextBytes(count: Int): ByteArray = ByteArray(count).also { secureRandom.nextBytes(it) }
},
dispatch = { action -> mainHandler.post(action) },
platform = AndroidProtocolPlatform,
)
@Synchronized fun setRequestListener(listener: Runnable?) { requestListener = listener }
fun setRequestListener(listener: Runnable?) { session.setRequestListener(listener?.let { { it.run() } }) }
private fun notifyRequestReady(id: Long) {
// Called with this monitor held; posting avoids entering JS on the socket thread.
mainHandler.post {
val callback: Runnable?
synchronized(this) {
if (requestId != id || pending == null || System.currentTimeMillis() >= deadline) return@post
callback = requestListener
}
callback?.run()
}
}
fun start(port: Int): String { return startAddress(port, "127.0.0.1") }
fun startAddress(port: Int, address: String): String = session.start(port, address)
fun stop() = session.stop()
fun nextRequest(): String? = session.nextRequest()
fun complete(id: Long, value: String?) = session.complete(id, value)
fun randomSecret(): String = session.randomSecret()
fun hash(value: String): String = RemoteInputSession.tokenHash(value)
}
@Synchronized fun start(port: Int): String { return startAddress(port, "127.0.0.1") }
@Synchronized fun startAddress(port: Int, address: String): String {
if (server != null) return ""
try {
// Numeric local addresses only; never resolve hostnames or bind a wildcard.
if (!address.matches(Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?"))) return "Invalid local address."
private object AndroidLocalServerPort : LocalServerPort {
override fun isLocalUnicastAddress(address: String): Boolean {
return try {
// Numeric literals only reach here, so getByName never resolves a hostname.
val local = InetAddress.getByName(address)
if (local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null)
return "Address is not assigned to a local interface."
val listener = ServerSocket(port, 8, local)
server = listener
val thread = Thread({ serve(listener) }, "FaceclawInput")
thread.isDaemon = true
thread.start()
return ""
} catch (e: Exception) { return "Could not open local input port $port." }
}
@Synchronized fun stop() {
try { server?.close() } catch (ignored: Exception) {}
try { client?.close() } catch (ignored: Exception) {}
server = null; client = null; pending = null; requestId = 0; reply = null
(this as java.lang.Object).notifyAll()
}
@Synchronized fun nextRequest(): String? {
if (System.currentTimeMillis() >= deadline) { pending = null; return null }
val result = pending; pending = null; return result
}
@Synchronized fun complete(id: Long, value: String?) {
if (id == requestId && System.currentTimeMillis() < deadline) { reply = value; (this as java.lang.Object).notifyAll() }
}
private fun serve(listener: ServerSocket) {
while (!listener.isClosed) {
try {
listener.accept().use { socket ->
synchronized(this) { if (server !== listener) return; client = socket }
socket.soTimeout = 5000
socket.tcpNoDelay = true
val readDeadline = System.currentTimeMillis() + 5000
val body = ByteArrayOutputStream()
var b = -1
while (true) {
val remaining = readDeadline - System.currentTimeMillis()
if (remaining <= 0) throw Exception("Read timeout")
socket.soTimeout = remaining.toInt()
b = socket.getInputStream().read()
if (b == -1 || b == '\n'.code) break
if (body.size() >= 65536) throw Exception("Invalid frame")
body.write(b)
}
if (b != '\n'.code) return@use
val text = StandardCharsets.UTF_8.newDecoder().decode(ByteBuffer.wrap(body.toByteArray())).toString()
val response: String
synchronized(this) {
if (server !== listener) return
requestId = ++nextId
deadline = System.currentTimeMillis() + 5000
reply = null
pending = JSONObject().put("id", requestId).put("expiresAt", deadline)
.put("body", text).toString()
notifyRequestReady(requestId)
while (reply == null && server === listener && System.currentTimeMillis() < deadline) {
(this as java.lang.Object).wait(Math.max(1L, deadline - System.currentTimeMillis()))
}
response = reply ?: "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"
if (server === listener) { pending = null; reply = null; requestId = 0 }
}
socket.getOutputStream().write((response + "\n").toByteArray(StandardCharsets.UTF_8))
}
} catch (ignored: Exception) {
// Never log request bodies: they contain credentials and possibly private text.
} finally {
synchronized(this) { if (server === listener) client = null }
}
!(local.isAnyLocalAddress || local.isMulticastAddress || NetworkInterface.getByInetAddress(local) == null)
} catch (e: Exception) {
false
}
}
fun randomSecret(): String {
val bytes = ByteArray(32); SecureRandom().nextBytes(bytes); return hex(bytes)
}
fun hash(value: String): String {
try { return hex(MessageDigest.getInstance("SHA-256").digest(value.toByteArray(StandardCharsets.UTF_8))) }
catch (e: Exception) { throw IllegalStateException("SHA-256 unavailable", e) }
override fun bind(address: String, port: Int): RemoteInputListener {
val server = ServerSocket(port, 8, InetAddress.getByName(address))
return object : RemoteInputListener {
override fun accept(): RemoteInputConnection? {
val socket = try { server.accept() } catch (e: IOException) { return null }
return AndroidConnection(socket)
}
override val isClosed: Boolean get() = server.isClosed
override fun close() = server.close()
}
}
}
private class AndroidConnection(private val socket: Socket) : RemoteInputConnection {
init {
socket.soTimeout = 5000
socket.tcpNoDelay = true
}
override fun readByte(timeoutMs: Long): Int {
return try {
socket.soTimeout = timeoutMs.coerceIn(1L, Int.MAX_VALUE.toLong()).toInt()
socket.getInputStream().read()
} catch (e: IOException) {
-1
}
}
override fun write(bytes: ByteArray) {
socket.getOutputStream().write(bytes)
socket.getOutputStream().flush()
}
override fun close() = socket.close()
}
@@ -4,9 +4,6 @@ import android.content.Context
import android.content.SharedPreferences
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.util.concurrent.CopyOnWriteArrayList
/**
* App settings store shared by every JS isolate (main thread and app
@@ -22,16 +19,26 @@ import java.util.concurrent.CopyOnWriteArrayList
* NativeScript worker threads run a message loop, so both the main thread
* and workers have a Looper; a listener registered from a Looper-less thread
* is accepted but never notified (it can still read fresh values on demand).
* The reads/writes and the fan-out live in the shared SettingsChangeHub.
*/
class FaceclawSettings private constructor(context: Context) {
private val prefs: SharedPreferences = context.applicationContext
.getSharedPreferences(PREFS_NAME, Context.MODE_PRIVATE)
private val listeners = CopyOnWriteArrayList<ListenerEntry>()
private val hub = SettingsChangeHub(object : SettingsStorage {
override fun getString(key: String?, defaultValue: String?): String? = prefs.getString(key, defaultValue)
private class ListenerEntry(val listener: FaceclawSettingsListener, val handler: Handler?)
override fun putString(key: String?, value: String?) {
prefs.edit().putString(key, value).apply()
}
override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = prefs.getBoolean(key, defaultValue)
override fun putBoolean(key: String?, value: Boolean) {
prefs.edit().putBoolean(key, value).apply()
}
})
companion object {
private const val TAG = "FaceclawSettings"
private const val PREFS_NAME = "faceclaw_settings"
@Volatile
private var instance: FaceclawSettings? = null
@@ -58,23 +65,13 @@ class FaceclawSettings private constructor(context: Context) {
}
}
fun getString(key: String?, defaultValue: String?): String? {
return prefs.getString(key, defaultValue)
}
fun getString(key: String?, defaultValue: String?): String? = hub.getString(key, defaultValue)
fun setString(key: String?, value: String?) {
prefs.edit().putString(key, value).apply()
notifyChanged(key)
}
fun setString(key: String?, value: String?) = hub.setString(key, value)
fun getBoolean(key: String?, defaultValue: Boolean): Boolean {
return prefs.getBoolean(key, defaultValue)
}
fun getBoolean(key: String?, defaultValue: Boolean): Boolean = hub.getBoolean(key, defaultValue)
fun setBoolean(key: String?, value: Boolean) {
prefs.edit().putBoolean(key, value).apply()
notifyChanged(key)
}
fun setBoolean(key: String?, value: Boolean) = hub.setBoolean(key, value)
/**
* Register a change listener. Must be called from the thread whose
@@ -83,30 +80,9 @@ class FaceclawSettings private constructor(context: Context) {
*/
fun registerListener(listener: FaceclawSettingsListener) {
val looper = Looper.myLooper()
if (looper == null) {
Log.w(TAG, "settings listener registered from a Looper-less thread; it will never be notified")
}
listeners.add(ListenerEntry(listener, if (looper != null) Handler(looper) else null))
val handler = if (looper != null) Handler(looper) else null
hub.register(listener, if (handler != null) SettingsDispatcher { action -> handler.post(action) } else null)
}
fun unregisterListener(listener: FaceclawSettingsListener?) {
for (entry in listeners) {
if (entry.listener === listener) {
listeners.remove(entry)
}
}
}
private fun notifyChanged(key: String?) {
for (entry in listeners) {
val handler = entry.handler ?: continue
handler.post {
try {
entry.listener.onSettingChanged(key)
} catch (e: Exception) {
Log.w(TAG, "settings listener failed for key $key", e)
}
}
}
}
fun unregisterListener(listener: FaceclawSettingsListener?) = hub.unregister(listener)
}
@@ -11,28 +11,9 @@ import com.k2fsa.sherpa.onnx.SpeakerEmbeddingExtractorConfig
* utterance; matching/enrollment happens on the TS side (cosine similarity
* against the stored speaker profiles).
*/
class FaceclawSpeakerId(private val modelPath: String?) {
class FaceclawSpeakerId(private val modelPath: String?) : SpeakerEmbedder {
companion object {
private const val TAG = "FaceclawSpeakerId"
@JvmStatic
private fun l2Normalize(v: FloatArray?): FloatArray? {
if (v == null) {
return null
}
var sum = 0.0
for (x in v) {
sum += x.toDouble() * x
}
val norm = Math.sqrt(sum)
if (norm <= 0) {
return v
}
for (i in v.indices) {
v[i] = (v[i] / norm).toFloat()
}
return v
}
}
private var extractor: SpeakerEmbeddingExtractor? = null
@@ -71,20 +52,18 @@ class FaceclawSpeakerId(private val modelPath: String?) {
* is too short to embed (under ~0.5 s).
*/
@Synchronized
fun embed(pcm16le: ByteArray?, sampleRate: Int): FloatArray? {
if (pcm16le == null || pcm16le.size < sampleRate) {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedNullable(pcm16le, sampleRate)
@Synchronized
fun embedNullable(pcm16le: ByteArray?, sampleRate: Int): FloatArray? {
if (pcm16le == null || !VoicePrint.isLongEnoughToEmbed(pcm16le.size, sampleRate)) {
return null
}
if (!ensureLoaded()) {
return null
}
val extractor = this.extractor!!
val count = pcm16le.size / 2
val samples = FloatArray(count)
for (i in 0 until count) {
val s = ((pcm16le[i * 2].toInt() and 0xff) or (pcm16le[i * 2 + 1].toInt() shl 8)).toShort()
samples[i] = s / 32768.0f
}
val samples = AudioSegmentation.pcm16leToFloat(pcm16le)
var stream: OnlineStream? = null
try {
stream = extractor.createStream()
@@ -94,7 +73,7 @@ class FaceclawSpeakerId(private val modelPath: String?) {
return null
}
val embedding = extractor.compute(stream)
return l2Normalize(embedding)
return VoicePrint.l2Normalize(embedding)
} catch (t: Throwable) {
Log.w(TAG, "embedding failed", t)
return null
@@ -105,6 +84,8 @@ class FaceclawSpeakerId(private val modelPath: String?) {
}
}
override fun release() = close()
@Synchronized
fun close() {
val extractor = this.extractor
@@ -4,7 +4,6 @@ import android.os.Handler
import android.os.Looper
import android.util.Log
import java.io.BufferedReader
import java.io.IOException
import java.util.concurrent.TimeUnit
@@ -23,6 +22,10 @@ import okhttp3.Response
* is delivered line by line as it arrives, so the JS side can parse SSE
* events incrementally. Like FaceclawWebSocket, listener callbacks are posted
* to the Looper of the thread that constructed this object.
*
* The line splitting, error-body and overflow policy live in the shared
* SseStream; this class only supplies okhttp as the transport and the Handler
* delivery.
*/
class FaceclawSseRequest
/**
@@ -32,7 +35,8 @@ class FaceclawSseRequest
*/
constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener: FaceclawSseListener?) {
private val callbackHandler: Handler
private val call: Call
private val stream = SseStream()
private val transfer: Cancellable
@Volatile
private var cancelled = false
@@ -45,63 +49,49 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
}
val looper = Looper.myLooper()
callbackHandler = Handler(looper ?: Looper.getMainLooper())
val builder = Request.Builder()
.url(url.trim())
.post((jsonBody ?: "").toRequestBody(JSON))
if (headers != null) {
var i = 0
while (i + 1 < headers.size) {
val name = headers[i]
val value = headers[i + 1]
if (name != null && !name.isEmpty() && value != null) {
builder.addHeader(name, value)
}
i += 2
}
}
call = getClient().newCall(builder.build())
call.enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
postFailure(listener, e.toString())
val sink = object : ChunkSink {
override fun onStatus(code: Int) {
stream.onStatus(code)
}
override fun onResponse(call: Call, response: Response) {
try {
response.body.use { body ->
if (!response.isSuccessful) {
val code = response.code
var errorBody = ""
try {
errorBody = body?.string() ?: ""
} catch (e: IOException) {
Log.w(TAG, "error body read failed", e)
}
val finalBody = errorBody
post { listener.onHttpError(code, finalBody) }
return
}
if (body == null) {
post { listener.onComplete() }
return
}
val reader = BufferedReader(body.charStream())
var line: String?
while (reader.readLine().also { line = it } != null) {
val finalLine = line
post { listener.onLine(finalLine) }
}
post { listener.onComplete() }
}
} catch (e: IOException) {
postFailure(listener, e.toString())
}
override fun onChunk(bytes: ByteArray, offset: Int, length: Int) {
stream.onChunk(bytes, offset, length)
drain(listener)
}
})
override fun onComplete() {
stream.onComplete()
drain(listener)
}
override fun onFailure(message: String) {
stream.onFailure(message)
drain(listener)
}
}
transfer = OkHttpStreaming.post(url.trim(), jsonBody ?: "", SseStream.unpackHeaders(headers), sink)
}
fun cancel() {
cancelled = true
call.cancel()
stream.cancel()
transfer.cancel()
}
/** Hands every queued event to the listener on the constructing thread, in order. */
private fun drain(listener: FaceclawSseListener) {
val events = stream.takeEvents()
if (events.isEmpty()) return
post {
for (event in events) {
if (cancelled) return@post
try {
SseStream.deliver(event, listener)
} catch (t: Throwable) {
Log.w(TAG, "listener callback failed", t)
}
}
}
}
private fun post(runnable: Runnable) {
@@ -112,19 +102,47 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
if (cancelled) {
return@post
}
try {
runnable.run()
} catch (t: Throwable) {
Log.w(TAG, "listener callback failed", t)
}
runnable.run()
}
}
private fun postFailure(listener: FaceclawSseListener, message: String) {
if (cancelled) {
return
/** okhttp transport: enqueues the call and feeds the body to the sink in chunks. */
private object OkHttpStreaming : StreamingHttp {
override fun post(url: String, body: String, headers: List<Pair<String, String>>, sink: ChunkSink): Cancellable {
val builder = Request.Builder()
.url(url)
.post(body.toRequestBody(JSON))
for ((name, value) in headers) builder.addHeader(name, value)
val call = getClient().newCall(builder.build())
call.enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
sink.onFailure(e.toString())
}
override fun onResponse(call: Call, response: Response) {
try {
response.use {
sink.onStatus(response.code)
val input = response.body?.byteStream()
if (input != null) {
val buffer = ByteArray(8192)
while (true) {
val read = input.read(buffer)
if (read < 0) break
if (read > 0) sink.onChunk(buffer, 0, read)
}
}
sink.onComplete()
}
} catch (e: IOException) {
sink.onFailure(e.toString())
}
}
})
return object : Cancellable {
override fun cancel() = call.cancel()
}
}
post { listener.onFailure(message) }
}
companion object {
@@ -141,9 +159,14 @@ constructor(url: String?, jsonBody: String?, headers: Array<String?>?, listener:
// Streaming responses can pause between events (e.g. while
// the model thinks), so the read timeout is generous. The
// Anthropic API sends periodic ping events well within it.
// Redirects are not followed: provider credentials in the
// headers must never be forwarded to another endpoint (a
// 3xx surfaces to the listener as an HTTP error instead).
sharedClient = OkHttpClient.Builder()
.connectTimeout(15, TimeUnit.SECONDS)
.readTimeout(180, TimeUnit.SECONDS)
.followRedirects(false)
.followSslRedirects(false)
.addInterceptor(FaceclawHttp.userAgentInterceptor())
.build()
}
@@ -14,10 +14,10 @@ import android.util.Log
import java.io.ByteArrayOutputStream
import java.lang.reflect.Method
/** Reads Wi-Fi/cell/hotspot state; the icon artwork is the shared StatusIconArt. */
class FaceclawSystemStatusIconProvider private constructor() {
companion object {
private const val TAG = "FaceclawStatusIcons"
private const val ICON_SIZE = 24
@JvmStatic
fun getSystemStatusIconGrays(context: Context?, iconSize: Int): ByteArray {
@@ -30,16 +30,16 @@ class FaceclawSystemStatusIconProvider private constructor() {
val wifiLevel = getWifiLevel(appContext)
if (wifiLevel >= 0) {
append(out, scaleIcon(drawWifiIcon(wifiLevel), size))
append(out, StatusIconArt.scale(StatusIconArt.wifi(wifiLevel), size))
}
val cellLevel = getCellLevel(appContext)
if (cellLevel >= 0) {
append(out, scaleIcon(drawCellIcon(cellLevel), size))
append(out, StatusIconArt.scale(StatusIconArt.cell(cellLevel), size))
}
if (isHotspotEnabled(appContext)) {
append(out, scaleIcon(drawHotspotIcon(), size))
append(out, StatusIconArt.scale(StatusIconArt.hotspot(), size))
}
return out.toByteArray()
@@ -106,86 +106,6 @@ class FaceclawSystemStatusIconProvider private constructor() {
return rssi > -127 && rssi < 0
}
private fun drawWifiIcon(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
val cx = 12
val cy = 18
fillRect(icon, cx - 1, cy - 1, 3, 3, 230)
val radii = intArrayOf(5, 9, 13, 17)
for (index in radii.indices) {
if (level <= index) {
continue
}
drawWifiArc(icon, cx, cy, radii[index], 210)
}
return icon
}
private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) {
val inner = (radius - 1) * (radius - 1)
val outer = (radius + 1) * (radius + 1)
for (y in 0 until ICON_SIZE) {
for (x in 0 until ICON_SIZE) {
val dx = x - cx
val dy = y - cy
val dist = dx * dx + dy * dy
if (dy <= 0 && dist >= inner && dist <= outer && Math.abs(dx) <= radius) {
setPixel(icon, x, y, value)
}
}
}
}
private fun drawCellIcon(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
for (bar in 0 until 4) {
val x = 5 + bar * 4
val height = 5 + bar * 4
val value = if (level > bar) 220 else 55
fillRect(icon, x, 20 - height, 3, height, value)
}
return icon
}
private fun drawHotspotIcon(): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
fillRect(icon, 11, 17, 3, 3, 230)
drawWifiArc(icon, 12, 19, 7, 210)
drawWifiArc(icon, 12, 19, 12, 210)
fillRect(icon, 6, 5, 12, 2, 170)
return icon
}
private fun scaleIcon(source: ByteArray, size: Int): ByteArray {
if (size == ICON_SIZE) {
return source
}
val scaled = ByteArray(size * size)
for (y in 0 until size) {
val sy = Math.min(ICON_SIZE - 1, (y * ICON_SIZE) / size)
for (x in 0 until size) {
val sx = Math.min(ICON_SIZE - 1, (x * ICON_SIZE) / size)
scaled[y * size + x] = source[sy * ICON_SIZE + sx]
}
}
return scaled
}
private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) {
for (row in y until y + height) {
for (col in x until x + width) {
setPixel(icon, col, row, value)
}
}
}
private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) {
if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) {
return
}
icon[y * ICON_SIZE + x] = Math.max(0, Math.min(255, value)).toByte()
}
private fun append(out: ByteArrayOutputStream, bytes: ByteArray) {
out.write(bytes, 0, bytes.size)
}
File diff suppressed because it is too large. Load diff
@@ -11,7 +11,6 @@ import android.util.Log
import java.io.File
import java.io.RandomAccessFile
import java.nio.ByteBuffer
import java.nio.ByteOrder
import java.util.LinkedHashMap
/**
@@ -20,7 +19,9 @@ import java.util.LinkedHashMap
* minikin/HarfBuzz. Output uses the same grayscale packet format as
* ImageFileLoader ([widthLo, widthHi, heightLo, heightHi, pixels...], one
* byte per pixel, row-major), which the TS side turns into a GrayImage and
* the compositor later quantizes to the 4bpp the firmware wants.
* the compositor later quantizes to the 4bpp the firmware wants. The packet
* layouts, gamma mapping and ink cropping are shared with iOS in GlyphPacket;
* only rasterization lives here.
*
* The gamma parameter maps antialiased coverage to output shade:
* out = 255 * (coverage/255)^gamma. The G2 display response looks roughly
@@ -88,27 +89,13 @@ class FontFileRenderer private constructor() {
// keep at least the advance width so spacing stays truthful.
var left = pad
var right = pad + advance
val painted = paintedColumnRange(pixels, width, height)
val painted = GlyphPacket.paintedColumnRange(pixels, width, height)
if (painted != null) {
left = Math.min(left, painted[0])
right = Math.max(right, painted[1] + 1)
}
val outWidth = Math.max(1, right - left)
val lut = gammaLut(gamma)
val out = ByteArray(4 + outWidth * height)
out[0] = (outWidth and 0xff).toByte()
out[1] = ((outWidth shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
for (y in 0 until height) {
val srcRow = y * width
val dstRow = 4 + y * outWidth
for (x in 0 until outWidth) {
out[dstRow + x] = lut[pixels[srcRow + left + x].toInt() and 0xff]
}
}
return out
return GlyphPacket.packImageRegion(pixels, width, left, 0, outWidth, height, gamma)
} catch (e: Exception) {
Log.w(TAG, "text render failed: $path", e)
return ByteArray(0)
@@ -159,16 +146,7 @@ class FontFileRenderer private constructor() {
val pixels = alpha8Pixels(bitmap, maxWidth, height)
bitmap.recycle()
val lut = gammaLut(gamma)
val out = ByteArray(4 + pixels.size)
out[0] = (maxWidth and 0xff).toByte()
out[1] = ((maxWidth shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
for (i in pixels.indices) {
out[4 + i] = lut[pixels[i].toInt() and 0xff]
}
return out
return GlyphPacket.packImage(pixels, maxWidth, height, gamma)
} catch (e: Exception) {
Log.w(TAG, "wrapped render failed: $path", e)
return ByteArray(0)
@@ -206,7 +184,7 @@ class FontFileRenderer private constructor() {
val bounds = android.graphics.Rect()
paint.getTextBounds(text, 0, text.length, bounds)
if (bounds.isEmpty) {
return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null)
return GlyphPacket.emptyGlyphCell(advanceFixed)
}
// getTextBounds can be off by a hair on AA edges; render with
// padding, then trim to the actually painted box.
@@ -224,40 +202,11 @@ class FontFileRenderer private constructor() {
val pixels = alpha8Pixels(bitmap, bw, bh)
bitmap.recycle()
var x0 = bw
var x1 = -1
var y0 = bh
var y1 = -1
for (y in 0 until bh) {
val row = y * bw
for (x in 0 until bw) {
if (pixels[row + x].toInt() != 0) {
if (x < x0) x0 = x
if (x > x1) x1 = x
if (y < y0) y0 = y
if (y > y1) y1 = y
}
}
}
if (x1 < x0) {
return glyphCellPacket(advanceFixed, 0, 0, 0, 0, null)
}
val width = x1 - x0 + 1
val height = y1 - y0 + 1
// Trim to the tight ink box; bearing/inkTop become pen- and
// line-relative inside the shared packer.
val fm = paint.fontMetricsInt
val bearingX = x0 - penX
val inkTop = (-fm.ascent) + (y0 - baselineY)
val lut = gammaLut(gamma)
val coverage = ByteArray(width * height)
for (y in 0 until height) {
val src = (y0 + y) * bw + x0
val dst = y * width
for (x in 0 until width) {
coverage[dst + x] = lut[pixels[src + x].toInt() and 0xff]
}
}
return glyphCellPacket(advanceFixed, bearingX, inkTop, width, height, coverage)
return GlyphPacket.packGlyphCell(
advanceFixed, pixels, bw, bh, penX, baselineY, -fm.ascent, gamma)
} catch (e: Exception) {
Log.w(TAG, "glyph render failed: " + path + " U+" + Integer.toHexString(codePoint), e)
return ByteArray(0)
@@ -294,26 +243,6 @@ class FontFileRenderer private constructor() {
return paint
}
@JvmStatic
private fun glyphCellPacket(
advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray {
val out = ByteArray(10 + (coverage?.size ?: 0))
out[0] = (advanceFixed and 0xff).toByte()
out[1] = ((advanceFixed shr 8) and 0xff).toByte()
out[2] = (bearingX and 0xff).toByte()
out[3] = ((bearingX shr 8) and 0xff).toByte()
out[4] = (inkTop and 0xff).toByte()
out[5] = ((inkTop shr 8) and 0xff).toByte()
out[6] = (width and 0xff).toByte()
out[7] = ((width shr 8) and 0xff).toByte()
out[8] = (height and 0xff).toByte()
out[9] = ((height shr 8) and 0xff).toByte()
if (coverage != null) {
System.arraycopy(coverage, 0, out, 10, coverage.size)
}
return out
}
/**
* Line metrics for a font at a pixel size, as "ascent descent lineGap"
* (integers, pixels). Empty string if the font cannot be loaded.
@@ -361,8 +290,21 @@ class FontFileRenderer private constructor() {
*/
@JvmStatic
fun getFontName(path: String?): String {
if (path == null) {
return ""
}
try {
return parseNameTable(path)
RandomAccessFile(path, "r").use { raf ->
return OpenTypeNames.parse(object : RandomAccessBytes {
override val length: Long
get() = raf.length()
override fun read(offset: Long, into: ByteArray, count: Int) {
raf.seek(offset)
raf.readFully(into, 0, count)
}
})
}
} catch (e: Exception) {
Log.w(TAG, "name table parse failed: $path", e)
return ""
@@ -431,144 +373,5 @@ class FontFileRenderer private constructor() {
}
return pixels
}
/** [first, last] painted (nonzero) column indexes, or null if blank. */
@JvmStatic
private fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? {
var first = width
var last = -1
for (y in 0 until height) {
val row = y * width
var x = 0
while (x < first) {
if (pixels[row + x].toInt() != 0) {
first = x
break
}
x++
}
x = width - 1
while (x > last) {
if (pixels[row + x].toInt() != 0) {
last = x
break
}
x--
}
}
return if (last >= first) intArrayOf(first, last) else null
}
@JvmStatic
private fun gammaLut(gammaIn: Double): ByteArray {
val lut = ByteArray(256)
var gamma = gammaIn
if (gamma <= 0) {
gamma = 1.0
}
for (i in 0 until 256) {
lut[i] = Math.max(0, Math.min(255,
Math.round(255.0 * Math.pow(i / 255.0, gamma)).toInt())).toByte()
}
return lut
}
// --- Minimal 'name' table reader (TTF/OTF/TTC) for preview titles ---
@JvmStatic
@Throws(Exception::class)
private fun parseNameTable(path: String?): String {
RandomAccessFile(path, "r").use { raf ->
val fileSize = raf.length()
if (fileSize < 12) {
return ""
}
var offset = 0L
val tag = readU32(raf, 0)
if (tag == 0x74746366) { // 'ttcf': use the first face
if (readU32(raf, 8) < 1) {
return ""
}
offset = readU32(raf, 12).toLong() and 0xffffffffL
}
val numTables = readU16(raf, offset + 4)
var nameOffset = -1L
for (i in 0 until numTables) {
val rec = offset + 12 + i * 16L
if (rec + 16 > fileSize) {
return ""
}
if (readU32(raf, rec) == 0x6e616d65) { // 'name'
nameOffset = readU32(raf, rec + 8).toLong() and 0xffffffffL
break
}
}
if (nameOffset < 0 || nameOffset + 6 > fileSize) {
return ""
}
val count = readU16(raf, nameOffset + 2)
val stringStorage = nameOffset + readU16(raf, nameOffset + 4)
var family: String? = null
var style: String? = null
var preferredFamily: String? = null
var preferredStyle: String? = null
for (i in 0 until count) {
val rec = nameOffset + 6 + i * 12L
if (rec + 12 > fileSize) {
break
}
val platform = readU16(raf, rec)
val nameId = readU16(raf, rec + 6)
if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) {
continue
}
val length = readU16(raf, rec + 8)
val strOffset = stringStorage + readU16(raf, rec + 10)
if (strOffset + length > fileSize || length <= 0 || length > 512) {
continue
}
val data = ByteArray(length)
raf.seek(strOffset)
raf.readFully(data)
// Platform 0 (Unicode) and 3 (Windows) store UTF-16BE;
// platform 1 (Mac) is close enough to Latin-1 for names.
var value = if (platform == 1)
String(data, java.nio.charset.StandardCharsets.ISO_8859_1)
else String(data, java.nio.charset.StandardCharsets.UTF_16BE)
value = value.trim()
if (value.isEmpty()) {
continue
}
if (nameId == 1 && family == null) family = value
if (nameId == 2 && style == null) style = value
if (nameId == 16 && preferredFamily == null) preferredFamily = value
if (nameId == 17 && preferredStyle == null) preferredStyle = value
}
val outFamily = preferredFamily ?: family
val outStyle = preferredStyle ?: style
if (outFamily == null) {
return ""
}
return outFamily + "\n" + (outStyle ?: "")
}
}
@JvmStatic
@Throws(Exception::class)
private fun readU16(raf: RandomAccessFile, offset: Long): Int {
raf.seek(offset)
val b = ByteArray(2)
raf.readFully(b)
return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getShort().toInt() and 0xffff
}
@JvmStatic
@Throws(Exception::class)
private fun readU32(raf: RandomAccessFile, offset: Long): Int {
raf.seek(offset)
val b = ByteArray(4)
raf.readFully(b)
return ByteBuffer.wrap(b).order(ByteOrder.BIG_ENDIAN).getInt()
}
}
}
@@ -9,33 +9,13 @@ import java.io.FileOutputStream
import java.io.OutputStreamWriter
import java.io.Writer
import java.nio.charset.StandardCharsets
import java.text.SimpleDateFormat
import java.util.ArrayDeque
import java.util.ArrayList
import java.util.Collections
import java.util.Date
import java.util.LinkedHashMap
import java.util.Locale
/**
* Per-frame timing instrumentation, shared between the Java BLE layer and the
* Typescript UI layer. A "frame" starts when we receive an input event (or a
* timer fires and we decide to redraw), and finishes when the resulting screen
* update has been fully transmitted to the glasses, is discarded unsent, or
* times out because nobody reported finishing it.
*
* Frames form a tree. One input event can fan out into several renders (the
* focused app's window plus the shell chrome, say), and each of those is its
* own frame linked back to the input frame that caused it via
* [startFrame]. The export nests descendants under their
* root and renders every line against the root's clock, so a single block
* shows the whole input-to-pixels story. Latency percentiles are measured over
* roots: root start to the first descendant that actually reached the glasses.
*
* All public methods are thread-safe and cheap enough to call from the BLE
* worker thread, the Android main thread, the JS thread, and app worker
* threads. Frame IDs are positive ints; 0 means "no frame" and is silently
* ignored everywhere, so callers do not need to null-check.
* Android facade over the shared [FrameTimingsCore] (per-frame latency
* instrumentation; see that class for the frame-tree model). This class keeps
* the process-wide singleton the BLE layer and TypeScript call by name, and owns
* the Android-only parts: the export directory derived from the Context and the
* daemon thread that sweeps timed-out frames and writes the export file.
*
* Statistics and full log lines for recent and slowest frames are periodically
* exported to getExternalFilesDir()/frame-timings.txt, retrievable via
@@ -43,131 +23,19 @@ import java.util.Locale
*/
class FrameTimings private constructor() {
companion object {
private const val TAG = "FrameTimings"
private const val TAG = FrameTimingsCore.TAG
private val INSTANCE = FrameTimings()
/** Frames still open after this long are finished as "timeout". */
private const val FRAME_TIMEOUT_MS = 30_000L
private const val EXPORT_INTERVAL_MS = 15_000L
private const val SWEEP_INTERVAL_MS = 5_000L
private const val RECENT_ROOTS_KEPT = 40
private const val SLOWEST_ROOTS_KEPT = 20
private const val SENT_DURATIONS_WINDOW = 512
private const val MAX_LINES_PER_FRAME = 200
/**
* Frames retained for ID lookup. Well above the number any export can
* reference, so a late log/span/finish call always finds its frame; frames
* older than this are unreachable and get collected.
*/
private const val FRAMES_RETAINED = 1024
private const val EXPORT_FILE_NAME = "frame-timings.txt"
@JvmStatic
fun getInstance(): FrameTimings {
return INSTANCE
}
@JvmStatic
private fun percentileOfSorted(sorted: List<Long>, percentile: Int): Long {
val index = Math.ceil(percentile / 100.0 * sorted.size).toInt() - 1
return sorted[Math.max(0, Math.min(sorted.size - 1, index))]
}
@JvmStatic
private fun repeat(unit: String, times: Int): String {
val out = StringBuilder(unit.length * times)
for (i in 0 until times) {
out.append(unit)
}
return out.toString()
}
}
private class Line(
val atMs: Long, // SystemClock.elapsedRealtime()
val thread: String,
val message: String?
)
private class Frame(
val id: Int,
var reason: String,
val startedAtMs: Long, // SystemClock.elapsedRealtime()
val startedWallClockMs: Long,
val parent: Frame?
) {
val children: MutableList<Frame> = ArrayList(0)
val lines: MutableList<Line> = ArrayList()
val openSpans: MutableMap<String, Long> = LinkedHashMap()
var finishedAtMs: Long = 0
var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..."
/** Set on the root once some frame in its subtree reached the glasses. */
var subtreeSentRecorded = false
fun root(): Frame {
var frame: Frame = this
while (frame.parent != null) {
frame = frame.parent!!
}
return frame
}
fun isOpen(): Boolean {
return outcome == null
}
fun durationMs(): Long {
return finishedAtMs - startedAtMs
}
fun wasSent(): Boolean {
val outcome = this.outcome
return outcome != null && outcome.startsWith("sent")
}
/** Last moment anything in this subtree happened, for whole-tree duration. */
fun subtreeEndAtMs(nowMs: Long): Long {
var end = if (isOpen()) nowMs else finishedAtMs
for (child in children) {
end = Math.max(end, child.subtreeEndAtMs(nowMs))
}
return end
}
fun subtreeHasOpenFrame(): Boolean {
if (isOpen()) {
return true
}
for (child in children) {
if (child.subtreeHasOpenFrame()) {
return true
}
}
return false
}
}
/** The shared core; the glasses session core records frames into it directly. */
val core = FrameTimingsCore(AndroidProtocolPlatform)
private val lock = Any()
/**
* Every frame we still retain, keyed by ID: open frames plus enough
* history that late calls and the export can still resolve one.
*/
private val framesById: LinkedHashMap<Int, Frame> =
object : LinkedHashMap<Int, Frame>(64, 0.75f, false) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<Int, Frame>): Boolean {
return size > FRAMES_RETAINED && !eldest.value.isOpen()
}
}
private val recentRoots = ArrayDeque<Frame>()
private val slowestRoots: MutableList<Frame> = ArrayList()
private val inputLatencies = ArrayDeque<Long>()
private val renderLatencies = ArrayDeque<Long>()
private var nextFrameId = 1
private var framesStarted: Long = 0
private var framesSent: Long = 0
private var framesDiscarded: Long = 0
private var framesTimedOut: Long = 0
private var exportDirty = false
private var exportDir: File? = null
private var exportThread: Thread? = null
@@ -189,245 +57,47 @@ class FrameTimings private constructor() {
}
/** Begin a root frame; reason is a short label like "input:sys-event type=0". */
fun startFrame(reason: String?): Int {
return startFrame(reason, 0)
}
fun startFrame(reason: String?): Int = core.startFrame(reason)
/**
* Begin a frame caused by an existing one (parentFrameId; 0 for a root).
* The child is nested under its root in the export and its latency counts
* against the root, so an input event that fans out into an app render and
* a shell-chrome render reads as one timeline instead of three.
*/
fun startFrame(reason: String?, parentFrameId: Int): Int {
val now = SystemClock.elapsedRealtime()
val frame: Frame
synchronized(lock) {
val parent = framesById[parentFrameId]
frame = Frame(nextFrameId++, reason ?: "", now,
System.currentTimeMillis(), parent)
framesById[frame.id] = frame
framesStarted++
if (parent != null) {
parent.children.add(frame)
addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")")
}
}
return frame.id
}
/** Begin a frame caused by an existing one (parentFrameId; 0 for a root). */
fun startFrame(reason: String?, parentFrameId: Int): Int = core.startFrame(reason, parentFrameId)
/**
* Append to a frame's label, for facts only known after it started (which
* app is in the foreground, which window a render is for). Shows up in the
* frame's header line, so the export is scannable without reading bodies.
*/
fun annotate(frameId: Int, text: String?) {
if (text == null || text.isEmpty()) {
return
}
synchronized(lock) {
val frame = framesById[frameId]
if (frame == null) {
return
}
frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text
}
}
/** Append to a frame's label, for facts only known after it started. */
fun annotate(frameId: Int, text: String?) = core.annotate(frameId, text)
fun log(frameId: Int, message: String?) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
addLineLocked(frame, now, message)
}
}
fun log(frameId: Int, message: String?) = core.log(frameId, message)
fun spanStart(frameId: Int, name: String) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
frame.openSpans[name] = now
addLineLocked(frame, now, "span $name start")
}
}
fun spanStart(frameId: Int, name: String) = core.spanStart(frameId, name)
fun spanEnd(frameId: Int, name: String) {
val now = SystemClock.elapsedRealtime()
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
val startedAt = frame.openSpans.remove(name)
if (startedAt == null) {
addLineLocked(frame, now, "span $name end (start not recorded)")
} else {
addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)")
}
}
}
fun spanEnd(frameId: Int, name: String) = core.spanEnd(frameId, name)
/**
* Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything
* starting with "discarded", or "timeout". First finish wins; later calls for
* the same frame are ignored, so racing completion paths are safe.
* starting with "discarded", or "timeout". First finish wins.
*/
fun finishFrame(frameId: Int, outcome: String?) {
val now = SystemClock.elapsedRealtime()
val finished: Frame
synchronized(lock) {
val frame = openFrameLocked(frameId)
if (frame == null) {
return
}
finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given")
finished = frame
}
Log.i(TAG, "frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome
+ " in " + finished.durationMs() + "ms")
}
fun finishFrame(frameId: Int, outcome: String?) = core.finishFrame(frameId, outcome)
/** One-line stats summary, e.g. for showing in the phone UI. */
fun statsSummary(): String {
synchronized(lock) {
val input = percentilesLocked(inputLatencies)
return "frames started=" + framesStarted + " sent=" + framesSent +
" discarded=" + framesDiscarded + " timeout=" + framesTimedOut +
(if (input == null)
""
else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] +
"ms p99=" + input[2] + "ms max=" + input[3] + "ms")
}
}
fun statsSummary(): String = core.statsSummary()
// ---------------------------------------------------------------------
/** The frame with this ID if it exists and is still open, else null. */
private fun openFrameLocked(frameId: Int): Frame? {
if (frameId <= 0) {
return null
}
val frame = framesById[frameId]
return if (frame != null && frame.isOpen()) frame else null
}
private fun addLineLocked(frame: Frame, now: Long, message: String?) {
if (frame.lines.size >= MAX_LINES_PER_FRAME) {
if (frame.lines.size == MAX_LINES_PER_FRAME) {
frame.lines.add(Line(now, Thread.currentThread().name, "... line cap reached"))
}
return
}
frame.lines.add(Line(now, Thread.currentThread().name, message))
}
private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) {
frame.finishedAtMs = now
frame.outcome = outcome
for (open in frame.openSpans.entries) {
frame.lines.add(Line(now, Thread.currentThread().name,
"span " + open.key + " never ended (started at +" +
(open.value - frame.startedAtMs) + "ms)"))
}
frame.openSpans.clear()
addLineLocked(frame, now, "finished: $outcome")
val root = frame.root()
if (frame.wasSent()) {
framesSent++
// The user-visible latency is input (or timer) to the first pixels
// that actually reached the glasses, wherever in the tree that was.
if (!root.subtreeSentRecorded) {
root.subtreeSentRecorded = true
recordLatencyLocked(root, now - root.startedAtMs)
insertSlowestLocked(root)
}
} else if (outcome.startsWith("timeout")) {
framesTimedOut++
} else {
framesDiscarded++
}
if (root === frame) {
recentRoots.addLast(frame)
while (recentRoots.size > RECENT_ROOTS_KEPT) {
recentRoots.removeFirst()
}
}
exportDirty = true
}
private fun recordLatencyLocked(root: Frame, latencyMs: Long) {
val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies
bucket.addLast(latencyMs)
while (bucket.size > SENT_DURATIONS_WINDOW) {
bucket.removeFirst()
}
}
private fun insertSlowestLocked(root: Frame) {
if (!slowestRoots.contains(root)) {
slowestRoots.add(root)
}
// Sorted and trimmed at export time, when every subtree duration is final.
}
/** {p50, p90, p99, max} over a rolling latency window, or null if empty. */
private fun percentilesLocked(window: ArrayDeque<Long>): LongArray? {
if (window.isEmpty()) {
return null
}
val sorted: MutableList<Long> = ArrayList(window)
Collections.sort(sorted)
return longArrayOf(
percentileOfSorted(sorted, 50),
percentileOfSorted(sorted, 90),
percentileOfSorted(sorted, 99),
sorted[sorted.size - 1],
)
}
private fun sweepTimedOutFrames() {
val now = SystemClock.elapsedRealtime()
val timedOut: MutableList<Frame> = ArrayList()
synchronized(lock) {
for (frame in ArrayList(framesById.values)) {
if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) {
finishFrameLocked(frame, now, "timeout: never reported finished")
timedOut.add(frame)
}
}
}
for (frame in timedOut) {
Log.w(TAG, "frame#" + frame.id + " [" + frame.reason + "] timed out after " +
frame.durationMs() + "ms without being finished")
}
}
private fun exportLoop() {
var lastExportAtMs = 0L
while (true) {
try {
Thread.sleep(SWEEP_INTERVAL_MS)
Thread.sleep(FrameTimingsCore.SWEEP_INTERVAL_MS)
} catch (e: InterruptedException) {
return
}
try {
sweepTimedOutFrames()
core.sweepTimedOut()
val now = SystemClock.elapsedRealtime()
val shouldExport: Boolean
synchronized(lock) {
shouldExport = exportDirty && exportDir != null && now - lastExportAtMs >= EXPORT_INTERVAL_MS
}
if (shouldExport) {
val dir: File?
synchronized(lock) { dir = exportDir }
if (dir != null && core.isExportDirty && now - lastExportAtMs >= FrameTimingsCore.EXPORT_INTERVAL_MS) {
lastExportAtMs = now
exportToFile()
exportToFile(dir)
}
} catch (t: Throwable) {
Log.w(TAG, "export loop error", t)
@@ -435,14 +105,8 @@ class FrameTimings private constructor() {
}
}
private fun exportToFile() {
val content: String
val dir: File
synchronized(lock) {
dir = exportDir ?: return
content = buildExportLocked()
exportDirty = false
}
private fun exportToFile(dir: File) {
val content = core.takeExport() ?: return
val target = File(dir, EXPORT_FILE_NAME)
val temp = File(dir, "$EXPORT_FILE_NAME.tmp")
try {
@@ -456,99 +120,4 @@ class FrameTimings private constructor() {
Log.w(TAG, "failed to export frame timings", t)
}
}
private fun buildExportLocked(): String {
val now = SystemClock.elapsedRealtime()
val wallClockFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS", Locale.US)
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(wallClockFormat.format(Date())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
Collections.sort(slowestRoots, Comparator { a, b ->
java.lang.Long.compare(b.subtreeEndAtMs(now) - b.startedAtMs, a.subtreeEndAtMs(now) - a.startedAtMs) })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, wallClockFormat, now)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, wallClockFormat, now)
}
return out.toString()
}
private fun statsSummaryLocked(): String {
val input = percentilesLocked(inputLatencies)
val render = percentilesLocked(renderLatencies)
val out = StringBuilder()
out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent)
.append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut)
.append(" open=").append(openFrameCountLocked())
appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input)
appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render)
return out.toString()
}
private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) {
if (percentiles == null) {
return
}
out.append('\n').append(label).append(" (last ").append(count).append("): p50=")
.append(percentiles[0]).append("ms p90=").append(percentiles[1])
.append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms")
}
private fun openFrameCountLocked(): Int {
var open = 0
for (frame in framesById.values) {
if (frame.isOpen()) {
open++
}
}
return open
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(
out: StringBuilder, frame: Frame, root: Frame, wallClockFormat: SimpleDateFormat, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
val indent = repeat(" ", depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(wallClockFormat.format(Date(frame.startedWallClockMs)))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
if (!frame.children.isEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
out.append(indent).append(String.format(Locale.US, " %+7dms [%s] %s",
line.atMs - root.startedAtMs, line.thread, line.message)).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, wallClockFormat, now)
}
}
}
@@ -8,9 +8,6 @@ import android.util.Log
import androidx.core.graphics.PathParser
import java.util.regex.Matcher
import java.util.regex.Pattern
/**
* Renders a small subset of SVG (the shapes used by Lucide / simple Noun
* Project icons: path, circle, ellipse, rect, line, polyline, polygon) to a
@@ -18,13 +15,14 @@ import java.util.regex.Pattern
* (Lucide style: fill="none", 2px stroke, round caps/joins); the returned
* bytes are one coverage value per pixel (0=transparent .. 255=opaque white),
* row-major, size*size. Called once per icon and cached on the TS side.
*
* Parsing, placement math and the coverage quantization are shared with iOS
* (SvgIconSource); this class only maps shapes onto android.graphics.Path and
* rasterizes them.
*/
class IconRenderer private constructor() {
companion object {
private const val TAG = "IconRenderer"
private val TAG_RE: Pattern = Pattern.compile("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>")
private val VIEWBOX_RE: Pattern = Pattern.compile("viewBox\\s*=\\s*\"([^\"]*)\"")
private val SEPARATOR_RE: Pattern = Pattern.compile("[\\s,]+")
@JvmStatic
fun renderSvgGray(svg: String?, size: Int, strokeWidth: Float): ByteArray {
@@ -32,27 +30,10 @@ class IconRenderer private constructor() {
return ByteArray(0)
}
try {
var minX = 0f
var minY = 0f
var viewW = 24f
var viewH = 24f
val vb = VIEWBOX_RE.matcher(svg)
if (vb.find()) {
val parts = SEPARATOR_RE.split(vb.group(1)!!.trim())
if (parts.size == 4) {
minX = java.lang.Float.parseFloat(parts[0])
minY = java.lang.Float.parseFloat(parts[1])
viewW = java.lang.Float.parseFloat(parts[2])
viewH = java.lang.Float.parseFloat(parts[3])
}
}
// Lucide-style icons are stroked outlines (fill="none"); everything
// else (Noun Project glyphs, brand logos) is a filled shape.
val stroked = svg.contains("fill=\"none\"")
val icon = SvgIconSource.parse(svg) ?: return ByteArray(0)
val path = Path()
val m = TAG_RE.matcher(svg)
while (m.find()) {
appendElement(path, m.group(1)!!, m.group(2)!!)
for (shape in icon.shapes) {
appendShape(path, shape)
}
if (path.isEmpty) {
return ByteArray(0)
@@ -60,14 +41,14 @@ class IconRenderer private constructor() {
val bitmap = Bitmap.createBitmap(size, size, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
val scale = Math.min(size / viewW, size / viewH)
// Center the (square) viewBox in the bitmap, then map its origin.
canvas.translate((size - viewW * scale) / 2f, (size - viewH * scale) / 2f)
canvas.scale(scale, scale)
canvas.translate(-minX, -minY)
val placement = icon.placement(size)
canvas.translate(placement[0], placement[1])
canvas.scale(placement[2], placement[2])
canvas.translate(-icon.minX, -icon.minY)
val paint = Paint(Paint.ANTI_ALIAS_FLAG)
paint.color = 0xffffffff.toInt()
if (stroked) {
if (icon.stroked) {
paint.style = Paint.Style.STROKE
paint.strokeWidth = strokeWidth
paint.strokeCap = Paint.Cap.ROUND
@@ -79,117 +60,47 @@ class IconRenderer private constructor() {
val pixels = IntArray(size * size)
bitmap.getPixels(pixels, 0, size, 0, 0, size, size)
val gray = ByteArray(size * size)
for (i in pixels.indices) {
gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte() // alpha = coverage
}
return gray
return SvgIconSource.quantizeArgb(pixels) // alpha = coverage
} catch (e: Exception) {
Log.w(TAG, "icon render failed", e)
return ByteArray(0)
}
}
/**
* Snap antialiased coverage toward hard edges so the 4bpp frames sent
* over BLE deflate better: mostly-transparent and mostly-opaque pixels
* become exactly 0/255 (long runs, and 0 stays color-key transparent for
* bitBlt), and only genuinely half-covered edge pixels keep one mid gray.
* The packed frame then uses a 3-symbol alphabet instead of all 16
* levels, at icon sizes this is visually near-identical.
*/
@JvmStatic
private fun quantizeCoverage(alpha: Int): Int {
if (alpha < 64) return 0
if (alpha >= 192) return 255
return 128
}
@JvmStatic
private fun appendElement(path: Path, tag: String, attrs: String) {
when (tag) {
"path" -> {
val d = attr(attrs, "d")
if (d != null) {
val sub = PathParser.createPathFromPathData(d)
if (sub != null) {
path.addPath(sub)
}
private fun appendShape(path: Path, shape: SvgShape) {
when (shape) {
is SvgShape.PathData -> {
val sub = PathParser.createPathFromPathData(shape.d)
if (sub != null) {
path.addPath(sub)
}
}
"circle" -> {
val cx = num(attrs, "cx", 0f)
val cy = num(attrs, "cy", 0f)
val r = num(attrs, "r", 0f)
if (r > 0) path.addCircle(cx, cy, r, Path.Direction.CW)
}
"ellipse" -> {
val cx = num(attrs, "cx", 0f)
val cy = num(attrs, "cy", 0f)
val rx = num(attrs, "rx", 0f)
val ry = num(attrs, "ry", 0f)
if (rx > 0 && ry > 0) path.addOval(cx - rx, cy - ry, cx + rx, cy + ry, Path.Direction.CW)
}
"rect" -> {
val x = num(attrs, "x", 0f)
val y = num(attrs, "y", 0f)
val w = num(attrs, "width", 0f)
val h = num(attrs, "height", 0f)
var rx = num(attrs, "rx", 0f)
var ry = num(attrs, "ry", if (rx > 0) rx else 0f)
if (rx <= 0) rx = ry
if (ry <= 0) ry = rx
if (w > 0 && h > 0) {
if (rx > 0 || ry > 0) {
path.addRoundRect(x, y, x + w, y + h, rx, ry, Path.Direction.CW)
} else {
path.addRect(x, y, x + w, y + h, Path.Direction.CW)
}
is SvgShape.Circle -> path.addCircle(shape.cx, shape.cy, shape.r, Path.Direction.CW)
is SvgShape.Ellipse -> path.addOval(
shape.cx - shape.rx, shape.cy - shape.ry, shape.cx + shape.rx, shape.cy + shape.ry, Path.Direction.CW)
is SvgShape.Rect -> {
if (shape.rx > 0 || shape.ry > 0) {
path.addRoundRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height,
shape.rx, shape.ry, Path.Direction.CW)
} else {
path.addRect(shape.x, shape.y, shape.x + shape.width, shape.y + shape.height, Path.Direction.CW)
}
}
"line" -> {
path.moveTo(num(attrs, "x1", 0f), num(attrs, "y1", 0f))
path.lineTo(num(attrs, "x2", 0f), num(attrs, "y2", 0f))
is SvgShape.Line -> {
path.moveTo(shape.x1, shape.y1)
path.lineTo(shape.x2, shape.y2)
}
"polyline", "polygon" -> {
val points = attr(attrs, "points")
if (points != null) {
val nums = SEPARATOR_RE.split(points.trim())
var started = false
var i = 0
while (i + 1 < nums.size) {
val px = java.lang.Float.parseFloat(nums[i])
val py = java.lang.Float.parseFloat(nums[i + 1])
if (!started) {
path.moveTo(px, py)
started = true
} else {
path.lineTo(px, py)
}
i += 2
}
if (tag == "polygon") path.close()
is SvgShape.Polyline -> {
val p = shape.points
path.moveTo(p[0], p[1])
var i = 2
while (i + 1 < p.size) {
path.lineTo(p[i], p[i + 1])
i += 2
}
if (shape.closed) path.close()
}
else -> {
}
}
}
@JvmStatic
private fun attr(attrs: String, name: String): String? {
val m: Matcher = Pattern.compile("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").matcher(attrs)
return if (m.find()) m.group(1) else null
}
@JvmStatic
private fun num(attrs: String, name: String, fallback: Float): Float {
val value = attr(attrs, name)
if (value == null) return fallback
try {
return java.lang.Float.parseFloat(value.trim())
} catch (e: NumberFormatException) {
return fallback
}
}
}
@@ -10,7 +10,8 @@ import android.util.Log
* pixels...] with one byte per pixel, row-major, or an empty array on
* failure. FaceclawMediaController's album art shares bitmapToGrayPacket;
* the TS side (app/native/image-files.ts) owns the shared photo-tone preset
* passed as (gamma, dither).
* passed as (gamma, dither). The pixel math (tone curve, alpha darkening,
* dither) lives in the shared GrayPacket so iOS produces identical bytes.
*/
class ImageFileLoader private constructor() {
companion object {
@@ -151,24 +152,7 @@ class ImageFileLoader private constructor() {
}
val pixels = IntArray(width * height)
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
val out = ByteArray(4 + width * height)
out[0] = (width and 0xff).toByte()
out[1] = ((width shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
val tone = toneCurve(gamma)
for (i in pixels.indices) {
val p = pixels[i]
val a = (p ushr 24) and 0xff
val r = (p shr 16) and 0xff
val g = (p shr 8) and 0xff
val b = p and 0xff
out[4 + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte()
}
if (dither) {
ditherToDisplayLevels(out, 4, width, height)
}
return out
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
} catch (e: Exception) {
Log.w(TAG, "bitmap conversion failed", e)
return ByteArray(0)
@@ -177,70 +161,5 @@ class ImageFileLoader private constructor() {
return ByteArray(0)
}
}
/** 256-entry map of source gray to displayed gray for out = in^gamma. */
@JvmStatic
private fun toneCurve(gamma: Float): IntArray {
val curve = IntArray(256)
for (v in 0 until 256) {
curve[v] = if (gamma == 1f)
v
else Math.round(255f * Math.pow((v / 255f).toDouble(), gamma.toDouble()).toFloat())
}
return curve
}
/**
* Floyd-Steinberg error diffusion onto the display's 16 gray levels, in
* place over an 8bpp plane. Serpentine scanning keeps the diffusion from
* building up a directional texture across wide flat areas.
*
* Level n is written back as n * 16 rather than the n * 17 it stands for
* so that BmpUtil's (v + 8) >> 4 reproduces n exactly; level 0 is written
* as 1, not 0, because 0 is the shell's color-key for transparent.
*/
@JvmStatic
private fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) {
var curr = FloatArray(width)
var next = FloatArray(width)
for (x in 0 until width) {
curr[x] = (plane[offset + x].toInt() and 0xff).toFloat()
}
for (y in 0 until height) {
val rowStart = offset + y * width
val nextStart = rowStart + width
val hasNext = y + 1 < height
for (x in 0 until width) {
next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f
}
val leftToRight = (y and 1) == 0
val start = if (leftToRight) 0 else width - 1
val step = if (leftToRight) 1 else -1
for (i in 0 until width) {
val x = start + i * step
val wanted = curr[x]
val level = Math.round(Math.min(255f, Math.max(0f, wanted)) / 17f)
val error = wanted - level * 17f
plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte()
val ahead = x + step
if (ahead >= 0 && ahead < width) {
curr[ahead] += error * (7f / 16f)
}
if (hasNext) {
if (ahead >= 0 && ahead < width) {
next[ahead] += error * (1f / 16f)
}
next[x] += error * (5f / 16f)
val behind = x - step
if (behind >= 0 && behind < width) {
next[behind] += error * (3f / 16f)
}
}
}
val swap = curr
curr = next
next = swap
}
}
}
}
@@ -9,34 +9,11 @@ import java.nio.ByteBuffer
* grayscale buffer arrives as a ByteBuffer (NativeScript marshals a JS
* ArrayBuffer to one without copying element-by-element); doing the
* gray-to-ARGB expansion here keeps the 165KB-per-frame loop out of the
* JS/Java bridge, where it used to cost ~150ms per preview.
* JS/Java bridge, where it used to cost ~150ms per preview. The palette
* itself is the shared PreviewPalette.
*/
class PreviewBitmapUtil private constructor() {
companion object {
private var cachedLut: IntArray? = null
private var cachedGammaBits: Long = 0
private var cachedGreen: Boolean = false
/** Reuse the current preview palette; rebuild only when gamma or color changes. */
@JvmStatic
@Synchronized
private fun lookupTable(brightenGamma: Double, green: Boolean): IntArray {
val gammaBits = java.lang.Double.doubleToLongBits(brightenGamma)
val cached = cachedLut
if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) {
return cached
}
val lut = IntArray(256)
for (g in 0 until 256) {
val v = Math.max(0L, Math.min(255L, Math.round(255 * Math.pow(g / 255.0, brightenGamma)))).toInt()
lut[g] = if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v)
}
cachedGammaBits = gammaBits
cachedGreen = green
cachedLut = lut
return lut
}
@JvmStatic
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double): Bitmap {
return fromGray(gray, width, height, brightenGamma, false)
@@ -48,13 +25,9 @@ class PreviewBitmapUtil private constructor() {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw IllegalArgumentException("invalid gray preview buffer")
}
val lut = lookupTable(brightenGamma, green)
val bytes = ByteArray(width * height)
gray.get(bytes)
val colors = IntArray(width * height)
for (i in colors.indices) {
colors[i] = lut[bytes[i].toInt() and 0xff]
}
val colors = PreviewPalette.expand(bytes, 0, width * height, brightenGamma, green)
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
}
}
@@ -1,48 +0,0 @@
package com.faceclaw.app
import android.os.SystemClock
class InterruptibleSleep {
private val lock = java.lang.Object()
private var interrupted = false
fun sleep(ms: Long): Boolean {
val delayMs = Math.max(1L, ms)
val deadline = SystemClock.elapsedRealtime() + delayMs
synchronized(lock) {
// Consume (do not clear-and-ignore) an interrupt delivered before this
// sleep began. interrupt() runs on other threads with no lock held by
// the sleeper between driveSession() returning and sleep() starting, so
// clearing the flag at entry lost that wake and forced the worker to
// sleep the full interval — up to 250ms per idle frame. Honoring it
// costs at most one extra (cheap) driveSession pass.
if (interrupted) {
interrupted = false
return false
}
while (true) {
val remaining = deadline - SystemClock.elapsedRealtime()
if (remaining <= 0) {
return true
}
try {
lock.wait(remaining)
} catch (ignored: InterruptedException) {
interrupted = false
return false
}
if (interrupted) {
interrupted = false
return false
}
}
}
}
fun interrupt() {
synchronized(lock) {
interrupted = true
lock.notifyAll()
}
}
}
+18 -2
View File
@@ -8,7 +8,21 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
- `commonMain/kotlin`: wire protocol, fragmentation/reassembly, event decoding,
image/RLE/texture planning, retained surface composition, glyph/image atlases,
LVGL font parsing, spectral noise suppression, PNG/GIF/WAV encodings and
callback interfaces.
callback interfaces. Since 2026-09-22 also the platform-neutral cores behind
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, GlyphPacket,
OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
`alarms/` (AlarmSchedule, AlarmRingingPolicy), `settings/SettingsChangeHub`,
the stock-firmware flows over a `StockLink` GATT port (`g2protocol/StockLinkSession`,
`DeviceInfoProbeFlow`, `FlashPromptFlow`, `OtaFlashFlow`), the full glasses session
(`g2protocol/session/GlassesSessionCore` over `SessionLink`/`SessionHost`) and the
voice/caption pipelines (`audio/VoiceCaptureSession`, `CaptionEngineCore`).
Cores take port interfaces (GATT link, storage, transport, dispatcher) and call
listeners synchronously; the Android adapters in `App_Resources` own threads,
Handler marshalling and OS APIs. See `notes/stage2-shared-kotlin-plan.md`.
- `androidMain/kotlin`: monotonic clock, reentrant locks, JVM zlib and file I/O,
zero-copy `ByteBuffer` readers, firmware hashing/file writes and WAV file storage.
- `iosMain/kotlin`: monotonic clock, recursive locks, native zlib and POSIX file
@@ -22,7 +36,9 @@ bulk buffer crossing without putting Java APIs in common code. iOS uses
## Platform boundaries
`ProtocolPlatform` supplies clocks, locks and persistent deflaters. `expect` /
`ProtocolPlatform` supplies clocks, locks, persistent deflaters and condition
variables (`ProtocolCondition`, NOT reentrant on iOS: NSCondition). `InterruptibleSleep`,
`Latch` and `BlockingQueue` in commonMain build on it. `expect` /
`actual` functions select default platform services at compilation. Stateful
transports and caches retain reentrant locking; callbacks still run on their
caller's thread. The migration does not introduce implicit UI-thread dispatch.
+1 -1
View File
@@ -409,6 +409,6 @@
},
{
"java": "com/faceclaw/app/util/InterruptibleSleep.java",
"kotlin": "App_Resources/Android/src/main/java/com/faceclaw/app/util/InterruptibleSleep.kt"
"kotlin": "native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/util/Concurrency.kt"
}
]
@@ -0,0 +1,57 @@
package com.faceclaw.app
import java.security.MessageDigest
import java.text.SimpleDateFormat
import java.util.Date
import java.util.Locale
actual class Sha256Digest actual constructor() {
private val digest = MessageDigest.getInstance("SHA-256")
actual fun update(bytes: ByteArray, offset: Int, length: Int) = digest.update(bytes, offset, length)
actual fun digest(): ByteArray = digest.digest()
}
actual fun currentTimeMillis(): Long = System.currentTimeMillis()
actual fun formatLocalTime(epochMs: Long, pattern: String): String =
SimpleDateFormat(pattern, Locale.US).format(Date(epochMs))
actual object PlatformLog {
actual fun i(tag: String, message: String) {
android.util.Log.i(tag, message)
}
actual fun w(tag: String, message: String) {
android.util.Log.w(tag, message)
}
actual fun e(tag: String, message: String, error: Throwable?) {
if (error != null) android.util.Log.e(tag, message, error) else android.util.Log.e(tag, message)
}
}
actual fun startThread(name: String, daemon: Boolean, body: () -> Unit) {
val thread = Thread(body, name)
thread.isDaemon = daemon
thread.start()
}
actual fun sleepMs(ms: Long) {
try {
Thread.sleep(ms)
} catch (ignored: InterruptedException) {
Thread.currentThread().interrupt()
}
}
actual fun writeFileData(path: String, bytes: ByteArray) = java.io.File(path).writeBytes(bytes)
actual fun fileExists(path: String): Boolean = java.io.File(path).exists()
actual fun fileLength(path: String): Long = java.io.File(path).let { if (it.exists()) it.length() else -1L }
actual fun deleteFile(path: String): Boolean = java.io.File(path).delete()
actual fun renameFile(from: String, to: String): Boolean = java.io.File(from).renameTo(java.io.File(to))
@@ -20,6 +20,22 @@ object AndroidProtocolPlatform : ProtocolPlatform {
override fun createDeflater(): ProtocolDeflater = AndroidDeflater()
override fun createCondition(): ProtocolCondition =
object : ProtocolCondition {
private val lock = ReentrantLock()
private val condition = lock.newCondition()
override fun lock() = lock.lock()
override fun unlock() = lock.unlock()
override fun awaitMs(timeoutMs: Long) {
if (timeoutMs > 0) condition.await(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS)
}
override fun signalAll() = condition.signalAll()
}
@JvmStatic
fun writeType(mode: GattWriteMode): Int =
when (mode) {
@@ -8,3 +8,15 @@ internal expect fun deflateData(bytes: ByteArray): ByteArray
internal expect fun inflateData(bytes: ByteArray, size: Int): ByteArray
internal expect fun readFileData(path: String): ByteArray
/** Whole-file write (truncating); parent directories must exist. */
expect fun writeFileData(path: String, bytes: ByteArray)
expect fun fileExists(path: String): Boolean
/** Size in bytes, or -1 when the file does not exist. */
expect fun fileLength(path: String): Long
expect fun deleteFile(path: String): Boolean
expect fun renameFile(from: String, to: String): Boolean
@@ -5,7 +5,7 @@ class SettingsDocument private constructor(private val settings: Map<String, Any
fun containsString(key: String, value: String): Boolean = settings[key] == structuredValue(key, value)
fun getString(key: String, fallback: String): String = when (val value = settings[key]) {
is String -> value
is Map<*, *>, is List<*> -> SettingsJson.write(value)
is Map<*, *>, is List<*> -> Json.write(value)
else -> fallback
}
fun getBoolean(key: String, fallback: Boolean): Boolean = settings[key] as? Boolean ?: fallback
@@ -17,7 +17,7 @@ class SettingsDocument private constructor(private val settings: Map<String, Any
SettingsDocument(settings + (key to structuredValue(key, value)))
fun replacingBoolean(key: String, value: Boolean): SettingsDocument = SettingsDocument(settings + (key to value))
fun merged(other: SettingsDocument): SettingsDocument = SettingsDocument(settings + other.settings)
fun encode(): String = SettingsJson.write(mapOf("schema" to JsonNumber(CURRENT_SCHEMA.toString()), "settings" to settings), pretty = true) + "\n"
fun encode(): String = Json.write(mapOf("schema" to JsonNumber(CURRENT_SCHEMA.toString()), "settings" to settings), pretty = true) + "\n"
// A mutation/merge must never commit a document that the next launch rejects.
fun encodeForStorage(): String? {
@@ -36,7 +36,7 @@ class SettingsDocument private constructor(private val settings: Map<String, Any
private fun structuredValue(key: String, value: Any?): Any? {
if ((key !in structuredKeys && !key.startsWith("ios.ble.peripheral.")) || value !is String) return value
// Preserve empty/invalid old values: callers already provide their own fallback.
val parsed = runCatching { SettingsJson.parse(value) }.getOrNull()
val parsed = runCatching { Json.parse(value) }.getOrNull()
return if (parsed is Map<*, *> || parsed is List<*>) parsed else value
}
// Each entry migrates version N to N+1. Never rewrite unknown future schemas.
@@ -45,7 +45,7 @@ class SettingsDocument private constructor(private val settings: Map<String, Any
)
internal fun decode(text: String): SettingsDocument {
require(text.encodeToByteArray().size <= 2 * 1024 * 1024) { "Config exceeds 2 MiB" }
val root = SettingsJson.parse(text) as? Map<*, *> ?: error("Expected config object")
val root = Json.parse(text) as? Map<*, *> ?: error("Expected config object")
var schema = (root["schema"] as? JsonNumber)?.text?.toIntOrNull() ?: error("Missing schema")
require(schema in 1..CURRENT_SCHEMA) { "Unsupported settings schema" }
val raw = root["settings"] as? Map<*, *> ?: error("Missing settings")
@@ -64,103 +64,7 @@ class SettingsDocument private constructor(private val settings: Map<String, Any
/** Nullable decode keeps invalid files recoverable across Objective-C/JVM bridges. */
class SettingsCodec {
fun needsMigration(text: String): Boolean = runCatching {
((SettingsJson.parse(text) as Map<*, *>)["schema"] as JsonNumber).text != SettingsDocument.CURRENT_SCHEMA.toString()
((Json.parse(text) as Map<*, *>)["schema"] as JsonNumber).text != SettingsDocument.CURRENT_SCHEMA.toString()
}.getOrDefault(false)
fun decode(text: String): SettingsDocument? = runCatching { SettingsDocument.decode(text) }.getOrNull()
}
private data class JsonNumber(val text: String)
/** Strict JSON plus // and block comments and trailing commas, on both platforms. */
private object SettingsJson {
private val numberPattern = Regex("-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
fun parse(text: String): Any? = Parser(text).parse()
fun write(value: Any?, pretty: Boolean = false, depth: Int = 0): String {
fun quoted(text: String): String = buildString {
append('"')
for (c in text) when (c) {
'"' -> append("\\\""); '\\' -> append("\\\\"); '\n' -> append("\\n"); '\r' -> append("\\r"); '\t' -> append("\\t")
else -> if (c < ' ') append("\\u" + c.code.toString(16).padStart(4, '0')) else append(c)
}
append('"')
}
fun collection(open: String, close: String, entries: List<String>): String {
if (entries.isEmpty()) return open + close
return if (pretty) open + "\n" + entries.joinToString(",\n") { " ".repeat(depth + 1) + it } + "\n" + " ".repeat(depth) + close
else open + entries.joinToString(",") + close
}
return when (value) {
null -> "null"
is String -> quoted(value)
is Boolean -> value.toString()
is JsonNumber -> value.text
is List<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) })
is Map<*, *> -> collection("{", "}", value.entries.map { quoted(it.key as String) + (if (pretty) ": " else ":") + write(it.value, pretty, depth + 1) })
else -> error("Unsupported JSON value")
}
}
private class Parser(val text: String) {
var i = 0
fun parse(): Any? { val result = value(0); space(); require(i == text.length); return result }
fun space() {
while (i < text.length) {
if (text[i] in " \r\n\t\uFEFF") { i++; continue }
if (text.startsWith("//", i)) { while (i < text.length && text[i] != '\n' && text[i] != '\r') i++; continue }
if (text.startsWith("/*", i)) { val end = text.indexOf("*/", i + 2); require(end >= 0); i = end + 2; continue }
break
}
}
fun take(c: Char): Boolean { space(); return if (i < text.length && text[i] == c) { i++; true } else false }
fun value(depth: Int): Any? {
require(depth < 64); space(); require(i < text.length)
return when (text[i]) {
'"' -> string()
'{' -> {
i++; val result = linkedMapOf<String, Any?>()
if (!take('}')) while (true) {
space(); val key = string(); require(!result.containsKey(key)); require(take(':'))
result[key] = value(depth + 1)
if (take('}')) break
require(take(',')); if (take('}')) break
}
result
}
'[' -> {
i++; val result = mutableListOf<Any?>()
if (!take(']')) while (true) {
result.add(value(depth + 1)); if (take(']')) break
require(take(',')); if (take(']')) break
}
result
}
't' -> literal("true", true)
'f' -> literal("false", false)
'n' -> literal("null", null)
else -> {
val match = numberPattern.find(text, i)
require(match != null && match.range.first == i)
require(match.value.toDouble().isFinite()); i += match.value.length; JsonNumber(match.value)
}
}
}
fun literal(word: String, result: Any?): Any? { require(text.startsWith(word, i)); i += word.length; return result }
fun string(): String {
require(i < text.length && text[i++] == '"')
return buildString {
while (true) {
require(i < text.length); val c = text[i++]
if (c == '"') break
require(c >= ' ')
if (c != '\\') { append(c); continue }
require(i < text.length)
append(when (val escaped = text[i++]) {
'"', '\\', '/' -> escaped
'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t'
'u' -> { require(i + 4 <= text.length); val hex = text.substring(i, i + 4); require(hex.all { it in "0123456789abcdefABCDEF" }); val code = hex.toInt(16); i += 4; code.toChar() }
else -> error("Invalid escape")
})
}
}
}
}
}
@@ -0,0 +1,275 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/** One item currently ringing. Subclassed by the Android service for its own bookkeeping. */
open class RingingItem(
@JvmField val id: Long,
title: String?,
text: String?,
kind: String?,
snoozeMinutes: Int,
@JvmField val ringAtMs: Long,
) {
@JvmField val title: String = if (title == null || title.trim().isEmpty()) "Alarm" else title
@JvmField val text: String = text ?: ""
@JvmField val kind: String = kind ?: AlarmSchedule.KIND_ALARM
@JvmField val snoozeMinutes: Int = maxOf(1, snoozeMinutes)
@JvmField var deliveredAtMs: Long = 0
@JvmField var escalated: Boolean = false
@JvmField var silenced: Boolean = false
}
/** Delayed callbacks for the escalation timers; the owner's main thread on Android. */
interface AlarmScheduler {
fun postDelayed(token: Any, delayMs: Long, action: () -> Unit)
fun cancel(token: Any)
fun cancelAll()
}
/** Platform effects of ringing: notifications, sound, the schedule, and the journal. */
interface RingingSink {
/** Show or refresh the item's notification (title/status wording comes from the item). */
fun showNotification(item: RingingItem)
/** The item is gone: drop its notification and any per-item resources. */
fun itemRemoved(item: RingingItem)
/** Something started ringing: keep the device awake for the escalation window. */
fun ringingStarted()
fun startSound()
fun stopSound()
/** Re-arm the item (a snooze) at [atMs]. */
fun reschedule(item: RingingItem, atMs: Long, minutes: Int)
/** Forget the item's schedule entry (a dismiss). */
fun cancelSchedule(id: Long)
fun recordPhoneAction(id: Long, action: String, minutes: Int)
fun log(line: String)
}
/**
* Ringing escalation. Each item starts silent on the phone (a notification only, the glasses do
* the ringing) and escalates to phone sound when the glasses cannot carry it: not connected,
* not on a head, charging, no delivery confirmation from the JS side within [DELIVERY_WAIT_MS],
* or no acknowledgement within [ACK_WAIT_MS] of delivery. Sound stops after [AUTO_SILENCE_MS]
* while the notification stays. The policy is process-wide; the platform host (a service on
* Android) attaches its scheduler and sink while it is alive.
*/
class AlarmRingingPolicy(
private val gate: GlassesStatusGate,
private val clock: () -> Long = ::currentTimeMillis,
platform: ProtocolPlatform = protocolPlatform(),
) {
private val lock = platform.createLock()
/** Ringing items, oldest first. */
private val ringing = LinkedHashMap<Long, RingingItem>()
/**
* Signals that arrived before the ring request was processed (the JS engine fires, launches
* its window and reports delivery on the same thread the host queues behind), keyed by id
* with their arrival time. Consumed by [ring].
*/
private val deliveredEarly = LinkedHashMap<Long, Long>()
private val stoppedEarly = LinkedHashMap<Long, Long>()
private var scheduler: AlarmScheduler? = null
private var sink: RingingSink? = null
fun attach(scheduler: AlarmScheduler, sink: RingingSink) {
lock.withLock {
this.scheduler = scheduler
this.sink = sink
}
}
fun detach() {
lock.withLock {
scheduler?.cancelAll()
scheduler = null
sink = null
}
}
fun isRinging(id: Long): Boolean = lock.withLock { ringing.containsKey(id) }
fun isAnythingRinging(): Boolean = lock.withLock { ringing.isNotEmpty() }
fun snapshot(): List<RingingItem> = lock.withLock { ArrayList(ringing.values) }
/**
* The glasses report showing [id]. True when the item is ringing (the host should process
* the delivery); false when it is not yet known, in which case the signal is kept for a
* ring request that follows within [EARLY_SIGNAL_MAX_AGE_MS].
*/
fun deliveredOrDefer(id: Long): Boolean =
lock.withLock {
if (ringing.containsKey(id)) true
else {
deliveredEarly[id] = clock()
false
}
}
/** As [deliveredOrDefer], for an acknowledgement / cancellation. */
fun stopOrDefer(id: Long): Boolean =
lock.withLock {
if (ringing.containsKey(id)) true
else {
stoppedEarly[id] = clock()
false
}
}
/** An item came due (idempotent per id: a duplicate ring only refreshes the notification). */
fun ring(item: RingingItem) {
if (item.id == 0L) return
lock.withLock {
val sink = sink ?: return
val existing = ringing[item.id]
if (existing != null) {
// The engine's JS timeout and the platform alarm both fired: one ring.
sink.showNotification(existing)
return
}
if (takeEarlySignal(stoppedEarly, item.id)) {
// Acknowledged or cancelled before this request was processed.
sink.log("ring " + item.id + " already acknowledged")
return
}
ringing[item.id] = item
sink.log("ring " + item.kind + " " + item.id + " (" + gate.description() + ")")
sink.ringingStarted()
sink.showNotification(item)
if (!gate.canCarryAlarm()) {
escalate(item, gate.description())
return
}
if (takeEarlySignal(deliveredEarly, item.id)) {
markDelivered(item.id)
return
}
// The glasses could carry it: give the JS side a moment to confirm they are
// actually showing it, then hold for the acknowledgement.
scheduler?.postDelayed("delivery:" + item.id, DELIVERY_WAIT_MS) {
lock.withLock {
val current = ringing[item.id]
if (current != null && !current.escalated && current.deliveredAtMs == 0L) {
escalate(current, "not delivered to the glasses")
}
}
}
}
}
fun markDelivered(id: Long) {
lock.withLock {
val item = ringing[id]
if (item == null || item.deliveredAtMs != 0L) return
item.deliveredAtMs = clock()
sink?.log("delivered $id to glasses")
scheduler?.postDelayed("ack:$id", ACK_WAIT_MS) {
lock.withLock {
val current = ringing[id]
if (current != null && !current.escalated) {
escalate(current, "not acknowledged on the glasses")
}
}
}
}
}
private fun escalate(item: RingingItem, reason: String) {
if (item.escalated) return
item.escalated = true
val sink = sink ?: return
sink.log("phone sound for " + item.id + ": " + reason)
sink.showNotification(item)
sink.startSound()
scheduler?.postDelayed("silence:" + item.id, AUTO_SILENCE_MS) {
lock.withLock {
val current = ringing[item.id]
if (current != null && current.escalated && !current.silenced) {
current.silenced = true
this.sink?.log("auto-silenced " + current.id)
this.sink?.showNotification(current)
syncSound()
}
}
}
}
/** Dismissed on the phone: journal it, stop it, and drop the schedule entry. */
fun dismiss(id: Long) {
lock.withLock {
val item = ringing[id] ?: return
val sink = sink ?: return
sink.recordPhoneAction(id, "dismiss", 0)
removeItem(item)
// A one-off is over; the engine re-arms repeats.
sink.cancelSchedule(id)
}
}
/** Snoozed on the phone: journal it, stop it, and re-arm on the phone directly so the snooze holds even with the JS side gone. */
fun snooze(id: Long) {
lock.withLock {
val item = ringing[id] ?: return
val sink = sink ?: return
val minutes = item.snoozeMinutes
sink.recordPhoneAction(id, "snooze", minutes)
removeItem(item)
sink.reschedule(item, clock() + minutes * 60_000L, minutes)
}
}
/** Stop an item without a phone action (acknowledged on the glasses / cancelled by the engine). */
fun stopQuietly(id: Long, reason: String) {
lock.withLock {
val item = ringing[id] ?: return
sink?.log("$reason $id")
removeItem(item)
}
}
/** Start or stop the sound to match whether any escalated, not-yet-silenced item exists. */
fun syncSound() {
lock.withLock {
val sink = sink ?: return
if (ringing.values.any { it.escalated && !it.silenced }) sink.startSound() else sink.stopSound()
}
}
fun cancelTimers() {
lock.withLock { scheduler?.cancelAll() }
}
private fun removeItem(item: RingingItem) {
ringing.remove(item.id)
sink?.itemRemoved(item)
syncSound()
}
/** Consume an early signal for the id; stale entries are dropped. */
private fun takeEarlySignal(signals: MutableMap<Long, Long>, id: Long): Boolean {
val at = signals.remove(id)
val now = clock()
signals.entries.removeAll { now - it.value > EARLY_SIGNAL_MAX_AGE_MS }
return at != null && now - at <= EARLY_SIGNAL_MAX_AGE_MS
}
companion object {
/** How long to wait for the JS side to confirm the glasses are showing the item. */
const val DELIVERY_WAIT_MS = 5_000L
/** How long after delivery to the glasses before the phone joins in. */
const val ACK_WAIT_MS = 30_000L
/** Sound and vibration stop after this; the notification stays. */
const val AUTO_SILENCE_MS = 10 * 60_000L
const val EARLY_SIGNAL_MAX_AGE_MS = 15_000L
}
}
@@ -0,0 +1,235 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/** Key/value persistence for the alarm schedule, journal and log (device-protected prefs on Android). */
interface AlarmStore {
fun read(key: String): String?
fun write(key: String, value: String?)
}
/** One scheduled item: the JS engine's item id, its wall-clock trigger time, and what to show. */
class AlarmEntry(
@JvmField val id: Long,
@JvmField val at: Long,
title: String?,
text: String?,
kind: String?,
@JvmField val snoozeMinutes: Int,
) {
@JvmField val title: String = title ?: ""
@JvmField val text: String = text ?: ""
@JvmField val kind: String = kind ?: AlarmSchedule.KIND_ALARM
fun toJson(): Map<String, Any?> =
linkedMapOf(
"id" to JsonNumber.of(id),
"at" to JsonNumber.of(at),
"title" to title,
"text" to text,
"kind" to kind,
"snoozeMinutes" to JsonNumber.of(snoozeMinutes),
)
companion object {
@JvmStatic
fun fromJson(value: Any?): AlarmEntry? {
val map = value.asObject() ?: return null
return AlarmEntry(
map["id"].asLong() ?: 0L,
map["at"].asLong() ?: 0L,
map["title"].asString() ?: "",
map["text"].asString() ?: "",
map["kind"].asString() ?: AlarmSchedule.KIND_ALARM,
map["snoozeMinutes"].asInt() ?: 10,
)
}
}
}
/**
* The persisted alarm schedule, the phone-action journal and the diagnostics log, plus the
* replay / staleness policies. Everything the platform's alarm clock API does not own.
* Compound read-modify-write operations are serialized on one lock; the store itself only
* needs to be atomic per key.
*/
class AlarmSchedule(
private val store: AlarmStore,
private val clock: () -> Long = ::currentTimeMillis,
platform: ProtocolPlatform = protocolPlatform(),
) {
private val lock = platform.createLock()
/** All entries, in stored order. */
fun entries(): List<AlarmEntry> = lock.withLock { readEntries() }
fun find(id: Long): AlarmEntry? = lock.withLock { readEntries().firstOrNull { it.id == id } }
/** Add or replace the entry with the same id (moved to the end). */
fun put(entry: AlarmEntry) {
lock.withLock {
val next = readEntries().filter { it.id != entry.id } + entry
writeEntries(next)
}
}
fun remove(id: Long) {
lock.withLock { writeEntries(readEntries().filter { it.id != id }) }
}
/**
* Replay the schedule after a reboot, package update or clock change: entries already due
* ring now if inside the grace window, older ones are dropped as missed (and logged), the
* rest are handed to [arm]. Logs the outcome.
*/
fun replay(reason: String?, ring: (AlarmEntry) -> Unit, arm: (AlarmEntry) -> Unit) {
val scheduled = entries()
val now = clock()
var armed = 0
for (entry in scheduled) {
if (entry.at <= now) {
if (now - entry.at <= LATE_GRACE_MS) {
ring(entry)
} else {
log("missed " + entry.kind + " " + entry.id + " (" + reason + ")")
remove(entry.id)
}
continue
}
arm(entry)
armed++
}
log("rescheduled $armed ($reason)")
}
/** The soonest future trigger time, or null when nothing is pending. */
fun soonestPendingAt(now: Long = clock()): Long? =
entries().filter { it.at > now }.minOfOrNull { it.at }
/**
* The schedule and the system alarm clock can disagree (a force-stop clears alarms but not
* the file). True when the soonest stored entry is not what the system reports as its next
* alarm clock; another app's sooner alarm is fine, only a later (or absent) one proves ours
* is missing. [systemNextAlarmAt] is null when the system has no alarm clock set.
*/
fun looksStale(systemNextAlarmAt: Long?, now: Long = clock()): Boolean {
val soonest = soonestPendingAt(now) ?: return false
return systemNextAlarmAt == null || systemNextAlarmAt > soonest + 1000
}
// ------------------------------------------------------------------
// Phone-action journal (dismiss / snooze done on the phone, replayed by the JS engine)
fun appendJournal(id: Long, action: String, minutes: Int) {
val event = linkedMapOf<String, Any?>(
"id" to JsonNumber.of(id),
"action" to action,
"minutes" to JsonNumber.of(minutes),
"at" to JsonNumber.of(clock()),
)
lock.withLock {
val journal = readArray(KEY_JOURNAL) + event
store.write(KEY_JOURNAL, Json.write(journal))
}
log("phone " + action + " " + id + (if (minutes > 0) " ($minutes min)" else ""))
}
/** Hand the journal over as a JSON array and clear it. */
fun drainJournal(): String =
lock.withLock {
val text = Json.write(readArray(KEY_JOURNAL))
store.write(KEY_JOURNAL, "[]")
text
}
// ------------------------------------------------------------------
// Diagnostics log (bounded, newest last)
fun log(line: String) {
lock.withLock {
val existing = readArray(KEY_LOG).map { it.asString() ?: it?.toString() ?: "" }
val kept = existing.drop(maxOf(0, existing.size - (LOG_LINES - 1)))
store.write(KEY_LOG, Json.write(kept + (clock().toString() + " " + line)))
}
}
fun readLog(): String =
lock.withLock { readArray(KEY_LOG).joinToString("\n") { it.asString() ?: it?.toString() ?: "" } }
// ------------------------------------------------------------------
private fun readArray(key: String): List<Any?> {
val raw = store.read(key) ?: return emptyList()
return runCatching { Json.parseArray(raw) }.getOrDefault(emptyList())
}
private fun readEntries(): List<AlarmEntry> = readArray(KEY_SCHEDULED).mapNotNull { AlarmEntry.fromJson(it) }
private fun writeEntries(entries: List<AlarmEntry>) {
store.write(KEY_SCHEDULED, Json.write(entries.map { it.toJson() }))
}
companion object {
const val KIND_TIMER = "timer"
const val KIND_ALARM = "alarm"
const val KEY_SCHEDULED = "scheduled"
const val KEY_JOURNAL = "journal"
const val KEY_LOG = "log"
const val LOG_LINES = 60
/**
* A schedule entry found already past due (after a reboot, or when the process was
* asleep) rings if it is at most this late; older ones are dropped as missed. Matches
* the JS engine's grace window.
*/
const val LATE_GRACE_MS = 5 * 60_000L
/** A stable per-item request code for platform alarm/notification identities. */
@JvmStatic
fun requestCode(id: Long): Int = (id xor (id ushr 32)).toInt() and 0x7fffffff
}
}
/**
* The last glasses status report pushed by the JS side. In memory only, so a dead process
* reads as "no glasses"; a report older than [MAX_AGE_MS] is not trusted either.
*/
class GlassesStatusGate(private val clock: () -> Long = ::currentTimeMillis) {
@Volatile private var connected = false
@Volatile private var worn = false
@Volatile private var charging = false
@Volatile private var statusAtMs = 0L
fun set(connected: Boolean, worn: Boolean, charging: Boolean) {
this.connected = connected
this.worn = worn
this.charging = charging
statusAtMs = clock()
}
/** True only when a fresh report says the glasses are connected, on a head, and not charging. */
fun canCarryAlarm(): Boolean {
val at = statusAtMs
if (at == 0L || clock() - at > MAX_AGE_MS) return false
return connected && worn && !charging
}
fun description(): String {
val at = statusAtMs
if (at == 0L) return "no glasses status"
if (clock() - at > MAX_AGE_MS) return "glasses status stale"
if (!connected) return "glasses not connected"
if (charging) return "glasses charging"
if (!worn) return "glasses not worn"
return "glasses worn"
}
companion object {
/** A glasses status report older than this is not trusted (the JS side is probably gone). */
const val MAX_AGE_MS = 3 * 60_000L
}
}
@@ -0,0 +1,127 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.abs
import kotlin.math.min
import kotlin.math.sqrt
/**
* Sample-level helpers shared by the push-to-talk voice controller and the caption engine
* (previously two private copies each). All PCM is 16 kHz mono S16LE unless stated.
*/
object AudioSegmentation {
/**
* Where to end a decode segment: the center of the quietest [windowSamples]-long window
* within the last [searchSamples] of `samples[offset until offset + length]`, so the cut
* lands between words rather than splitting one. Returns [length] when the region is
* too short to search.
*/
@JvmStatic
fun quietestCutPoint(samples: FloatArray, length: Int, searchSamples: Int, windowSamples: Int): Int =
quietestCutPoint(length, searchSamples, windowSamples) { samples[it].toDouble() }
@JvmStatic
fun quietestCutPoint(samples: ShortArray, offset: Int, length: Int, searchSamples: Int, windowSamples: Int): Int =
quietestCutPoint(length, searchSamples, windowSamples) { samples[offset + it].toDouble() }
private inline fun quietestCutPoint(length: Int, searchSamples: Int, windowSamples: Int, sample: (Int) -> Double): Int {
val searchStart = maxOf(0, length - searchSamples)
val win = windowSamples
if (length - searchStart <= win) return length
var sum = 0.0
for (i in searchStart until searchStart + win) {
val s = sample(i)
sum += s * s
}
var best = sum
var bestStart = searchStart
var start = searchStart + 1
while (start + win <= length) {
val dropped = sample(start - 1)
val added = sample(start + win - 1)
sum += added * added - dropped * dropped
if (sum < best) {
best = sum
bestStart = start
}
start++
}
return bestStart + win / 2
}
@JvmStatic
fun peakAmplitude(samples: FloatArray): Float = peakAmplitude(samples, samples.size)
@JvmStatic
fun peakAmplitude(samples: FloatArray, count: Int): Float {
var peak = 0f
for (i in 0 until count) {
val a = abs(samples[i])
if (a > peak) peak = a
}
return peak
}
/**
* Scales [samples] in place so the peak reaches [targetPeak], with the gain capped at
* [maxGain]; audio already at or above the target is left untouched.
*/
@JvmStatic
fun normalizePeak(samples: FloatArray, targetPeak: Float, maxGain: Float) {
val peak = peakAmplitude(samples)
if (peak <= 0f) return
val gain = min(targetPeak / peak, maxGain)
if (gain <= 1f) return
for (i in samples.indices) samples[i] *= gain
}
/** S16 samples to [-1, 1] floats. */
@JvmStatic
fun pcm16ToFloat(samples: ShortArray, offset: Int, count: Int): FloatArray {
val out = FloatArray(count)
for (i in 0 until count) out[i] = samples[offset + i] / 32768.0f
return out
}
/** S16LE bytes to [-1, 1] floats (an odd trailing byte is ignored). */
@JvmStatic
fun pcm16leToFloat(bytes: ByteArray, offset: Int, byteCount: Int): FloatArray {
val count = byteCount / 2
val out = FloatArray(count)
for (i in 0 until count) {
val base = offset + i * 2
out[i] = pcm16le(bytes[base], bytes[base + 1]) / 32768.0f
}
return out
}
@JvmStatic
fun pcm16leToFloat(bytes: ByteArray): FloatArray = pcm16leToFloat(bytes, 0, bytes.size)
/** One little-endian S16 sample from its two bytes. */
@JvmStatic
fun pcm16le(low: Byte, high: Byte): Short = ((low.toInt() and 0xff) or (high.toInt() shl 8)).toShort()
/** S16 samples to S16LE bytes. */
@JvmStatic
fun pcm16ToLittleEndian(samples: ShortArray, offset: Int, count: Int): ByteArray {
val le = ByteArray(count * 2)
for (i in 0 until count) {
val s = samples[offset + i].toInt()
le[i * 2] = (s and 0xff).toByte()
le[i * 2 + 1] = ((s shr 8) and 0xff).toByte()
}
return le
}
/** Root-mean-square of `samples[offset until offset + count]` in S16 units. */
@JvmStatic
fun rms(samples: ShortArray, offset: Int, count: Int): Double {
var sumSquares = 0.0
for (i in 0 until count) {
val s = samples[offset + i].toDouble()
sumSquares += s * s
}
return sqrt(sumSquares / maxOf(1, count))
}
}
@@ -0,0 +1,380 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.math.max
import kotlin.math.min
import kotlin.math.sqrt
/** Receives segmenter events synchronously on the thread feeding [CaptionSegmenter.processChunk]. */
interface CaptionSegmentSink {
fun onSpeechStart(startMs: Long)
/** [samples] is the segmenter's own buffer, valid only for the duration of the call. */
fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double)
}
/**
* Splits decoded mic PCM into utterances with an adaptive energy gate: the noise floor is a
* rolling mean tracked while idle, onset fires at ONSET_FACTOR above it (keeping PRE_ROLL_MS of
* idle audio so leading consonants survive), and the utterance ends after [silenceMs] below
* RELEASE_FACTOR or at MAX_UTTERANCE_MS. Boundaries are measured on the 16 kHz sample clock,
* not the wall clock: BLE delivers mic packets in bursts.
*/
class CaptionSegmenter(private val sink: CaptionSegmentSink) {
companion object {
const val SAMPLE_RATE = 16000
const val ONSET_FACTOR = 3.0
const val RELEASE_FACTOR = 1.8
const val MIN_RMS = 220.0
const val NOISE_EMA_ALPHA = 0.05
const val PRE_ROLL_MS = 400
const val DEFAULT_SILENCE_MS = 800
const val MIN_SILENCE_MS = 200
const val MAX_SILENCE_MS = 3000
const val MIN_UTTERANCE_MS = 350
const val MAX_UTTERANCE_MS = 15000
}
@Volatile
var silenceMs = DEFAULT_SILENCE_MS
set(value) {
field = max(MIN_SILENCE_MS, min(MAX_SILENCE_MS, value))
}
private var totalSamples: Long = 0
private var noiseFloor = 0.0
private var inUtterance = false
private var utteranceStartSample: Long = 0
private var silenceRunSamples: Long = 0
private var utterancePeakRms = 0.0
private val utterance = ShortArray(SAMPLE_RATE * (MAX_UTTERANCE_MS / 1000))
private var utteranceLength = 0
private val preRoll = ShortArray(SAMPLE_RATE * PRE_ROLL_MS / 1000)
private var preRollLength = 0
val isInUtterance: Boolean
get() = inUtterance
fun reset() {
totalSamples = 0
noiseFloor = 0.0
inUtterance = false
utteranceLength = 0
preRollLength = 0
silenceRunSamples = 0
utterancePeakRms = 0.0
}
/** Feeds one chunk of S16LE PCM bytes (any chunking). */
fun processChunk(chunk: ByteArray) {
val count = chunk.size / 2
val pcm = ShortArray(count)
for (i in 0 until count) {
pcm[i] = AudioSegmentation.pcm16le(chunk[i * 2], chunk[i * 2 + 1])
}
processChunk(pcm, count)
}
fun processChunk(pcm: ShortArray, count: Int) {
var sumSquares = 0.0
for (i in 0 until count) {
val s = pcm[i].toDouble()
sumSquares += s * s
}
val rms = sqrt(sumSquares / max(1, count))
totalSamples += count.toLong()
if (!inUtterance) {
// Track the noise floor only while idle so speech doesn't raise it.
noiseFloor = if (noiseFloor == 0.0) rms else noiseFloor * (1 - NOISE_EMA_ALPHA) + rms * NOISE_EMA_ALPHA
val threshold = max(noiseFloor, MIN_RMS)
if (rms >= threshold * ONSET_FACTOR) {
inUtterance = true
utteranceLength = 0
silenceRunSamples = 0
utterancePeakRms = rms
appendUtterance(preRoll, preRollLength)
utteranceStartSample = max(0L, totalSamples - count - preRollLength)
appendUtterance(pcm, count)
sink.onSpeechStart(utteranceStartSample * 1000 / SAMPLE_RATE)
} else {
appendPreRoll(pcm, count)
}
return
}
appendUtterance(pcm, count)
utterancePeakRms = max(utterancePeakRms, rms)
val threshold = max(noiseFloor, MIN_RMS)
if (rms < threshold * RELEASE_FACTOR) {
silenceRunSamples += count.toLong()
} else {
silenceRunSamples = 0
}
val utteranceMs = utteranceLength.toLong() * 1000 / SAMPLE_RATE
val silenceEnded = silenceRunSamples * 1000L / SAMPLE_RATE >= silenceMs
if (silenceEnded || utteranceMs >= MAX_UTTERANCE_MS) {
finalizeUtterance()
inUtterance = false
preRollLength = 0
}
}
/** Emits a trailing utterance (if any) so its text isn't lost on stop. */
fun flush() {
if (inUtterance && utteranceLength > 0) {
finalizeUtterance()
}
inUtterance = false
preRollLength = 0
}
private fun appendPreRoll(pcm: ShortArray, count: Int) {
// Keep the last PRE_ROLL_MS of idle audio so onset consonants survive.
if (count >= preRoll.size) {
pcm.copyInto(preRoll, 0, count - preRoll.size, count)
preRollLength = preRoll.size
return
}
val keep = min(preRollLength, preRoll.size - count)
preRoll.copyInto(preRoll, 0, preRollLength - keep, preRollLength)
pcm.copyInto(preRoll, keep, 0, count)
preRollLength = keep + count
}
private fun appendUtterance(pcm: ShortArray, count: Int) {
val room = utterance.size - utteranceLength
val copied = min(room, count)
if (copied > 0) {
pcm.copyInto(utterance, utteranceLength, 0, copied)
utteranceLength += copied
}
}
private fun finalizeUtterance() {
val length = utteranceLength
val startMs = utteranceStartSample * 1000 / SAMPLE_RATE
val endMs = (utteranceStartSample + length) * 1000 / SAMPLE_RATE
if (endMs - startMs < MIN_UTTERANCE_MS) {
return
}
sink.onUtterance(utterance, length, startMs, endMs, utterancePeakRms)
}
}
/**
* Transcribes one utterance in model-safe segments cut at the quietest window (Moonshine v2
* fails past ~9.1 s of input) and attaches a voice-print from at most its first 10 s.
*/
class UtteranceDecoder(private val transcriber: OfflineTranscriber?, private val embedder: SpeakerEmbedder?) {
companion object {
const val SAMPLE_RATE = 16000
const val DECODE_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8
const val CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2
const val CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000
const val NORMALIZE_TARGET_PEAK = 0.9f
const val NORMALIZE_MAX_GAIN = 30f
// Voice-prints degrade on very long inputs; embed at most the first 10 s.
const val EMBED_MAX_SAMPLES = SAMPLE_RATE * 10
}
val hasTranscriber: Boolean
get() = transcriber != null
fun recognize(utterance: ShortArray, length: Int): String {
val recognizer = transcriber
if (recognizer == null || length <= 0) {
return ""
}
val joined = StringBuilder()
var offset = 0
while (offset < length) {
val remaining = length - offset
var segment = min(remaining, DECODE_SEGMENT_MAX_SAMPLES)
if (remaining > DECODE_SEGMENT_MAX_SAMPLES) {
segment = AudioSegmentation.quietestCutPoint(utterance, offset, segment, CUT_SEARCH_SAMPLES, CUT_WINDOW_SAMPLES)
}
val samples = AudioSegmentation.pcm16ToFloat(utterance, offset, segment)
AudioSegmentation.normalizePeak(samples, NORMALIZE_TARGET_PEAK, NORMALIZE_MAX_GAIN)
val part = recognizer.recognize(samples, samples.size).trim()
if (part.isNotEmpty()) {
if (joined.isNotEmpty() && ".,!?;:%)]}".indexOf(part[0]) < 0) {
joined.append(' ')
}
joined.append(part)
}
offset += segment
}
return joined.toString().trim()
}
fun embed(utterance: ShortArray, length: Int): FloatArray? {
val currentEmbedder = embedder
if (currentEmbedder == null || length <= 0) {
return null
}
val count = min(length, EMBED_MAX_SAMPLES)
return currentEmbedder.embed(AudioSegmentation.pcm16ToLittleEndian(utterance, 0, count), SAMPLE_RATE)
}
fun release() {
transcriber?.release()
embedder?.release()
}
}
/** Platform services for [CaptionSession]. */
interface CaptionHost {
val dispatcher: CallbackDispatcher
/** Loads the caption recognizer from [modelDir], or null when the model files are absent. */
fun loadTranscriber(modelDir: String): OfflineTranscriber?
/** Loads the speaker-embedding model, or null when the file is absent. */
fun loadEmbedder(modelPath: String): SpeakerEmbedder?
}
/**
* Continuous captioning over pushed mic PCM: queue, worker thread (owned here, via the shared
* [startThread]), model lifecycle, segmentation and per-utterance decode/embedding, with
* listener callbacks marshalled through the host dispatcher.
*/
class CaptionSession(
private val host: CaptionHost,
private val platform: ProtocolPlatform = protocolPlatform(),
private val threadName: String = "FaceclawCaptionEngine",
private val logError: (String, Throwable) -> Unit = { message, error -> PlatformLog.e(TAG, message, error) },
) : CaptionSegmentSink {
companion object {
const val TAG = "FaceclawCaptions"
const val MAX_QUEUE_PACKETS = 200
}
private val lock = platform.createLock()
private val queue = DroppingPacketQueue<ByteArray>(MAX_QUEUE_PACKETS, platform)
private val segmenter = CaptionSegmenter(this)
@Volatile
private var listener: FaceclawCaptionEngineListener? = null
@Volatile
private var asrModelDir: String? = null
@Volatile
private var speakerModelPath: String? = null
@Volatile
private var started = false
private var workerFinished: Latch? = null
@Volatile
private var workerThreadName: String? = null
private var decoder: UtteranceDecoder? = null
fun setListener(listener: FaceclawCaptionEngineListener?) {
this.listener = listener
}
fun setAsrModelDir(dir: String?) {
this.asrModelDir = dir
}
fun setSpeakerModelPath(path: String?) {
this.speakerModelPath = path
}
fun setSilenceMs(ms: Int) {
segmenter.silenceMs = ms
}
fun start() {
lock.withLock {
if (started) {
return
}
started = true
val finished = Latch(1, platform)
workerFinished = finished
startThread(threadName, true) { runLoop(finished) }
}
}
/** Stops and waits up to 2 s for the worker unless called from it. */
fun stop() {
var finished: Latch? = null
var workerName: String? = null
lock.withLock {
if (!started) {
return
}
started = false
finished = workerFinished
workerName = workerThreadName
}
queue.wake()
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
finished.await(2000)
}
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
fun acceptPcm(pcm16le: ByteArray?) {
if (!started || pcm16le == null || pcm16le.size < 2) {
return
}
queue.put(pcm16le)
}
private fun runLoop(finished: Latch) {
workerThreadName = currentThreadName()
try {
loadModels()
segmenter.reset()
while (started) {
val chunk = queue.take(250) { started } ?: continue
segmenter.processChunk(chunk)
}
// Flush a trailing utterance so its text isn't lost on stop.
segmenter.flush()
} catch (t: Throwable) {
logError("caption engine failed", t)
emitStatus("Captions failed: " + t.message)
} finally {
decoder?.release()
decoder = null
lock.withLock {
workerFinished = null
workerThreadName = null
}
finished.countDown()
}
}
private fun loadModels() {
val modelDir = asrModelDir
val transcriber = if (modelDir != null) {
emitStatus("Loading caption model...")
host.loadTranscriber(modelDir)
} else {
null
}
val embedModel = speakerModelPath
val embedder = if (embedModel != null) host.loadEmbedder(embedModel) else null
decoder = UtteranceDecoder(transcriber, embedder)
emitStatus(if (transcriber != null) "Captions listening..." else "Captions listening (no ASR model)...")
}
override fun onSpeechStart(startMs: Long) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeechStart(startMs) }
}
override fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) {
val currentDecoder = decoder ?: return
val text = currentDecoder.recognize(samples, length)
val embedding = currentDecoder.embed(samples, length)
val currentListener = listener ?: return
val embeddingCopy = embedding?.copyOf()
host.dispatcher.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) }
}
private fun emitStatus(status: String?) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onStatus(status) }
}
}
@@ -0,0 +1,93 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/** Decodes the LC3 frames of one G2 packet into [pcmOut]; returns samples written, <= 0 on error. */
interface Lc3Codec {
fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int
}
/**
* G2 microphone packet framing shared by the Android JNI decoder and the iOS port: length
* check, counter-based duplicate/late/missing accounting, trailer parsing and statistics.
* The codec itself (liblc3) is injected.
*
* Both arms may relay the same packet, so a counter gap of 0 or >= 128 (a late copy, including
* across the 255 -> 0 wrap) is dropped before it can disturb the decoder's history. The
* former Android decoder only dropped gap == 0; this adopts the wrap-safe rule the iOS
* decoder already used.
*/
class Lc3PacketFramer(private val codec: Lc3Codec) {
companion object {
const val SAMPLE_RATE = 16000
const val FRAME_US = 10000
const val FRAME_BYTES = 40
const val FRAMES_PER_PACKET = 5
const val PACKET_BYTES = 205
const val COUNTER_OFFSET = 204
const val SAMPLES_PER_FRAME = 160
const val SAMPLES_PER_PACKET = FRAMES_PER_PACKET * SAMPLES_PER_FRAME
const val LC3_BYTES = FRAME_BYTES * FRAMES_PER_PACKET
// Trailer layout (firmware service_audio.c): after the 200 LC3 bytes the
// glasses DSP appends two signed 16-bit LE values computed on the raw
// stereo capture before the mono downmix — a signal-strength ratio and the
// asin-derived direction-of-arrival in degrees — then the packet counter.
const val SSR_OFFSET = 200
const val ANGLE_OFFSET = 202
/** Counter gaps at or beyond this are late copies, not losses. */
const val LATE_GAP = 128
@JvmStatic
fun readTrailerS16(packet: ByteArray, offset: Int): Int =
((packet[offset].toInt() and 0xff) or (packet[offset + 1].toInt() shl 8)).toShort().toInt()
}
private var lastCounter = -1
var realPackets: Long = 0
private set
var duplicatePackets: Long = 0
private set
var missingPackets: Long = 0
private set
var decodeErrors: Long = 0
private set
/** Firmware signal-strength ratio from the most recent decoded packet. */
var lastSsr: Int = 0
private set
/** Firmware direction-of-arrival (signed degrees) from the most recent decoded packet. */
var lastAngleDegrees: Int = 0
private set
/**
* Returns the number of PCM samples written to [pcmOut], or 0 when the packet was dropped
* (wrong length, duplicate/late, or codec failure; see the counters).
*/
fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
if (packet == null || packet.size != PACKET_BYTES) {
decodeErrors++
return 0
}
if (pcmOut == null || pcmOut.size < SAMPLES_PER_PACKET) {
throw IllegalArgumentException("pcmOut must hold $SAMPLES_PER_PACKET samples")
}
val counter = packet[COUNTER_OFFSET].toInt() and 0xff
if (lastCounter >= 0) {
val gap = (counter - lastCounter) and 0xff
if (gap == 0 || gap >= LATE_GAP) {
duplicatePackets++
return 0
}
missingPackets += (gap - 1).toLong()
}
val decoded = codec.decodeFrames(packet, 0, LC3_BYTES, pcmOut)
if (decoded <= 0) {
decodeErrors++
return 0
}
realPackets++
lastCounter = counter
lastSsr = readTrailerS16(packet, SSR_OFFSET)
lastAngleDegrees = readTrailerS16(packet, ANGLE_OFFSET)
return decoded
}
}
@@ -0,0 +1,833 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmStatic
import kotlin.math.abs
import kotlin.math.max
import kotlin.math.min
/** Platform services the voice-capture session needs; supplied by the Android/iOS adapter. */
interface VoiceCaptureHost {
val dispatcher: CallbackDispatcher
/** Runs first on the worker thread of every session (Android deletes legacy model files here). */
fun onSessionStarting()
/** True when a glasses session can carry microphone packets (false when no communicator is set). */
fun isG2SessionReady(): Boolean
/** Whether the glasses are still delivering packets; see [VoiceCaptureSession.isCapturing]. */
fun isG2AudioCaptureActive(): Boolean
/** The glasses packet source, or null when no communicator is available. */
fun g2AudioSource(): PacketAudioSource?
fun createPacketDecoder(): AudioPacketDecoder
/** Opens the phone microphone at 16 kHz mono PCM16, or null when it cannot start. */
fun openPhoneMic(): PcmAudioSource?
/** Whether the on-device model files for [kind] are present (downloaded). */
fun hasTranscriberModel(kind: VoiceModelKind): Boolean
/** Loads the recognizer for [kind]; throws when the model cannot be loaded. */
fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber
/** The speaker-embedding model at [modelPath] (may be shared/cached across sessions). */
fun speakerEmbedder(modelPath: String): SpeakerEmbedder
/** Where a saved recording goes, or null when recordings cannot be stored. */
fun recordingSink(): RecordingSink?
}
/**
* Voice-input capture pipeline shared by both platforms: LC3 packet queue with drop/late
* accounting, firmware-DoA beam gate, per-chunk processing (suppression, recording, speaker
* verification buffer, endpointing, PCM emit), the segmented on-device transcript machine with
* REPLACE-semantics partials, and speaker verification at session end. The session owns its
* worker thread (started through the shared [startThread]); listener callbacks go through the
* host's [CallbackDispatcher] so the adapter decides the delivery thread.
*/
class VoiceCaptureSession(
private val host: VoiceCaptureHost,
private val platform: ProtocolPlatform = protocolPlatform(),
private val threadName: String = "FaceclawVoiceController",
private val logInfo: (String) -> Unit = { PlatformLog.i(TAG, it) },
private val logWarn: (String) -> Unit = { PlatformLog.w(TAG, it) },
private val logError: (String, Throwable) -> Unit = { message, error -> PlatformLog.e(TAG, message, error) },
) {
companion object {
const val TAG = "FaceclawVoice"
const val SAMPLE_RATE = 16000
const val MAX_AUDIO_QUEUE_PACKETS = 80
const val EXPECTED_PACKET_INTERVAL_MS = 50
const val LATE_PACKET_INTERVAL_MS = 90
const val STATS_INTERVAL_MS = 5_000
// Push-to-talk utterance boundaries come from the button. We re-decode the
// current audio segment in full for each live partial and emit the complete
// utterance text (REPLACE, never a delta). The sherpa Moonshine v2 decoder
// used here fails once a single input grows past roughly 9.1 seconds, so
// longer utterances are committed in model-safe segments.
const val TRANSCRIPT_DECODE_INTERVAL_MS = 700
const val TRANSCRIPT_MIN_SAMPLES = SAMPLE_RATE / 3
const val TRANSCRIPT_SEGMENT_MAX_SAMPLES = SAMPLE_RATE * 8
// When a segment fills, cut at the quietest window within the last
// TRANSCRIPT_CUT_SEARCH_SAMPLES rather than mid-word at the 8s mark; the
// audio after the cut carries over into the next segment.
const val TRANSCRIPT_CUT_SEARCH_SAMPLES = SAMPLE_RATE * 2
const val TRANSCRIPT_CUT_WINDOW_SAMPLES = SAMPLE_RATE * 30 / 1000
// Glasses-mic PCM peaks around 0.1 full scale, and at that level the
// quantized Moonshine model often returns empty or garbled text. Boost
// each decode window toward this peak, with a gain cap so near-silent
// buffers aren't amplified into pure noise.
const val TRANSCRIPT_NORMALIZE_TARGET_PEAK = 0.9f
const val TRANSCRIPT_NORMALIZE_MAX_GAIN = 30f
const val TRANSCRIPT_LOG_PREVIEW_CHARS = 80
// Below this peak amplitude (pre-normalization, of a full-scale +/-1.0f
// buffer) a segment is treated as silence and never reaches the Whisper
// recognizer at all, rather than risking a hallucinated non-answer. Picked
// conservatively low (well under typical mic noise floor already seen in
// this pipeline's normalization target) -- UNTESTED on real hardware, tune
// against real glasses captures rather than trusting this number.
const val WHISPER_SILENCE_PEAK_THRESHOLD = 0.01f
// Speaker verification against the enrolled wearer voice-print ("my voice
// only" command gating). Configured before start(); the utterance PCM is
// buffered (capped) and verified once at session end.
const val VERIFY_MAX_SAMPLES = SAMPLE_RATE * 10
const val VERIFY_MIN_SAMPLES = SAMPLE_RATE
const val DEFAULT_VERIFY_THRESHOLD = 0.8f
/** Samples per G2 packet; the decode buffer the worker reuses. */
const val PACKET_SAMPLES = Lc3PacketFramer.SAMPLES_PER_PACKET
/** Joins transcript pieces, attaching leading punctuation to the previous text. */
@JvmStatic
fun joinTranscript(prefix: String?, suffix: String?): String {
if (prefix.isNullOrEmpty()) {
return suffix ?: ""
}
if (suffix.isNullOrEmpty()) {
return prefix
}
val first = suffix[0]
val attachesToPrevious = ".,!?;:%)]}".indexOf(first) >= 0
return prefix + (if (attachesToPrevious) "" else " ") + suffix
}
@JvmStatic
fun parseModelKind(kind: String?): VoiceModelKind =
if ("whisper" == kind) VoiceModelKind.WHISPER else VoiceModelKind.MOONSHINE
}
enum class Mode {
ONBOARD, // on-phone transcription (Moonshine or Whisper; see onboardModelKind)
CLOUD, // decode locally, emit PCM for a cloud recognizer on the TS side
}
private class AudioPacket(val data: ByteArray, arm: String?, val arrivalMs: Long) {
val arm: String = arm ?: "?"
}
private val lock = platform.createLock()
private val audioQueue = DroppingPacketQueue<AudioPacket>(MAX_AUDIO_QUEUE_PACKETS, platform)
@Volatile
private var listener: FaceclawVoiceControllerListener? = null
@Volatile
private var started = false
// Set once the worker has the mic enabled for this session. Read and
// written under `lock`, so it flips with `started` atomically.
private var audioStarted = false
private var workerFinished: Latch? = null
@Volatile
private var workerThreadName: String? = null
// Capture from the phone's own microphone instead of the G2 over BLE
// (preview-only mode). Latched into activePhoneMic at start() (under `lock`)
// so a mid-session setter call can't switch pipelines underneath the worker.
@Volatile
private var usePhoneMic = false
private var activePhoneMic = false
private var mode = Mode.CLOUD
@Volatile
private var onboardModelKind = VoiceModelKind.MOONSHINE
private var recognizer: OfflineTranscriber? = null
private var packetDecoder: AudioPacketDecoder? = null
private val transcriptSamples = FloatArray(TRANSCRIPT_SEGMENT_MAX_SAMPLES)
private var transcriptSampleCount = 0
private var committedTranscriptSampleCount: Long = 0
private var committedTranscript = ""
private var currentSegmentTranscript = ""
private var lastTranscriptDecodeAtMs: Long = 0
private var lastTranscript = ""
@Volatile
private var saveRecordings = false
@Volatile
private var endpointing = false
private val endpointDetector = VoiceEndpointDetector()
private var recordingPcm: ByteSink? = null
@Volatile
private var verifySpeakerModelPath: String? = null
@Volatile
private var verifyWearerEmbedding: FloatArray? = null
@Volatile
private var verifyThreshold = DEFAULT_VERIFY_THRESHOLD
private var verifyBuffer: ShortArray? = null
private var verifyCount = 0
// Global mic processing (Microphones app config): spectral noise
// suppression and firmware-DoA beam gating, applied to every capture
// session that opts in. The raw tap opts out.
@Volatile
private var suppressionEnabled = false
@Volatile
private var beamFilterEnabled = false
@Volatile
private var beamCenterDeg = 0
@Volatile
private var beamHalfWidthDeg = 180
private var suppressor: FaceclawNoiseSuppressor? = null
private var queuedPackets: Long = 0
private var decodedSamples: Long = 0
private var latePackets: Long = 0
private var wrongArmPackets: Long = 0
private var lastPacketArrivalMs: Long = 0
private var maxInterPacketMs: Long = 0
private var lastStatsAtMs: Long = 0
fun setListener(listener: FaceclawVoiceControllerListener?) {
this.listener = listener
}
fun setUsePhoneMic(usePhoneMic: Boolean) {
this.usePhoneMic = usePhoneMic
}
fun setSaveRecordings(saveRecordings: Boolean) {
this.saveRecordings = saveRecordings
}
fun setOnboardModelKind(kind: String?) {
this.onboardModelKind = parseModelKind(kind)
}
fun setEndpointing(endpointing: Boolean) {
this.endpointing = endpointing
}
fun setSpeakerVerification(speakerModelPath: String?, wearerEmbedding: FloatArray?, threshold: Float) {
this.verifySpeakerModelPath = speakerModelPath
this.verifyWearerEmbedding = wearerEmbedding
this.verifyThreshold = if (threshold > 0) threshold else DEFAULT_VERIFY_THRESHOLD
}
fun clearSpeakerVerification() {
this.verifySpeakerModelPath = null
this.verifyWearerEmbedding = null
}
fun setNoiseSuppression(enabled: Boolean) {
this.suppressionEnabled = enabled
}
fun setBeamFilter(enabled: Boolean, centerDeg: Int, halfWidthDeg: Int) {
this.beamFilterEnabled = enabled
this.beamCenterDeg = centerDeg
this.beamHalfWidthDeg = max(5, min(180, halfWidthDeg))
}
/** Direction gating: true when [angleDegrees] lies within centerDeg ± halfWidthDeg (wrap-safe). */
fun withinBeam(angleDegrees: Int): Boolean {
var delta = angleDegrees - beamCenterDeg
while (delta > 180) delta -= 360
while (delta < -180) delta += 360
return abs(delta) <= beamHalfWidthDeg
}
private fun applySuppression(pcm: ShortArray, count: Int) {
try {
if (suppressor == null) {
suppressor = FaceclawNoiseSuppressor(SAMPLE_RATE)
}
val le = AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count)
val cleaned = suppressor!!.process(le)!!
val cleanedCount = min(count, cleaned.size / 2)
for (i in 0 until cleanedCount) {
pcm[i] = AudioSegmentation.pcm16le(cleaned[i * 2], cleaned[i * 2 + 1])
}
} catch (t: Throwable) {
logWarn("noise suppression failed; passing audio through (${t.message})")
suppressionEnabled = false
}
}
fun start(requestedMode: String?) {
start(requestedMode, 0)
}
fun start(requestedMode: String?, captureId: Int) {
lock.withLock {
if (started) {
emitStatus("Voice control is already listening.")
emitStopped(captureId)
return
}
if (!usePhoneMic && !host.isG2SessionReady()) {
emitStatus("Voice control needs an active G2 connection.")
emitStopped(captureId)
return
}
mode = parseMode(requestedMode)
activePhoneMic = usePhoneMic
started = true
audioStarted = false
val finished = Latch(1, platform)
workerFinished = finished
startThread(threadName, true) { runLoop(captureId, finished) }
}
}
/**
* Whether mic audio is actually flowing. start() only records intent: the
* enable lives in the glasses' EvenHub session, so a transport drop or a
* session suspend can leave this session started with a worker that will
* never see another packet.
*/
fun isCapturing(): Boolean {
var audioUp = false
var phoneMic = false
lock.withLock {
if (!started) {
return false
}
audioUp = audioStarted
phoneMic = activePhoneMic
}
if (!audioUp) {
// The worker is still bringing the mic up; report it as running so
// a concurrent request shares it instead of restarting it.
return true
}
if (phoneMic) {
// The phone mic has no session to lose the enable to; it runs until stop().
return true
}
return host.isG2AudioCaptureActive()
}
/**
* Stops the session and waits up to 1.5 s for the worker to finish unless
* called from the worker itself. The worker checks the stop flag between
* packets (every packet or 250 ms wait) and between phone-mic reads (50 ms).
*/
fun stop() {
var finished: Latch? = null
var workerName: String? = null
lock.withLock {
if (!started) {
return
}
started = false
finished = workerFinished
workerName = workerThreadName
}
stopG2Audio()
audioQueue.wake()
if (finished != null && (workerName == null || currentThreadName() != workerName)) {
finished.await(1500)
}
}
fun close() {
stop()
}
private fun parseMode(requestedMode: String?): Mode {
if ("cloud" == requestedMode) {
return Mode.CLOUD
}
return Mode.ONBOARD
}
private fun runLoop(captureId: Int, finished: Latch) {
workerThreadName = currentThreadName()
try {
host.onSessionStarting()
val currentMode = mode
val currentOnboardKind = onboardModelKind
if (currentMode == Mode.ONBOARD) {
if (!host.hasTranscriberModel(currentOnboardKind)) {
emitStatus("Voice model not downloaded (see Settings > Voice).")
return
}
emitStatus("Loading transcription model...")
recognizer = host.loadTranscriber(currentOnboardKind)
resetTranscriptState()
lastTranscript = ""
}
endpointDetector.reset()
suppressor?.reset()
recordingPcm = if (saveRecordings) ByteSink(SAMPLE_RATE * 2 * 4) else null
val verifying = verifySpeakerModelPath != null && verifyWearerEmbedding != null
verifyBuffer = if (verifying) ShortArray(VERIFY_MAX_SAMPLES) else null
verifyCount = 0
if (activePhoneMic) {
val record = host.openPhoneMic()
if (record == null) {
emitStatus("Could not start the phone microphone.")
return
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
try {
processPhoneAudio(record)
} finally {
record.close()
}
} else {
packetDecoder = host.createPacketDecoder()
if (!startG2Audio()) {
emitStatus("Could not start G2 microphone input.")
return
}
lock.withLock { audioStarted = true }
emitStatus(if (currentMode == Mode.CLOUD) "Listening (cloud)..." else "Listening...")
processG2Audio()
}
// Verification result must precede the final transcript so the
// TS bridge can suppress a non-wearer command before it is acted
// on (the callbacks are posted in order to the dispatcher).
runSpeakerVerification()
// Button released / stop requested: emit one final full-utterance
// transcript so the UI can freeze it.
if (currentMode == Mode.ONBOARD) {
decodeTranscript(true)
}
} catch (error: Throwable) {
logError("Voice control failed", error)
emitStatus("Voice control failed: " + error.message)
} finally {
stopG2Audio()
writeRecordingIfAny()
releaseRecognizer()
releaseDecoder()
lock.withLock {
started = false
audioStarted = false
workerFinished = null
workerThreadName = null
}
emitStopped(captureId)
finished.countDown()
}
}
private fun appendRecording(pcm: ShortArray, count: Int) {
val out = recordingPcm ?: return
out.write(AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count))
}
private fun writeRecordingIfAny() {
val out = recordingPcm
recordingPcm = null
if (out == null || out.size() == 0) {
return
}
val sink = host.recordingSink() ?: return
try {
sink.save(out.toByteArray(), SAMPLE_RATE)
} catch (t: Throwable) {
logWarn("failed to save voice recording (${t.message})")
}
}
private fun startG2Audio(): Boolean {
val source = host.g2AudioSource() ?: return false
resetAudioStats()
audioQueue.clear()
return source.start(object : FaceclawAudioPacketListener {
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
queueAudioPacket(data, arm, arrivalMs)
}
})
}
private fun processG2Audio() {
val pcm = ShortArray(PACKET_SAMPLES)
while (started) {
val currentDecoder = packetDecoder ?: return
val packet = takeAudioPacket()
if (packet == null) {
continue
}
val count = currentDecoder.decodePacket(packet.data, pcm)
if (count <= 0) {
maybeEmitAudioStats(false)
continue
}
decodedSamples += count.toLong()
// The beam filter drops packets whose firmware direction-of-arrival
// falls outside the listening wedge (isolating the aimed talker for
// every consumer, recognition included); the spectral noise
// suppressor then cleans what remains before it reaches the
// recognizer, cloud PCM, endpointing, or speaker verification.
val angleDegrees = currentDecoder.lastAngleDegrees
val ssr = currentDecoder.lastSsr
if (beamFilterEnabled && ssr > 0 && !withinBeam(angleDegrees)) {
emitFrameMeta(angleDegrees, ssr)
maybeEmitAudioStats(false)
continue
}
processPcmChunk(pcm, count, angleDegrees, ssr, true)
maybeEmitAudioStats(false)
}
}
/**
* Per-chunk processing shared by the G2 and phone-mic paths, downstream of
* decode and the beam filter. hasFrameMeta is false for the phone mic,
* which has no firmware DSP metadata to report.
*/
internal fun processPcmChunk(pcm: ShortArray, count: Int, angleDegrees: Int, ssr: Int, hasFrameMeta: Boolean) {
if (suppressionEnabled) {
applySuppression(pcm, count)
}
if (recordingPcm != null) {
appendRecording(pcm, count)
}
val verifyBuffer = this.verifyBuffer
if (verifyBuffer != null && verifyCount < VERIFY_MAX_SAMPLES) {
val copied = min(count, VERIFY_MAX_SAMPLES - verifyCount)
pcm.copyInto(verifyBuffer, verifyCount, 0, copied)
verifyCount += copied
}
if (endpointing && endpointDetector.accept(pcm, count)) {
emitSpeechEnd()
}
// PCM and frame metadata flow in every mode so levels, recording,
// and the Microphones radar keep working alongside onboard ASR.
emitPcm(pcm, count)
if (hasFrameMeta) {
emitFrameMeta(angleDegrees, ssr)
}
if (mode != Mode.CLOUD) {
processRecognizer(AudioSegmentation.pcm16ToFloat(pcm, 0, count))
}
}
/**
* Phone-mic capture loop: no LC3 decode, no arm bookkeeping, no frame
* metadata. The blocking read returns every chunk (50 ms), which bounds how
* long a stop() waits for the loop to notice `started` dropped.
*/
private fun processPhoneAudio(record: PcmAudioSource) {
val pcm = ShortArray(SAMPLE_RATE / 20)
while (started) {
val read = record.read(pcm)
if (read < 0) {
logWarn("phone mic read failed: $read")
return
}
if (read == 0) {
continue
}
decodedSamples += read.toLong()
processPcmChunk(pcm, read, 0, 0, false)
}
}
private fun processRecognizer(samples: FloatArray) {
appendTranscriptSamples(samples)
// Each Whisper call re-encodes the whole buffer plus ~10s of tail padding,
// so it skips the live-partial redecode Moonshine does on this interval and
// only decodes when a segment commits (8s buffer fill) or the utterance ends
// (decodeTranscript(true) in runLoop()). No live preview text in Whisper
// mode -- status stays "Listening..." until release. Deliberate tradeoff.
if (onboardModelKind == VoiceModelKind.WHISPER) {
return
}
val now = platform.elapsedRealtimeMs()
if (transcriptSampleCount >= TRANSCRIPT_MIN_SAMPLES &&
now - lastTranscriptDecodeAtMs >= TRANSCRIPT_DECODE_INTERVAL_MS
) {
decodeTranscript(false)
lastTranscriptDecodeAtMs = now
}
}
private fun appendTranscriptSamples(samples: FloatArray) {
var sourceOffset = 0
while (sourceOffset < samples.size) {
val available = TRANSCRIPT_SEGMENT_MAX_SAMPLES - transcriptSampleCount
val count = min(available, samples.size - sourceOffset)
samples.copyInto(transcriptSamples, transcriptSampleCount, sourceOffset, sourceOffset + count)
transcriptSampleCount += count
sourceOffset += count
if (transcriptSampleCount == TRANSCRIPT_SEGMENT_MAX_SAMPLES) {
commitTranscriptSegment()
}
}
}
/**
* Decode the current model-safe segment and emit the best transcript of the
* complete utterance (REPLACE semantics — the caller displays it as-is).
*/
private fun decodeTranscript(isFinal: Boolean) {
if (recognizer == null || transcriptSampleCount <= 0) {
if (isFinal) {
emitTranscript(lastTranscript, true)
}
return
}
val segmentSampleCount = transcriptSampleCount
var segmentText = recognizeTranscriptSegment(segmentSampleCount)
if (segmentText.isNotEmpty()) {
currentSegmentTranscript = segmentText
} else {
segmentText = currentSegmentTranscript
}
val text = joinTranscript(committedTranscript, segmentText)
lastTranscript = text
logTranscriptDecode(isFinal, segmentSampleCount, text)
emitTranscript(text, isFinal)
}
/**
* Finalize a full segment before accepting more audio. This keeps every
* Moonshine invocation below its failing sequence length while retaining
* all earlier text in the replace-semantics preview.
*/
private fun commitTranscriptSegment() {
val cut = AudioSegmentation.quietestCutPoint(
transcriptSamples, transcriptSampleCount, TRANSCRIPT_CUT_SEARCH_SAMPLES, TRANSCRIPT_CUT_WINDOW_SAMPLES,
)
var segmentText = recognizeTranscriptSegment(cut)
if (segmentText.isEmpty()) {
// Fallback text came from partial decodes of the full buffer, so it
// may include words from the carried-over tail; rare now that decode
// windows are peak-normalized.
segmentText = currentSegmentTranscript
}
committedTranscript = joinTranscript(committedTranscript, segmentText)
currentSegmentTranscript = ""
committedTranscriptSampleCount += cut.toLong()
val tail = transcriptSampleCount - cut
transcriptSamples.copyInto(transcriptSamples, 0, cut, cut + tail)
transcriptSampleCount = tail
lastTranscript = committedTranscript
lastTranscriptDecodeAtMs = platform.elapsedRealtimeMs()
logTranscriptDecode(false, cut, committedTranscript)
emitTranscript(committedTranscript, false)
}
private fun recognizeTranscriptSegment(sampleCount: Int): String {
val currentRecognizer = recognizer
if (currentRecognizer == null || sampleCount <= 0) {
return ""
}
val segment = transcriptSamples.copyOf(sampleCount)
// Whisper hallucinates text on near-silent input; gate it on the
// PRE-normalization peak, since normalizePeak() below would otherwise
// amplify true silence right up to the target level. Moonshine does not
// share this failure mode in practice, so it is left unchanged.
if (onboardModelKind == VoiceModelKind.WHISPER &&
AudioSegmentation.peakAmplitude(segment) < WHISPER_SILENCE_PEAK_THRESHOLD
) {
return ""
}
AudioSegmentation.normalizePeak(segment, TRANSCRIPT_NORMALIZE_TARGET_PEAK, TRANSCRIPT_NORMALIZE_MAX_GAIN)
return currentRecognizer.recognize(segment, segment.size).trim()
}
private fun logTranscriptDecode(isFinal: Boolean, segmentSampleCount: Int, text: String) {
val totalAudioSec = (committedTranscriptSampleCount + transcriptSampleCount) / SAMPLE_RATE.toDouble()
val preview = if (text.length <= TRANSCRIPT_LOG_PREVIEW_CHARS) text
else text.substring(0, TRANSCRIPT_LOG_PREVIEW_CHARS) + "..."
logInfo(
(if (onboardModelKind == VoiceModelKind.WHISPER) "Whisper" else "Moonshine") + " decode final=" + isFinal +
" audioSec=" + formatFixed(totalAudioSec, 2) +
" segmentAudioSec=" + formatFixed(segmentSampleCount / SAMPLE_RATE.toDouble(), 2) +
" textLen=" + text.length + " text=\"" + preview + "\"",
)
}
private fun resetTranscriptState() {
transcriptSampleCount = 0
committedTranscriptSampleCount = 0
committedTranscript = ""
currentSegmentTranscript = ""
lastTranscriptDecodeAtMs = 0
}
private fun emitPcm(pcm: ShortArray, count: Int) {
val currentListener = listener
if (currentListener == null || count <= 0) {
return
}
val le = AudioSegmentation.pcm16ToLittleEndian(pcm, 0, count)
host.dispatcher.post { currentListener.onPcm(le) }
}
/**
* Embed the session's buffered utterance and compare it to the enrolled
* wearer voice-print. Fails open: a session too short to verify, or a
* model that will not load, counts as the wearer rather than silencing
* every command.
*/
private fun runSpeakerVerification() {
val buffer = verifyBuffer
val wearer = verifyWearerEmbedding
val modelPath = verifySpeakerModelPath
verifyBuffer = null
if (buffer == null || wearer == null || modelPath == null) {
return
}
if (verifyCount < VERIFY_MIN_SAMPLES) {
emitSpeakerVerified(true, 0f)
return
}
try {
val embedder = host.speakerEmbedder(modelPath)
val le = AudioSegmentation.pcm16ToLittleEndian(buffer, 0, verifyCount)
val embedding = embedder.embed(le, SAMPLE_RATE)
if (embedding == null || embedding.size != wearer.size) {
emitSpeakerVerified(true, 0f)
return
}
val dot = VoicePrint.dot(embedding, wearer)
val isWearer = dot >= verifyThreshold
logInfo(
"speaker verification similarity=" + formatFixed(dot, 3) +
" threshold=" + verifyThreshold + " isWearer=" + isWearer,
)
emitSpeakerVerified(isWearer, dot.toFloat())
} catch (t: Throwable) {
logWarn("speaker verification failed (${t.message})")
emitSpeakerVerified(true, 0f)
}
}
private fun emitSpeakerVerified(isWearer: Boolean, similarity: Float) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeakerVerified(isWearer, similarity) }
}
private fun emitFrameMeta(angleDegrees: Int, ssr: Int) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onFrameMeta(angleDegrees, ssr) }
}
private fun emitSpeechEnd() {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onSpeechEnd() }
}
private fun emitStopped(captureId: Int) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onStopped(captureId) }
}
private fun stopG2Audio() {
host.g2AudioSource()?.stop()
maybeEmitAudioStats(true)
}
private fun releaseRecognizer() {
recognizer?.release()
recognizer = null
}
private fun releaseDecoder() {
packetDecoder?.close()
packetDecoder = null
}
/** Called from the BLE notification thread for every glasses mic packet. */
fun queueAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
if (!started || data == null) {
return
}
if ("L" != arm) {
wrongArmPackets++
}
audioQueue.put(AudioPacket(data, arm, arrivalMs)) {
queuedPackets++
if (lastPacketArrivalMs > 0) {
val delta = arrivalMs - lastPacketArrivalMs
if (delta > maxInterPacketMs) {
maxInterPacketMs = delta
}
if (delta > LATE_PACKET_INTERVAL_MS) {
latePackets++
}
}
lastPacketArrivalMs = arrivalMs
}
}
private fun takeAudioPacket(): AudioPacket? {
val packet = audioQueue.take(250) { started }
if (packet == null) {
maybeEmitAudioStats(false)
}
return packet
}
private fun resetAudioStats() {
queuedPackets = 0
decodedSamples = 0
latePackets = 0
wrongArmPackets = 0
lastPacketArrivalMs = 0
maxInterPacketMs = 0
lastStatsAtMs = platform.elapsedRealtimeMs()
}
/** Diagnostic packet accounting for tests and adapters. */
class AudioStats(
val queuedPackets: Long,
val queueDroppedPackets: Long,
val latePackets: Long,
val wrongArmPackets: Long,
val maxInterPacketMs: Long,
val decodedSamples: Long,
)
fun audioStats(): AudioStats = AudioStats(
queuedPackets, audioQueue.droppedCount, latePackets, wrongArmPackets, maxInterPacketMs, decodedSamples,
)
private fun maybeEmitAudioStats(force: Boolean) {
val now = platform.elapsedRealtimeMs()
if (!force && now - lastStatsAtMs < STATS_INTERVAL_MS) {
return
}
lastStatsAtMs = now
val currentDecoder = packetDecoder
val real = currentDecoder?.realPackets ?: 0L
val duplicate = currentDecoder?.duplicatePackets ?: 0L
val missing = currentDecoder?.missingPackets ?: 0L
val decodeErrors = currentDecoder?.decodeErrors ?: 0L
val status = "G2 mic packets=" + queuedPackets +
" decoded=" + real +
" missing=" + missing +
" duplicate=" + duplicate +
" late=" + latePackets +
" maxGapMs=" + maxInterPacketMs +
" queueDrop=" + audioQueue.droppedCount +
" decodeErrors=" + decodeErrors +
" wrongArm=" + wrongArmPackets +
" audioSec=" + formatFixed(decodedSamples / SAMPLE_RATE.toDouble(), 1)
// Audio-pipeline stats are diagnostic; keep them in the log only, out of
// the on-glasses voice UI.
logInfo(status + " expectedIntervalMs=" + EXPECTED_PACKET_INTERVAL_MS)
}
private fun emitStatus(status: String?) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onStatus(status) }
}
private fun emitTranscript(text: String?, isFinal: Boolean) {
val currentListener = listener ?: return
logInfo("Emit transcript final=" + isFinal + " textLen=" + (text?.trim()?.length ?: 0))
host.dispatcher.post { currentListener.onTranscript(text, isFinal) }
}
}
@@ -0,0 +1,137 @@
package com.faceclaw.app
import kotlin.math.abs
import kotlin.math.round
/** Which on-device model ONBOARD voice input loads. */
enum class VoiceModelKind {
MOONSHINE,
WHISPER,
}
/**
* Whole-utterance (offline) recognizer: each call decodes one complete buffer. The Android
* implementation wraps sherpa-onnx; a streaming recognizer (Apple Speech) could satisfy this
* by buffering and re-running, but the session's REPLACE-semantics partials assume re-decoding
* the same buffer is cheap enough on the MOONSHINE cadence.
*/
interface OfflineTranscriber {
/** Returns the trimmed text for [count] float samples at 16 kHz, or "" when nothing was recognized. */
fun recognize(samples: FloatArray, count: Int): String
fun release()
}
/** Speaker voice-print extraction over 16 kHz mono S16LE PCM; null when unavailable or too short. */
interface SpeakerEmbedder {
fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray?
fun release()
}
/** Marshals listener callbacks to the platform's UI/JS thread (Android: the main Handler). */
fun interface CallbackDispatcher {
fun post(action: () -> Unit)
}
/** Receives the session's decoded PCM once capture ends (Android saves a WAV file). */
fun interface RecordingSink {
fun save(pcm16le: ByteArray, sampleRate: Int)
}
/** A G2 microphone packet decoder (the shared [Lc3PacketFramer] over a platform LC3 codec). */
interface AudioPacketDecoder {
fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int
val lastAngleDegrees: Int
val lastSsr: Int
val realPackets: Long
val duplicatePackets: Long
val missingPackets: Long
val decodeErrors: Long
fun close()
}
/** Glasses microphone packets over BLE (Android: FaceclawBleCommunicator's capture API). */
interface PacketAudioSource {
fun start(listener: FaceclawAudioPacketListener): Boolean
fun stop()
/** Whether the glasses are still delivering packets for this capture. */
fun isActive(): Boolean
}
/** The phone's own microphone delivering 16 kHz mono PCM16 in blocking reads. */
interface PcmAudioSource {
/** Blocking read into [pcm]; returns samples read, 0 to retry, or a negative error. */
fun read(pcm: ShortArray): Int
fun close()
}
/** Formats [value] with exactly [decimals] fraction digits (the shared stand-in for %.Nf). */
fun formatFixed(value: Double, decimals: Int): String {
var scale = 1L
repeat(decimals) { scale *= 10 }
val scaled = round(abs(value) * scale).toLong()
val whole = scaled / scale
val fraction = scaled % scale
val sign = if (value < 0 && scaled != 0L) "-" else ""
return if (decimals == 0) "$sign$whole" else "$sign$whole." + fraction.toString().padStart(decimals, '0')
}
/**
* Bounded FIFO between a producer thread and one consumer: when full the oldest item is
* dropped (counted). [take] waits up to [waitMs] and returns null on timeout or when [open] is
* false, so the consumer can poll a stop flag and emit periodic diagnostics.
*/
class DroppingPacketQueue<T : Any>(private val capacity: Int, platform: ProtocolPlatform = protocolPlatform()) {
private val condition = platform.createCondition()
private val items = ArrayDeque<T>()
var droppedCount: Long = 0
private set
fun put(item: T) {
condition.withLock {
if (items.size >= capacity) {
items.removeFirst()
droppedCount++
}
items.addLast(item)
condition.signalAll()
}
}
/** Runs [action] while holding the queue lock, after enqueuing (used to update stats atomically). */
fun put(item: T, action: () -> Unit) {
condition.withLock {
if (items.size >= capacity) {
items.removeFirst()
droppedCount++
}
items.addLast(item)
action()
condition.signalAll()
}
}
fun take(waitMs: Long, open: () -> Boolean): T? {
condition.withLock {
if (items.isEmpty() && open()) condition.awaitMs(waitMs)
return items.removeFirstOrNull()
}
}
fun clear() {
condition.withLock {
items.clear()
droppedCount = 0
}
}
fun wake() {
condition.withLock { condition.signalAll() }
}
}
@@ -0,0 +1,104 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.sqrt
/** Voice-print (speaker embedding) vector math shared by extraction, verification and storage. */
object VoicePrint {
/** L2-normalizes [v] in place and returns it; a null or zero vector is returned unchanged. */
@JvmStatic
fun l2Normalize(v: FloatArray?): FloatArray? {
if (v == null) return null
var sum = 0.0
for (x in v) sum += x.toDouble() * x
val norm = sqrt(sum)
if (norm <= 0) return v
for (i in v.indices) v[i] = (v[i] / norm).toFloat()
return v
}
/** Dot product; on L2-normalized vectors this is the cosine similarity. */
@JvmStatic
fun dot(a: FloatArray, b: FloatArray): Double {
require(a.size == b.size) { "embedding sizes differ: ${a.size} vs ${b.size}" }
var sum = 0.0
for (i in a.indices) sum += a[i].toDouble() * b[i]
return sum
}
/** Utterances shorter than this (~0.5 s of S16 mono at [sampleRate]) produce unreliable prints. */
@JvmStatic
fun isLongEnoughToEmbed(pcm16leBytes: Int, sampleRate: Int): Boolean = pcm16leBytes >= sampleRate
}
/** Persistence-side voice-print math: float32-LE blobs and centroid updates. */
object SpeakerEmbeddings {
/** Decodes a float32-LE blob; null for a null, empty or misaligned blob. */
@JvmStatic
fun blobToFloats(blob: ByteArray?): FloatArray? {
if (blob == null || blob.size < 4 || blob.size % 4 != 0) return null
val out = FloatArray(blob.size / 4)
for (i in out.indices) {
val base = i * 4
val bits = (blob[base].toInt() and 0xff) or
((blob[base + 1].toInt() and 0xff) shl 8) or
((blob[base + 2].toInt() and 0xff) shl 16) or
((blob[base + 3].toInt() and 0xff) shl 24)
out[i] = Float.fromBits(bits)
}
return out
}
@JvmStatic
fun floatsToBlob(values: FloatArray): ByteArray {
val out = ByteArray(values.size * 4)
for (i in values.indices) {
val bits = values[i].toRawBits()
val base = i * 4
out[base] = (bits and 0xff).toByte()
out[base + 1] = ((bits shr 8) and 0xff).toByte()
out[base + 2] = ((bits shr 16) and 0xff).toByte()
out[base + 3] = ((bits shr 24) and 0xff).toByte()
}
return out
}
class Centroid(val embedding: FloatArray, val count: Int)
/**
* Running-mean update of a speaker's centroid with one new print, with the existing
* weight capped at [maxCount] so a long session cannot drown the enrolled voice. A
* missing or differently sized current centroid is replaced by [update] (count 1).
*/
@JvmStatic
fun runningMean(current: FloatArray?, currentCount: Int, update: FloatArray, maxCount: Int): Centroid {
if (current == null || current.size != update.size) return Centroid(update, 1)
val effective = minOf(currentCount, maxOf(1, maxCount))
val merged = FloatArray(current.size)
for (i in merged.indices) merged[i] = (current[i] * effective + update[i]) / (effective + 1)
VoicePrint.l2Normalize(merged)
return Centroid(merged, currentCount + 1)
}
/**
* Blends two centroids weighted by their sample counts when both exist with the same
* size; otherwise keeps [into], falling back to [from] when [into] is missing. Null when
* neither exists.
*/
@JvmStatic
fun blend(from: FloatArray?, fromCount: Int, into: FloatArray?, intoCount: Int): FloatArray? {
if (from != null && into != null && from.size == into.size) {
val blended = FloatArray(into.size)
val total = maxOf(1, fromCount + intoCount)
for (i in blended.indices) blended[i] = (into[i] * intoCount + from[i] * fromCount) / total
VoicePrint.l2Normalize(blended)
return blended
}
return into ?: from
}
/** When merging speakers, the target keeps its conversation insights; it adopts the source's only when it has none. */
@JvmStatic
fun shouldAdoptInsights(targetHasInsights: Boolean, sourceHasInsights: Boolean): Boolean =
!targetHasInsights && sourceHasInsights
}
@@ -0,0 +1,66 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/** RIFF/WAVE header walk shared by the AAC transcoder and the diarizer (16-bit PCM only). */
object WavPcmReader {
/** Bytes of a WAV file the header walk needs at most; chunk payloads beyond it are skipped by offset. */
const val HEADER_PREFIX_BYTES = 65536
class WavInfo(
@JvmField val sampleRate: Int,
@JvmField val channels: Int,
@JvmField val bitsPerSample: Int,
/** Byte offset of the first PCM sample. */
@JvmField val dataOffset: Long,
/** PCM byte count; a zero-length data chunk (header never patched after a crash) is recovered from the file length. */
@JvmField val dataBytes: Long,
)
/**
* Parses the fmt/data chunks from [prefix], the first bytes of a WAV file of [fileLength]
* bytes. Throws IllegalArgumentException for anything that is not 16-bit PCM WAV or whose
* data chunk header lies beyond the prefix.
*/
@JvmStatic
fun parseHeader(prefix: ByteArray, fileLength: Long): WavInfo {
require(prefix.size >= 12 && prefix[0] == 'R'.code.toByte() && prefix[1] == 'I'.code.toByte() &&
prefix[2] == 'F'.code.toByte() && prefix[3] == 'F'.code.toByte() &&
prefix[8] == 'W'.code.toByte() && prefix[9] == 'A'.code.toByte()) { "not a WAV file" }
var sampleRate = 0
var channels = 0
var bits = 0
var offset = 12L
while (offset + 8 <= prefix.size) {
val at = offset.toInt()
val id = u32(prefix, at)
val size = u32(prefix, at + 4).toLong() and 0xffffffffL
offset += 8
when (id) {
0x20746d66 -> { // "fmt "
val available = minOf(size, 16L).toInt()
require(offset + available <= prefix.size) { "truncated fmt chunk" }
val f = offset.toInt()
val audioFormat = u16(prefix, f)
channels = u16(prefix, f + 2)
sampleRate = u32(prefix, f + 4)
bits = u16(prefix, f + 14)
require(audioFormat == 1 && bits == 16) { "only 16-bit PCM WAV is supported" }
offset += size
}
0x61746164 -> { // "data"
require(sampleRate != 0) { "missing fmt or data chunk" }
val dataBytes = if (size > 0) size else fileLength - offset
return WavInfo(sampleRate, channels, bits, offset, maxOf(0L, dataBytes))
}
else -> offset += size
}
}
throw IllegalArgumentException("missing fmt or data chunk")
}
private fun u16(bytes: ByteArray, at: Int): Int = (bytes[at].toInt() and 0xff) or ((bytes[at + 1].toInt() and 0xff) shl 8)
private fun u32(bytes: ByteArray, at: Int): Int = u16(bytes, at) or (u16(bytes, at + 2) shl 16)
}
@@ -0,0 +1,94 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/**
* Offline asset resolution for an unpacked EvenHub app served from a fake per-app origin.
* Shared by the Android WebViewClient interceptor and the iOS WKURLSchemeHandler so both apply
* the same path rules, MIME table and bridge-shim injection.
*/
object EvenHubAssetServer {
/** A resolved request: [relativePath] is root-relative with no leading slash. */
data class Asset(val relativePath: String, val mime: String) {
val isHtml: Boolean
get() = mime == "text/html"
}
/** A loaded response body. */
class Response(val mime: String, val bytes: ByteArray)
/**
* Maps a decoded URL path to a root-relative file path, or null when it escapes the root.
* Empty and "/" paths serve index.html. Segments are normalised lexically ("." and "..");
* platform adapters must still canonicalise the final file to defeat symlinks.
*/
@JvmStatic
fun resolve(urlPath: String?): Asset? {
val path = if (urlPath.isNullOrEmpty() || urlPath == "/") "/index.html" else urlPath
val segments = ArrayList<String>()
for (segment in path.split('/')) {
when (segment) {
"", "." -> continue
".." -> if (segments.isEmpty()) return null else segments.removeAt(segments.size - 1)
else -> segments.add(segment)
}
}
if (segments.isEmpty()) return null
val relative = segments.joinToString("/")
return Asset(relative, mimeTypeFor(segments.last()))
}
/** True when [canonicalFile] lies inside [canonicalRoot] (both already canonicalised, no trailing slash). */
@JvmStatic
fun isInsideRoot(canonicalRoot: String, canonicalFile: String): Boolean =
canonicalFile == canonicalRoot || canonicalFile.startsWith(canonicalRoot.trimEnd('/') + "/")
/**
* Inserts the bridge shim right after `<head...>` when present, else before everything, so
* window.flutter_inappwebview exists before any app script runs.
*/
@JvmStatic
fun injectIntoHtml(html: String, injectScript: String): String {
val tag = "<script>$injectScript</script>"
val headIndex = html.lowercase().indexOf("<head")
if (headIndex >= 0) {
val close = html.indexOf('>', headIndex)
if (close >= 0) return html.substring(0, close + 1) + tag + html.substring(close + 1)
}
return tag + html
}
/** Loads [asset] under [rootDir] with the shim spliced into HTML; null when the file is unreadable. */
@JvmStatic
fun load(rootDir: String, asset: Asset, injectScript: String): Response? {
val path = rootDir.trimEnd('/') + "/" + asset.relativePath
if (!fileExists(path)) return null
val bytes = try { readFileData(path) } catch (e: Exception) { return null }
if (!asset.isHtml) return Response(asset.mime, bytes)
return Response(asset.mime, injectIntoHtml(bytes.decodeToString(), injectScript).encodeToByteArray())
}
/** Extension-based MIME lookup (the union of the Android and iOS tables). */
@JvmStatic
fun mimeTypeFor(name: String): String {
val lower = name.lowercase()
val dot = lower.lastIndexOf('.')
val ext = if (dot >= 0) lower.substring(dot + 1) else ""
return MIME_TYPES[ext] ?: "application/octet-stream"
}
private val MIME_TYPES = mapOf(
"html" to "text/html", "htm" to "text/html",
"js" to "text/javascript", "mjs" to "text/javascript",
"css" to "text/css",
"json" to "application/json",
"wasm" to "application/wasm",
"svg" to "image/svg+xml",
"png" to "image/png", "jpg" to "image/jpeg", "jpeg" to "image/jpeg", "gif" to "image/gif",
"webp" to "image/webp", "ico" to "image/x-icon",
"woff" to "font/woff", "woff2" to "font/woff2", "ttf" to "font/ttf", "otf" to "font/otf",
"txt" to "text/plain", "map" to "text/plain",
"xml" to "application/xml", "mp3" to "audio/mpeg", "wav" to "audio/wav", "ogg" to "audio/ogg",
"mp4" to "video/mp4", "webm" to "video/webm",
)
}
@@ -0,0 +1,219 @@
package com.faceclaw.app
/**
* Connects to the glasses (stock-firmware compatible), reads the device-info / settings
* response, and reports the firmware versions plus the firmware-extension string (empty on
* stock firmware). Used by onboarding to decide whether to flash. Owns its own [StockLink] and
* runs [run] on the caller's worker thread; listener callbacks are synchronous on that thread
* (the platform adapter marshals them). Shows nothing on the lens.
*/
class DeviceInfoProbeFlow(
link: StockLink,
rightAddress: String,
leftAddress: String,
private val listener: FaceclawDeviceInfoProbeListener,
private val timings: StockFlowTimings = StockFlowTimings(),
platform: ProtocolPlatform = protocolPlatform(),
) {
companion object {
const val TAG = "FaceclawDeviceInfo"
/** Connect + auth attempts per arm; a link that drops mid-pairing is retried once the bond settles. */
const val ARM_ATTEMPTS = 3
}
private val rightAddress = rightAddress.trim()
private val leftAddress = leftAddress.trim()
private val sleeper = InterruptibleSleep(platform)
private val session = StockLinkSession(link, ::emitLog, timings, traceFrames = true, platform = platform)
val cancelled: Boolean
get() = session.cancelled
fun cancel() {
session.cancel()
sleeper.interrupt()
}
fun run() {
try {
if (rightAddress.isEmpty()) {
emitError("No right-arm address configured.")
return
}
val haveLeft = leftAddress.isNotEmpty() && !leftAddress.equals(rightAddress, ignoreCase = true)
// Bring up and authenticate BOTH arms before asking for versions. Each arm is its
// own peripheral with its own bond, so on a fresh phone each raises its own pairing
// prompt; completing both here means the flash step (and the app proper) find both
// bonds in place. Strictly one arm at a time: Android pairs with one device at a
// time, and a second arm connected while the first was still pairing has been seen
// to drop within a second and never pair (2026-09-11 logcat).
var rightAuthenticated = bringUpArm(rightAddress, "right")
var leftAuthenticated = false
if (haveLeft && !cancelled) {
leftAuthenticated = bringUpArm(leftAddress, "left")
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (!session.isConnected(rightAddress)) {
// The right lens may have dropped while the left one paired; it is bonded by
// now, so this comes back quickly.
emitLog("right lens disconnected while the left lens was set up; reconnecting")
rightAuthenticated = bringUpArm(rightAddress, "right")
if (cancelled) {
emitError("Cancelled.")
return
}
}
var ack: ByteArray? = null
var rightFailure: String? = null
try {
ack = queryArm(rightAddress, "right", rightAuthenticated)
} catch (e: IllegalStateException) {
rightFailure = StockLinkSession.messageOf(e)
emitLog("right-lens query failed: $rightFailure")
}
// The right lens is the documented control endpoint, but a silent right lens has
// been observed on stock 2.2.9 even after a successful security auth — run the same
// probe against the left lens rather than giving up, and log which lens answered.
if (ack == null && !cancelled && haveLeft) {
emitLog("right lens did not answer the settings query; probing the left lens")
try {
ack = queryArm(leftAddress, "left", leftAuthenticated)
} catch (e: Exception) {
emitLog("left-lens probe failed: " + StockLinkSession.messageOf(e))
}
}
if (cancelled) {
emitError("Cancelled.")
return
}
if (ack == null) {
throw IllegalStateException(
rightFailure
?: ("no response to the device-info query on either lens — check `adb logcat -s " + TAG + "` for the frame trace")
)
}
val info = BleProtocol.parseSettingsFirmwareInfo(ack)
val left = info?.leftVersion ?: ""
val right = info?.rightVersion ?: ""
val extension = info?.extension ?: ""
emitLog("device-info: L=$left R=$right ext=[$extension]")
listener.onResult(left, right, extension)
} catch (e: Exception) {
emitError(if (cancelled) "Cancelled." else StockLinkSession.messageOf(e))
} finally {
session.closeQuietly()
}
}
/**
* Connect to one arm and complete its security-auth exchange, retrying when the link drops
* mid-pairing. Returns whether the auth success was seen (false = connected but
* unconfirmed, tolerated because the custom firmware's response to the exchange is not yet
* hardware-verified); throws when the arm cannot be brought up at all.
*/
private fun bringUpArm(address: String, label: String): Boolean {
var lastFailure: String? = null
var attempt = 1
while (attempt <= ARM_ATTEMPTS && !cancelled) {
if (attempt > 1) {
// A link the OS dropped while it was still pairing comes back once the bond
// settles; reconnecting sooner just fails again.
session.waitForBondToSettle(address, label, sleeper)
if (cancelled) {
break
}
emitLog("retrying the $label lens (attempt $attempt/$ARM_ATTEMPTS): $lastFailure")
sleeper.sleep(timings.armRetryDelayMs.toLong())
}
listener.onState("connecting", label)
try {
session.bringUp(address)
} catch (e: IllegalStateException) {
lastFailure = StockLinkSession.messageOf(e)
emitLog("$label lens: $lastFailure")
attempt++
continue
}
if (cancelled) {
break
}
// Firmware 2.2.9 answers no queries until the security-auth exchange completes over
// an encrypted link; on a phone with no existing bond this is also what triggers SMP
// pairing (and its OS prompt), so it must come before the prelude and query.
listener.onState("authenticating", label)
when (session.authenticate(address, label)) {
StockLinkSession.AuthResult.SUCCESS -> return true
StockLinkSession.AuthResult.UNCONFIRMED -> return false
StockLinkSession.AuthResult.LINK_DROPPED -> {
lastFailure = "link dropped during authentication"
attempt++
}
}
}
if (cancelled) {
return false
}
throw IllegalStateException(
"could not connect to the $label lens ($address): " + lastFailure +
" — if the phone showed a Bluetooth pairing request, accept it and try again"
)
}
/**
* Prelude + settings read on an already connected/authenticated lens. Returns the settings
* ack protobuf, an unsolicited settings push that carried firmware versions, or null when
* the lens never answered the read. Throws on prelude failure; the caller wraps the fallback
* lens's attempt so its failure cannot mask the primary lens's outcome.
*/
private fun queryArm(address: String, label: String, authenticated: Boolean): ByteArray? {
if (cancelled) {
return null
}
listener.onState("querying", label)
// Session prelude, then a settings/device-info read (both arms' versions and the
// firmware-extension string ride back in one response).
session.sendPrelude(
address,
" ($label lens)" +
(if (authenticated) "" else "; authentication did not complete — if the phone shows a Bluetooth pairing request, accept it and try again"),
)
// Two attempts: the first read straight after a fresh pairing has been seen to go
// unanswered while a later one succeeds.
var attempt = 0
while (attempt < 2 && !cancelled) {
val ack = session.readSettings(address, timings.queryTimeoutMs)
if (ack != null) {
return ack
}
// A push with the firmware versions on the device's own magic is as good as the
// ack we asked for.
val pushed = session.unsolicitedSettingsPb
if (pushed != null) {
emitLog("using unsolicited settings push instead of the read ack ($label lens)")
return pushed
}
emitLog("settings query attempt " + (attempt + 1) + " unanswered (" + label + " lens)")
attempt++
}
return null
}
// Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG).
private fun emitLog(line: String) {
listener.onLog(line)
}
private fun emitError(message: String) {
listener.onError(message)
}
}
@@ -0,0 +1,167 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/**
* Stock EVENOTA container layout, validation and CRC-32C, shared by the Android flasher and
* (via stage 3) the iOS port of app/g2/firmware-ota.ts. Mirrors g2flash.py: a 0x40-byte
* header (component count at +8), a 16-byte table-of-contents entry per component
* (offset at +4, CRC at +12), and per component a 128-byte subheader (payload size at +8,
* CRC at +12, NUL-terminated name at +48) followed by the payload.
*
* Validation takes the stricter of the two former implementations: the Android checks
* (component count, CRC in both TOC and subheader, main-app preamble/MRAM ceiling) plus the
* TypeScript ones (TOC/subheader/payload bounds, no overlapping components).
*/
object FirmwareImage {
const val HEADER_SIZE = 0x40
const val TOC_ENTRY_SIZE = 16
const val SUBHEADER_SIZE = 128
const val NAME_OFFSET = 48
const val NAME_MAX = 80
// Firmware 2.2.4 has 5 components; 2.2.6 has 6.
const val MIN_SEGMENTS = 5
const val MAX_SEGMENTS = 6
const val REQUIRED_SEGMENT = "ota/s200_firmware_ota.bin"
const val APP_LOAD_ADDR = 0x00438000L
const val APP_MAX_END = 0x007F0000L
const val APP_PREAMBLE = 0x20
class Segment(
/** Offset of the 128-byte subheader within the image. */
@JvmField val off: Int,
/** Payload size in bytes (payload starts at off + SUBHEADER_SIZE). */
@JvmField val ps: Int,
/** CRC-32C recorded in the table of contents. */
@JvmField val crc: Long,
@JvmField val name: String,
) {
val payloadStart: Int get() = off + SUBHEADER_SIZE
val end: Int get() = payloadStart + ps
}
private val CRC32C_TABLE: IntArray = IntArray(256).also { table ->
for (b in 0 until 256) {
var c = b shl 24
for (i in 0 until 8) {
c = if ((c and 0x80000000.toInt()) != 0) (c shl 1) xor 0x1edc6f41 else c shl 1
}
table[b] = c
}
}
/** CRC-32C, MSB-first, init 0, no final xor (matches g2flash.py crc32c_msb). */
@JvmStatic
fun crc32cMsb(data: ByteArray, offset: Int = 0, length: Int = data.size - offset): Int {
require(offset >= 0 && length >= 0 && offset + length <= data.size)
var crc = 0
for (i in offset until offset + length) {
crc = (crc shl 8) xor CRC32C_TABLE[((crc ushr 24) xor (data[i].toInt() and 0xff)) and 0xff]
}
return crc
}
/** Unsigned CRC-32C as a Long, for comparison with recorded values. */
@JvmStatic
fun crc32cUnsigned(data: ByteArray, offset: Int = 0, length: Int = data.size - offset): Long =
crc32cMsb(data, offset, length).toLong() and 0xffffffffL
@JvmStatic
fun readU32(buf: ByteArray, offset: Int): Long {
if (offset < 0 || offset + 4 > buf.size) throw IllegalStateException("firmware header is truncated")
return (buf[offset].toLong() and 0xffL) or
((buf[offset + 1].toLong() and 0xffL) shl 8) or
((buf[offset + 2].toLong() and 0xffL) shl 16) or
((buf[offset + 3].toLong() and 0xffL) shl 24)
}
@JvmStatic
fun readCString(buf: ByteArray, offset: Int, maxLen: Int): String {
var end = offset
val limit = minOf(buf.size, offset + maxLen)
while (end < limit && buf[end].toInt() != 0) end++
return buildString(end - offset) { for (i in offset until end) append((buf[i].toInt() and 0xff).toChar()) }
}
/** Parses the table of contents without checking CRCs; throws on bounds violations. */
@JvmStatic
fun parseSegments(img: ByteArray): List<Segment> {
if (img.size < HEADER_SIZE) throw IllegalStateException("file is too small to be a firmware image")
val n = readU32(img, 8)
if (n <= 0 || n > 64) throw IllegalStateException("implausible component count $n (corrupt image?)")
val count = n.toInt()
val tocEnd = HEADER_SIZE + count * TOC_ENTRY_SIZE
if (tocEnd > img.size) throw IllegalStateException("table of contents runs past end of file")
val segs = ArrayList<Segment>(count)
for (i in 0 until count) {
val base = HEADER_SIZE + i * TOC_ENTRY_SIZE
val crc = readU32(img, base + 12)
val offLong = readU32(img, base + 4)
if (offLong < tocEnd || offLong + SUBHEADER_SIZE > img.size) {
throw IllegalStateException("segment $i subheader is out of bounds")
}
val off = offLong.toInt()
val psLong = readU32(img, off + 8)
if (psLong <= 0 || off + SUBHEADER_SIZE + psLong > img.size) {
throw IllegalStateException("segment $i payload is truncated")
}
val name = readCString(img, off + NAME_OFFSET, NAME_MAX)
val seg = Segment(off, psLong.toInt(), crc, name)
for (other in segs) {
if (seg.off < other.end && other.off < seg.end) {
throw IllegalStateException("components ${other.name} and ${seg.name} overlap")
}
}
segs.add(seg)
}
return segs
}
/** Full validation: layout, component count, CRCs, and the main-app preamble/MRAM check. */
@JvmStatic
fun validate(img: ByteArray): List<Segment> {
val segs = parseSegments(img)
if (segs.size < MIN_SEGMENTS || segs.size > MAX_SEGMENTS) {
throw IllegalStateException("expected $MIN_SEGMENTS-$MAX_SEGMENTS components, found ${segs.size}")
}
var main: Segment? = null
for (s in segs) {
val calc = crc32cUnsigned(img, s.payloadStart, s.ps)
val subCrc = readU32(img, s.off + 12)
if (calc != s.crc || calc != subCrc) {
throw IllegalStateException("component ${s.name} CRC32C is stale (image not checksum-fixed)")
}
if (REQUIRED_SEGMENT == s.name) {
if (main != null) throw IllegalStateException("required component $REQUIRED_SEGMENT is duplicated")
main = s
}
}
if (main == null) throw IllegalStateException("required component $REQUIRED_SEGMENT not found")
checkMainAppFitsMram(img, main)
return segs
}
@JvmStatic
fun checkMainAppFitsMram(img: ByteArray, main: Segment) {
if (main.ps < APP_PREAMBLE) throw IllegalStateException("main-app payload is smaller than its preamble")
val loadAddr = readU32(img, main.payloadStart + 0x14)
val preLen = readU32(img, main.payloadStart) and 0xFFFFFFL
if (loadAddr != APP_LOAD_ADDR) {
throw IllegalStateException(
"main-app preamble load address is 0x" + loadAddr.toString(16) +
", expected 0x" + APP_LOAD_ADDR.toString(16))
}
if (preLen != main.ps.toLong()) {
throw IllegalStateException("main-app preamble length ($preLen) != staged payload size (${main.ps})")
}
val progEnd = APP_LOAD_ADDR + main.ps - APP_PREAMBLE
if (progEnd > APP_MAX_END) {
val over = progEnd - APP_MAX_END
throw IllegalStateException(
"main-app is too large: programmed region ends at 0x" + progEnd.toString(16) +
", " + over + " bytes past the safe MRAM ceiling — refusing to flash (brick risk)")
}
}
}
@@ -0,0 +1,353 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
/**
* A minimal, stock-firmware-compatible flow used only to show the pre-flash confirmation
* prompt on the glasses and read back the user's Yes/No choice. It speaks only the stock subset
* — session prelude, a text container, a list container, and heartbeats. No images.
*
* Both arms are connected and security-authenticated up front, even though the prompt itself
* only needs the right arm (the lenses relay messages to each other, and acks/events only ever
* come from the right arm): on an unbonded phone the auth exchange is what raises the OS
* pairing prompt, and doing both here means both prompts appear at the start rather than one
* popping up half way through flashing the second lens. While connected, each arm's battery
* level is read (they are independent batteries) so the caller can refuse to flash on a low
* charge.
*
* With `skipPrompt` the on-glasses confirmation is not shown: the flow is just connect + auth +
* battery read + result(approved). Used to re-check the battery after the user has already
* confirmed once. [run] executes on the caller's worker thread; the heartbeat runs on its own
* thread while the prompt is up.
*/
class FlashPromptFlow(
link: StockLink,
rightAddress: String,
leftAddress: String,
warningText: String,
private val skipPrompt: Boolean,
private val listener: FaceclawFlashPromptListener,
private val timings: StockFlowTimings = StockFlowTimings(),
platform: ProtocolPlatform = protocolPlatform(),
) {
companion object {
const val TAG = "FaceclawFlashPrompt"
const val TEXT_NAME = "flashwarn"
const val LIST_NAME = "flashmenu"
const val TEXT_CONTAINER_ID = 1
const val LIST_CONTAINER_ID = 2
// Index 0 = decline, index 1 = approve. Kept short for the ~50-col grid.
val ITEMS = arrayOf("No, cancel", "Yes, flash")
}
private val rightAddress = rightAddress.trim()
private val leftAddress = leftAddress.trim()
private val warningText = warningText
private val platform = platform
private val lock = platform.createLock()
private val sleeper = InterruptibleSleep(platform)
private val selectionLatch = Latch(1, platform)
private val session = StockLinkSession(link, ::emitLog, timings, platform = platform)
@Volatile
private var finished = false
@Volatile
private var rightConnected = false
@Volatile
private var leftConnected = false
@Volatile
private var rightLost = false
@Volatile
private var approved: Boolean? = null
/** The running heartbeat's sleeper; interrupting it ends that thread. */
@Volatile
private var heartbeat: InterruptibleSleep? = null
val cancelled: Boolean
get() = session.cancelled
init {
session.onEvent = { _, frame -> handleEvent(frame) }
session.onDisconnected = ::handleDisconnected
}
/** Abort the prompt (e.g. the user backed out on the phone). */
fun cancel() {
session.cancel()
selectionLatch.countDown()
sleeper.interrupt()
stopHeartbeat()
}
fun run() {
try {
if (rightAddress.isEmpty()) {
listener.onState("error", "No right-arm address configured.")
return
}
listener.onState("connecting", "")
connectArm(rightAddress, "right")
rightConnected = true
if (cancelled) {
teardown()
return
}
if (leftAddress.isNotEmpty()) {
connectArm(leftAddress, "left")
leftConnected = true
if (cancelled) {
teardown()
return
}
}
listener.onState("connected", "")
// Auth both arms now (pairing prompts, if any, happen here) before anything else,
// so both bonds exist by the time flashing starts.
authenticateArm(rightAddress, "right")
if (leftConnected) {
authenticateArm(leftAddress, "left")
}
if (cancelled) {
teardown()
return
}
session.sendPrelude(rightAddress, ": $rightAddress")
if (!skipPrompt) {
showPrompt(rightAddress)
startHeartbeat()
listener.onState("prompting", "")
selectionLatch.await(timings.selectionTimeoutMs.toLong())
stopHeartbeat()
if (cancelled) {
listener.onState("cancelled", "")
teardown()
return
}
if (approved == null) {
if (rightLost) {
listener.onState("disconnected", "Lost connection to the glasses.")
} else {
listener.onState("timeout", "No response from the glasses.")
}
teardown()
return
}
try {
sendShutdown()
} catch (ignored: Exception) {
}
if (approved != true) {
listener.onResult(false)
listener.onState("result", "declined")
teardown()
return
}
}
// Approved (or prompt skipped): read each arm's battery while the links are still
// up, then report the result.
listener.onState("battery", "")
readBatteries()
if (cancelled) {
listener.onState("cancelled", "")
teardown()
return
}
if (rightLost) {
listener.onState("disconnected", "Lost connection to the glasses.")
teardown()
return
}
finished = true
listener.onResult(true)
listener.onState("result", "approved")
teardown()
} catch (e: Exception) {
stopHeartbeat()
if (!cancelled) {
listener.onState("error", StockLinkSession.messageOf(e))
}
teardown()
}
}
private fun connectArm(address: String, arm: String) {
try {
session.bringUp(address)
} catch (e: IllegalStateException) {
throw IllegalStateException(StockLinkSession.messageOf(e) + ": $arm arm ($address)")
}
emitLog("connected $arm arm")
}
/**
* Security-auth exchange on sid=0x80: firmware 2.2.9 will not run a session (and closes
* the link after ~30 s) without it, and on an unbonded phone this is what triggers SMP
* pairing, so it may sit waiting on an OS pairing prompt.
*/
private fun authenticateArm(address: String, arm: String) {
if (session.authenticate(address, arm) != StockLinkSession.AuthResult.SUCCESS) {
throw IllegalStateException(
"could not authenticate with the $arm arm ($address" +
") — if the phone shows a Bluetooth pairing request, accept it and try again"
)
}
emitLog("security auth complete: $arm arm")
}
private fun showPrompt(address: String) {
// Create the prompt page (warning text + No/Yes list) on sid=0xe0 Cmd=0. Two attempts:
// on 2.2.9 the first request sent right after the prelude can be dropped while the lens
// emits its own sid-0x80 notifications (observed with the device-info settings read).
var pageAck: ByteArray? = null
var attempt = 0
while (attempt < 2 && pageAck == null && !cancelled) {
val magic = session.allocMagic()
val page = BleProtocol.buildCreatePromptPage(magic, TEXT_NAME, TEXT_CONTAINER_ID, warningText, LIST_NAME, LIST_CONTAINER_ID, ITEMS)
pageAck = session.writeAndAwaitAck(address, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, magic, page, timings.createAckTimeoutMs)
if (pageAck == null) {
emitLog("prompt page attempt " + (attempt + 1) + " unacked: " + address)
}
attempt++
}
if (pageAck == null) {
throw IllegalStateException("prompt page not acked: $address")
}
emitLog("prompt page shown on $address")
}
/**
* Read each arm's battery via a sid-0x09 settings read on that arm's own link (the arms
* have independent batteries and each answers with its own). An arm that does not answer
* reports -1; the caller decides what to do with a partial reading.
*/
private fun readBatteries() {
val right = if (rightConnected) readBattery(rightAddress, "right") else -1
val left = if (leftConnected) readBattery(leftAddress, "left") else -1
emitLog("battery R=" + (if (right < 0) "?" else "$right%") + " L=" + (if (left < 0) "?" else "$left%"))
listener.onBattery(right, left)
}
private fun readBattery(address: String, arm: String): Int {
var attempt = 0
while (attempt < 2 && !cancelled) {
val ack = session.readSettings(address, timings.batteryAckTimeoutMs)
if (ack == null) {
emitLog("battery read attempt " + (attempt + 1) + " unacked: " + arm + " arm")
attempt++
continue
}
val snapshot = BleProtocol.parseSettingsBattery(ack)
if (snapshot != null) {
return snapshot.battery
}
emitLog("battery read ack had no battery field: $arm arm")
attempt++
}
return -1
}
private fun startHeartbeat() {
stopHeartbeat()
val ticker = InterruptibleSleep(platform)
heartbeat = ticker
startThread("faceclaw-flash-hb", true) {
while (heartbeat === ticker && ticker.sleep(timings.heartbeatIntervalMs.toLong())) {
if (heartbeat !== ticker) break
sendHeartbeat()
}
}
}
private fun sendHeartbeat() {
if (finished || cancelled) {
return
}
try {
val heartbeat = BleProtocol.buildHeartbeat(session.allocMagic())
if (rightConnected) {
session.writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, heartbeat)
}
} catch (e: Exception) {
emitLog("heartbeat failed: " + e.message)
}
}
private fun stopHeartbeat() {
val ticker = heartbeat ?: return
heartbeat = null
ticker.interrupt()
}
private fun sendShutdown() {
val shutdown = BleProtocol.buildShutdown(session.allocMagic(), 0)
if (rightConnected) {
session.writeFrame(rightAddress, BleProtocol.SID_EVENHUB, BleProtocol.FLAG_REQUEST, shutdown)
}
}
private fun teardown() {
finished = true
stopHeartbeat()
session.closeQuietly()
rightConnected = false
leftConnected = false
}
private fun handleEvent(frame: BleProtocol.ParsedFrame) {
val selection = BleProtocol.parseListSelection(frame) ?: return
if (LIST_NAME != selection.containerName) {
return
}
// Only a confirmed click is a decision; scroll/highlight changes are ignored.
if (selection.eventType != BleProtocol.EVENT_CLICK) {
return
}
val itemName = selection.itemName
val yes = selection.itemIndex == 1 || (itemName != null && itemName.lowercase().startsWith("yes"))
lock.withLock {
if (finished) {
return
}
finished = true
approved = yes
}
emitLog("selection: " + (if (yes) "flash" else "cancel") + " (index " + selection.itemIndex + ")")
selectionLatch.countDown()
}
private fun handleDisconnected(address: String) {
if (address.equals(leftAddress, ignoreCase = true)) {
// The prompt itself only needs the right arm; a dropped left link just means no
// left battery reading.
leftConnected = false
emitLog("left arm disconnected")
return
}
if (address.equals(rightAddress, ignoreCase = true)) {
rightConnected = false
lock.withLock {
if (!finished && !cancelled) {
rightLost = true
finished = true
selectionLatch.countDown()
}
}
}
}
// Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG).
private fun emitLog(line: String) {
listener.onLog(line)
}
}
@@ -0,0 +1,312 @@
package com.faceclaw.app
/**
* Ports g2flash.py's OTA flash procedure. Streams the EVENOTA firmware container to each lens
* over the firmware-data service using the aa21 envelope (BleProtocol.framePb) and the c0/c1
* control/data protocol.
*
* Each lens connection completes the sid-0x80 security-auth exchange before OTA BEGIN —
* firmware 2.2.9 closes unauthenticated links ~30 s in, killing the transfer mid-component. No
* control-channel traffic is injected during the transfer itself: official OTA captures show
* none between BEGIN and the final END, heartbeats demonstrably do not satisfy the 2.2.9
* deadline, and a concurrent control writer would interleave with the marker/data
* transactions. (See notes/ble-connections-2.2.9.md.)
*
* Lenses are flashed one at a time (left, then right). Finishing either lens reboots BOTH
* lenses, so the second lens is briefly unreachable — the reconnect for it uses a generous
* window rather than failing fast. [run] executes on the caller's worker thread.
*/
class OtaFlashFlow(
link: StockLink,
rightAddress: String,
leftAddress: String,
private val firmwarePath: String,
private val listener: FaceclawFirmwareFlasherListener,
private val timings: StockFlowTimings = StockFlowTimings(),
platform: ProtocolPlatform = protocolPlatform(),
/** Test seam: the firmware container bytes; defaults to reading [firmwarePath]. */
private val readImage: (String) -> ByteArray = ::readFirmwareFile,
) {
companion object {
const val TAG = "FaceclawFlasher"
// OTA message types (envelope sid byte) and control opcodes.
const val SID_CTRL = 0xc0
const val SID_DATA = 0xc1
const val FLAG_OTA = 0x00
const val OP_BEGIN = 0x00
const val OP_FILE_CHECK = 0x01
const val OP_BLOCK = 0x02
const val OP_END = 0x03
const val BLOCK_SIZE = 4096
const val BLOCK_NAK_RETRIES = 3
const val COMPONENT_RETRIES = 3
// END ack statuses that mean "component accepted": SUCCESS, UPDATING, SYS_RESTART.
private val END_OK = intArrayOf(0, 8, 9)
private val EMPTY = ByteArray(0)
fun isEndOk(status: Int): Boolean = END_OK.contains(status)
private fun readFirmwareFile(path: String): ByteArray =
try {
readFileData(path)
} catch (e: Exception) {
throw IllegalStateException("could not read firmware: " + e.message, e)
}
}
private val rightAddress = rightAddress.trim()
private val leftAddress = leftAddress.trim()
private val sleeper = InterruptibleSleep(platform)
// The auth exchange uses its own magic range (0x60..0x7f) and the OTA envelope counter
// starts at 1, mirroring the stock sequence.
private val session = StockLinkSession(link, ::emitLog, timings, magicStart = 0x60, magicEnd = 0x7f, seqStart = 1, platform = platform)
val cancelled: Boolean
get() = session.cancelled
fun cancel() {
session.cancel()
sleeper.interrupt()
}
fun run() {
try {
if (leftAddress.isEmpty() || rightAddress.isEmpty()) {
throw IllegalStateException("Both lens addresses are required to flash.")
}
listener.onState("validating", "")
val img = readImage(firmwarePath)
val segs = FirmwareImage.validate(img)
emitLog("firmware validated: " + segs.size + " components, " + img.size + " bytes")
flashLens("left", leftAddress, img, segs, timings.firstLensConnectWindowMs)
if (cancelled) {
listener.onState("error", "Cancelled.")
listener.onComplete(false, "Cancelled after the left lens.")
return
}
listener.onState("rebooting", "Left lens done. Both lenses reboot briefly; reconnecting for the right lens.")
sleeper.sleep(timings.rebootSettleMs.toLong())
flashLens("right", rightAddress, img, segs, timings.secondLensConnectWindowMs)
listener.onState("done", "")
listener.onComplete(true, "Both lenses flashed. The glasses are rebooting into the custom firmware.")
} catch (e: Exception) {
val message = if (cancelled) "Cancelled." else StockLinkSession.messageOf(e)
listener.onState("error", message)
listener.onComplete(false, message)
} finally {
session.closeQuietly()
}
}
private fun flashLens(lens: String, address: String, img: ByteArray, segs: List<FirmwareImage.Segment>, connectWindowMs: Int) {
listener.onState("connecting", lens)
connectLensResilient(lens, address, connectWindowMs)
sleeper.sleep(timings.notifySettleMs.toLong())
session.otaAcks.clear()
// Fresh envelope counter for the OTA stream (the auth exchange during bring-up used its
// own); mirrors the stock sequence.
session.resetSeq(1)
listener.onState("flashing", lens)
val beginStatus = sendCtrlAndWait(address, OP_BEGIN, EMPTY, timings.ctrlAckTimeoutMs)
if (!isEndOk(beginStatus)) {
emitLog("warning: unexpected begin status $beginStatus; continuing")
}
// Progress is reported in bytes across the whole image: the segments are one large
// firmware blob plus several small ones, so a bar stepped per segment would sit still
// through the big one and then sprint through the rest.
var totalBytes = 0L
for (seg in segs) {
totalBytes += seg.ps
}
var bytesBefore = 0L
for (i in segs.indices) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
flashComponentWithRetry(lens, address, i, segs.size, segs[i], img, bytesBefore, totalBytes)
bytesBefore += segs[i].ps
}
emitLog("$lens lens: all components verified")
session.disconnectQuietly(address)
}
private fun flashComponentWithRetry(
lens: String,
address: String,
index: Int,
count: Int,
seg: FirmwareImage.Segment,
img: ByteArray,
bytesBefore: Long,
totalBytes: Long,
) {
for (attempt in 0 until COMPONENT_RETRIES) {
if (attempt > 0) {
emitLog(seg.name + ": re-flash attempt " + (attempt + 1) + "/" + COMPONENT_RETRIES)
}
var endStatus: Int
try {
endStatus = flashComponent(lens, address, index, count, seg, img, bytesBefore, totalBytes)
} catch (e: StockTimeoutException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
} catch (e: RuntimeException) {
emitLog(seg.name + ": block phase failed: " + e.message)
endStatus = -1
}
if (isEndOk(endStatus)) {
emitLog(seg.name + ": END verify OK (status " + endStatus + ")")
return
}
if (endStatus >= 0) {
emitLog(seg.name + ": END verify FAILED (status " + endStatus + ")")
}
session.otaAcks.clear()
sleeper.sleep(timings.componentRetryDelayMs.toLong())
}
throw IllegalStateException("component " + seg.name + " failed after " + COMPONENT_RETRIES + " attempts")
}
private fun flashComponent(
lens: String,
address: String,
index: Int,
count: Int,
seg: FirmwareImage.Segment,
img: ByteArray,
bytesBefore: Long,
totalBytes: Long,
): Int {
val sub = img.copyOfRange(seg.off, seg.off + FirmwareImage.SUBHEADER_SIZE)
val ps = seg.ps
val payloadStart = seg.off + FirmwareImage.SUBHEADER_SIZE
val checkStatus = sendCtrlAndWait(address, OP_FILE_CHECK, sub, timings.ctrlAckTimeoutMs)
if (checkStatus != 0) {
throw IllegalStateException("FILE_CHECK rejected status=$checkStatus")
}
val blockCount = (ps + BLOCK_SIZE - 1) / BLOCK_SIZE
for (b in 0 until blockCount) {
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
val start = payloadStart + b * BLOCK_SIZE
val end = minOf(start + BLOCK_SIZE, payloadStart + ps)
val block = img.copyOfRange(start, end)
var accepted = false
for (tries in 0 until BLOCK_NAK_RETRIES) {
val status = sendBlock(address, block) // throws StockTimeoutException -> component re-flash
if (status == 0) {
accepted = true
break
}
emitLog(seg.name + ": block " + b + "/" + blockCount + " NAK=" + status + " resend " + (tries + 1) + "/" + BLOCK_NAK_RETRIES)
}
if (!accepted) {
throw IllegalStateException("block " + b + " NAK'd " + BLOCK_NAK_RETRIES + " times")
}
if (b % 20 == 0 || b == blockCount - 1) {
val bytesSent = bytesBefore + minOf((b + 1).toLong() * BLOCK_SIZE, ps.toLong())
listener.onProgress(lens, index + 1, count, b + 1, blockCount, bytesSent, totalBytes)
}
}
emitLog(seg.name + ": data phase done; sending END")
return sendCtrlAndWait(address, OP_END, EMPTY, timings.ctrlAckTimeoutMs)
}
/** Send one 4 KB block as a marker + data pair sharing one envelope seq. */
private fun sendBlock(address: String, block: ByteArray): Int {
val seq = session.nextSeq()
session.otaAcks.clear()
writeOta(address, SID_CTRL, byteArrayOf(OP_BLOCK.toByte()), seq)
writeOta(address, SID_DATA, block, seq)
return waitAck(OP_BLOCK, timings.blockAckTimeoutMs)
}
private fun sendCtrlAndWait(address: String, op: Int, data: ByteArray, timeoutMs: Int): Int {
val seq = session.nextSeq()
session.otaAcks.clear()
val payload = ByteArray(1 + data.size)
payload[0] = op.toByte()
data.copyInto(payload, 1)
writeOta(address, SID_CTRL, payload, seq)
return waitAck(op, timeoutMs)
}
private fun writeOta(address: String, sid: Int, payload: ByteArray, seq: Int): Boolean =
session.writeFrame(address, BleProtocol.OTA_DATA_WRITE_UUID, sid, FLAG_OTA, payload, seq)
private fun waitAck(wantOp: Int, timeoutMs: Int): Int {
val deadline = session.now() + timeoutMs
while (true) {
val remaining = deadline - session.now()
if (remaining <= 0 || cancelled) {
break
}
val pb = session.otaAcks.poll(remaining) ?: break
if (pb.size >= 2 && (pb[0].toInt() and 0xff) == wantOp) {
return pb[1].toInt() and 0xff
}
// Different opcode (e.g. a late ack from a prior stage) — ignore, keep waiting.
}
throw StockTimeoutException("no ack op=0x" + wantOp.toString(16) + " within " + timeoutMs + "ms")
}
// ---- connection ----------------------------------------------------------
private fun connectLensResilient(lens: String, address: String, windowMs: Int) {
val deadline = session.now() + windowMs
var lastError: String? = "unknown"
var attempt = 0
while (session.now() < deadline && !cancelled) {
attempt++
try {
if (bringUpLens(address)) {
emitLog("connected $lens lens (attempt $attempt)")
return
}
lastError = "connect/discover incomplete"
} catch (e: Exception) {
lastError = StockLinkSession.messageOf(e)
}
emitLog("$lens connect attempt $attempt failed ($lastError); retrying...")
session.disconnectQuietly(address)
sleeper.sleep(timings.otaRetryDelayMs.toLong())
}
if (cancelled) {
throw IllegalStateException("Cancelled.")
}
throw IllegalStateException("could not reach $lens lens: $lastError")
}
private fun bringUpLens(address: String): Boolean {
try {
// Control-channel notify carries the auth response; the OTA notify carries the acks.
session.bringUp(address, otaChannel = true)
} catch (e: IllegalStateException) {
return false
}
// Complete the sid-0x80 security-auth exchange on this connection. On an unbonded
// phone this triggers SMP pairing (possibly with an OS prompt), so the wait is
// generous. Must succeed before any OTA traffic.
if (session.authenticate(address, "ota") != StockLinkSession.AuthResult.SUCCESS) {
throw IllegalStateException("security auth not acknowledged — if the phone shows a Bluetooth pairing request, accept it")
}
return true
}
// Log lines reach the platform log through the adapter's onLog (Android: logcat under TAG).
private fun emitLog(line: String) {
listener.onLog(line)
}
}
@@ -8,6 +8,34 @@ interface ProtocolPlatform {
fun createLock(): ProtocolLock
fun createDeflater(): ProtocolDeflater
/** A mutex plus condition variable for blocking waits (see [ProtocolCondition]). */
fun createCondition(): ProtocolCondition
}
/**
* Mutex + condition variable. NOT reentrant: iOS backs it with NSCondition, so a thread that
* already holds the lock must not lock it again (Android's implementation would tolerate it
* and hide the bug). Hold the lock while calling [awaitMs] and [signalAll].
*/
interface ProtocolCondition {
fun lock()
fun unlock()
/** Waits up to [timeoutMs] (spurious wake-ups possible); the lock is held again on return. */
fun awaitMs(timeoutMs: Long)
fun signalAll()
}
inline fun <T> ProtocolCondition.withLock(action: () -> T): T {
lock()
try {
return action()
} finally {
unlock()
}
}
/** Must be reentrant: transport encoding can reset its history under the lock. */
@@ -0,0 +1,91 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
/** Android bond state of a peripheral; iOS reports UNKNOWN (pairing is invisible to the app). */
enum class BondState {
NONE,
BONDING,
BONDED,
UNKNOWN,
}
/** Inbound side of a [StockLink]; called on whatever thread the platform stack delivers on. */
interface StockLinkListener {
fun onNotification(address: String, characteristicUuid: String, data: ByteArray)
fun onConnectionStateChange(address: String, connected: Boolean)
}
/**
* The GATT boundary used by the stock-firmware flows (device-info probe, flash prompt, OTA
* flasher). Calls block on the caller's thread with explicit timeouts; addresses are opaque
* strings (Android MAC, iOS peripheral identifier). Android wraps FaceclawBleManager; an iOS
* implementation sits on CoreBluetooth. Methods other than [isConnected]/[bondState]/
* [disconnect]/[close] may throw IllegalStateException when the link is not connected.
*/
interface StockLink {
fun setListener(listener: StockLinkListener?)
fun connect(address: String, timeoutMs: Int): Boolean
/** Best-effort link tuning after connect (connection priority, MTU). */
fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int)
fun discoverServices(address: String, timeoutMs: Int): Boolean
fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean
fun writeFrames(
address: String,
characteristicUuid: String,
frames: List<ByteArray>,
mode: GattWriteMode,
timeoutMs: Int,
): Boolean
fun isConnected(address: String): Boolean
fun bondState(address: String): BondState
fun disconnect(address: String)
fun close()
}
/** A wait for a lens reply expired. */
class StockTimeoutException(message: String) : Exception(message)
/**
* Timeouts and pacing shared by the stock flows. Production uses the defaults; tests shrink
* them. Values are milliseconds unless named otherwise.
*/
class StockFlowTimings(
@JvmField val connectTimeoutMs: Int = ConnectionOptions.CONNECT_TIMEOUT_MS,
@JvmField val servicesTimeoutMs: Int = ConnectionOptions.SERVICES_TIMEOUT_MS,
@JvmField val descriptorTimeoutMs: Int = ConnectionOptions.DESCRIPTOR_TIMEOUT_MS,
@JvmField val writeTimeoutMs: Int = ConnectionOptions.WRITE_TIMEOUT_MS,
@JvmField val preludeTimeoutMs: Int = ConnectionOptions.PRELUDE_TIMEOUT_MS,
@JvmField val securityAuthTimeoutMs: Int = ConnectionOptions.SECURITY_AUTH_TIMEOUT_MS,
@JvmField val securityAuthSoftTimeoutMs: Int = ConnectionOptions.SECURITY_AUTH_SOFT_TIMEOUT_MS,
/** Upper bound on waiting while the OS reports pairing in progress (a dialog may be up). */
@JvmField val pairingWaitCapMs: Int = 90_000,
/** After a fresh bond, how long the firmware gets to notify success before the auth request is re-sent. */
@JvmField val postBondGraceMs: Int = 2_000,
@JvmField val pollMs: Int = 250,
/** Delay between connect/auth attempts on one arm and between auth re-sends. */
@JvmField val armRetryDelayMs: Int = 1_000,
@JvmField val queryTimeoutMs: Int = 4_000,
@JvmField val heartbeatIntervalMs: Int = 4_000,
@JvmField val selectionTimeoutMs: Int = 120_000,
@JvmField val createAckTimeoutMs: Int = 3_000,
@JvmField val batteryAckTimeoutMs: Int = 3_000,
@JvmField val blockAckTimeoutMs: Int = 4_000,
@JvmField val ctrlAckTimeoutMs: Int = 8_000,
@JvmField val firstLensConnectWindowMs: Int = 30_000,
@JvmField val secondLensConnectWindowMs: Int = 120_000,
@JvmField val rebootSettleMs: Int = 5_000,
@JvmField val notifySettleMs: Int = 2_500,
@JvmField val otaRetryDelayMs: Int = 2_500,
@JvmField val componentRetryDelayMs: Int = 1_500,
)
@@ -0,0 +1,443 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
/**
* The stock-firmware request/reply plumbing that the device-info probe, the flash prompt and
* the OTA flasher all need on top of a [StockLink]: connect + subscribe, envelope sequence and
* magic allocation, frame demux, ack matching keyed by (sid, magic), the bond-aware security
* auth exchange (only the firmware's SUCCESS satisfies it), the unsolicited settings capture,
* async-event and disconnect hooks, and the OTA ack queue. Blocking calls run on the owning
* flow's worker thread; notifications arrive on the platform's callback thread.
*/
class StockLinkSession(
private val link: StockLink,
private val log: (String) -> Unit,
private val timings: StockFlowTimings = StockFlowTimings(),
magicStart: Int = 100,
private val magicEnd: Int = 255,
seqStart: Int = 0x40,
/** Log every parsed control frame with a payload prefix (the probe's 2.2.9 diagnostics). */
private val traceFrames: Boolean = false,
private val platform: ProtocolPlatform = protocolPlatform(),
) : StockLinkListener {
enum class AuthResult {
SUCCESS,
UNCONFIRMED,
LINK_DROPPED,
}
private val lock = platform.createLock()
private val magicStart = magicStart
private var nextMagic = magicStart
private var nextSeq = seqStart
private val pendingAcks = HashMap<String, Latch>()
private val ackPayloads = HashMap<String, ByteArray>()
private val authMagics = HashSet<Int>()
private var authLatch: Latch? = null
/** Latest sid-0x09 frame carrying firmware versions, whatever magic it came with. */
@Volatile
var unsolicitedSettingsPb: ByteArray? = null
private set
/** OTA acks (first two payload bytes: opcode, status) from the data-notify characteristic. */
val otaAcks = BlockingQueue<ByteArray>(platform)
/** Async events (flag 0x01/0x06) on the control characteristic, e.g. list selections. */
@Volatile
var onEvent: ((address: String, frame: BleProtocol.ParsedFrame) -> Unit)? = null
@Volatile
var onDisconnected: ((address: String) -> Unit)? = null
@Volatile
var cancelled = false
private set
init {
link.setListener(this)
}
val platformServices: ProtocolPlatform
get() = platform
fun now(): Long = platform.elapsedRealtimeMs()
/** Wake every pending wait; subsequent waits return immediately as timeouts. */
fun cancel() {
cancelled = true
lock.withLock {
for (latch in pendingAcks.values) latch.countDown()
authLatch?.countDown()
}
}
fun allocMagic(): Int =
lock.withLock {
val magic = nextMagic
nextMagic = if (nextMagic >= magicEnd) magicStart else nextMagic + 1
magic
}
fun nextSeq(): Int =
lock.withLock {
val seq = nextSeq and 0xff
nextSeq = (nextSeq + 1) and 0xff
seq
}
fun resetSeq(value: Int) {
lock.withLock { nextSeq = value and 0xff }
}
/** Connect, tune, discover and subscribe to the control notify (plus the OTA notify when asked). Throws on failure. */
fun bringUp(address: String, otaChannel: Boolean = false) {
if (!link.connect(address, timings.connectTimeoutMs)) {
throw IllegalStateException("connect failed")
}
link.prepareLink(address, ConnectionOptions.DESIRED_MTU, timings.connectTimeoutMs)
if (!link.discoverServices(address, timings.servicesTimeoutMs)) {
throw IllegalStateException("service discovery failed")
}
if (!link.enableNotifications(address, BleProtocol.NOTIFY_CHAR_UUID, timings.descriptorTimeoutMs)) {
throw IllegalStateException("could not subscribe to notifications")
}
if (otaChannel && !link.enableNotifications(address, BleProtocol.OTA_DATA_NOTIFY_UUID, timings.descriptorTimeoutMs)) {
throw IllegalStateException("could not subscribe to OTA notifications")
}
}
fun isConnected(address: String): Boolean = link.isConnected(address)
fun disconnectQuietly(address: String) {
try {
link.disconnect(address)
} catch (ignored: Exception) {
}
}
fun closeQuietly() {
try {
link.close()
} catch (ignored: Exception) {
}
}
/** Frame and write one message on the control characteristic; false when the write failed. */
fun writeFrame(address: String, sid: Int, flag: Int, payload: ByteArray): Boolean =
writeFrame(address, BleProtocol.WRITE_CHAR_UUID, sid, flag, payload, nextSeq())
/** @throws IllegalStateException when the link reports "not connected". */
fun writeFrame(address: String, characteristicUuid: String, sid: Int, flag: Int, payload: ByteArray, seq: Int): Boolean {
val frames = BleProtocol.framePb(payload, sid, flag, seq)
return link.writeFrames(address, characteristicUuid, frames, ConnectionOptions.WRITE_MODE, timings.writeTimeoutMs)
}
/**
* Write a request and wait for the ack carrying the same sid and magic. Returns the ack's
* protobuf (possibly empty), or null on write failure / timeout / cancel.
*/
fun writeAndAwaitAck(address: String, sid: Int, flag: Int, magic: Int, payload: ByteArray, timeoutMs: Int): ByteArray? {
val key = ackKey(sid, magic)
val latch = Latch(1, platform)
lock.withLock {
pendingAcks[key] = latch
ackPayloads.remove(key)
}
try {
val seq = nextSeq()
log(
"tx " + address + " sid=0x" + hex2(sid) + " flag=0x" + hex2(flag) + " magic=" + magic +
" seq=0x" + hex2(seq) + " len=" + payload.size
)
val written =
try {
writeFrame(address, BleProtocol.WRITE_CHAR_UUID, sid, flag, payload, seq)
} catch (e: IllegalStateException) {
// "Not connected" — the link went away underneath us.
false
}
if (!written) {
log("tx write FAILED sid=0x" + sid.toString(16) + " (" + address + ")")
return null
}
if (!latch.await(timeoutMs.toLong()) || cancelled) {
return null
}
return lock.withLock { ackPayloads[key] ?: ByteArray(0) }
} finally {
lock.withLock {
if (pendingAcks[key] === latch) pendingAcks.remove(key)
ackPayloads.remove(key)
}
}
}
/** Mandatory session prelude on sid=0x01; throws when it is not acked. */
fun sendPrelude(address: String, failureDetail: String = "") {
if (writeAndAwaitAck(
address,
BleProtocol.PRELUDE_ACK_SID,
BleProtocol.FLAG_REQUEST,
BleProtocol.PRELUDE_ACK_MAGIC,
BleProtocol.PRELUDE_F5872_PAYLOAD,
timings.preludeTimeoutMs,
) == null
) {
throw IllegalStateException("session prelude not acked$failureDetail")
}
}
/** One sid-0x09 settings read; the ack protobuf or null when unanswered. */
fun readSettings(address: String, timeoutMs: Int): ByteArray? {
val magic = allocMagic()
return writeAndAwaitAck(address, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_REQUEST, magic, BleProtocol.buildSettingsQuery(magic), timeoutMs)
}
/**
* Send the sid-0x80 authentication request and wait for the firmware's SUCCESS notification,
* which only arrives once the link is encrypted. On an unbonded phone that means waiting
* through the OS pairing flow (BOND_BONDING, possibly with a dialog up), so the wait is
* extended while the OS reports pairing in progress and the request is re-sent once after a
* fresh bond in case the pre-pairing write was dropped. Bond state is polled; 250 ms
* granularity is plenty here. Platforms that cannot see bonding (UNKNOWN) get the plain
* timeout. The first reply with a non-success result is expected before encryption and does
* not satisfy the wait.
*/
fun authenticate(address: String, label: String): AuthResult {
val start = now()
var deadline = start + timings.securityAuthTimeoutMs
val hardDeadline = start + timings.pairingWaitCapMs
val latch = Latch(1, platform)
lock.withLock {
authMagics.clear()
authLatch = latch
}
val initialBond = link.bondState(address)
log("security auth: $label lens, bond state " + bondStateName(initialBond))
try {
var sends = 0
var lastSendFailed = false
var nextSendAt = start
var bondedAt = -1L
var resentAfterBond = false
var loggedBonding = false
while (!cancelled) {
val now = now()
val bond = link.bondState(address)
if (bond == BondState.BONDING) {
// OS pairing in progress — the firmware will answer once the link is
// encrypted. Don't write, don't give up, don't move on to the other arm.
if (!loggedBonding) {
log("the OS is pairing with the $label lens; waiting for the user to accept")
loggedBonding = true
}
if (now >= hardDeadline) {
break
}
} else {
if (bond == BondState.BONDED && initialBond != BondState.BONDED && bondedAt < 0) {
bondedAt = now
log("bond established with the $label lens; waiting for the auth success")
deadline = maxOf(deadline, now + timings.securityAuthSoftTimeoutMs)
}
val wantSend = sends == 0 ||
(lastSendFailed && sends < 3 && now >= nextSendAt) ||
(bondedAt >= 0 && !resentAfterBond && now - bondedAt >= timings.postBondGraceMs)
if (wantSend) {
if (bondedAt >= 0) {
resentAfterBond = true
if (sends > 0) {
log("re-sending the auth request after pairing ($label lens)")
}
}
lastSendFailed = !sendAuthRequest(address)
sends++
nextSendAt = now() + timings.armRetryDelayMs
if (lastSendFailed && !link.isConnected(address)) {
return AuthResult.LINK_DROPPED
}
}
if (now >= deadline) {
break
}
}
if (latch.await(timings.pollMs.toLong()) && !cancelled) {
log("security auth complete: $label lens ($address)")
return AuthResult.SUCCESS
}
if (!link.isConnected(address)) {
log(
"security auth: $label lens disconnected while waiting" +
(if (bond == BondState.BONDING) " (the OS was still pairing)" else "")
)
return AuthResult.LINK_DROPPED
}
}
if (cancelled) {
return AuthResult.UNCONFIRMED
}
log("security auth unconfirmed: $label lens ($address), bond state " + bondStateName(link.bondState(address)))
return AuthResult.UNCONFIRMED
} finally {
lock.withLock {
authLatch = null
authMagics.clear()
}
}
}
private fun sendAuthRequest(address: String): Boolean {
val magic = allocMagic()
lock.withLock { authMagics.add(magic) }
val seq = nextSeq()
val payload = BleProtocol.buildAuthenticationRequest(magic)
log(
"tx " + address + " sid=0x" + hex2(BleProtocol.SID_SECURITY_AUTH) + " flag=0x" + hex2(BleProtocol.FLAG_SECURITY_AUTH) +
" magic=" + magic + " seq=0x" + hex2(seq) + " len=" + payload.size
)
val written =
try {
writeFrame(address, BleProtocol.WRITE_CHAR_UUID, BleProtocol.SID_SECURITY_AUTH, BleProtocol.FLAG_SECURITY_AUTH, payload, seq)
} catch (e: IllegalStateException) {
false
}
if (!written) {
log("tx write FAILED sid=0x80 ($address)")
}
return written
}
/** Block while the OS reports this arm as BONDING (pairing dialog up / SMP in flight). */
fun waitForBondToSettle(address: String, label: String, sleeper: InterruptibleSleep) {
val cap = now() + timings.pairingWaitCapMs
var logged = false
while (!cancelled && link.bondState(address) == BondState.BONDING && now() < cap) {
if (!logged) {
log("waiting for the OS to finish pairing with the $label lens before reconnecting")
logged = true
}
sleeper.sleep(timings.pollMs.toLong())
}
}
// ---- inbound --------------------------------------------------------------
override fun onNotification(address: String, characteristicUuid: String, data: ByteArray) {
if (BleProtocol.OTA_DATA_NOTIFY_UUID.equals(characteristicUuid, ignoreCase = true)) {
val frame = BleProtocol.parseFrame(data)
if (frame.ok && frame.pb.size >= 2) {
otaAcks.put(frame.pb.copyOf(minOf(frame.pb.size, 2)))
}
return
}
if (!BleProtocol.NOTIFY_CHAR_UUID.equals(characteristicUuid, ignoreCase = true)) {
return
}
// One notification value can carry several envelope frames back to back; reading only
// the first would silently drop the rest.
val frames = BleProtocol.splitFrames(data)
if (traceFrames && frames.size > 1) {
log("rx " + address + " value carries " + frames.size + " frames (raw " + data.size + " bytes)")
}
for (buf in frames) {
handleFrame(address, buf, data.size)
}
}
private fun handleFrame(address: String, buf: ByteArray, rawValueLength: Int) {
val frame = BleProtocol.parseFrame(buf)
if (!frame.ok) {
if (traceFrames) {
log(
"rx " + address + " unparseable frame len=" + buf.size + " (raw value " + rawValueLength + ")" +
" head=" + BinaryEncoding.bytesToHex(buf.copyOf(minOf(16, buf.size)))
)
}
return
}
if (traceFrames) {
// Log every control frame while diagnosing 2.2.9: sid/flag/type/magic plus a
// payload prefix is enough to reconstruct what the lens said.
val declared = if (buf.size > 3) buf[3].toInt() and 0xff else 0
val truncated = if (buf.size < 8 + declared) " TRUNCATED(declared=$declared)" else ""
log(
"rx " + address + " sid=0x" + hex2(frame.sid) + " flag=0x" + hex2(frame.flag) + " type=" + frame.msgType +
" magic=" + frame.msgSeq + " frag=" + (if (buf.size > 5) buf[5].toInt() and 0xff else 0) + "/" +
(if (buf.size > 4) buf[4].toInt() and 0xff else 0) + " len=" + frame.pb.size + truncated +
" pb=" + BinaryEncoding.bytesToHex(frame.pb.copyOf(minOf(48, frame.pb.size)))
)
}
if (frame.sid == BleProtocol.SID_UI_SETTING && BleProtocol.parseSettingsFirmwareInfo(frame.pb) != null) {
// Any settings frame carrying firmware versions answers the probe's question,
// whether or not it matches the magic we asked with.
unsolicitedSettingsPb = frame.pb
}
if (frame.flag == BleProtocol.FLAG_NOTIFY || frame.flag == BleProtocol.FLAG_NOTIFY_ALT) {
// Async event, not an ack. This includes the lens's own periodic sid-0x80 notifies
// (device-chosen magic, non-empty result).
onEvent?.invoke(address, frame)
return
}
if (frame.sid == BleProtocol.SID_SECURITY_AUTH) {
var success = false
var ours = false
lock.withLock {
val currentLatch = authLatch
if (currentLatch != null) {
ours = authMagics.contains(frame.msgSeq)
for (magic in authMagics) {
if (BleProtocol.isAuthenticationSuccess(frame.pb, magic)) {
success = true
currentLatch.countDown()
break
}
}
}
}
if (ours && !success) {
// Expected before the link is encrypted: the firmware answers the request
// straight away with a non-success result and sends the real success once
// pairing/encryption completes.
log("auth reply for magic " + frame.msgSeq + " is not the success result; still waiting")
}
return
}
if (frame.msgSeq >= 0) {
val key = ackKey(frame.sid, frame.msgSeq)
lock.withLock {
val latch = pendingAcks[key]
if (latch != null) {
ackPayloads[key] = frame.pb
latch.countDown()
}
}
}
}
override fun onConnectionStateChange(address: String, connected: Boolean) {
if (!connected) {
log("disconnected: $address")
onDisconnected?.invoke(address)
}
}
companion object {
private fun ackKey(sid: Int, magic: Int): String = "$sid:$magic"
internal fun hex2(value: Int): String = (value and 0xff).toString(16).padStart(2, '0')
internal fun bondStateName(state: BondState): String =
when (state) {
BondState.NONE -> "none"
BondState.BONDING -> "bonding"
BondState.BONDED -> "bonded"
BondState.UNKNOWN -> "unknown"
}
internal fun messageOf(e: Throwable): String = e.message ?: e.toString()
}
}
@@ -0,0 +1,695 @@
package com.faceclaw.app
import com.faceclaw.app.GlassesSessionCore.Companion.DASHBOARD_TILE
import com.faceclaw.app.GlassesSessionCore.Companion.FACECLAW_WAKE_CONTROL_WAIT_MS
import com.faceclaw.app.GlassesSessionCore.Companion.TAG
/*
* GlassesSessionCore, part 3: CFW lease controls, EvenHub shutdown, the bandwidth
* benchmark stream, transport-failure handling and the listener emits (all
* marshalled through SessionHost.postToMain, or a subscriber's own dispatcher
* for compass events).
*/
// ---------------------------------------------------------------------------
// Bandwidth benchmark
/** Pending + in-flight benchmark no-op messages. */
internal fun GlassesSessionCore.benchmarkOutstandingLocked(): Int {
var count = 0
for (message in pendingMessages) {
if ("bandwidth" == message.kind) count++
}
for (message in inFlightMessages) {
if ("bandwidth" == message.kind) count++
}
return count
}
/**
* Keep the benchmark stream fed: top the pending queue up so the send
* window never starves, stop enqueueing once the run expires (or a message
* times out), and finish the run when the last outstanding message drains.
*/
internal fun GlassesSessionCore.maintainBenchmarkLocked(now: Long) {
if (!sessionReady || !fixedLayoutCreated || shutdownRequested) {
finishBenchmarkLocked(true, "session no longer ready")
return
}
if (benchmarkLinkPending || now < benchmarkReadyAtMs) {
return
}
if (benchmarkAborted || now >= benchmarkDeadlineAtMs) {
// The run is over: drop queued-but-unsent no-ops (sending them
// would stretch the run past its deadline) and finish once the
// in-flight tail has acked or timed out.
val pendingIterator = pendingMessages.iterator()
while (pendingIterator.hasNext()) {
val queued = pendingIterator.next()
if ("bandwidth" == queued.kind) {
pendingIterator.remove()
magicPool.release(queued.sid, queued.magic, queued.label, "benchmark over")
}
}
if (benchmarkOutstandingLocked() == 0) {
finishBenchmarkLocked(false, "complete")
}
return
}
// One more than the window so a fresh message is always ready to write
// the moment an ack frees a slot.
var outstanding = benchmarkOutstandingLocked()
while (outstanding <= benchmarkWindowSize) {
enqueueBenchmarkMessageLocked()
outstanding++
}
}
internal fun GlassesSessionCore.finishBenchmarkLocked(aborted: Boolean, reason: String) {
if (!benchmarkActive) {
return
}
benchmarkActive = false
benchmarkAborted = benchmarkAborted or aborted
// Prefer the last ack as the end time so drain lag after the deadline
// doesn't dilute the throughput figure.
benchmarkEndAtMs = if (benchmarkLastAckAtMs != 0L) benchmarkLastAckAtMs else now()
logLine("bandwidth benchmark " + (if (benchmarkAborted) "aborted" else "finished") + " (" + reason + "): "
+ benchmarkMessagesAcked + "/" + benchmarkMessagesSent + " acked, "
+ benchmarkPayloadBytesAcked + "B payload, " + benchmarkTimeouts + " timeouts")
}
internal fun GlassesSessionCore.enqueueBenchmarkMessageLocked() {
// Fresh bytes per message: the transport's persistent compression history
// would compress even a random payload if we reused it across the run.
val payload = ByteArray(benchmarkMessageSize)
kotlin.random.Random.nextBytes(payload)
payload[0] = CFW_MSG_DIAGNOSTICS.toByte() // CFW diagnostic-control mode...
payload[1] = 0x7f.toByte() // ...with an unused sub-op: acked, no effect
val message = messageBuilder.imagePayload(
"bandwidth",
DASHBOARD_TILE,
nextMapSessionId(),
payload,
"bandwidth no-op " + payload.size + "B",
connectionOptions.sendImagesToLeft)
val payloadBytes = payload.size
// Logical custom-message bytes; excludes length tag and packet framing.
val wireBytes = message.message.size
message.onSent = MessageCallback {
benchmarkMessagesSent++
if (benchmarkStartAtMs == 0L) {
benchmarkStartAtMs = now()
benchmarkDeadlineAtMs = benchmarkStartAtMs + benchmarkDurationMs
}
}
message.onAck = MessageCallback {
val ackedAtMs = now()
benchmarkMessagesAcked++
benchmarkPayloadBytesAcked += payloadBytes
benchmarkWireBytesAcked += wireBytes
benchmarkLastAckAtMs = ackedAtMs
// No-op payloads ride the image path, so the firmware resets its
// heartbeat timer on them just like real image messages.
lastHeartbeatAckedAtMs = ackedAtMs
}
message.onTimeout = MessageCallback {
benchmarkTimeouts++
benchmarkAborted = true
logLine("bandwidth benchmark ack timeout")
}
pendingMessages.addLast(message)
}
// ---------------------------------------------------------------------------
// EvenHub shutdown and audio control waits
internal fun GlassesSessionCore.sendShutdownInternal(exitMode: Int, reconnectOnTimeout: Boolean): Boolean {
val magic: Int
val startedAtMs = now()
monitor.withLock {
if (!running || !sessionReady) {
logLine("skip shutdown; session not ready")
return false
}
if (shutdownRequested) {
return true
}
shutdownRequested = true
// Magic values wrap, so an ACK from a much older suspend must not
// satisfy this request after enough sleep/wake cycles.
lastShutdownAckMagic = 0
clearPendingMessagesLocked("shutdown requested")
val message = messageBuilder.shutdown(exitMode)
magic = message.magic
message.onAck = MessageCallback {
lastShutdownAckMagic = message.magic
fixedLayoutCreated = false
displayedFingerprint = ""
}
message.onTimeout = MessageCallback {
if (reconnectOnTimeout) {
handleTransportFailure("shutdown ack timeout")
} else {
logLine("EvenHub shutdown ack timeout; keeping BLE connected")
}
}
pendingMessages.addFirst(message)
logLine("queue shutdown")
// The stock compass keeps the magnetometer sampling independently of
// the plugin task, so ending the page does not stop it. Force a
// disable ahead of the shutdown command whenever it may be running:
// this also covers a disable that was wiped by the queue flush above
// or whose ack was lost, and the charging-mode/exit paths where the
// Compass window never got a chance to release it.
if (compassMaybeOn && fixedLayoutCreated) {
enqueueCompassControlLocked(true, false)
}
}
interruptibleSleep.interrupt()
val ackDeadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + 500
monitor.withLock {
while (running
&& sessionReady
&& lastShutdownAckMagic != magic
&& lastShutdownExitAtMs < startedAtMs
&& hasPendingOrInflightMagicLocked(magic)) {
val remaining = ackDeadline - now()
if (remaining <= 0) {
break
}
monitor.awaitMs(minOf(remaining, 100L))
}
val acked = lastShutdownAckMagic == magic
if (acked) {
val exitDeadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + 500
while (running && sessionReady && lastShutdownExitAtMs < startedAtMs) {
val remaining = exitDeadline - now()
if (remaining <= 0) {
break
}
monitor.awaitMs(minOf(remaining, 100L))
}
}
return acked || lastShutdownExitAtMs >= startedAtMs
}
}
internal fun GlassesSessionCore.waitForAudioControlAck(magic: Int, operation: String): Boolean {
val deadline = now() + ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 500
monitor.withLock {
while (running && sessionReady && lastAudioControlAckMagic != magic && hasPendingOrInflightMagicLocked(magic)) {
val remaining = deadline - now()
if (remaining <= 0) {
break
}
monitor.awaitMs(minOf(remaining, 100L))
}
val acked = lastAudioControlAckMagic == magic
if (!acked) {
if ("enable" == operation) {
audioPacketListener = null
audioCaptureActive = false
}
logLine("G2 mic $operation ack timeout")
}
return acked
}
}
// ---------------------------------------------------------------------------
// CFW wake and framebuffer leases
/**
* Replace any stale lease control with one right-arm and one left-arm
* fire-and-forget write. With priority=true the right arm is sent first so
* CLAIM reaches the lens that originated the deferred wake immediately.
*/
internal fun GlassesSessionCore.enqueueFaceclawWakeControlLocked(operation: Int, nonce: Int, priority: Boolean): Int {
clearMessagesOfKindLocked("wake-lease-control")
val generation = ++faceclawWakeControlGeneration
faceclawWakeControlSentCount = 0
if (operation == BleProtocol.FACECLAW_WAKE_OP_ACQUIRE) {
lastFaceclawWakeLeaseQueuedAtMs = now()
}
val onSent = MessageCallback {
if (faceclawWakeControlGeneration == generation) {
faceclawWakeControlSentCount += 1
}
}
val right = messageBuilder.faceclawWakeControl(operation, nonce, false)
val left = messageBuilder.faceclawWakeControl(operation, nonce, true)
right.onSent = onSent
left.onSent = onSent
if (priority) {
pendingMessages.addFirst(left)
pendingMessages.addFirst(right)
} else {
pendingMessages.addLast(right)
pendingMessages.addLast(left)
}
logLine("queue " + right.label + " + L")
return generation
}
internal fun GlassesSessionCore.waitForFaceclawWakeControlDelivery(generation: Int, timeoutMs: Long): Boolean {
val deadline = now() + maxOf(0L, timeoutMs)
monitor.withLock {
while (running
&& sessionReady
&& faceclawWakeControlGeneration == generation
&& faceclawWakeControlSentCount < 2) {
val remaining = deadline - now()
if (remaining <= 0) {
break
}
monitor.awaitMs(minOf(remaining, 100L))
}
return faceclawWakeControlGeneration == generation
&& faceclawWakeControlSentCount >= 2
}
}
/**
* Acquire/renew or release CFW's independent direct-framebuffer repaint
* guard on both arms. It is separate from the optional idle-wake lease:
* every Faceclaw display session needs this guard while its EvenHub layout
* contains swipe-capturing stock widgets.
*/
internal fun GlassesSessionCore.enqueueFaceclawFramebufferControlLocked(operation: Int, priority: Boolean): Int {
clearMessagesOfKindLocked("framebuffer-lease-control")
val generation = ++faceclawFramebufferControlGeneration
faceclawFramebufferControlSentCount = 0
if (operation == BleProtocol.FACECLAW_FB_OP_ACQUIRE) {
lastFaceclawFramebufferLeaseQueuedAtMs = now()
}
val onSent = MessageCallback {
if (faceclawFramebufferControlGeneration == generation) {
faceclawFramebufferControlSentCount += 1
}
}
val right = messageBuilder.faceclawFramebufferControl(operation, false)
val left = messageBuilder.faceclawFramebufferControl(operation, true)
right.onSent = onSent
left.onSent = onSent
if (priority) {
pendingMessages.addFirst(left)
pendingMessages.addFirst(right)
} else {
pendingMessages.addLast(right)
pendingMessages.addLast(left)
}
logLine("queue " + right.label + " + L")
return generation
}
internal fun GlassesSessionCore.waitForFaceclawFramebufferControlDelivery(generation: Int, timeoutMs: Long): Boolean {
val deadline = now() + maxOf(0L, timeoutMs)
monitor.withLock {
while (running
&& sessionReady
&& faceclawFramebufferControlGeneration == generation
&& faceclawFramebufferControlSentCount < 2) {
val remaining = deadline - now()
if (remaining <= 0) {
break
}
monitor.awaitMs(minOf(remaining, 100L))
}
return faceclawFramebufferControlGeneration == generation
&& faceclawFramebufferControlSentCount >= 2
}
}
internal fun GlassesSessionCore.releaseFaceclawFramebufferLease(): Boolean {
val generation: Int
monitor.withLock {
if (!running || !sessionReady) {
return true
}
generation = enqueueFaceclawFramebufferControlLocked(
BleProtocol.FACECLAW_FB_OP_RELEASE,
true
)
}
interruptibleSleep.interrupt()
return waitForFaceclawFramebufferControlDelivery(
generation,
FACECLAW_WAKE_CONTROL_WAIT_MS
)
}
// ---------------------------------------------------------------------------
// Failure handling
/** The address of a configured arm the OS no longer holds a bond for, or null. */
internal fun GlassesSessionCore.firstUnpairedArm(): String? {
if (!link.isBonded(rightAddress)) return rightAddress
if (!link.isBonded(leftAddress)) return leftAddress
return null
}
/**
* A connect failure while an arm's bond is missing (the pairing was
* forgotten in the OS settings, or the address was typed in by hand and
* never paired) will repeat forever, so instead of scheduling a retry,
* park the worker loop and tell the user to re-pair. Only an explicit
* connect (which builds a fresh communicator) starts a new attempt.
*/
internal fun GlassesSessionCore.handleUnpairedFailure(address: String) {
logError("Connect failed and $address is not paired; suspending reconnect")
monitor.withLock {
reconnectHalted = true
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
shutdownRequested = false
chargingMode = false
imageRetryAfterMs = 0
displayedFingerprint = ""
faceclawWakePendingNonce = -1
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked("arm not paired: $address")
reconnectAfterMs = Long.MAX_VALUE
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
if (!userDisconnectRequested) {
setStateDisplay(
"unpaired",
"The glasses (" + address + ") are not paired with this phone."
+ " Use \"Pair glasses\" to pair them again, then connect."
)
}
interruptibleSleep.interrupt()
}
internal fun GlassesSessionCore.handleTransportFailure(reason: String?) {
logError("Transport failure: $reason")
monitor.withLock {
maybeEmitEvenAppConflictLocked(reason)
finishBenchmarkLocked(true, "transport failure")
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
shutdownRequested = false
chargingMode = false
imageRetryAfterMs = 0
displayedFingerprint = ""
faceclawWakePendingNonce = -1
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
clearAllMessagesLocked("transport failure: $reason")
reconnectAfterMs = now() + ConnectionOptions.RECONNECT_DELAY_MS
link.disconnect(rightAddress)
link.disconnect(leftAddress)
}
if (!userDisconnectRequested) {
setStateDisplay("retrying", if (reason == null || reason.isEmpty()) "Reconnecting..." else "Reconnecting after $reason")
}
interruptibleSleep.interrupt()
}
internal fun GlassesSessionCore.resetSessionStateLocked() {
ringBattery = -1
ringCharging = -1
sessionReady = false
shutdownRequested = false
fixedLayoutCreated = false
chargingMode = false
rightConnected = false
leftConnected = false
ringConnected = false
ringNotificationsReady = false
reconnectAfterMs = 0
reconnectHalted = false
ringReconnectAfterMs = 0
lastAckAtMs = 0
lastIncomingAtMs = 0
lastHeartbeatSentAtMs = 0
lastSessionReadyAtMs = 0
consecutiveAckTimeouts = 0
lastAudioControlAckMagic = 0
audioCaptureActive = false
audioPacketListener = null
compassControlLastSent = -1
// A dead transport orphans any glasses-side compass state; the fresh
// session re-asserts the desired state once its layout is ready.
compassMaybeOn = false
faceclawWakePendingNonce = -1
lastFaceclawWakeLeaseQueuedAtMs = 0
faceclawWakeControlSentCount = 0
cfwCleanupDelivered = false
for (transport in cfwTransports) transport.reset()
lastCfwCleanupAckMagic = 0
wearState = -1
displayedFingerprint = ""
// Deliberately not clearing silentMode: it is a property of the glasses,
// not of our session, and silent mode blocks app launches, so it can be
// the very cause of the session teardown that got us here.
}
// ---------------------------------------------------------------------------
// Listener emits (main thread via the host)
internal fun GlassesSessionCore.emitAmbientLight(body: ByteArray) {
if (ambientLightListeners.isEmpty()) {
return
}
val copy = body.copyOf()
host.postToMain {
for (alsListener in ambientLightListeners) {
try {
alsListener.onAmbientLight(copy)
} catch (t: Throwable) {
logWarn("ambient light listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitMicStatus(body: ByteArray, address: String) {
if (micStatusListeners.isEmpty()) {
return
}
val arm = if (address.equals(leftAddress, ignoreCase = true)) "L"
else if (address.equals(rightAddress, ignoreCase = true)) "R" else "?"
val copy = body.copyOf()
host.postToMain {
for (micListener in micStatusListeners) {
try {
micListener.onMicStatus(copy, arm)
} catch (t: Throwable) {
logWarn("mic status listener failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitRingEvent(event: G2Event, frameId: Int) {
val current = listener
if (current == null) {
frameTimings.finishFrame(frameId, "discarded: no listener attached")
return
}
val containerNameSnapshot = event.containerName
host.postToMain {
frameTimings.log(frameId, "dispatching input event on main thread")
try {
current.onRingEvent(event.kind, containerNameSnapshot, event.eventType, event.eventSource, event.systemExitReasonCode, frameId,
event.ringTick, event.ringType, event.ringAux, event.ringSpeed)
} catch (t: Throwable) {
logWarn("listener onRingEvent failed", t)
frameTimings.finishFrame(frameId, "discarded: listener onRingEvent failed")
}
}
}
/** Finish a frame owned by the communicator and tell the TS side, which may be awaiting it. */
internal fun GlassesSessionCore.finishFrame(frameId: Int, outcome: String) {
if (frameId <= 0) {
return
}
frameTimings.finishFrame(frameId, outcome)
val current = listener ?: return
host.postToMain {
try {
current.onFrameFinished(frameId, outcome)
} catch (t: Throwable) {
logWarn("listener onFrameFinished failed", t)
}
}
}
internal fun GlassesSessionCore.emitImuData(x: Double, y: Double, z: Double, eventSource: Int) {
if (imuListeners.isEmpty()) {
return
}
host.postToMain {
for (imuListener in imuListeners) {
try {
imuListener.onImuData(x, y, z, eventSource)
} catch (t: Throwable) {
logWarn("listener onImuData failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitCompassEvent(event: BleProtocol.CompassEvent) {
if (event.diagnosticFlags >= 0) {
val sources = arrayOf("unknown", "GRV", "GMRV", "RV")
host.log(SessionLogLevel.INFO, "FaceclawCompass", "heading=" + event.headingDegrees
+ " magneticAccuracy=" + event.magneticAccuracy
+ " magneticAnomalies=" + event.magneticAnomalies
+ " orientationSource=" + sources[event.orientationSource]
+ " flags=0x" + event.diagnosticFlags.toString(16)
+ " sampleTimeMs=" + event.sampleTimeMs, null)
}
for (subscription in compassSubscriptions) {
subscription.dispatcher.post {
try {
subscription.listener.onCompassEvent(event.command, event.headingDegrees,
event.magneticAccuracy, event.magneticAnomalies, event.orientationSource,
event.diagnosticFlags, event.sampleTimeMs)
} catch (t: Throwable) {
logWarn("listener onCompassEvent failed", t)
}
}
}
}
internal fun GlassesSessionCore.emitSilentMode(silent: Boolean) {
val current = listener ?: return
host.postToMain {
try {
current.onSilentMode(silent)
} catch (t: Throwable) {
logWarn("listener onSilentMode failed", t)
}
}
}
internal fun GlassesSessionCore.emitWearState(wearing: Boolean) {
val current = listener ?: return
host.postToMain {
try {
current.onWearState(wearing)
} catch (t: Throwable) {
logWarn("listener onWearState failed", t)
}
}
}
internal fun GlassesSessionCore.emitPhoneLockStateIfChanged(force: Boolean) {
val locked: Boolean
monitor.withLock {
val now = now()
if (!force && now - lastPhoneLockCheckAtMs < 1_000) return
lastPhoneLockCheckAtMs = now
locked = host.isPhoneLocked()
val value = if (locked) 1 else 0
if (!force && value == phoneLockState) return
phoneLockState = value
}
val current = listener ?: return
host.postToMain {
try {
current.onPhoneLockState(locked)
} catch (t: Throwable) {
logWarn("listener onPhoneLockState failed", t)
}
}
}
internal fun GlassesSessionCore.emitBatteryState(headsetBattery: Int, headsetCharging: Int) {
val reportedRingBattery = ringBattery
val reportedRingCharging = ringCharging
val current = listener ?: return
host.postToMain {
try {
current.onBatteryState(headsetBattery, headsetCharging, reportedRingBattery, reportedRingCharging)
} catch (t: Throwable) {
logWarn("listener onBatteryState failed", t)
}
}
}
internal fun GlassesSessionCore.emitFirmwareInfo(info: BleProtocol.FirmwareInfo) {
val current = listener ?: return
host.postToMain {
try {
current.onFirmwareInfo(info.leftVersion, info.rightVersion, info.extension)
} catch (t: Throwable) {
logWarn("listener onFirmwareInfo failed", t)
}
}
}
internal fun GlassesSessionCore.emitFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) {
val current = listener ?: return
host.postToMain {
try {
current.onFrameMetrics(paintMs, transmitMs, tileCount)
} catch (t: Throwable) {
logWarn("listener onFrameMetrics failed", t)
}
}
}
internal fun GlassesSessionCore.maybeEmitEvenAppConflictLocked(reason: String?) {
if ("write failed" != reason) {
return
}
val now = now()
if (lastSessionReadyAtMs <= 0 || now - lastSessionReadyAtMs > ConnectionOptions.EVEN_APP_WRITE_FAILURE_WINDOW_MS) {
return
}
if (lastEvenAppConflictAtMs > 0 && now - lastEvenAppConflictAtMs < 60_000) {
return
}
if (!host.isEvenAppActive()) {
return
}
lastEvenAppConflictAtMs = now
emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw.")
}
internal fun GlassesSessionCore.emitEvenAppConflict(message: String?) {
val current = listener ?: return
val messageSnapshot = message ?: ""
host.postToMain {
try {
current.onEvenAppConflict(messageSnapshot)
} catch (t: Throwable) {
logWarn("listener onEvenAppConflict failed", t)
}
}
}
internal fun GlassesSessionCore.setStateDisplay(nextPhase: String, nextStatus: String) {
monitor.withLock {
phase = nextPhase
status = nextStatus
}
emitState()
}
internal fun GlassesSessionCore.emitState() {
val current = listener ?: return
val phaseSnapshot: String
val statusSnapshot: String
monitor.withLock {
phaseSnapshot = phase
statusSnapshot = status
}
host.postToMain {
try {
current.onStateChange(phaseSnapshot, statusSnapshot)
} catch (t: Throwable) {
logWarn("listener onStateChange failed", t)
}
}
}
@@ -0,0 +1,165 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.contracts.ExperimentalContracts
import kotlin.contracts.InvocationKind
import kotlin.contracts.contract
/**
* GATT transport used by [GlassesSessionCore]. Addresses are opaque strings (Android MAC
* addresses, CoreBluetooth identifiers on iOS); every call blocks the calling thread until the
* operation completes or [timeoutMs] elapses, exactly like Android's FaceclawBleManager. Inbound
* traffic reaches the core through its FaceclawBleListener methods, which the implementation
* must call from whatever thread the platform stack delivers on.
*/
interface SessionLink {
fun connect(address: String, timeoutMs: Int): Boolean
/** Best effort; no completion point (see connectArm). */
fun requestHighPriority(address: String)
fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean
fun negotiatedMtu(address: String): Int
fun discoverServices(address: String, timeoutMs: Int): Boolean
fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean
fun writeFrames(
address: String,
characteristicUuid: String,
frames: List<ByteArray>,
mode: GattWriteMode,
timeoutMs: Int,
): Boolean
fun disconnect(address: String)
fun close()
/** Whether the OS still holds a pairing bond for the address (iOS: always true). */
fun isBonded(address: String): Boolean
/** Bandwidth-benchmark link tuning; 0: current link, 1: HIGH priority, 2: 2M PHY, 3: both. */
fun prepareBenchmarkLink(address: String, mode: Int)
/** Traffic counter hook: one display frame fully acknowledged. */
fun recordDisplayFrameSent()
}
/** Delivers an action on a specific thread (a Looper, a dispatch queue, ...). */
fun interface SessionDispatcher {
fun post(action: () -> Unit)
}
enum class SessionLogLevel {
DEBUG,
INFO,
WARN,
ERROR,
}
/**
* Platform side effects the session core needs. Listener callbacks are marshalled through
* [postToMain]; everything else is a synchronous query or command on the calling thread.
*/
interface SessionHost {
fun postToMain(action: () -> Unit)
/** A dispatcher for the calling thread (its Looper/queue), falling back to the main thread. */
fun currentThreadDispatcher(): SessionDispatcher
fun isPhoneLocked(): Boolean
/** Hold (true) or release (false) the "glasses screen on" wake lock; idempotent. */
fun setScreenWakeLock(on: Boolean)
/** Whether the stock Even app appears active (it can hold the BLE link). */
fun isEvenAppActive(): Boolean
/** Start the worker thread that runs [body] (the core's run loop) once. */
fun startWorker(body: () -> Unit)
/** Interrupt (where supported) and wait up to [timeoutMs] for the worker started by [startWorker]. */
fun joinWorker(timeoutMs: Long)
fun log(level: SessionLogLevel, tag: String, message: String, error: Throwable?)
}
/**
* Java-monitor replacement: a reentrant state lock plus a separate condition used only at the
* blocking wait sites. Waiters must hold [lock] exactly once when calling [awaitMs]; notifiers
* may hold it (they normally do). Ordering is always lock -> signal, and the waiter never holds
* the signal while acquiring the lock, so the pair cannot deadlock and cannot lose a wake-up:
* the predicate is checked under [lock], the signal is taken before [lock] is released, and
* [ProtocolCondition.awaitMs] releases the signal atomically.
*/
internal class SessionMonitor(platform: ProtocolPlatform) {
val lock: ProtocolLock = platform.createLock()
private val signal: ProtocolCondition = platform.createCondition()
fun signalAll() {
signal.lock()
try {
signal.signalAll()
} finally {
signal.unlock()
}
}
/** Caller holds [lock] once; returns with it held again. Spurious wake-ups are possible. */
fun awaitMs(timeoutMs: Long) {
signal.lock()
lock.unlock()
try {
signal.awaitMs(timeoutMs)
} finally {
signal.unlock()
}
lock.lock()
}
}
/** Snapshot-iterated listener list (java.util.concurrent.CopyOnWriteArrayList stand-in). */
internal class CopyOnWriteList<T>(platform: ProtocolPlatform) : Iterable<T> {
private val lock = platform.createLock()
@Volatile private var items: List<T> = emptyList()
fun add(item: T) {
lock.withLock { items = items + item }
}
fun remove(item: T) {
lock.withLock { items = items - item }
}
fun isEmpty(): Boolean = items.isEmpty()
override fun iterator(): Iterator<T> = items.iterator()
}
/** Runs [action] under the monitor's state lock; the contract lets `val`s be assigned inside, like `synchronized`. */
@OptIn(ExperimentalContracts::class)
internal inline fun <T> SessionMonitor.withLock(action: () -> T): T {
contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
lock.lock()
try {
return action()
} finally {
lock.unlock()
}
}
/** [ProtocolLock.withLock] with a call-in-place contract. */
@OptIn(ExperimentalContracts::class)
internal inline fun <T> ProtocolLock.locked(action: () -> T): T {
contract { callsInPlace(action, InvocationKind.EXACTLY_ONCE) }
lock()
try {
return action()
} finally {
unlock()
}
}
@@ -0,0 +1,168 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.pow
import kotlin.math.roundToInt
/**
* Wire packets produced by the platform text rasterizers (Android FontFileRenderer over
* minikin/HarfBuzz, iOS over CoreText) and consumed by the TypeScript side
* (app/native/font-files.ts, app/graphics/ttf-font.ts). Rasterization stays per platform;
* the packet layouts, gamma mapping and ink-box cropping live here so both platforms agree.
*
* Image packet (renderText / renderWrapped): [widthLo, widthHi, heightLo, heightHi, pixels...],
* one gamma-mapped coverage byte per pixel, row-major.
*
* Glyph-cell packet (renderGlyphCell), little-endian:
* [advance u16, 26.6 fixed point]
* [bearingX s16] ink left relative to the pen position
* [inkTop s16] ink top relative to the line top (ascent above baseline)
* [width u16][height u16]
* [width*height coverage bytes, gamma-mapped]
* An ink-free glyph (space) has width = height = 0 but a valid advance.
*/
object GlyphPacket {
const val IMAGE_HEADER_BYTES = 4
const val GLYPH_CELL_HEADER_BYTES = 10
/** out = 255 * (coverage/255)^gamma; non-positive gamma means linear. */
@JvmStatic
fun gammaLut(gammaIn: Double): ByteArray {
val gamma = if (gammaIn > 0 && gammaIn.isFinite()) gammaIn else 1.0
val lut = ByteArray(256)
for (i in 0 until 256) {
lut[i] = (255.0 * (i / 255.0).pow(gamma)).roundToInt().coerceIn(0, 255).toByte()
}
return lut
}
/** [first, last] painted (nonzero) column indexes, or null if blank. */
@JvmStatic
fun paintedColumnRange(pixels: ByteArray, width: Int, height: Int): IntArray? {
var first = width
var last = -1
for (y in 0 until height) {
val row = y * width
var x = 0
while (x < first) {
if (pixels[row + x].toInt() != 0) {
first = x
break
}
x++
}
x = width - 1
while (x > last) {
if (pixels[row + x].toInt() != 0) {
last = x
break
}
x--
}
}
return if (last >= first) intArrayOf(first, last) else null
}
/** Tight [x0, y0, x1, y1] (inclusive) box of nonzero pixels, or null if blank. */
@JvmStatic
fun inkBounds(pixels: ByteArray, width: Int, height: Int): IntArray? {
var x0 = width
var x1 = -1
var y0 = height
var y1 = -1
for (y in 0 until height) {
val row = y * width
for (x in 0 until width) {
if (pixels[row + x].toInt() != 0) {
if (x < x0) x0 = x
if (x > x1) x1 = x
if (y < y0) y0 = y
if (y > y1) y1 = y
}
}
}
return if (x1 >= x0) intArrayOf(x0, y0, x1, y1) else null
}
/** Image packet of the whole width*height alpha raster. */
@JvmStatic
fun packImage(alpha8: ByteArray, width: Int, height: Int, gamma: Double): ByteArray =
packImageRegion(alpha8, width, 0, 0, width, height, gamma)
/** Image packet of the [x, y, width, height] region of a raster with row [stride]. */
@JvmStatic
fun packImageRegion(alpha8: ByteArray, stride: Int, x: Int, y: Int, width: Int, height: Int, gamma: Double): ByteArray {
require(width >= 0 && height >= 0 && x >= 0 && y >= 0 && x + width <= stride)
val lut = gammaLut(gamma)
val out = ByteArray(IMAGE_HEADER_BYTES + width * height)
putU16(out, 0, width)
putU16(out, 2, height)
for (row in 0 until height) {
val src = (y + row) * stride + x
val dst = IMAGE_HEADER_BYTES + row * width
for (col in 0 until width) {
out[dst + col] = lut[alpha8[src + col].toInt() and 0xff]
}
}
return out
}
/** 26.6 fixed-point advance clamped to u16. */
@JvmStatic
fun advanceFixed(advancePx: Double): Int = (advancePx * 64.0).roundToInt().coerceIn(0, 0xffff)
/** Glyph cell with no ink (space): valid advance, zero-size box. */
@JvmStatic
fun emptyGlyphCell(advanceFixed: Int): ByteArray = glyphCellPacket(advanceFixed, 0, 0, 0, 0, null)
/** Glyph cell from already gamma-mapped coverage. */
@JvmStatic
fun glyphCellPacket(advanceFixed: Int, bearingX: Int, inkTop: Int, width: Int, height: Int, coverage: ByteArray?): ByteArray {
val out = ByteArray(GLYPH_CELL_HEADER_BYTES + (coverage?.size ?: 0))
putU16(out, 0, advanceFixed)
putU16(out, 2, bearingX)
putU16(out, 4, inkTop)
putU16(out, 6, width)
putU16(out, 8, height)
coverage?.copyInto(out, GLYPH_CELL_HEADER_BYTES)
return out
}
/**
* Glyph cell cropped to the tight ink box of a padded raster. [penX]/[baselineY] are the
* pen position inside the raster and [ascent] the font ascent above the baseline; the
* returned bearing/inkTop are relative to those, as the wire format requires.
*/
@JvmStatic
fun packGlyphCell(
advanceFixed: Int,
alpha8: ByteArray,
rasterWidth: Int,
rasterHeight: Int,
penX: Int,
baselineY: Int,
ascent: Int,
gamma: Double,
): ByteArray {
val box = inkBounds(alpha8, rasterWidth, rasterHeight) ?: return emptyGlyphCell(advanceFixed)
val x0 = box[0]
val y0 = box[1]
val width = box[2] - x0 + 1
val height = box[3] - y0 + 1
val lut = gammaLut(gamma)
val coverage = ByteArray(width * height)
for (y in 0 until height) {
val src = (y0 + y) * rasterWidth + x0
val dst = y * width
for (x in 0 until width) {
coverage[dst + x] = lut[alpha8[src + x].toInt() and 0xff]
}
}
return glyphCellPacket(advanceFixed, x0 - penX, ascent + (y0 - baselineY), width, height, coverage)
}
private fun putU16(out: ByteArray, offset: Int, value: Int) {
out[offset] = (value and 0xff).toByte()
out[offset + 1] = ((value shr 8) and 0xff).toByte()
}
}
@@ -0,0 +1,117 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.pow
import kotlin.math.roundToInt
/**
* Converts ARGB pixels to the grayscale packet format used across the TS bridge:
* [widthLo, widthHi, heightLo, heightHi, pixels...] with one byte per pixel, row-major.
* Transparent pixels darken toward black (the on-glasses background). Platform code owns
* decoding and scaling (Bitmap / ImageIO); this object owns the pixel math so both platforms
* produce identical tone and dither.
*/
object GrayPacket {
const val HEADER_SIZE = 4
/**
* gamma > 1 darkens midtones (out = 255 * (in/255)^gamma). Source pixels are sRGB-encoded
* but the G2 drives its 16 levels roughly linearly, so mid-gray otherwise displays far
* brighter than intended; 2.2 undoes the sRGB encoding outright.
*
* dither error-diffuses against the 16 levels the frame is eventually packed to
* (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to round on its own, which turns
* smooth gradients into visible bands. Dithered output is already quantized: every byte is
* a level times 16, which BmpUtil re-quantizes back to exactly that level.
*/
@JvmStatic
fun fromArgb(pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean): ByteArray {
require(width > 0 && height > 0 && pixels.size >= width * height) { "bad pixel plane" }
val out = ByteArray(HEADER_SIZE + width * height)
writeHeader(out, width, height)
val tone = toneCurve(gamma)
for (i in 0 until width * height) {
val p = pixels[i]
val a = (p ushr 24) and 0xff
val r = (p shr 16) and 0xff
val g = (p shr 8) and 0xff
val b = p and 0xff
out[HEADER_SIZE + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte()
}
if (dither) {
ditherToDisplayLevels(out, HEADER_SIZE, width, height)
}
return out
}
@JvmStatic
fun writeHeader(out: ByteArray, width: Int, height: Int) {
out[0] = (width and 0xff).toByte()
out[1] = ((width shr 8) and 0xff).toByte()
out[2] = (height and 0xff).toByte()
out[3] = ((height shr 8) and 0xff).toByte()
}
/** 256-entry map of source gray to displayed gray for out = in^gamma. */
@JvmStatic
fun toneCurve(gamma: Float): IntArray {
val curve = IntArray(256)
for (v in 0 until 256) {
curve[v] = if (gamma == 1f) v else (255f * (v / 255f).pow(gamma)).roundToInt()
}
return curve
}
/**
* Floyd-Steinberg error diffusion onto the display's 16 gray levels, in place over an 8bpp
* plane. Serpentine scanning keeps the diffusion from building up a directional texture
* across wide flat areas.
*
* Level n is written back as n * 16 rather than the n * 17 it stands for so that BmpUtil's
* (v + 8) >> 4 reproduces n exactly; level 0 is written as 1, not 0, because 0 is the shell's
* color-key for transparent.
*/
@JvmStatic
fun ditherToDisplayLevels(plane: ByteArray, offset: Int, width: Int, height: Int) {
var curr = FloatArray(width)
var next = FloatArray(width)
for (x in 0 until width) {
curr[x] = (plane[offset + x].toInt() and 0xff).toFloat()
}
for (y in 0 until height) {
val rowStart = offset + y * width
val nextStart = rowStart + width
val hasNext = y + 1 < height
for (x in 0 until width) {
next[x] = if (hasNext) (plane[nextStart + x].toInt() and 0xff).toFloat() else 0f
}
val leftToRight = (y and 1) == 0
val start = if (leftToRight) 0 else width - 1
val step = if (leftToRight) 1 else -1
for (i in 0 until width) {
val x = start + i * step
val wanted = curr[x]
val level = (minOf(255f, maxOf(0f, wanted)) / 17f).roundToInt()
val error = wanted - level * 17f
plane[rowStart + x] = (if (level == 0) 1 else level * 16).toByte()
val ahead = x + step
if (ahead >= 0 && ahead < width) {
curr[ahead] += error * (7f / 16f)
}
if (hasNext) {
if (ahead >= 0 && ahead < width) {
next[ahead] += error * (1f / 16f)
}
next[x] += error * (5f / 16f)
val behind = x - step
if (behind >= 0 && behind < width) {
next[behind] += error * (3f / 16f)
}
}
}
val swap = curr
curr = next
next = swap
}
}
}
@@ -0,0 +1,110 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/** Random access over font bytes, so large files are not read whole just for a name. */
interface RandomAccessBytes {
val length: Long
/** Reads exactly [count] bytes at [offset] into [into]; throws when out of range. */
fun read(offset: Long, into: ByteArray, count: Int)
}
class ArrayRandomAccessBytes(private val bytes: ByteArray) : RandomAccessBytes {
override val length: Long
get() = bytes.size.toLong()
override fun read(offset: Long, into: ByteArray, count: Int) {
require(offset >= 0 && count >= 0 && offset + count <= bytes.size) { "read out of range" }
bytes.copyInto(into, 0, offset.toInt(), offset.toInt() + count)
}
}
/**
* Minimal OpenType 'name' table reader (TTF/OTF/TTC, first face) for font titles.
* Returns "Family\nStyle" ("Style" may be empty), preferring the typographic names
* (ids 16/17) over the legacy ones (1/2), or "" when the table cannot be parsed.
*/
object OpenTypeNames {
private const val TTCF = 0x74746366
private const val NAME = 0x6e616d65
@JvmStatic
fun parseFile(path: String): String = parse(ArrayRandomAccessBytes(readFileData(path)))
@JvmStatic
fun parse(bytes: ByteArray): String = parse(ArrayRandomAccessBytes(bytes))
@JvmStatic
fun parse(source: RandomAccessBytes): String {
val fileSize = source.length
if (fileSize < 12) return ""
var offset = 0L
if (readU32(source, 0) == TTCF) { // 'ttcf': use the first face
if (readU32(source, 8) < 1) return ""
offset = readU32(source, 12).toLong() and 0xffffffffL
}
val numTables = readU16(source, offset + 4)
var nameOffset = -1L
for (i in 0 until numTables) {
val rec = offset + 12 + i * 16L
if (rec + 16 > fileSize) return ""
if (readU32(source, rec) == NAME) {
nameOffset = readU32(source, rec + 8).toLong() and 0xffffffffL
break
}
}
if (nameOffset < 0 || nameOffset + 6 > fileSize) return ""
val count = readU16(source, nameOffset + 2)
val stringStorage = nameOffset + readU16(source, nameOffset + 4)
var family: String? = null
var style: String? = null
var preferredFamily: String? = null
var preferredStyle: String? = null
for (i in 0 until count) {
val rec = nameOffset + 6 + i * 12L
if (rec + 12 > fileSize) break
val platform = readU16(source, rec)
val nameId = readU16(source, rec + 6)
if (nameId != 1 && nameId != 2 && nameId != 16 && nameId != 17) continue
val length = readU16(source, rec + 8)
val strOffset = stringStorage + readU16(source, rec + 10)
if (strOffset + length > fileSize || length <= 0 || length > 512) continue
val data = ByteArray(length)
source.read(strOffset, data, length)
// Platform 0 (Unicode) and 3 (Windows) store UTF-16BE;
// platform 1 (Mac) is close enough to Latin-1 for names.
val value = (if (platform == 1) latin1(data) else utf16be(data)).trim()
if (value.isEmpty()) continue
if (nameId == 1 && family == null) family = value
if (nameId == 2 && style == null) style = value
if (nameId == 16 && preferredFamily == null) preferredFamily = value
if (nameId == 17 && preferredStyle == null) preferredStyle = value
}
val outFamily = preferredFamily ?: family ?: return ""
return outFamily + "\n" + (preferredStyle ?: style ?: "")
}
private fun latin1(data: ByteArray): String =
CharArray(data.size) { (data[it].toInt() and 0xff).toChar() }.concatToString()
private fun utf16be(data: ByteArray): String {
val chars = CharArray(data.size / 2) {
(((data[2 * it].toInt() and 0xff) shl 8) or (data[2 * it + 1].toInt() and 0xff)).toChar()
}
return chars.concatToString()
}
private fun readU16(source: RandomAccessBytes, offset: Long): Int {
val b = ByteArray(2)
source.read(offset, b, 2)
return ((b[0].toInt() and 0xff) shl 8) or (b[1].toInt() and 0xff)
}
private fun readU32(source: RandomAccessBytes, offset: Long): Int {
val b = ByteArray(4)
source.read(offset, b, 4)
return ((b[0].toInt() and 0xff) shl 24) or ((b[1].toInt() and 0xff) shl 16) or
((b[2].toInt() and 0xff) shl 8) or (b[3].toInt() and 0xff)
}
}
@@ -0,0 +1,52 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.pow
import kotlin.math.roundToLong
/**
* Palette for the phone-UI preview of an 8bpp grayscale frame: gray -> ARGB with a brightening
* gamma, either grayscale or green-on-black (matching the physical glasses). Both platforms
* should build their preview images from this table so they brighten identically.
*/
object PreviewPalette {
private var cachedLut: IntArray? = null
private var cachedGammaBits: Long = 0
private var cachedGreen: Boolean = false
private val lock = protocolPlatform().createLock()
/** ARGB value for [gray] under [brightenGamma]: 255 * (gray/255)^gamma per channel. */
@JvmStatic
fun argb(gray: Int, brightenGamma: Double, green: Boolean): Int {
val v = (255 * (gray / 255.0).pow(brightenGamma)).roundToLong().coerceIn(0L, 255L).toInt()
return if (green) (0xff000000.toInt() or (v shl 8)) else (0xff000000.toInt() or (v shl 16) or (v shl 8) or v)
}
/** Reuse the current preview palette; rebuild only when gamma or color changes. */
@JvmStatic
fun lookupTable(brightenGamma: Double, green: Boolean): IntArray {
val gammaBits = brightenGamma.toBits()
lock.withLock {
val cached = cachedLut
if (cached != null && cachedGammaBits == gammaBits && cachedGreen == green) {
return cached
}
val lut = IntArray(256) { argb(it, brightenGamma, green) }
cachedGammaBits = gammaBits
cachedGreen = green
cachedLut = lut
return lut
}
}
/** Expands [count] gray bytes starting at [offset] to ARGB pixels through the palette. */
@JvmStatic
fun expand(gray: ByteArray, offset: Int, count: Int, brightenGamma: Double, green: Boolean): IntArray {
val lut = lookupTable(brightenGamma, green)
val colors = IntArray(count)
for (i in 0 until count) {
colors[i] = lut[gray[offset + i].toInt() and 0xff]
}
return colors
}
}
@@ -0,0 +1,98 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.abs
/**
* Pure 24x24 8bpp icons for the system status strip (Wi-Fi level, cellular level, hotspot).
* Platform code reads the levels; this object only draws, so both platforms render the same
* artwork.
*/
object StatusIconArt {
const val ICON_SIZE = 24
@JvmStatic
fun wifi(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
val cx = 12
val cy = 18
fillRect(icon, cx - 1, cy - 1, 3, 3, 230)
val radii = intArrayOf(5, 9, 13, 17)
for (index in radii.indices) {
if (level <= index) {
continue
}
drawWifiArc(icon, cx, cy, radii[index], 210)
}
return icon
}
@JvmStatic
fun cell(level: Int): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
for (bar in 0 until 4) {
val x = 5 + bar * 4
val height = 5 + bar * 4
val value = if (level > bar) 220 else 55
fillRect(icon, x, 20 - height, 3, height, value)
}
return icon
}
@JvmStatic
fun hotspot(): ByteArray {
val icon = ByteArray(ICON_SIZE * ICON_SIZE)
fillRect(icon, 11, 17, 3, 3, 230)
drawWifiArc(icon, 12, 19, 7, 210)
drawWifiArc(icon, 12, 19, 12, 210)
fillRect(icon, 6, 5, 12, 2, 170)
return icon
}
/** Nearest-neighbour resample of a 24x24 icon to size x size. */
@JvmStatic
fun scale(source: ByteArray, size: Int): ByteArray {
if (size == ICON_SIZE) {
return source
}
val scaled = ByteArray(size * size)
for (y in 0 until size) {
val sy = minOf(ICON_SIZE - 1, (y * ICON_SIZE) / size)
for (x in 0 until size) {
val sx = minOf(ICON_SIZE - 1, (x * ICON_SIZE) / size)
scaled[y * size + x] = source[sy * ICON_SIZE + sx]
}
}
return scaled
}
private fun drawWifiArc(icon: ByteArray, cx: Int, cy: Int, radius: Int, value: Int) {
val inner = (radius - 1) * (radius - 1)
val outer = (radius + 1) * (radius + 1)
for (y in 0 until ICON_SIZE) {
for (x in 0 until ICON_SIZE) {
val dx = x - cx
val dy = y - cy
val dist = dx * dx + dy * dy
if (dy <= 0 && dist >= inner && dist <= outer && abs(dx) <= radius) {
setPixel(icon, x, y, value)
}
}
}
}
private fun fillRect(icon: ByteArray, x: Int, y: Int, width: Int, height: Int, value: Int) {
for (row in y until y + height) {
for (col in x until x + width) {
setPixel(icon, col, row, value)
}
}
}
private fun setPixel(icon: ByteArray, x: Int, y: Int, value: Int) {
if (x < 0 || y < 0 || x >= ICON_SIZE || y >= ICON_SIZE) {
return
}
icon[y * ICON_SIZE + x] = value.coerceIn(0, 255).toByte()
}
}
@@ -0,0 +1,150 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
import kotlin.math.min
/** One drawable element of a parsed icon; coordinates are in viewBox units. */
sealed class SvgShape {
class PathData(@JvmField val d: String) : SvgShape()
class Circle(@JvmField val cx: Float, @JvmField val cy: Float, @JvmField val r: Float) : SvgShape()
class Ellipse(@JvmField val cx: Float, @JvmField val cy: Float, @JvmField val rx: Float, @JvmField val ry: Float) : SvgShape()
/** rx/ry are both 0 for a sharp rectangle, otherwise both positive. */
class Rect(
@JvmField val x: Float,
@JvmField val y: Float,
@JvmField val width: Float,
@JvmField val height: Float,
@JvmField val rx: Float,
@JvmField val ry: Float,
) : SvgShape()
class Line(@JvmField val x1: Float, @JvmField val y1: Float, @JvmField val x2: Float, @JvmField val y2: Float) : SvgShape()
/** [points] is x0,y0,x1,y1,... with at least one pair; [closed] for polygon. */
class Polyline(@JvmField val points: FloatArray, @JvmField val closed: Boolean) : SvgShape()
}
class SvgIcon(
@JvmField val minX: Float,
@JvmField val minY: Float,
@JvmField val viewWidth: Float,
@JvmField val viewHeight: Float,
/** Lucide-style outline icons (fill="none") are stroked; everything else is filled. */
@JvmField val stroked: Boolean,
@JvmField val shapes: List<SvgShape>,
) {
/**
* Mapping from viewBox to a size*size raster: [translateX, translateY, scale]. Apply as
* translate(tx, ty), scale(s), translate(-minX, -minY), i.e. the viewBox centered in the box.
*/
fun placement(size: Int): FloatArray {
val scale = min(size / viewWidth, size / viewHeight)
return floatArrayOf((size - viewWidth * scale) / 2f, (size - viewHeight * scale) / 2f, scale)
}
}
/**
* Parses the small subset of SVG used by Lucide / simple Noun Project icons (path, circle,
* ellipse, rect, line, polyline, polygon) into shapes the platform path builders draw.
* Also owns the 3-level coverage quantization applied after rasterization on both platforms.
*/
object SvgIconSource {
private val TAG_RE = Regex("<(path|circle|ellipse|rect|line|polyline|polygon)\\b([^>]*)>")
private val VIEWBOX_RE = Regex("viewBox\\s*=\\s*\"([^\"]*)\"")
private val SEPARATOR_RE = Regex("[\\s,]+")
/** Null when nothing drawable was found. */
@JvmStatic
fun parse(svg: String): SvgIcon? {
var minX = 0f
var minY = 0f
var viewW = 24f
var viewH = 24f
val vb = VIEWBOX_RE.find(svg)
if (vb != null) {
val parts = SEPARATOR_RE.split(vb.groupValues[1].trim())
if (parts.size == 4) {
minX = parts[0].toFloat()
minY = parts[1].toFloat()
viewW = parts[2].toFloat()
viewH = parts[3].toFloat()
}
}
val stroked = svg.contains("fill=\"none\"")
val shapes = ArrayList<SvgShape>()
for (m in TAG_RE.findAll(svg)) {
parseElement(m.groupValues[1], m.groupValues[2])?.let { shapes.add(it) }
}
if (shapes.isEmpty()) return null
return SvgIcon(minX, minY, viewW, viewH, stroked, shapes)
}
/**
* Snap antialiased coverage toward hard edges so the 4bpp frames sent over BLE deflate
* better: mostly-transparent and mostly-opaque pixels become exactly 0/255 (long runs, and
* 0 stays color-key transparent for bitBlt), and only genuinely half-covered edge pixels
* keep one mid gray. At icon sizes this is visually near-identical.
*/
@JvmStatic
fun quantizeCoverage(alpha: Int): Int {
if (alpha < 64) return 0
if (alpha >= 192) return 255
return 128
}
/** Quantized coverage bytes from ARGB pixels (alpha channel = coverage). */
@JvmStatic
fun quantizeArgb(pixels: IntArray): ByteArray {
val gray = ByteArray(pixels.size)
for (i in pixels.indices) gray[i] = quantizeCoverage((pixels[i] ushr 24) and 0xff).toByte()
return gray
}
private fun parseElement(tag: String, attrs: String): SvgShape? =
when (tag) {
"path" -> attr(attrs, "d")?.let { SvgShape.PathData(it) }
"circle" -> {
val r = num(attrs, "r", 0f)
if (r > 0) SvgShape.Circle(num(attrs, "cx", 0f), num(attrs, "cy", 0f), r) else null
}
"ellipse" -> {
val rx = num(attrs, "rx", 0f)
val ry = num(attrs, "ry", 0f)
if (rx > 0 && ry > 0) SvgShape.Ellipse(num(attrs, "cx", 0f), num(attrs, "cy", 0f), rx, ry) else null
}
"rect" -> {
val w = num(attrs, "width", 0f)
val h = num(attrs, "height", 0f)
var rx = num(attrs, "rx", 0f)
var ry = num(attrs, "ry", if (rx > 0) rx else 0f)
if (rx <= 0) rx = ry
if (ry <= 0) ry = rx
if (w > 0 && h > 0) {
val rounded = rx > 0 || ry > 0
SvgShape.Rect(num(attrs, "x", 0f), num(attrs, "y", 0f), w, h, if (rounded) rx else 0f, if (rounded) ry else 0f)
} else null
}
"line" -> SvgShape.Line(num(attrs, "x1", 0f), num(attrs, "y1", 0f), num(attrs, "x2", 0f), num(attrs, "y2", 0f))
"polyline", "polygon" -> {
val points = attr(attrs, "points") ?: return null
val nums = SEPARATOR_RE.split(points.trim())
val pairs = nums.size / 2
if (pairs == 0) return null
val values = FloatArray(pairs * 2) { nums[it].toFloat() }
SvgShape.Polyline(values, tag == "polygon")
}
else -> null
}
private fun attr(attrs: String, name: String): String? =
Regex("\\b" + name + "\\s*=\\s*\"([^\"]*)\"").find(attrs)?.groupValues?.get(1)
private fun num(attrs: String, name: String, fallback: Float): Float {
val value = attr(attrs, name) ?: return fallback
return value.trim().toFloatOrNull() ?: fallback
}
}
@@ -0,0 +1,26 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.jvm.JvmStatic
/**
* Shared HTTP identity for the native networking helpers on both platforms. The version
* lives in app/version.ts, so the TypeScript side pushes the real string in at startup
* (see installNativeUserAgent in app/util/http.ts); the default is only a fallback for
* requests that somehow beat that call.
*/
object HttpIdentity {
const val DEFAULT_USER_AGENT = "Faceclaw"
@Volatile
private var userAgent = DEFAULT_USER_AGENT
@JvmStatic
fun setUserAgent(value: String?) {
val trimmed = value?.trim()
if (!trimmed.isNullOrEmpty()) userAgent = trimmed
}
@JvmStatic
fun getUserAgent(): String = userAgent
}
@@ -0,0 +1,229 @@
package com.faceclaw.app
import kotlin.jvm.JvmField
import kotlin.jvm.JvmStatic
/** One accepted client connection of the local input port. */
interface RemoteInputConnection {
/** Reads one byte, blocking at most [timeoutMs]; returns -1 on end of stream, timeout or error. */
fun readByte(timeoutMs: Long): Int
fun write(bytes: ByteArray)
fun close()
}
/** A bound, listening socket. */
interface RemoteInputListener {
/** Blocks for the next client; returns null when the listener is closed or accept fails. */
fun accept(): RemoteInputConnection?
val isClosed: Boolean
fun close()
}
/** OS socket access for [RemoteInputSession]; a POSIX actual is the iOS counterpart. */
interface LocalServerPort {
/** True when [address] is a unicast address assigned to a local interface (not any/multicast). */
fun isLocalUnicastAddress(address: String): Boolean
/** Binds and listens on [address]:[port]; throws when the port cannot be opened. */
fun bind(address: String, port: Int): RemoteInputListener
}
interface RandomSource {
fun nextBytes(count: Int): ByteArray
}
/** One network address of a local interface, as enumerated by the platform. */
data class InterfaceAddress(
val interfaceName: String,
val address: String,
val up: Boolean,
val pointToPoint: Boolean,
val loopback: Boolean,
val linkLocal: Boolean,
val anyLocal: Boolean,
)
/**
* Explicit-address, bounded local input port shared by Android (java.net) and iOS (POSIX).
* Exactly one request is in flight: a client sends one newline-terminated UTF-8 frame of at most
* 64 KiB, the session publishes it as a pending JSON request, notifies the app through
* [dispatch] (which must run the callback on the app's UI thread, outside the lock), and blocks
* the serving thread until [complete] supplies the reply or the request deadline passes.
*/
class RemoteInputSession(
private val port: LocalServerPort,
private val random: RandomSource,
private val dispatch: (() -> Unit) -> Unit,
platform: ProtocolPlatform = protocolPlatform(),
private val wallClock: () -> Long = ::currentTimeMillis,
private val requestTimeoutMs: Long = REQUEST_TIMEOUT_MS,
private val readTimeoutMs: Long = READ_TIMEOUT_MS,
) {
private val condition = platform.createCondition()
private var server: RemoteInputListener? = null
private var client: RemoteInputConnection? = null
private var pending: String? = null
private var reply: String? = null
private var nextId = 0L
private var requestId = 0L
private var deadline = 0L
private var requestListener: (() -> Unit)? = null
fun setRequestListener(listener: (() -> Unit)?) {
condition.withLock { requestListener = listener }
}
/** Returns "" on success or a user-facing error. Never resolves hostnames or binds a wildcard. */
fun start(port: Int, address: String): String {
condition.withLock {
if (server != null) return ""
if (!isNumericLocalAddress(address)) return "Invalid local address."
if (!this.port.isLocalUnicastAddress(address)) return "Address is not assigned to a local interface."
val listener = try {
this.port.bind(address, port)
} catch (e: Exception) {
return "Could not open local input port $port."
}
server = listener
startThread("FaceclawInput", true) { serve(listener) }
return ""
}
}
fun stop() {
condition.withLock {
try { server?.close() } catch (ignored: Exception) {}
try { client?.close() } catch (ignored: Exception) {}
server = null
client = null
pending = null
requestId = 0
reply = null
condition.signalAll()
}
}
/** The pending request JSON (id, expiresAt, body), consumed once; null when none or expired. */
fun nextRequest(): String? {
condition.withLock {
if (wallClock() >= deadline) {
pending = null
return null
}
val result = pending
pending = null
return result
}
}
fun complete(id: Long, value: String?) {
condition.withLock {
if (id == requestId && wallClock() < deadline) {
reply = value
condition.signalAll()
}
}
}
fun randomSecret(): String = BinaryEncoding.bytesToHex(random.nextBytes(32))
private fun notifyRequestReady(id: Long) {
// Called with the condition held; dispatching keeps JS off the socket thread and outside the lock.
dispatch {
val callback = condition.withLock {
if (requestId != id || pending == null || wallClock() >= deadline) null else requestListener
}
callback?.invoke()
}
}
private fun serve(listener: RemoteInputListener) {
while (!listener.isClosed) {
val connection = listener.accept() ?: continue
try {
condition.withLock {
if (server !== listener) return
client = connection
}
val text = readFrame(connection) ?: continue
val response: String = condition.withLock {
if (server !== listener) return
requestId = ++nextId
deadline = wallClock() + requestTimeoutMs
reply = null
pending = Json.write(linkedMapOf("id" to requestId, "expiresAt" to deadline, "body" to text))
notifyRequestReady(requestId)
while (reply == null && server === listener && wallClock() < deadline) {
condition.awaitMs(maxOf(1L, deadline - wallClock()))
}
val result = reply ?: TIMEOUT_RESPONSE
if (server === listener) {
pending = null
reply = null
requestId = 0
}
result
}
connection.write((response + "\n").encodeToByteArray())
} catch (ignored: Exception) {
// Never log request bodies: they contain credentials and possibly private text.
} finally {
try { connection.close() } catch (ignored: Exception) {}
condition.withLock { if (server === listener) client = null }
}
}
}
/** One newline-terminated frame, or null when the client hung up, stalled or overflowed. */
private fun readFrame(connection: RemoteInputConnection): String? {
val readDeadline = wallClock() + readTimeoutMs
val body = ByteSink()
while (true) {
val remaining = readDeadline - wallClock()
if (remaining <= 0) return null
val b = connection.readByte(remaining)
if (b == -1) return null
if (b == '\n'.code) break
if (body.size() >= MAX_FRAME_BYTES) return null
body.write(b)
}
return try {
body.toByteArray().decodeToString(throwOnInvalidSequence = true)
} catch (e: Exception) {
null
}
}
companion object {
const val MAX_FRAME_BYTES = 65536
const val REQUEST_TIMEOUT_MS = 5000L
const val READ_TIMEOUT_MS = 5000L
@JvmField
val TIMEOUT_RESPONSE = "{\"ok\":false,\"error\":\"timeout\",\"message\":\"Faceclaw did not respond in time.\"}"
private val numericAddress = Regex("[0-9.]+|[0-9a-fA-F]*:[0-9a-fA-F:.]*(%[a-zA-Z0-9_.-]+)?")
/** Numeric IPv4/IPv6 literals only (optionally zone-scoped); hostnames are rejected. */
@JvmStatic fun isNumericLocalAddress(address: String): Boolean = numericAddress.matches(address)
/** Addresses a remote client could reach: up interfaces, minus loopback, link-local and wildcard. */
@JvmStatic
fun reachableAddresses(all: List<InterfaceAddress>): List<InterfaceAddress> =
all.filter { it.up && !it.loopback && !it.linkLocal && !it.anyLocal }
/** JSON array of {name, address, pointToPoint} consumed by app/remote/listeners.ts. */
@JvmStatic
fun interfacesJson(all: List<InterfaceAddress>): String =
Json.write(reachableAddresses(all).map {
linkedMapOf("name" to it.interfaceName, "address" to it.address, "pointToPoint" to it.pointToPoint)
})
/** Lowercase hex SHA-256 of the UTF-8 token; the stored form of remote-input tokens. */
@JvmStatic fun tokenHash(value: String): String = sha256Hex(value.encodeToByteArray())
}
}
@@ -0,0 +1,178 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
/** One open HTTP response body; [read] returns bytes read or -1 at end. */
interface DownloadResponse {
val code: Int
/** Body length if known, else -1. */
val contentLength: Long
fun read(buffer: ByteArray): Int
fun close()
}
/** Blocking GET; `rangeFrom > 0` asks for `Range: bytes=<rangeFrom>-`. */
interface DownloadTransport {
fun open(url: String, rangeFrom: Long): DownloadResponse
}
interface FileSink {
fun write(bytes: ByteArray, offset: Int, length: Int)
fun close()
}
/** Storage port; paths are platform file paths. Multi-GB files must stream, hence no whole-file reads. */
interface DownloadStorage {
/** Size in bytes, or -1 when missing. */
fun length(path: String): Long
fun prepareParent(path: String)
fun openWrite(path: String, append: Boolean): FileSink
/** Streams an existing file through [consumer] (bytes, count); stops early when it returns false. Returns bytes consumed. */
fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long
fun rename(from: String, to: String): Boolean
fun delete(path: String): Boolean
}
/**
* Large-file download to "<dest>.part" with HTTP Range resume across app restarts, a pinned
* sha256 hashed incrementally (including the resumed prefix), then a rename into place.
* [run] blocks on the calling thread; listener callbacks are invoked synchronously on it, so
* the platform adapter marshals them (Android posts to the constructing Looper).
*/
class ResumableDownload(
private val url: String,
private val destPath: String,
expectedSha256: String?,
private val expectedTotalBytes: Long,
private val transport: DownloadTransport,
private val storage: DownloadStorage,
private val listener: FaceclawModelDownloaderListener,
private val now: () -> Long = { currentTimeMillis() },
) {
private val partPath = "$destPath.part"
private val expectedSha256 = expectedSha256?.lowercase() ?: ""
@Volatile
private var cancelled = false
@Volatile
private var response: DownloadResponse? = null
/** Stops the download but keeps the .part file so a later run resumes. */
fun cancel() {
cancelled = true
try {
response?.close()
} catch (ignored: Exception) {
}
}
fun run() {
try {
if (storage.length(destPath) > 0) {
listener.onDone(destPath)
return
}
storage.prepareParent(destPath)
var digest = Sha256Digest()
var offset = 0L
if (storage.length(partPath) >= 0) {
offset = hashExistingPrefix(digest)
if (cancelled) return
}
val opened = transport.open(url, offset)
response = opened
var done: Long
try {
if (opened.code < 200 || opened.code >= 300) {
listener.onError("Model download failed: HTTP " + opened.code)
return
}
val resumed = opened.code == 206
if (offset > 0 && !resumed) {
// Server ignored the Range header; start over.
offset = 0L
digest = Sha256Digest()
}
val total =
if (expectedTotalBytes > 0) expectedTotalBytes
else if (opened.contentLength > 0) offset + opened.contentLength
else -1L
val out = storage.openWrite(partPath, resumed)
try {
val buffer = ByteArray(1 shl 16)
done = offset
var lastProgressAt = 0L
while (true) {
val read = opened.read(buffer)
if (read < 0) break
if (cancelled) return
out.write(buffer, 0, read)
digest.update(buffer, 0, read)
done += read
val time = now()
if (time - lastProgressAt >= PROGRESS_INTERVAL_MS) {
lastProgressAt = time
listener.onProgress(done, total)
}
}
} finally {
out.close()
}
} finally {
try {
opened.close()
} catch (ignored: Exception) {
}
}
val partLength = storage.length(partPath)
if (expectedTotalBytes > 0 && partLength != expectedTotalBytes) {
listener.onError("Model download ended early (" + partLength + " of " + expectedTotalBytes + " bytes); try again to resume")
return
}
val actualSha = BinaryEncoding.bytesToHex(digest.digest())
if (expectedSha256.isNotEmpty() && actualSha != expectedSha256) {
storage.delete(partPath)
listener.onError("Model download was corrupted (checksum mismatch); download it again")
return
}
if (!storage.rename(partPath, destPath)) {
listener.onError("Could not move the downloaded model into place")
return
}
listener.onDone(destPath)
} catch (e: Exception) {
if (cancelled) return
PlatformLog.w(TAG, "download failed: $e")
listener.onError("Model download failed: $e")
}
}
/** Hash the already-downloaded prefix so the final digest covers the whole file. */
private fun hashExistingPrefix(digest: Sha256Digest): Long =
storage.readAll(partPath) { bytes, count ->
if (cancelled) false
else {
digest.update(bytes, 0, count)
true
}
}
companion object {
private const val TAG = "FaceclawModelDl"
const val PROGRESS_INTERVAL_MS = 500L
}
}
@@ -0,0 +1,209 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/** Streaming POST transport; the platform (okhttp, NSURLSession) implements it. */
interface StreamingHttp {
fun post(url: String, body: String, headers: List<Pair<String, String>>, sink: ChunkSink): Cancellable
}
interface Cancellable {
fun cancel()
}
/** Raw response feed from a [StreamingHttp]; may be called from any thread, one call at a time. */
interface ChunkSink {
fun onStatus(code: Int)
fun onChunk(bytes: ByteArray, offset: Int, length: Int)
fun onComplete()
fun onFailure(message: String)
}
sealed class SseEvent {
data class Line(val text: String) : SseEvent()
data class HttpError(val code: Int, val body: String) : SseEvent()
object Complete : SseEvent()
data class Failure(val message: String) : SseEvent()
}
/**
* Server-sent-events response policy shared by both platforms. Feeds raw bytes in through
* [ChunkSink] and queues [SseEvent]s for [takeEvents]. Behavior (matching the stricter iOS
* implementation): a UTF-8 BOM is stripped from the first line; lines split on LF, CR or CRLF;
* non-2xx responses buffer at most [ERROR_BODY_LIMIT] bytes of body and end in [SseEvent.HttpError];
* a single line over [LINE_LIMIT] bytes, invalid UTF-8, or more than [QUEUE_BYTE_LIMIT] bytes /
* [QUEUE_EVENT_LIMIT] events waiting to be taken end the stream with a [SseEvent.Failure].
* After the terminal event nothing further is queued; [cancel] drops everything.
*/
class SseStream : ChunkSink {
private val lock = protocolPlatform().createLock()
private val events = ArrayList<SseEvent>()
private var queuedBytes = 0L
private var buffer = ByteArray(4096)
private var bufferLength = 0
private var status = 0
private var firstLine = true
private var finished = false
private var cancelled = false
val isFinished: Boolean
get() = lock.withLock { finished }
override fun onStatus(code: Int) {
lock.withLock { status = code }
}
override fun onChunk(bytes: ByteArray, offset: Int, length: Int) {
lock.withLock {
if (finished || cancelled || length <= 0) return
if (status < 200 || status >= 300) {
val remaining = ERROR_BODY_LIMIT - bufferLength
append(bytes, offset, minOf(remaining, length))
return
}
append(bytes, offset, length)
var start = 0
var i = 0
while (i < bufferLength) {
val c = buffer[i]
if (c != CR && c != LF) {
i++
continue
}
if (c == CR && i + 1 == bufferLength) break
line(start, i - start)
if (finished) return
if (c == CR && buffer[i + 1] == LF) i++
start = i + 1
i++
}
if (start > 0) {
buffer.copyInto(buffer, 0, start, bufferLength)
bufferLength -= start
}
if (bufferLength > LINE_LIMIT) finish(SseEvent.Failure("Streaming response line too large"))
}
}
override fun onComplete() {
lock.withLock {
if (finished || cancelled) return
if (status < 200 || status >= 300) {
finish(SseEvent.HttpError(status, decode(0, bufferLength) ?: ""))
return
}
if (bufferLength > 0) {
if (buffer[bufferLength - 1] == CR) bufferLength--
line(0, bufferLength)
if (finished) return
}
finish(SseEvent.Complete)
}
}
override fun onFailure(message: String) {
lock.withLock { finish(SseEvent.Failure(message)) }
}
/** Drains queued events in order. Empty once cancelled. */
fun takeEvents(): List<SseEvent> =
lock.withLock {
if (events.isEmpty()) return emptyList()
val taken = ArrayList(events)
events.clear()
queuedBytes = 0
taken
}
fun cancel() {
lock.withLock {
cancelled = true
events.clear()
queuedBytes = 0
bufferLength = 0
}
}
private fun line(start: Int, length: Int) {
var text = decode(start, length)
if (text == null) {
finish(SseEvent.Failure("Invalid UTF-8 in streaming response"))
return
}
if (firstLine && text.startsWith('')) text = text.substring(1)
firstLine = false
queuedBytes += length
if (queuedBytes > QUEUE_BYTE_LIMIT || events.size >= QUEUE_EVENT_LIMIT) {
events.clear()
finish(SseEvent.Failure("Streaming response queue overflow"))
return
}
events.add(SseEvent.Line(text))
}
private fun decode(start: Int, length: Int): String? =
try {
buffer.decodeToString(start, start + length, throwOnInvalidSequence = true)
} catch (e: CharacterCodingException) {
null
}
private fun finish(event: SseEvent) {
if (finished || cancelled) return
finished = true
events.add(event)
bufferLength = 0
}
private fun append(bytes: ByteArray, offset: Int, length: Int) {
if (length <= 0) return
if (bufferLength + length > buffer.size) buffer = buffer.copyOf(maxOf(bufferLength + length, buffer.size * 2))
bytes.copyInto(buffer, bufferLength, offset, offset + length)
bufferLength += length
}
companion object {
const val ERROR_BODY_LIMIT = 64 * 1024
const val LINE_LIMIT = 2 * 1024 * 1024
const val QUEUE_BYTE_LIMIT = 8L * 1024 * 1024
const val QUEUE_EVENT_LIMIT = 16384
private const val CR: Byte = '\r'.code.toByte()
private const val LF: Byte = '\n'.code.toByte()
/**
* Unpacks the flat alternating [name, value, name, value, ...] header array the
* TypeScript side builds (okhttp Headers cannot cross the JS bridge). Entries with a
* null/empty name or a null value are skipped, as before.
*/
@JvmStatic
fun unpackHeaders(flat: Array<String?>?): List<Pair<String, String>> {
if (flat == null) return emptyList()
val result = ArrayList<Pair<String, String>>()
var i = 0
while (i + 1 < flat.size) {
val name = flat[i]
val value = flat[i + 1]
if (!name.isNullOrEmpty() && value != null) result.add(name to value)
i += 2
}
return result
}
/** Delivers one event to a [FaceclawSseListener]. */
@JvmStatic
fun deliver(event: SseEvent, listener: FaceclawSseListener) {
when (event) {
is SseEvent.Line -> listener.onLine(event.text)
is SseEvent.HttpError -> listener.onHttpError(event.code, event.body)
SseEvent.Complete -> listener.onComplete()
is SseEvent.Failure -> listener.onFailure(event.message)
}
}
}
}
@@ -0,0 +1,71 @@
package com.faceclaw.app
/** Key/value backing store for app settings (SharedPreferences on Android, NSUserDefaults on iOS). */
interface SettingsStorage {
fun getString(key: String?, defaultValue: String?): String?
fun putString(key: String?, value: String?)
fun getBoolean(key: String?, defaultValue: Boolean): Boolean
fun putBoolean(key: String?, value: Boolean)
}
/** Runs an action on the thread that registered a listener (a Looper Handler, a dispatch queue). */
fun interface SettingsDispatcher {
fun post(action: () -> Unit)
}
/**
* Settings reads/writes plus change fan-out. Each JS isolate registers one listener together
* with a dispatcher for its own thread; every write notifies every listener through its own
* dispatcher, so callbacks never run on a foreign thread. A listener registered without a
* dispatcher is accepted but never notified (it can still read fresh values on demand).
*/
class SettingsChangeHub(
private val storage: SettingsStorage,
private val warn: (String) -> Unit = { PlatformLog.w("FaceclawSettings", it) },
platform: ProtocolPlatform = protocolPlatform(),
) {
private class Entry(val listener: FaceclawSettingsListener, val dispatcher: SettingsDispatcher?)
private val lock = platform.createLock()
private var listeners: List<Entry> = emptyList()
fun getString(key: String?, defaultValue: String?): String? = storage.getString(key, defaultValue)
fun setString(key: String?, value: String?) {
storage.putString(key, value)
notifyChanged(key)
}
fun getBoolean(key: String?, defaultValue: Boolean): Boolean = storage.getBoolean(key, defaultValue)
fun setBoolean(key: String?, value: Boolean) {
storage.putBoolean(key, value)
notifyChanged(key)
}
fun register(listener: FaceclawSettingsListener, dispatcher: SettingsDispatcher?) {
if (dispatcher == null) warn("settings listener registered from a thread without a dispatcher; it will never be notified")
lock.withLock { listeners = listeners + Entry(listener, dispatcher) }
}
fun unregister(listener: FaceclawSettingsListener?) {
lock.withLock { listeners = listeners.filter { it.listener !== listener } }
}
fun notifyChanged(key: String?) {
val current = lock.withLock { listeners }
for (entry in current) {
val dispatcher = entry.dispatcher ?: continue
dispatcher.post {
try {
entry.listener.onSettingChanged(key)
} catch (error: Exception) {
warn("settings listener failed for key $key: ${error.message}")
}
}
}
}
}
@@ -0,0 +1,105 @@
package com.faceclaw.app
/**
* Sleep that another thread can cut short. An interrupt delivered before the sleep begins is
* consumed by the next sleep (returns false immediately) rather than lost: interrupt() runs on
* other threads with no lock held by the sleeper between two work passes, so clearing the flag
* at entry would force a full sleep interval. Returns true when the full duration elapsed.
*/
class InterruptibleSleep(private val platform: ProtocolPlatform = protocolPlatform()) {
private val condition = platform.createCondition()
private var interrupted = false
fun sleep(ms: Long): Boolean {
val deadline = platform.elapsedRealtimeMs() + maxOf(1L, ms)
condition.lock()
try {
if (interrupted) {
interrupted = false
return false
}
while (true) {
val remaining = deadline - platform.elapsedRealtimeMs()
if (remaining <= 0) return true
condition.awaitMs(remaining)
if (interrupted) {
interrupted = false
return false
}
}
} finally {
condition.unlock()
}
}
fun interrupt() {
condition.withLock {
interrupted = true
condition.signalAll()
}
}
}
/** One-shot countdown; await returns true once the count reaches zero, false on timeout. */
class Latch(count: Int, private val platform: ProtocolPlatform = protocolPlatform()) {
private val condition = platform.createCondition()
private var remaining = count
fun countDown() {
condition.withLock {
if (remaining > 0) remaining--
if (remaining == 0) condition.signalAll()
}
}
fun await(timeoutMs: Long): Boolean {
val deadline = platform.elapsedRealtimeMs() + timeoutMs
condition.withLock {
while (remaining > 0) {
val left = deadline - platform.elapsedRealtimeMs()
if (left <= 0) return false
condition.awaitMs(left)
}
return true
}
}
}
/** Unbounded FIFO handoff between threads with a timed poll. */
class BlockingQueue<T : Any>(private val platform: ProtocolPlatform = protocolPlatform()) {
private val condition = platform.createCondition()
private val items = ArrayDeque<T>()
fun put(item: T) {
condition.withLock {
items.addLast(item)
condition.signalAll()
}
}
fun poll(timeoutMs: Long): T? {
val deadline = platform.elapsedRealtimeMs() + timeoutMs
condition.withLock {
while (items.isEmpty()) {
val left = deadline - platform.elapsedRealtimeMs()
if (left <= 0) return null
condition.awaitMs(left)
}
return items.removeFirst()
}
}
fun clear() {
condition.withLock { items.clear() }
}
val size: Int
get() = condition.withLock { items.size }
}
/** Starts a platform thread running [body]; daemon threads never block process exit on the JVM. */
expect fun startThread(name: String, daemon: Boolean, body: () -> Unit)
expect fun sleepMs(ms: Long)
fun currentThreadName(): String = com.faceclaw.shared.PlatformInfo.threadName()
@@ -0,0 +1,451 @@
package com.faceclaw.app
import kotlin.math.abs
import kotlin.math.ceil
import kotlin.math.max
import kotlin.math.min
/**
* Per-frame timing instrumentation, shared between the native BLE layer and the
* Typescript UI layer. A "frame" starts when we receive an input event (or a
* timer fires and we decide to redraw), and finishes when the resulting screen
* update has been fully transmitted to the glasses, is discarded unsent, or
* times out because nobody reported finishing it.
*
* Frames form a tree. One input event can fan out into several renders (the
* focused app's window plus the shell chrome, say), and each of those is its
* own frame linked back to the input frame that caused it via
* [startFrame]. The export nests descendants under their
* root and renders every line against the root's clock, so a single block
* shows the whole input-to-pixels story. Latency percentiles are measured over
* roots: root start to the first descendant that actually reached the glasses.
*
* All public methods are thread-safe and cheap enough to call from the BLE
* worker thread, the platform main thread, the JS thread, and app worker
* threads. Frame IDs are positive ints; 0 means "no frame" and is silently
* ignored everywhere, so callers do not need to null-check.
*
* This core owns the bookkeeping and the export text; the platform facade owns
* the export thread and file write (see the Android `FrameTimings`). The clock
* is [ProtocolPlatform.elapsedRealtimeMs] (monotonic, includes sleep); wall-clock,
* thread names and logging are injectable for tests.
*/
class FrameTimingsCore(
private val platform: ProtocolPlatform = protocolPlatform(),
private val wallClockMs: () -> Long = { currentTimeMillis() },
private val threadName: () -> String = { currentThreadName() },
private val formatWallClock: (Long) -> String = { formatLocalTime(it, "yyyy-MM-dd HH:mm:ss.SSS") },
private val logInfo: (String) -> Unit = { PlatformLog.i(TAG, it) },
private val logWarn: (String) -> Unit = { PlatformLog.w(TAG, it) },
) {
companion object {
const val TAG = "FrameTimings"
/** Frames still open after this long are finished as "timeout". */
const val FRAME_TIMEOUT_MS = 30_000L
const val EXPORT_INTERVAL_MS = 15_000L
const val SWEEP_INTERVAL_MS = 5_000L
private const val RECENT_ROOTS_KEPT = 40
private const val SLOWEST_ROOTS_KEPT = 20
private const val SENT_DURATIONS_WINDOW = 512
private const val MAX_LINES_PER_FRAME = 200
/**
* Frames retained for ID lookup. Well above the number any export can
* reference, so a late log/span/finish call always finds its frame; frames
* older than this are unreachable and get collected.
*/
private const val FRAMES_RETAINED = 1024
private fun percentileOfSorted(sorted: List<Long>, percentile: Int): Long {
val index = ceil(percentile / 100.0 * sorted.size).toInt() - 1
return sorted[max(0, min(sorted.size - 1, index))]
}
/** java.lang.String.format("%+7d"): explicit sign, right-aligned in 7 columns. */
private fun signedWidth7(value: Long): String {
val text = (if (value < 0) "-" else "+") + abs(value).toString()
return text.padStart(7)
}
}
private class Line(
val atMs: Long, // elapsedRealtimeMs
val thread: String,
val message: String?
)
private class Frame(
val id: Int,
var reason: String,
val startedAtMs: Long, // elapsedRealtimeMs
val startedWallClockMs: Long,
val parent: Frame?
) {
val children: MutableList<Frame> = ArrayList(0)
val lines: MutableList<Line> = ArrayList()
val openSpans: MutableMap<String, Long> = LinkedHashMap()
var finishedAtMs: Long = 0
var outcome: String? = null // null while open; "sent", "discarded: ...", "timeout: ..."
/** Set on the root once some frame in its subtree reached the glasses. */
var subtreeSentRecorded = false
fun root(): Frame {
var frame: Frame = this
while (frame.parent != null) {
frame = frame.parent!!
}
return frame
}
fun isOpen(): Boolean = outcome == null
fun durationMs(): Long = finishedAtMs - startedAtMs
fun wasSent(): Boolean {
val outcome = this.outcome
return outcome != null && outcome.startsWith("sent")
}
/** Last moment anything in this subtree happened, for whole-tree duration. */
fun subtreeEndAtMs(nowMs: Long): Long {
var end = if (isOpen()) nowMs else finishedAtMs
for (child in children) {
end = max(end, child.subtreeEndAtMs(nowMs))
}
return end
}
fun subtreeHasOpenFrame(): Boolean {
if (isOpen()) return true
for (child in children) {
if (child.subtreeHasOpenFrame()) return true
}
return false
}
}
private val lock = platform.createLock()
/**
* Every frame we still retain, keyed by ID, in insertion order: open frames
* plus enough history that late calls and the export can still resolve one.
*/
private val framesById: LinkedHashMap<Int, Frame> = LinkedHashMap()
private val recentRoots = ArrayDeque<Frame>()
private val slowestRoots: MutableList<Frame> = ArrayList()
private val inputLatencies = ArrayDeque<Long>()
private val renderLatencies = ArrayDeque<Long>()
private var nextFrameId = 1
private var framesStarted: Long = 0
private var framesSent: Long = 0
private var framesDiscarded: Long = 0
private var framesTimedOut: Long = 0
private var exportDirty = false
/** Begin a root frame; reason is a short label like "input:sys-event type=0". */
fun startFrame(reason: String?): Int = startFrame(reason, 0)
/**
* Begin a frame caused by an existing one (parentFrameId; 0 for a root).
* The child is nested under its root in the export and its latency counts
* against the root, so an input event that fans out into an app render and
* a shell-chrome render reads as one timeline instead of three.
*/
fun startFrame(reason: String?, parentFrameId: Int): Int {
val now = platform.elapsedRealtimeMs()
val frame = lock.withLock {
val parent = framesById[parentFrameId]
val frame = Frame(nextFrameId++, reason ?: "", now, wallClockMs(), parent)
framesById[frame.id] = frame
// Like LinkedHashMap.removeEldestEntry: drop at most the single eldest
// entry per insert, and only when it is already finished.
if (framesById.size > FRAMES_RETAINED) {
val eldest = framesById.entries.first()
if (!eldest.value.isOpen()) framesById.remove(eldest.key)
}
framesStarted++
if (parent != null) {
parent.children.add(frame)
addLineLocked(parent, now, "spawned frame#" + frame.id + " (" + frame.reason + ")")
}
frame
}
return frame.id
}
/**
* Append to a frame's label, for facts only known after it started (which
* app is in the foreground, which window a render is for). Shows up in the
* frame's header line, so the export is scannable without reading bodies.
*/
fun annotate(frameId: Int, text: String?) {
if (text == null || text.isEmpty()) return
lock.withLock {
val frame = framesById[frameId] ?: return
frame.reason = if (frame.reason.isEmpty()) text else frame.reason + " " + text
}
}
fun log(frameId: Int, message: String?) {
val now = platform.elapsedRealtimeMs()
lock.withLock {
val frame = openFrameLocked(frameId) ?: return
addLineLocked(frame, now, message)
}
}
fun spanStart(frameId: Int, name: String) {
val now = platform.elapsedRealtimeMs()
lock.withLock {
val frame = openFrameLocked(frameId) ?: return
frame.openSpans[name] = now
addLineLocked(frame, now, "span $name start")
}
}
fun spanEnd(frameId: Int, name: String) {
val now = platform.elapsedRealtimeMs()
lock.withLock {
val frame = openFrameLocked(frameId) ?: return
val startedAt = frame.openSpans.remove(name)
if (startedAt == null) {
addLineLocked(frame, now, "span $name end (start not recorded)")
} else {
addLineLocked(frame, now, "span " + name + " end (" + (now - startedAt) + "ms)")
}
}
}
/**
* Finish a frame. Outcomes: "sent" (counted in latency percentiles), anything
* starting with "discarded", or "timeout". First finish wins; later calls for
* the same frame are ignored, so racing completion paths are safe.
*/
fun finishFrame(frameId: Int, outcome: String?) {
val now = platform.elapsedRealtimeMs()
val finished = lock.withLock {
val frame = openFrameLocked(frameId) ?: return
finishFrameLocked(frame, now, outcome ?: "discarded: no outcome given")
frame
}
logInfo("frame#" + finished.id + " [" + finished.reason + "] -> " + finished.outcome
+ " in " + finished.durationMs() + "ms")
}
/** One-line stats summary, e.g. for showing in the phone UI. */
fun statsSummary(): String {
lock.withLock {
val input = percentilesLocked(inputLatencies)
return "frames started=" + framesStarted + " sent=" + framesSent +
" discarded=" + framesDiscarded + " timeout=" + framesTimedOut +
(if (input == null)
""
else " | input-to-display p50=" + input[0] + "ms p90=" + input[1] +
"ms p99=" + input[2] + "ms max=" + input[3] + "ms")
}
}
/** Finish every frame open longer than [FRAME_TIMEOUT_MS] as a timeout; returns how many. */
fun sweepTimedOut(): Int {
val now = platform.elapsedRealtimeMs()
val timedOut: MutableList<Frame> = ArrayList()
lock.withLock {
for (frame in ArrayList(framesById.values)) {
if (frame.isOpen() && now - frame.startedAtMs >= FRAME_TIMEOUT_MS) {
finishFrameLocked(frame, now, "timeout: never reported finished")
timedOut.add(frame)
}
}
}
for (frame in timedOut) {
logWarn("frame#" + frame.id + " [" + frame.reason + "] timed out after " +
frame.durationMs() + "ms without being finished")
}
return timedOut.size
}
/** True once a frame finished since the last [takeExport]/[buildExport] with clear. */
val isExportDirty: Boolean
get() = lock.withLock { exportDirty }
/** The full export text (does not clear the dirty flag). */
fun buildExport(): String = lock.withLock { buildExportLocked() }
/** The export text if anything changed since the last take, else null; clears the dirty flag. */
fun takeExport(): String? = lock.withLock {
if (!exportDirty) return null
exportDirty = false
buildExportLocked()
}
// ---------------------------------------------------------------------
/** The frame with this ID if it exists and is still open, else null. */
private fun openFrameLocked(frameId: Int): Frame? {
if (frameId <= 0) return null
val frame = framesById[frameId]
return if (frame != null && frame.isOpen()) frame else null
}
private fun addLineLocked(frame: Frame, now: Long, message: String?) {
if (frame.lines.size >= MAX_LINES_PER_FRAME) {
if (frame.lines.size == MAX_LINES_PER_FRAME) {
frame.lines.add(Line(now, threadName(), "... line cap reached"))
}
return
}
frame.lines.add(Line(now, threadName(), message))
}
private fun finishFrameLocked(frame: Frame, now: Long, outcome: String) {
frame.finishedAtMs = now
frame.outcome = outcome
for (open in frame.openSpans.entries) {
frame.lines.add(Line(now, threadName(),
"span " + open.key + " never ended (started at +" +
(open.value - frame.startedAtMs) + "ms)"))
}
frame.openSpans.clear()
addLineLocked(frame, now, "finished: $outcome")
val root = frame.root()
if (frame.wasSent()) {
framesSent++
// The user-visible latency is input (or timer) to the first pixels
// that actually reached the glasses, wherever in the tree that was.
if (!root.subtreeSentRecorded) {
root.subtreeSentRecorded = true
recordLatencyLocked(root, now - root.startedAtMs)
insertSlowestLocked(root)
}
} else if (outcome.startsWith("timeout")) {
framesTimedOut++
} else {
framesDiscarded++
}
if (root === frame) {
recentRoots.addLast(frame)
while (recentRoots.size > RECENT_ROOTS_KEPT) {
recentRoots.removeFirst()
}
}
exportDirty = true
}
private fun recordLatencyLocked(root: Frame, latencyMs: Long) {
val bucket = if (root.reason.startsWith("input:")) inputLatencies else renderLatencies
bucket.addLast(latencyMs)
while (bucket.size > SENT_DURATIONS_WINDOW) {
bucket.removeFirst()
}
}
private fun insertSlowestLocked(root: Frame) {
if (!slowestRoots.contains(root)) {
slowestRoots.add(root)
}
// Sorted and trimmed at export time, when every subtree duration is final.
}
/** {p50, p90, p99, max} over a rolling latency window, or null if empty. */
private fun percentilesLocked(window: ArrayDeque<Long>): LongArray? {
if (window.isEmpty()) return null
val sorted: MutableList<Long> = ArrayList(window)
sorted.sort()
return longArrayOf(
percentileOfSorted(sorted, 50),
percentileOfSorted(sorted, 90),
percentileOfSorted(sorted, 99),
sorted[sorted.size - 1],
)
}
private fun buildExportLocked(): String {
val now = platform.elapsedRealtimeMs()
val out = StringBuilder(64 * 1024)
out.append("FrameTimings export at ").append(formatWallClock(wallClockMs())).append('\n')
out.append(statsSummaryLocked()).append('\n')
out.append('\n')
out.append("Frames are trees: an input event's own frame is the root, and the renders it\n")
out.append("caused are indented under it with offsets measured from the root's start.\n")
out.append('\n')
// Stable sort, descending by whole-tree duration (matches Collections.sort).
slowestRoots.sortWith(compareByDescending { it.subtreeEndAtMs(now) - it.startedAtMs })
while (slowestRoots.size > SLOWEST_ROOTS_KEPT) {
slowestRoots.removeAt(slowestRoots.size - 1)
}
out.append("=== slowest frames that reached the glasses ===\n")
for (frame in slowestRoots) {
appendTreeLocked(out, frame, frame, now)
}
out.append("=== recent frames (oldest first) ===\n")
for (frame in recentRoots) {
appendTreeLocked(out, frame, frame, now)
}
return out.toString()
}
private fun statsSummaryLocked(): String {
val input = percentilesLocked(inputLatencies)
val render = percentilesLocked(renderLatencies)
val out = StringBuilder()
out.append("frames started=").append(framesStarted).append(" sent=").append(framesSent)
.append(" discarded=").append(framesDiscarded).append(" timeout=").append(framesTimedOut)
.append(" open=").append(openFrameCountLocked())
appendPercentilesLocked(out, "input-to-display latency", inputLatencies.size, input)
appendPercentilesLocked(out, "render-to-display latency", renderLatencies.size, render)
return out.toString()
}
private fun appendPercentilesLocked(out: StringBuilder, label: String, count: Int, percentiles: LongArray?) {
if (percentiles == null) return
out.append('\n').append(label).append(" (last ").append(count).append("): p50=")
.append(percentiles[0]).append("ms p90=").append(percentiles[1])
.append("ms p99=").append(percentiles[2]).append("ms max=").append(percentiles[3]).append("ms")
}
private fun openFrameCountLocked(): Int {
var open = 0
for (frame in framesById.values) {
if (frame.isOpen()) open++
}
return open
}
/** Print a frame and its descendants, all timed against root's start. */
private fun appendTreeLocked(out: StringBuilder, frame: Frame, root: Frame, now: Long) {
var depth = 0
var walk: Frame? = frame
while (walk !== root) {
depth++
walk = walk!!.parent
}
val indent = " ".repeat(depth)
val startOffsetMs = frame.startedAtMs - root.startedAtMs
out.append(indent).append("frame#").append(frame.id)
.append(" [").append(frame.reason).append(']')
if (frame === root) {
out.append(" started ").append(formatWallClock(frame.startedWallClockMs))
val totalMs = frame.subtreeEndAtMs(now) - frame.startedAtMs
out.append(" duration ").append(frame.durationMs()).append("ms")
if (frame.children.isNotEmpty()) {
out.append(" (tree ").append(totalMs).append("ms")
.append(if (frame.subtreeHasOpenFrame()) ", still open" else "").append(')')
}
} else {
out.append(" started +").append(startOffsetMs).append("ms")
.append(" duration ").append(if (frame.isOpen()) "open" else (frame.durationMs().toString() + "ms"))
}
out.append(" outcome ").append(frame.outcome ?: "(open)").append('\n')
for (line in frame.lines) {
out.append(indent).append(" ").append(signedWidth7(line.atMs - root.startedAtMs))
.append("ms [").append(line.thread).append("] ").append(line.message).append('\n')
}
for (child in frame.children) {
appendTreeLocked(out, child, root, now)
}
}
}
@@ -0,0 +1,161 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/** A JSON number kept as its source text so integers and decimals survive a round trip. */
data class JsonNumber(val text: String) {
fun toLong(): Long = text.toLongOrNull() ?: text.toDouble().toLong()
fun toInt(): Int = toLong().toInt()
fun toDouble(): Double = text.toDouble()
companion object {
@JvmStatic fun of(value: Long): JsonNumber = JsonNumber(value.toString())
@JvmStatic fun of(value: Int): JsonNumber = JsonNumber(value.toString())
@JvmStatic
fun of(value: Double): JsonNumber {
require(value.isFinite()) { "JSON numbers must be finite" }
val whole = value.toLong()
return JsonNumber(if (whole.toDouble() == value) whole.toString() else value.toString())
}
}
}
/**
* Minimal JSON codec shared by both platforms (org.json is JVM-only). Values are
* `Map<String, Any?>` (insertion ordered), `List<Any?>`, `String`, `Boolean`, `JsonNumber`
* and `null`. `parse` also accepts `//` and block comments plus trailing commas, which the
* settings documents rely on; `write` always emits strict JSON.
*/
object Json {
private val numberPattern = Regex("-?(?:0|[1-9][0-9]*)(?:\\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
@JvmStatic fun parse(text: String): Any? = Parser(text).parse()
@JvmStatic fun parseObject(text: String): Map<String, Any?> = parse(text).asObject() ?: error("JSON object expected")
@JvmStatic fun parseArray(text: String): List<Any?> = parse(text).asArray() ?: error("JSON array expected")
@JvmStatic
fun write(value: Any?, pretty: Boolean = false): String = write(value, pretty, 0)
@JvmStatic
fun quote(text: String): String = buildString {
append('"')
for (c in text) when (c) {
'"' -> append("\\\"")
'\\' -> append("\\\\")
'\n' -> append("\\n")
'\r' -> append("\\r")
'\t' -> append("\\t")
else -> if (c < ' ') append("\\u" + c.code.toString(16).padStart(4, '0')) else append(c)
}
append('"')
}
private fun write(value: Any?, pretty: Boolean, depth: Int): String {
fun collection(open: String, close: String, entries: List<String>): String {
if (entries.isEmpty()) return open + close
return if (pretty) open + "\n" + entries.joinToString(",\n") { " ".repeat(depth + 1) + it } + "\n" + " ".repeat(depth) + close
else open + entries.joinToString(",") + close
}
return when (value) {
null -> "null"
is String -> quote(value)
is Boolean -> value.toString()
is JsonNumber -> value.text
is Int -> value.toString()
is Long -> value.toString()
is Float -> JsonNumber.of(value.toDouble()).text
is Double -> JsonNumber.of(value).text
is List<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) })
is Array<*> -> collection("[", "]", value.map { write(it, pretty, depth + 1) })
is Map<*, *> -> collection("{", "}", value.entries.map { quote(it.key as String) + (if (pretty) ": " else ":") + write(it.value, pretty, depth + 1) })
else -> error("Unsupported JSON value: ${value::class.simpleName}")
}
}
private class Parser(val text: String) {
var i = 0
fun parse(): Any? { val result = value(0); space(); require(i == text.length) { "Trailing characters in JSON" }; return result }
fun space() {
while (i < text.length) {
if (text[i] in " \r\n\t") { i++; continue }
if (text.startsWith("//", i)) { while (i < text.length && text[i] != '\n' && text[i] != '\r') i++; continue }
if (text.startsWith("/*", i)) { val end = text.indexOf("*/", i + 2); require(end >= 0); i = end + 2; continue }
break
}
}
fun take(c: Char): Boolean { space(); return if (i < text.length && text[i] == c) { i++; true } else false }
fun value(depth: Int): Any? {
require(depth < 64); space(); require(i < text.length) { "Unexpected end of JSON" }
return when (text[i]) {
'"' -> string()
'{' -> {
i++; val result = linkedMapOf<String, Any?>()
if (!take('}')) while (true) {
space(); val key = string(); require(!result.containsKey(key)) { "Duplicate key $key" }; require(take(':'))
result[key] = value(depth + 1)
if (take('}')) break
require(take(',')); if (take('}')) break
}
result
}
'[' -> {
i++; val result = mutableListOf<Any?>()
if (!take(']')) while (true) {
result.add(value(depth + 1)); if (take(']')) break
require(take(',')); if (take(']')) break
}
result
}
't' -> literal("true", true)
'f' -> literal("false", false)
'n' -> literal("null", null)
else -> {
val match = numberPattern.find(text, i)
require(match != null && match.range.first == i) { "Invalid JSON value at $i" }
require(match.value.toDouble().isFinite()); i += match.value.length; JsonNumber(match.value)
}
}
}
fun literal(word: String, result: Any?): Any? { require(text.startsWith(word, i)); i += word.length; return result }
fun string(): String {
require(i < text.length && text[i++] == '"')
return buildString {
while (true) {
require(i < text.length); val c = text[i++]
if (c == '"') break
require(c >= ' ')
if (c != '\\') { append(c); continue }
require(i < text.length)
append(when (val escaped = text[i++]) {
'"', '\\', '/' -> escaped
'b' -> '\b'; 'f' -> '\u000c'; 'n' -> '\n'; 'r' -> '\r'; 't' -> '\t'
'u' -> { require(i + 4 <= text.length); val hex = text.substring(i, i + 4); require(hex.all { it in "0123456789abcdefABCDEF" }); val code = hex.toInt(16); i += 4; code.toChar() }
else -> error("Invalid escape")
})
}
}
}
}
}
// Typed accessors for parsed values; each returns null when the value has another type.
@Suppress("UNCHECKED_CAST")
fun Any?.asObject(): Map<String, Any?>? = this as? Map<String, Any?>
fun Any?.asArray(): List<Any?>? = this as? List<Any?>
fun Any?.asString(): String? = this as? String
fun Any?.asBoolean(): Boolean? = this as? Boolean
fun Any?.asLong(): Long? = (this as? JsonNumber)?.toLong() ?: (this as? Long) ?: (this as? Int)?.toLong()
fun Any?.asInt(): Int? = asLong()?.toInt()
fun Any?.asDouble(): Double? = (this as? JsonNumber)?.toDouble() ?: (this as? Double) ?: (this as? Int)?.toDouble() ?: (this as? Long)?.toDouble()
@@ -0,0 +1,30 @@
package com.faceclaw.app
/** Incremental SHA-256; [digest] finishes the hash. */
expect class Sha256Digest() {
fun update(bytes: ByteArray, offset: Int, length: Int)
fun digest(): ByteArray
}
fun Sha256Digest.update(bytes: ByteArray) = update(bytes, 0, bytes.size)
fun sha256(bytes: ByteArray): ByteArray = Sha256Digest().also { it.update(bytes) }.digest()
fun sha256Hex(bytes: ByteArray): String = BinaryEncoding.bytesToHex(sha256(bytes))
/** Wall-clock milliseconds since the Unix epoch (may jump; use ProtocolPlatform for intervals). */
expect fun currentTimeMillis(): Long
/** Formats [epochMs] in local time with a Unicode/SimpleDateFormat-style [pattern] (e.g. "yyyyMMdd-HHmmss-SSS"). */
expect fun formatLocalTime(epochMs: Long, pattern: String): String
expect object PlatformLog {
fun i(tag: String, message: String)
fun w(tag: String, message: String)
fun e(tag: String, message: String, error: Throwable?)
}
fun PlatformLog.e(tag: String, message: String) = e(tag, message, null)
@@ -0,0 +1,105 @@
@file:OptIn(kotlinx.cinterop.ExperimentalForeignApi::class)
package com.faceclaw.app
import kotlinx.cinterop.addressOf
import kotlinx.cinterop.alloc
import kotlinx.cinterop.convert
import kotlinx.cinterop.free
import kotlinx.cinterop.memScoped
import kotlinx.cinterop.nativeHeap
import kotlinx.cinterop.ptr
import kotlinx.cinterop.reinterpret
import kotlinx.cinterop.usePinned
import platform.CoreCrypto.CC_SHA256_CTX
import platform.CoreCrypto.CC_SHA256_DIGEST_LENGTH
import platform.CoreCrypto.CC_SHA256_Final
import platform.CoreCrypto.CC_SHA256_Init
import platform.CoreCrypto.CC_SHA256_Update
import platform.Foundation.NSDate
import platform.Foundation.NSDateFormatter
import platform.Foundation.NSLocale
import platform.Foundation.NSLog
import platform.Foundation.NSThread
import platform.Foundation.dateWithTimeIntervalSince1970
import platform.Foundation.timeIntervalSince1970
import platform.posix.F_OK
import platform.posix.access
import platform.posix.fclose
import platform.posix.fopen
import platform.posix.fwrite
import platform.posix.rename
import platform.posix.stat
import platform.posix.unlink
import platform.posix.usleep
actual class Sha256Digest actual constructor() {
private var context = nativeHeap.alloc<CC_SHA256_CTX>().also { CC_SHA256_Init(it.ptr) }
private var finished = false
actual fun update(bytes: ByteArray, offset: Int, length: Int) {
check(!finished) { "digest already finished" }
if (length == 0) return
bytes.usePinned { CC_SHA256_Update(context.ptr, it.addressOf(offset), length.convert()) }
}
actual fun digest(): ByteArray {
check(!finished) { "digest already finished" }
val out = ByteArray(CC_SHA256_DIGEST_LENGTH)
out.usePinned { CC_SHA256_Final(it.addressOf(0).reinterpret(), context.ptr) }
nativeHeap.free(context.ptr)
finished = true
return out
}
}
actual fun currentTimeMillis(): Long = (NSDate().timeIntervalSince1970 * 1000.0).toLong()
actual fun formatLocalTime(epochMs: Long, pattern: String): String {
val formatter = NSDateFormatter()
formatter.locale = NSLocale("en_US_POSIX")
formatter.dateFormat = pattern
return formatter.stringFromDate(NSDate.dateWithTimeIntervalSince1970(epochMs / 1000.0))
}
actual object PlatformLog {
actual fun i(tag: String, message: String) = NSLog("%@", "$tag: $message")
actual fun w(tag: String, message: String) = NSLog("%@", "$tag: [warn] $message")
actual fun e(tag: String, message: String, error: Throwable?) =
NSLog("%@", "$tag: [error] $message" + (error?.let { " (${it.message})" } ?: ""))
}
actual fun startThread(name: String, daemon: Boolean, body: () -> Unit) {
val thread = NSThread(block = body)
thread.name = name
thread.start()
}
actual fun sleepMs(ms: Long) {
if (ms > 0) usleep((ms * 1000).convert())
}
actual fun writeFileData(path: String, bytes: ByteArray) {
val file = fopen(path, "wb") ?: error("Cannot write $path")
try {
if (bytes.isNotEmpty())
bytes.usePinned {
check(fwrite(it.addressOf(0), 1u, bytes.size.convert(), file) == bytes.size.convert<platform.posix.size_t>()) { "Short write to $path" }
}
} finally {
fclose(file)
}
}
actual fun fileExists(path: String): Boolean = access(path, F_OK) == 0
actual fun fileLength(path: String): Long = memScoped {
val info = alloc<stat>()
if (stat(path, info.ptr) != 0) -1L else info.st_size
}
actual fun deleteFile(path: String): Boolean = unlink(path) == 0
actual fun renameFile(from: String, to: String): Boolean = rename(from, to) == 0
@@ -10,7 +10,10 @@ import kotlinx.cinterop.*
import platform.CoreBluetooth.CBCharacteristicWriteType
import platform.CoreBluetooth.CBCharacteristicWriteWithResponse
import platform.CoreBluetooth.CBCharacteristicWriteWithoutResponse
import platform.Foundation.NSCondition
import platform.Foundation.NSDate
import platform.Foundation.NSRecursiveLock
import platform.Foundation.dateWithTimeIntervalSinceNow
import platform.posix.CLOCK_MONOTONIC_RAW
import platform.posix.clock_gettime_nsec_np
import platform.posix.memset
@@ -32,6 +35,21 @@ object IosProtocolPlatform : ProtocolPlatform {
override fun createDeflater(): ProtocolDeflater = IosDeflater()
override fun createCondition(): ProtocolCondition =
object : ProtocolCondition {
private val condition = NSCondition()
override fun lock() = condition.lock()
override fun unlock() = condition.unlock()
override fun awaitMs(timeoutMs: Long) {
if (timeoutMs > 0) condition.waitUntilDate(NSDate.dateWithTimeIntervalSinceNow(timeoutMs / 1000.0))
}
override fun signalAll() = condition.broadcast()
}
fun writeType(mode: GattWriteMode): CBCharacteristicWriteType =
when (mode) {
GattWriteMode.WITH_RESPONSE -> CBCharacteristicWriteWithResponse
@@ -0,0 +1,33 @@
package com.faceclaw.app
import java.util.Locale
import kotlin.test.Test
import kotlin.test.assertEquals
/** The shared export must match what the Java-era String.format("%+7d") produced. */
class FrameTimingsFormatTest {
private class FakeClock : ProtocolPlatform by AndroidProtocolPlatform {
var nowMs = 5_000L
override fun elapsedRealtimeMs(): Long = nowMs
}
@Test
fun lineOffsetsMatchJavaFormat() {
val clock = FakeClock()
val core = FrameTimingsCore(clock, wallClockMs = { 0L }, threadName = { "worker" }, formatWallClock = { "w" }, logInfo = {}, logWarn = {})
val offsets = listOf(-1_234_567L, -12L, -1L, 0L, 1L, 42L, 999L, 1_000L, 12_345L, 1_234_567L, 12_345_678L)
val root = core.startFrame("input:fmt")
for (offset in offsets) {
clock.nowMs = 5_000L + offset
core.log(root, "o=$offset")
}
clock.nowMs = 5_000L
core.finishFrame(root, "sent")
val export = core.buildExport()
for (offset in offsets) {
val javaLine = String.format(Locale.US, " %+7dms [%s] %s", offset, "worker", "o=$offset")
assert(export.contains("\n$javaLine\n")) { "missing <$javaLine> in\n$export" }
}
assertEquals(2, export.split("\n").count { it.contains("finished: sent") })
}
}
@@ -0,0 +1,123 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
private class FakeScheduler : AlarmScheduler {
val pending = LinkedHashMap<Any, Pair<Long, () -> Unit>>()
override fun postDelayed(token: Any, delayMs: Long, action: () -> Unit) { pending[token] = delayMs to action }
override fun cancel(token: Any) { pending.remove(token) }
override fun cancelAll() { pending.clear() }
fun fire(token: Any) { val entry = pending.remove(token) ?: error("no timer $token"); entry.second() }
}
private class FakeSink : RingingSink {
val events = ArrayList<String>()
var sounding = false
override fun showNotification(item: RingingItem) { events.add("notify ${item.id} " + (if (item.silenced) "silenced" else if (item.escalated) "ringing" else "glasses")) }
override fun itemRemoved(item: RingingItem) { events.add("removed ${item.id}") }
override fun ringingStarted() { events.add("wake") }
override fun startSound() { if (!sounding) { sounding = true; events.add("sound on") } }
override fun stopSound() { if (sounding) { sounding = false; events.add("sound off") } }
override fun reschedule(item: RingingItem, atMs: Long, minutes: Int) { events.add("reschedule ${item.id} $atMs $minutes") }
override fun cancelSchedule(id: Long) { events.add("cancel $id") }
override fun recordPhoneAction(id: Long, action: String, minutes: Int) { events.add("action $id $action $minutes") }
override fun log(line: String) { events.add("log $line") }
}
class AlarmRingingPolicyTest {
private var now = 5_000_000L
private val gate = GlassesStatusGate { now }
private val scheduler = FakeScheduler()
private val sink = FakeSink()
private val policy = AlarmRingingPolicy(gate, { now }, testPlatform()).also { it.attach(scheduler, sink) }
private fun item(id: Long, kind: String = AlarmSchedule.KIND_ALARM) = RingingItem(id, "Wake", "", kind, 9, now)
@Test
fun glassesUnavailableEscalatesImmediately() {
policy.ring(item(1))
assertEquals(listOf("log ring alarm 1 (no glasses status)", "wake", "notify 1 glasses", "log phone sound for 1: no glasses status", "notify 1 ringing", "sound on"), sink.events)
assertTrue(policy.isRinging(1))
assertEquals(setOf<Any>("silence:1"), scheduler.pending.keys)
scheduler.fire("silence:1")
assertEquals(listOf("log auto-silenced 1", "notify 1 silenced", "sound off"), sink.events.drop(6))
}
@Test
fun deliveryAndAcknowledgementTimersEscalate() {
gate.set(connected = true, worn = true, charging = false)
policy.ring(item(2))
assertEquals(listOf("log ring alarm 2 (glasses worn)", "wake", "notify 2 glasses"), sink.events)
assertEquals(setOf<Any>("delivery:2"), scheduler.pending.keys)
policy.markDelivered(2)
assertEquals("log delivered 2 to glasses", sink.events.last())
assertEquals(setOf<Any>("delivery:2", "ack:2"), scheduler.pending.keys)
scheduler.fire("delivery:2") // delivered in time: no escalation
assertFalse(policy.snapshot().single().escalated)
scheduler.fire("ack:2")
assertTrue(policy.snapshot().single().escalated)
assertTrue(sink.sounding)
// Acknowledged on the glasses afterwards: quiet stop, sound off.
policy.stopQuietly(2, "acknowledged")
assertEquals(listOf("log acknowledged 2", "removed 2", "sound off"), sink.events.takeLast(3))
assertFalse(policy.isAnythingRinging())
}
@Test
fun undeliveredRingEscalatesAfterDeliveryWait() {
gate.set(connected = true, worn = true, charging = false)
policy.ring(item(3))
scheduler.fire("delivery:3")
assertEquals("log phone sound for 3: not delivered to the glasses", sink.events[3])
assertTrue(sink.sounding)
}
@Test
fun snoozeAndDismissRecordActionsAndRearm() {
policy.ring(item(4, AlarmSchedule.KIND_TIMER))
policy.snooze(4)
assertEquals(listOf("action 4 snooze 9", "removed 4", "sound off", "reschedule 4 ${now + 9 * 60_000L} 9"), sink.events.takeLast(4))
policy.ring(item(5))
policy.dismiss(5)
assertEquals(listOf("action 5 dismiss 0", "removed 5", "sound off", "cancel 5"), sink.events.takeLast(4))
policy.dismiss(99) // unknown: no-op
assertEquals("cancel 5", sink.events.last())
}
@Test
fun earlySignalsAreConsumedOrExpire() {
gate.set(connected = true, worn = true, charging = false)
assertFalse(policy.deliveredOrDefer(6))
policy.ring(item(6))
assertEquals("log delivered 6 to glasses", sink.events.last())
assertEquals(setOf<Any>("ack:6"), scheduler.pending.keys)
assertFalse(policy.stopOrDefer(7))
policy.ring(item(7))
assertEquals("log ring 7 already acknowledged", sink.events.last())
assertFalse(policy.isRinging(7))
assertFalse(policy.stopOrDefer(8))
now += AlarmRingingPolicy.EARLY_SIGNAL_MAX_AGE_MS + 1
policy.ring(item(8))
assertTrue(policy.isRinging(8))
assertTrue(policy.stopOrDefer(8))
assertTrue(policy.deliveredOrDefer(8))
}
@Test
fun duplicateRingOnlyRefreshesNotificationAndDetachDropsTimers() {
policy.ring(item(9))
val before = sink.events.size
policy.ring(item(9))
assertEquals(listOf("notify 9 ringing"), sink.events.drop(before))
assertEquals(1, policy.snapshot().size)
policy.detach()
assertTrue(scheduler.pending.isEmpty())
policy.ring(item(10)) // no host: ignored
assertFalse(policy.isRinging(10))
}
}
@@ -0,0 +1,109 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
private class MemoryStore : AlarmStore {
val values = HashMap<String, String?>()
override fun read(key: String): String? = values[key]
override fun write(key: String, value: String?) { values[key] = value }
}
class AlarmScheduleTest {
private val store = MemoryStore()
private var now = 1_700_000_000_000L
private val schedule = AlarmSchedule(store, { now }, testPlatform())
@Test
fun persistsEntriesAsJsonAndReplacesById() {
schedule.put(AlarmEntry(7, now + 60_000, "Tea", "steep", AlarmSchedule.KIND_TIMER, 5))
schedule.put(AlarmEntry(8, now + 120_000, null, null, null, 10))
assertEquals("""[{"id":7,"at":${now + 60_000},"title":"Tea","text":"steep","kind":"timer","snoozeMinutes":5},{"id":8,"at":${now + 120_000},"title":"","text":"","kind":"alarm","snoozeMinutes":10}]""", store.values[AlarmSchedule.KEY_SCHEDULED])
schedule.put(AlarmEntry(7, now + 90_000, "Tea", "steep", AlarmSchedule.KIND_TIMER, 5))
assertEquals(listOf(8L, 7L), schedule.entries().map { it.id })
assertEquals(now + 90_000, schedule.find(7)?.at)
schedule.remove(8)
assertNull(schedule.find(8))
// A corrupt file reads as empty rather than failing.
store.values[AlarmSchedule.KEY_SCHEDULED] = "{nope"
assertTrue(schedule.entries().isEmpty())
}
@Test
fun replayRingsRecentDropsMissedArmsFuture() {
schedule.put(AlarmEntry(1, now - 60_000, "recent", "", null, 10))
schedule.put(AlarmEntry(2, now - AlarmSchedule.LATE_GRACE_MS - 1, "old", "", AlarmSchedule.KIND_TIMER, 10))
schedule.put(AlarmEntry(3, now + 60_000, "future", "", null, 10))
val rang = ArrayList<Long>()
val armed = ArrayList<Long>()
schedule.replay("boot", { rang.add(it.id) }, { armed.add(it.id) })
assertEquals(listOf(1L), rang)
assertEquals(listOf(3L), armed)
assertEquals(listOf(1L, 3L), schedule.entries().map { it.id })
val log = schedule.readLog()
assertTrue(log.contains("$now missed timer 2 (boot)"), log)
assertTrue(log.endsWith("$now rescheduled 1 (boot)"), log)
}
@Test
fun stalenessComparesSoonestPendingWithSystemAlarm() {
assertFalse(schedule.looksStale(null))
schedule.put(AlarmEntry(1, now - 1, "past", "", null, 10))
assertNull(schedule.soonestPendingAt())
schedule.put(AlarmEntry(2, now + 50_000, "", "", null, 10))
schedule.put(AlarmEntry(3, now + 20_000, "", "", null, 10))
assertEquals(now + 20_000, schedule.soonestPendingAt())
assertTrue(schedule.looksStale(null))
assertTrue(schedule.looksStale(now + 30_000))
assertFalse(schedule.looksStale(now + 20_500))
assertFalse(schedule.looksStale(now + 5_000))
}
@Test
fun journalAppendsAndDrains() {
schedule.appendJournal(5, "snooze", 10)
now += 1000
schedule.appendJournal(6, "dismiss", 0)
assertEquals("""[{"id":5,"action":"snooze","minutes":10,"at":${now - 1000}},{"id":6,"action":"dismiss","minutes":0,"at":$now}]""", schedule.drainJournal())
assertEquals("[]", schedule.drainJournal())
val log = schedule.readLog().lines()
assertEquals("${now - 1000} phone snooze 5 (10 min)", log[0])
assertEquals("$now phone dismiss 6", log[1])
}
@Test
fun logKeepsOnlyTheLastLines() {
for (index in 0 until AlarmSchedule.LOG_LINES + 5) schedule.log("line $index")
val lines = schedule.readLog().lines()
assertEquals(AlarmSchedule.LOG_LINES, lines.size)
assertEquals("$now line 5", lines.first())
assertEquals("$now line ${AlarmSchedule.LOG_LINES + 4}", lines.last())
}
@Test
fun glassesGateRequiresFreshWornConnectedNotCharging() {
var clock = 10_000L
val gate = GlassesStatusGate { clock }
assertFalse(gate.canCarryAlarm())
assertEquals("no glasses status", gate.description())
gate.set(connected = true, worn = true, charging = false)
assertTrue(gate.canCarryAlarm())
assertEquals("glasses worn", gate.description())
gate.set(connected = true, worn = false, charging = false)
assertEquals("glasses not worn", gate.description())
gate.set(connected = true, worn = true, charging = true)
assertEquals("glasses charging", gate.description())
gate.set(connected = false, worn = true, charging = false)
assertEquals("glasses not connected", gate.description())
gate.set(connected = true, worn = true, charging = false)
clock += GlassesStatusGate.MAX_AGE_MS + 1
assertFalse(gate.canCarryAlarm())
assertEquals("glasses status stale", gate.description())
assertEquals(0x7fffffff and (0x1_0000_0002L xor 1L).toInt(), AlarmSchedule.requestCode(0x1_0000_0002L))
assertNotNull(AlarmEntry.fromJson(Json.parse("""{"id":1,"at":2}""")))
}
}
@@ -0,0 +1,130 @@
package com.faceclaw.app
import kotlin.math.abs
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
class AudioHelpersTest {
@Test
fun cutPointLandsInQuietestWindowForFloatsAndShorts() {
// 1000 loud samples with a silent gap at 700..760; search the last 500 with a 40 window.
val floats = FloatArray(1000) { if (it in 700 until 760) 0f else 0.5f }
val cut = AudioSegmentation.quietestCutPoint(floats, floats.size, 500, 40)
assertTrue(cut in 700..760, "cut=$cut")
val shorts = ShortArray(1100) { if (it - 100 in 700 until 760) 0 else 16000 }
assertEquals(cut, AudioSegmentation.quietestCutPoint(shorts, 100, 1000, 500, 40))
// Too short to search: returns the length unchanged.
assertEquals(30, AudioSegmentation.quietestCutPoint(FloatArray(30) { 0.1f }, 30, 500, 40))
}
@Test
fun normalizePeakScalesUpButNeverDown() {
val quiet = floatArrayOf(0.01f, -0.02f, 0.005f)
AudioSegmentation.normalizePeak(quiet, 0.9f, 30f)
assertTrue(abs(quiet[1] + 0.6f) < 1e-6f, "capped at maxGain 30: ${quiet[1]}")
val loud = floatArrayOf(0.95f, -0.5f)
AudioSegmentation.normalizePeak(loud, 0.9f, 30f)
assertEquals(0.95f, loud[0])
val silent = FloatArray(4)
AudioSegmentation.normalizePeak(silent, 0.9f, 30f)
assertEquals(0f, AudioSegmentation.peakAmplitude(silent))
}
@Test
fun pcmConversionsRoundTrip() {
val samples = shortArrayOf(0, 1, -1, 32767, -32768, 256)
val le = AudioSegmentation.pcm16ToLittleEndian(samples, 0, samples.size)
assertEquals(listOf(0, 0, 1, 0, 0xff, 0xff, 0xff, 0x7f, 0, 0x80, 0, 1), le.map { it.toInt() and 0xff })
val floats = AudioSegmentation.pcm16leToFloat(le)
assertEquals(AudioSegmentation.pcm16ToFloat(samples, 0, samples.size).toList(), floats.toList())
assertEquals(32767 / 32768.0f, floats[3])
assertEquals(-1f, floats[4])
assertEquals(2.0, AudioSegmentation.rms(shortArrayOf(2, -2, 2, -2), 0, 4))
}
@Test
fun voicePrintMathAndBlobCodec() {
val v = floatArrayOf(3f, 4f)
VoicePrint.l2Normalize(v)
assertEquals(listOf(0.6f, 0.8f), v.toList())
assertTrue(abs(VoicePrint.dot(v, v) - 1.0) < 1e-6)
assertNull(VoicePrint.l2Normalize(null))
assertTrue(VoicePrint.isLongEnoughToEmbed(16000, 16000))
assertTrue(!VoicePrint.isLongEnoughToEmbed(15999, 16000))
val blob = SpeakerEmbeddings.floatsToBlob(floatArrayOf(1.5f, -2f))
assertEquals(listOf(0, 0, 0xc0, 0x3f, 0, 0, 0, 0xc0), blob.map { it.toInt() and 0xff })
assertEquals(listOf(1.5f, -2f), SpeakerEmbeddings.blobToFloats(blob)!!.toList())
assertNull(SpeakerEmbeddings.blobToFloats(byteArrayOf(1, 2, 3)))
assertNull(SpeakerEmbeddings.blobToFloats(null))
}
@Test
fun centroidUpdatesAndBlends() {
val first = SpeakerEmbeddings.runningMean(null, 0, floatArrayOf(1f, 0f), 5)
assertEquals(1, first.count)
assertEquals(listOf(1f, 0f), first.embedding.toList())
val second = SpeakerEmbeddings.runningMean(first.embedding, first.count, floatArrayOf(0f, 1f), 5)
assertEquals(2, second.count)
assertTrue(abs(second.embedding[0] - second.embedding[1]) < 1e-6f)
assertTrue(abs(VoicePrint.dot(second.embedding, second.embedding) - 1.0) < 1e-6)
// The cap limits the existing weight: count 100 capped at 1 behaves like an equal blend.
val capped = SpeakerEmbeddings.runningMean(floatArrayOf(1f, 0f), 100, floatArrayOf(0f, 1f), 1)
assertTrue(abs(capped.embedding[0] - capped.embedding[1]) < 1e-6f)
assertEquals(101, capped.count)
// Mismatched sizes restart the centroid.
assertEquals(1, SpeakerEmbeddings.runningMean(floatArrayOf(1f), 4, floatArrayOf(0f, 1f), 5).count)
val blended = SpeakerEmbeddings.blend(floatArrayOf(1f, 0f), 1, floatArrayOf(0f, 1f), 3)
assertNotNull(blended)
assertTrue(blended[1] > blended[0])
assertEquals(listOf(0f, 1f), SpeakerEmbeddings.blend(null, 0, floatArrayOf(0f, 1f), 3)!!.toList())
assertEquals(listOf(1f, 0f), SpeakerEmbeddings.blend(floatArrayOf(1f, 0f), 2, null, 0)!!.toList())
assertNull(SpeakerEmbeddings.blend(null, 0, null, 0))
assertTrue(SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights = false, sourceHasInsights = true))
assertTrue(!SpeakerEmbeddings.shouldAdoptInsights(targetHasInsights = true, sourceHasInsights = true))
}
private fun wav(dataSize: Int, extraChunk: Boolean = false, channels: Int = 1, bits: Int = 16, format: Int = 1): ByteArray {
val out = ByteSink()
fun u16(v: Int) { out.write(v); out.write(v ushr 8) }
fun u32(v: Int) { u16(v); u16(v ushr 16) }
out.write("RIFF".encodeToByteArray()); u32(0); out.write("WAVE".encodeToByteArray())
if (extraChunk) { out.write("LIST".encodeToByteArray()); u32(4); u32(0) }
out.write("fmt ".encodeToByteArray()); u32(18)
u16(format); u16(channels); u32(16000); u32(16000 * channels * 2); u16(channels * 2); u16(bits); u16(0)
out.write("data".encodeToByteArray()); u32(dataSize)
return out.toByteArray()
}
@Test
fun wavHeaderParsesChunksAndRecoversZeroSize() {
val header = wav(4000)
val info = WavPcmReader.parseHeader(header, header.size + 4000L)
assertEquals(16000, info.sampleRate)
assertEquals(1, info.channels)
assertEquals(16, info.bitsPerSample)
assertEquals(header.size.toLong(), info.dataOffset)
assertEquals(4000L, info.dataBytes)
val withList = wav(100, extraChunk = true)
assertEquals(withList.size.toLong(), WavPcmReader.parseHeader(withList, withList.size + 100L).dataOffset)
val unpatched = wav(0)
assertEquals(1234L, WavPcmReader.parseHeader(unpatched, unpatched.size + 1234L).dataBytes)
assertFailsWith<IllegalArgumentException> { WavPcmReader.parseHeader("RIFX".encodeToByteArray() + ByteArray(20), 24) }
assertFailsWith<IllegalArgumentException> { WavPcmReader.parseHeader(wav(10, bits = 8), 100) }
assertFailsWith<IllegalArgumentException> { WavPcmReader.parseHeader(wav(10, format = 3), 100) }
assertFailsWith<IllegalArgumentException> { WavPcmReader.parseHeader(wav(10).copyOf(20), 100) }
// The shared header matches the shared writer.
val written = BinaryEncoding.wavHeader(640, 16000, 1)
val parsed = WavPcmReader.parseHeader(written, 44 + 640L)
assertEquals(44L, parsed.dataOffset)
assertEquals(640L, parsed.dataBytes)
}
}
@@ -0,0 +1,127 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNotNull
import kotlin.test.assertNull
import kotlin.test.assertTrue
class CaptionEngineCoreTest {
private class Sink : CaptionSegmentSink {
val starts = mutableListOf<Long>()
val utterances = mutableListOf<List<Any>>()
override fun onSpeechStart(startMs: Long) { starts.add(startMs) }
override fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) {
utterances.add(listOf(length, startMs, endMs, samples[0].toInt()))
}
}
private fun chunk(amplitude: Int, samples: Int = 800): ShortArray =
ShortArray(samples) { if (it % 2 == 0) amplitude.toShort() else (-amplitude).toShort() }
@Test
fun segmenterUsesPreRollOnsetReleaseAndSilence() {
val sink = Sink()
val segmenter = CaptionSegmenter(sink)
repeat(10) { segmenter.processChunk(chunk(100), 800) } // 500 ms idle, noise floor ~100 -> threshold 220
repeat(10) { segmenter.processChunk(chunk(3000), 800) } // 500 ms speech
assertEquals(listOf(100L), sink.starts) // 500 ms - 400 ms pre-roll
repeat(15) { segmenter.processChunk(chunk(100), 800) } // 750 ms < 800 ms silence: still open
assertTrue(sink.utterances.isEmpty())
segmenter.processChunk(chunk(100), 800) // 800 ms of silence closes it
assertEquals(1, sink.utterances.size)
val (length, startMs, endMs, first) = sink.utterances[0]
assertEquals(100L, startMs)
assertEquals(6400 + 10 * 800 + 16 * 800, length) // pre-roll + speech + trailing silence
assertEquals(startMs as Long + (length as Int) * 1000L / 16000, endMs)
assertEquals(100, first) // the pre-roll audio leads the utterance
assertTrue(segmenter.isInUtterance.not())
}
@Test
fun segmenterDropsTooShortUtterancesAndCapsLength() {
val sink = Sink()
val segmenter = CaptionSegmenter(sink)
segmenter.silenceMs = 100 // clamps to 200
segmenter.processChunk(chunk(10), 800) // seeds the noise floor; becomes the 50 ms pre-roll
segmenter.processChunk(chunk(3000), 800) // onset
repeat(4) { segmenter.processChunk(chunk(0), 800) } // 200 ms silence
assertEquals(1, sink.starts.size)
assertTrue(sink.utterances.isEmpty()) // 300 ms < 350 ms minimum
repeat(400) { segmenter.processChunk(chunk(3000), 800) } // 20 s of speech: capped at 15 s
assertEquals(1, sink.utterances.size)
assertEquals(16000 * 15, sink.utterances[0][0])
segmenter.flush()
assertEquals(2, sink.utterances.size) // the remainder is flushed on stop
}
@Test
fun utteranceDecoderSplitsLongAudioAndEmbedsTenSeconds() {
val calls = mutableListOf<Int>()
val transcriber = object : OfflineTranscriber {
override fun recognize(samples: FloatArray, count: Int): String { calls.add(count); return if (calls.size == 2) ", b" else "a" }
override fun release() {}
}
var embedded = 0
val embedder = object : SpeakerEmbedder {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? { embedded = pcm16le.size; return floatArrayOf(1f) }
override fun release() {}
}
val decoder = UtteranceDecoder(transcriber, embedder)
val utterance = ShortArray(16000 * 12) { if (it % 2 == 0) 2000 else -2000 }
assertEquals("a, b", decoder.recognize(utterance, utterance.size))
assertEquals(2, calls.size)
assertTrue(calls[0] <= UtteranceDecoder.DECODE_SEGMENT_MAX_SAMPLES)
assertEquals(utterance.size, calls.sum())
assertNotNull(decoder.embed(utterance, utterance.size))
assertEquals(16000 * 10 * 2, embedded)
assertEquals("", UtteranceDecoder(null, null).recognize(utterance, utterance.size))
assertNull(UtteranceDecoder(null, null).embed(utterance, utterance.size))
}
@Test
fun captionSessionRunsQueueThroughSegmenterAndDecoder() {
val startLatch = Latch(1, testPlatform())
val utteranceLatch = Latch(1, testPlatform())
val events = mutableListOf<String>()
val listener = object : FaceclawCaptionEngineListener {
override fun onUtterance(text: String?, embedding: FloatArray?, startMs: Long, endMs: Long, peakRms: Double) {
events.add("utterance:$text:${embedding?.size}:$startMs"); utteranceLatch.countDown()
}
override fun onSpeechStart(startMs: Long) { events.add("start:$startMs"); startLatch.countDown() }
override fun onStatus(status: String?) { events.add("status:$status") }
}
val host = object : CaptionHost {
override val dispatcher = CallbackDispatcher { it() }
override fun loadTranscriber(modelDir: String): OfflineTranscriber? = if (modelDir == "models") object : OfflineTranscriber {
override fun recognize(samples: FloatArray, count: Int): String = "hi"
override fun release() {}
} else null
override fun loadEmbedder(modelPath: String): SpeakerEmbedder? = object : SpeakerEmbedder {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = floatArrayOf(0.5f, 0.5f)
override fun release() {}
}
}
val session = CaptionSession(host, testPlatform(), "caption-test") { m, e -> error("$m: $e") }
session.setListener(listener)
session.setAsrModelDir("models")
session.setSpeakerModelPath("speaker")
session.setSilenceMs(300)
session.acceptPcm(ByteArray(1600)) // ignored before start
session.start()
val loud = AudioSegmentation.pcm16ToLittleEndian(chunk(3000), 0, 800)
val quiet = AudioSegmentation.pcm16ToLittleEndian(chunk(0), 0, 800)
val idle = AudioSegmentation.pcm16ToLittleEndian(chunk(10), 0, 800)
repeat(2) { session.acceptPcm(idle) } // seeds the noise floor and fills 100 ms of pre-roll
repeat(10) { session.acceptPcm(loud) }
assertTrue(startLatch.await(5000))
repeat(8) { session.acceptPcm(quiet) }
assertTrue(utteranceLatch.await(5000))
session.stop()
session.stop() // idempotent
assertEquals("status:Loading caption model...", events[0])
assertEquals("status:Captions listening...", events[1])
assertEquals("start:0", events[2])
assertEquals("utterance:hi:2:0", events[3])
}
}
@@ -0,0 +1,47 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
class EvenHubAssetServerTest {
@Test
fun resolvesPathsWithIndexDefaultAndEscapeChecks() {
assertEquals("index.html", EvenHubAssetServer.resolve(null)!!.relativePath)
assertEquals("index.html", EvenHubAssetServer.resolve("")!!.relativePath)
assertEquals("index.html", EvenHubAssetServer.resolve("/")!!.relativePath)
assertEquals("assets/app.js", EvenHubAssetServer.resolve("/assets/app.js")!!.relativePath)
assertEquals("assets/app.js", EvenHubAssetServer.resolve("/assets/./x/../app.js")!!.relativePath)
assertNull(EvenHubAssetServer.resolve("/../secret"))
assertNull(EvenHubAssetServer.resolve("/a/../../secret"))
assertNull(EvenHubAssetServer.resolve("/.."))
assertTrue(EvenHubAssetServer.resolve("/index.html")!!.isHtml)
assertTrue(EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/x/index.html"))
assertTrue(EvenHubAssetServer.isInsideRoot("/apps/x/", "/apps/x/a/b"))
assertEquals(false, EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/xy/index.html"))
assertEquals(false, EvenHubAssetServer.isInsideRoot("/apps/x", "/apps/index.html"))
}
@Test
fun mimeTableCoversBothPlatformsTables() {
assertEquals("text/html", EvenHubAssetServer.mimeTypeFor("Index.HTM"))
assertEquals("text/javascript", EvenHubAssetServer.mimeTypeFor("bundle.mjs"))
assertEquals("application/wasm", EvenHubAssetServer.mimeTypeFor("core.wasm"))
assertEquals("font/woff2", EvenHubAssetServer.mimeTypeFor("a.woff2"))
assertEquals("image/svg+xml", EvenHubAssetServer.mimeTypeFor("icon.svg"))
assertEquals("text/plain", EvenHubAssetServer.mimeTypeFor("bundle.js.map"))
assertEquals("application/octet-stream", EvenHubAssetServer.mimeTypeFor("blob"))
assertEquals("application/octet-stream", EvenHubAssetServer.mimeTypeFor("weird.xyz"))
}
@Test
fun injectsShimAfterHeadOrAtStart() {
assertEquals(
"<!doctype html><HEAD lang=\"x\"><script>S</script><title>t</title></HEAD>",
EvenHubAssetServer.injectIntoHtml("<!doctype html><HEAD lang=\"x\"><title>t</title></HEAD>", "S"),
)
assertEquals("<script>S</script><body>x</body>", EvenHubAssetServer.injectIntoHtml("<body>x</body>", "S"))
assertEquals("<script>S</script><head", EvenHubAssetServer.injectIntoHtml("<head", "S"), "unterminated head tag")
}
}
@@ -0,0 +1,118 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
import kotlin.test.assertTrue
class FirmwareImageTest {
private fun putU32(buf: ByteArray, offset: Int, value: Long) {
for (i in 0 until 4) buf[offset + i] = ((value ushr (8 * i)) and 0xff).toByte()
}
private class Component(val name: String, val payload: ByteArray)
/** Builds a checksum-fixed container; [tamper] runs before the CRCs are recorded. */
private fun container(components: List<Component>, tamper: (ByteArray) -> Unit = {}): ByteArray {
val count = components.size
val toc = FirmwareImage.HEADER_SIZE + count * FirmwareImage.TOC_ENTRY_SIZE
var offset = toc
val offsets = components.map { c -> offset.also { offset += FirmwareImage.SUBHEADER_SIZE + c.payload.size } }
val img = ByteArray(offset)
putU32(img, 8, count.toLong())
components.forEachIndexed { i, c ->
val off = offsets[i]
putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 4, off.toLong())
putU32(img, off + 8, c.payload.size.toLong())
c.name.encodeToByteArray().copyInto(img, off + FirmwareImage.NAME_OFFSET)
c.payload.copyInto(img, off + FirmwareImage.SUBHEADER_SIZE)
}
tamper(img)
components.forEachIndexed { i, c ->
val off = offsets[i]
val crc = FirmwareImage.crc32cUnsigned(img, off + FirmwareImage.SUBHEADER_SIZE, c.payload.size)
putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 12, crc)
putU32(img, off + 12, crc)
}
return img
}
private fun mainApp(size: Int): ByteArray {
val payload = ByteArray(size) { (it * 7).toByte() }
putU32(payload, 0, size.toLong())
putU32(payload, 0x14, FirmwareImage.APP_LOAD_ADDR)
return payload
}
private fun validComponents(): List<Component> =
listOf(
Component("ota/bootloader.bin", ByteArray(300) { it.toByte() }),
Component("ota/a.bin", ByteArray(64) { 1 }),
Component("ota/b.bin", ByteArray(128) { 2 }),
Component("ota/c.bin", ByteArray(32) { 3 }),
Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(600)),
)
@Test
fun crc32cMatchesKnownVector() {
// Castagnoli polynomial, MSB-first, init 0, no reflection or final xor (g2flash.py
// crc32c_msb); vectors from an independent Python implementation of that definition.
assertEquals(0xc052a8c8L, FirmwareImage.crc32cUnsigned("123456789".encodeToByteArray()))
assertEquals(0xd7b91914L, FirmwareImage.crc32cUnsigned(ByteArray(256) { it.toByte() }))
assertEquals(0L, FirmwareImage.crc32cUnsigned(ByteArray(0)))
assertEquals(FirmwareImage.crc32cUnsigned("23456789".encodeToByteArray()), FirmwareImage.crc32cUnsigned("123456789".encodeToByteArray(), 1, 8))
}
@Test
fun validImageParsesAllComponents() {
val segs = FirmwareImage.validate(container(validComponents()))
assertEquals(5, segs.size)
assertEquals(listOf(300, 64, 128, 32, 600), segs.map { it.ps })
assertEquals(FirmwareImage.REQUIRED_SEGMENT, segs[4].name)
assertTrue(segs.zipWithNext().all { (a, b) -> a.end <= b.off })
}
@Test
fun rejectsStaleCrc() {
val img = container(validComponents())
img[img.size - 1] = (img[img.size - 1] + 1).toByte()
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.validate(img) }.message!!.contains("CRC32C"))
}
@Test
fun rejectsOverlappingComponents() {
val img = container(validComponents()) { buf ->
// Point component 1's TOC entry at component 0, so both claim the same bytes.
val off0 = FirmwareImage.readU32(buf, FirmwareImage.HEADER_SIZE + 4)
putU32(buf, FirmwareImage.HEADER_SIZE + 16 + 4, off0)
}
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.parseSegments(img) }.message!!.contains("overlap"))
}
@Test
fun rejectsTruncatedPayload() {
val img = container(validComponents())
val truncated = img.copyOf(img.size - 10)
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.parseSegments(truncated) }.message!!.contains("truncated"))
assertFailsWith<IllegalStateException> { FirmwareImage.parseSegments(ByteArray(0x10)) }
}
@Test
fun rejectsWrongComponentCountAndMissingMain() {
val four = validComponents().dropLast(1)
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.validate(container(four)) }.message!!.contains("components"))
val noMain = validComponents().dropLast(1) + Component("ota/other.bin", ByteArray(40))
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.validate(container(noMain)) }.message!!.contains("not found"))
}
@Test
fun rejectsMainAppBeyondMramCeiling() {
val ceiling = (FirmwareImage.APP_MAX_END - FirmwareImage.APP_LOAD_ADDR + FirmwareImage.APP_PREAMBLE).toInt()
val fits = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(ceiling))
FirmwareImage.validate(container(fits))
val tooBig = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(ceiling + 1))
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.validate(container(tooBig)) }.message!!.contains("MRAM"))
val badAddr = validComponents().dropLast(1) + Component(FirmwareImage.REQUIRED_SEGMENT, mainApp(600).also { putU32(it, 0x14, 0x1234L) })
assertTrue(assertFailsWith<IllegalStateException> { FirmwareImage.validate(container(badAddr)) }.message!!.contains("load address"))
}
}
@@ -0,0 +1,69 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNull
import kotlin.test.assertTrue
class FoundationTest {
@Test
fun jsonRoundTripsObjectsArraysAndNumbers() {
val text = """{"a":1,"b":[true,null,"x\ny"],"c":{"d":2.5,"e":-3}}"""
val value = Json.parseObject(text)
assertEquals(1, value["a"].asInt())
assertEquals(listOf(true, null, "x\ny"), value["b"].asArray())
assertEquals(2.5, value["c"].asObject()!!["d"].asDouble())
assertEquals(-3L, value["c"].asObject()!!["e"].asLong())
assertEquals(text, Json.write(value))
assertEquals("[1,2,\"q\\\"\"]", Json.write(listOf(1, 2L, "q\"")))
assertEquals("1", JsonNumber.of(1.0).text)
assertEquals("1.5", JsonNumber.of(1.5).text)
assertNull("x".asInt())
}
@Test
fun sha256MatchesKnownVector() {
assertEquals("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", sha256Hex("abc".encodeToByteArray()))
val incremental = Sha256Digest()
incremental.update("a".encodeToByteArray())
incremental.update("bc".encodeToByteArray())
assertEquals(sha256("abc".encodeToByteArray()).toList(), incremental.digest().toList())
}
@Test
fun latchAndQueueHandOffAcrossThreads() {
val platform = testPlatform()
val latch = Latch(1, platform)
val queue = BlockingQueue<String>(platform)
assertFalse(latch.await(20))
startThread("foundation-test", true) {
sleepMs(20)
queue.put("hello")
latch.countDown()
}
assertEquals("hello", queue.poll(2000))
assertTrue(latch.await(2000))
assertNull(queue.poll(10))
}
@Test
fun interruptibleSleepConsumesEarlyInterrupt() {
val platform = testPlatform()
val sleep = InterruptibleSleep(platform)
sleep.interrupt()
val start = platform.elapsedRealtimeMs()
assertFalse(sleep.sleep(500))
assertTrue(platform.elapsedRealtimeMs() - start < 400)
assertTrue(sleep.sleep(5))
startThread("foundation-wake", true) { sleepMs(30); sleep.interrupt() }
assertFalse(sleep.sleep(5000))
}
@Test
fun wallClockFormatsPattern() {
val text = formatLocalTime(0L, "yyyy")
assertTrue(text == "1970" || text == "1969", text)
assertTrue(currentTimeMillis() > 1_600_000_000_000L)
}
}
@@ -0,0 +1,139 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertTrue
class FrameTimingsCoreTest {
private class FakeClock : ProtocolPlatform by testPlatform() {
var nowMs = 1_000L
override fun elapsedRealtimeMs(): Long = nowMs
}
private fun core(clock: FakeClock, logs: MutableList<String> = ArrayList()) = FrameTimingsCore(
platform = clock,
wallClockMs = { 1_700_000_000_000L + clock.nowMs },
threadName = { "t" },
formatWallClock = { "wall@$it" },
logInfo = { logs.add("I $it") },
logWarn = { logs.add("W $it") },
)
@Test
fun exportRendersTreeAgainstRootClockByteForByte() {
val clock = FakeClock()
val logs = ArrayList<String>()
val core = core(clock, logs)
val root = core.startFrame("input:sys-event type=0")
clock.nowMs += 5
core.annotate(root, "fg=terminal")
core.log(root, "input event received on JS side")
clock.nowMs += 1
core.spanStart(root, "handle-input")
clock.nowMs += 10
val child = core.startFrame("render:shell", root)
clock.nowMs += 4
core.spanEnd(root, "handle-input")
core.spanEnd(root, "missing")
core.spanStart(child, "paint")
clock.nowMs += 30
core.finishFrame(child, "discarded: superseded")
clock.nowMs += 50
core.finishFrame(root, "sent")
core.finishFrame(root, "sent") // ignored: first finish wins
core.log(root, "late line is dropped")
clock.nowMs += 100
assertEquals(listOf(
"I frame#2 [render:shell] -> discarded: superseded in 34ms",
"I frame#1 [input:sys-event type=0 fg=terminal] -> sent in 100ms",
), logs)
assertEquals("frames started=2 sent=1 discarded=1 timeout=0 | input-to-display p50=100ms p90=100ms p99=100ms max=100ms", core.statsSummary())
val expected = """
|FrameTimings export at wall@1700000001200
|frames started=2 sent=1 discarded=1 timeout=0 open=0
|input-to-display latency (last 1): p50=100ms p90=100ms p99=100ms max=100ms
|
|Frames are trees: an input event's own frame is the root, and the renders it
|caused are indented under it with offsets measured from the root's start.
|
|=== slowest frames that reached the glasses ===
|frame#1 [input:sys-event type=0 fg=terminal] started wall@1700000001000 duration 100ms (tree 100ms) outcome sent
| +5ms [t] input event received on JS side
| +6ms [t] span handle-input start
| +16ms [t] spawned frame#2 (render:shell)
| +20ms [t] span handle-input end (14ms)
| +20ms [t] span missing end (start not recorded)
| +100ms [t] finished: sent
| frame#2 [render:shell] started +16ms duration 34ms outcome discarded: superseded
| +20ms [t] span paint start
| +50ms [t] span paint never ended (started at +4ms)
| +50ms [t] finished: discarded: superseded
|=== recent frames (oldest first) ===
|frame#1 [input:sys-event type=0 fg=terminal] started wall@1700000001000 duration 100ms (tree 100ms) outcome sent
| +5ms [t] input event received on JS side
| +6ms [t] span handle-input start
| +16ms [t] spawned frame#2 (render:shell)
| +20ms [t] span handle-input end (14ms)
| +20ms [t] span missing end (start not recorded)
| +100ms [t] finished: sent
| frame#2 [render:shell] started +16ms duration 34ms outcome discarded: superseded
| +20ms [t] span paint start
| +50ms [t] span paint never ended (started at +4ms)
| +50ms [t] finished: discarded: superseded
|""".trimMargin()
assertTrue(core.isExportDirty)
assertEquals(expected, core.takeExport())
assertFalse(core.isExportDirty)
assertEquals(null, core.takeExport())
assertEquals(expected, core.buildExport())
}
@Test
fun sweepFinishesStaleFramesAsTimeoutsAndIgnoresZeroAndUnknownIds() {
val clock = FakeClock()
val logs = ArrayList<String>()
val core = core(clock, logs)
val stale = core.startFrame("render:app")
clock.nowMs += FrameTimingsCore.FRAME_TIMEOUT_MS - 1
val fresh = core.startFrame("render:late")
assertEquals(0, core.sweepTimedOut())
clock.nowMs += 1
assertEquals(1, core.sweepTimedOut())
assertEquals(listOf("W frame#$stale [render:app] timed out after 30000ms without being finished"), logs)
core.log(0, "ignored")
core.log(999, "ignored")
core.spanStart(0, "x")
core.finishFrame(0, "sent")
core.finishFrame(fresh, "sent")
assertEquals("frames started=2 sent=1 discarded=0 timeout=1", core.statsSummary())
val export = core.buildExport()
assertTrue(export.contains("frame#$stale [render:app] started wall@"))
assertTrue(export.contains("outcome timeout: never reported finished\n"))
assertTrue(export.contains("render-to-display latency (last 1): p50=1ms p90=1ms p99=1ms max=1ms"))
assertTrue(export.contains("open=0"))
}
@Test
fun lineCapAndNegativeOffsetsFormatLikeJava() {
val clock = FakeClock()
val core = core(clock)
val root = core.startFrame("input:x")
clock.nowMs -= 12 // a line stamped before the root's start still formats with a sign
core.log(root, "early")
clock.nowMs += 12
repeat(250) { core.log(root, "line $it") }
core.finishFrame(root, "sent")
val export = core.buildExport()
assertTrue(export.contains("\n -12ms [t] early\n"), export)
// Printed once under "slowest" and once under "recent".
assertEquals(2, Regex("\\.\\.\\. line cap reached").findAll(export).count())
assertTrue(export.contains("line 198"))
assertFalse(export.contains("line 199"))
// Like the Java original, lines past the cap (including the finish line) are dropped.
assertFalse(export.contains("finished: sent"))
assertTrue(export.contains("outcome sent\n"))
}
}
@@ -0,0 +1,421 @@
package com.faceclaw.app
import kotlin.concurrent.Volatile
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
private const val RIGHT = "AA:00:00:00:00:01"
private const val LEFT = "AA:00:00:00:00:02"
/** The test platform with a clock the tests can jump forward (the real waits keep real time). */
private class ClockPlatform(private val base: ProtocolPlatform) : ProtocolPlatform by base {
@Volatile var offsetMs = 0L
override fun elapsedRealtimeMs(): Long = base.elapsedRealtimeMs() + offsetMs
}
private fun waitUntil(timeoutMs: Long, condition: () -> Boolean): Boolean {
val platform = testPlatform()
val deadline = platform.elapsedRealtimeMs() + timeoutMs
while (platform.elapsedRealtimeMs() < deadline) {
if (condition()) return true
sleepMs(10)
}
return condition()
}
private fun ByteArray.containsSequence(needle: ByteArray): Boolean {
if (needle.isEmpty() || size < needle.size) return false
outer@ for (start in 0..size - needle.size) {
for (i in needle.indices) if (this[start + i] != needle[i]) continue@outer
return true
}
return false
}
/** One logical write as the fake link saw it: the frames, the decoded pb, and the in-flight message (if any). */
private class Write(
val address: String,
val sid: Int,
val frames: List<ByteArray>,
val pb: ByteArray,
val kind: String?,
val label: String?,
val magic: Int,
val message: ByteArray?,
)
/**
* A GATT link that records every write and, unless a kind is muted, answers it the way the
* firmware would: EvenHub/settings/auth requests get a matching ack frame, CFW stream
* messages get one ACK per lens.
*/
private class FakeLink(private val platform: ProtocolPlatform) : SessionLink {
lateinit var core: GlassesSessionCore
private val lock = platform.createLock()
private val writesList = ArrayList<Write>()
val muted = HashSet<String>()
val connected = HashSet<String>()
val disconnected = ArrayList<String>()
@Volatile var closed = false
var seq = 0
val writes: List<Write>
get() = lock.withLock { writesList.toList() }
/** Decoded CFW image messages (the drive loop's debug-flags control rides the same stream and is excluded). */
fun cfwMessages(): List<ByteArray> = writes.filter { it.sid == CfwTransport.SID && it.kind == "image" }.mapNotNull { it.message }
override fun connect(address: String, timeoutMs: Int): Boolean {
connected.add(address)
return true
}
override fun requestHighPriority(address: String) {}
override fun requestMtu(address: String, mtu: Int, timeoutMs: Int): Boolean = true
override fun negotiatedMtu(address: String): Int = 247
override fun discoverServices(address: String, timeoutMs: Int): Boolean = true
override fun enableNotifications(address: String, characteristicUuid: String, enable: Boolean, timeoutMs: Int): Boolean = true
override fun writeFrames(address: String, characteristicUuid: String, frames: List<ByteArray>, mode: GattWriteMode, timeoutMs: Int): Boolean {
val sid = frames[0][6].toInt() and 255
val magic = if (sid == CfwTransport.SID) frames[0][2].toInt() and 255 else BleProtocol.parseFrame(frames[0]).msgSeq
// The in-flight entry gives the label/kind and the decoded CFW message.
val inFlight = core.monitor.withLock { core.inFlightMessages.firstOrNull { it.sid == sid && it.magic == magic && magic != 0 } }
val pb = if (sid == CfwTransport.SID) ByteArray(0) else {
val joined = frames.flatMap { frame -> frame.slice(8 until 8 + (frame[3].toInt() and 255)) }.toByteArray()
joined.copyOf(maxOf(0, joined.size - 2))
}
val write = Write(address, sid, frames, pb, inFlight?.kind, inFlight?.label, magic, inFlight?.message?.copyOf())
lock.withLock { writesList.add(write) }
if (inFlight != null && inFlight.kind !in muted) {
if (sid == CfwTransport.SID) {
for (lens in 1..2) core.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, cfwAck(magic, lens, inFlight.message.size, inFlight.cfwChecksum))
} else {
core.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, ackFrame(sid, magic, pb))
}
}
return true
}
private fun ackFrame(sid: Int, magic: Int, request: ByteArray): ByteArray {
val body = if (sid == BleProtocol.SID_SECURITY_AUTH)
BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + byteArrayOf(0x1a, 0x00)
else
BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, magic)
return BleProtocol.framePb(body, sid, 0x00, seq++)[0]
}
override fun disconnect(address: String) {
disconnected.add(address)
connected.remove(address)
}
override fun close() {
closed = true
}
override fun isBonded(address: String): Boolean = true
override fun prepareBenchmarkLink(address: String, mode: Int) {}
override fun recordDisplayFrameSent() {}
}
private fun cfwAck(streamId: Int, lens: Int, size: Int, checksum: Int): ByteArray {
val packet = ByteArray(19)
packet[0] = 0xaa.toByte()
packet[1] = 0x12
packet[3] = 11
packet[4] = 1
packet[5] = 1
packet[6] = CfwTransport.SID.toByte()
packet[8] = 1
packet[9] = streamId.toByte()
packet[12] = lens.toByte()
packet[13] = size.toByte()
packet[14] = (size ushr 8).toByte()
packet[15] = checksum.toByte()
packet[16] = (checksum ushr 8).toByte()
val crc = CfwTransport.crc(packet.copyOfRange(8, 17))
packet[17] = crc.toByte()
packet[18] = (crc ushr 8).toByte()
return packet
}
private class FakeHost(private val platform: ProtocolPlatform) : SessionHost {
private val lock = platform.createLock()
private val lines = ArrayList<String>()
private var done: Latch? = null
var wakeLock = false
fun logs(): List<String> = lock.withLock { lines.toList() }
fun hasLog(fragment: String): Boolean = logs().any { it.contains(fragment) }
override fun postToMain(action: () -> Unit) = action()
override fun currentThreadDispatcher(): SessionDispatcher = SessionDispatcher { it() }
override fun isPhoneLocked(): Boolean = false
override fun setScreenWakeLock(on: Boolean) {
wakeLock = on
}
override fun isEvenAppActive(): Boolean = false
override fun startWorker(body: () -> Unit) {
val latch = Latch(1, platform)
done = latch
startThread("test-session", true) {
try {
body()
} finally {
latch.countDown()
}
}
}
override fun joinWorker(timeoutMs: Long) {
done?.await(timeoutMs)
}
override fun log(level: SessionLogLevel, tag: String, message: String, error: Throwable?) {
lock.withLock { lines.add("$level $tag: $message") }
}
}
private class FakeListener : FaceclawBleCommunicatorListener {
val phases = ArrayList<String>()
val events = ArrayList<String>()
val finished = ArrayList<String>()
val batteries = ArrayList<String>()
override fun onStateChange(phase: String?, status: String?) {
phases.add(phase ?: "")
}
override fun onRingEvent(kind: String?, containerName: String?, eventType: Int, eventSource: Int, systemExitReasonCode: Int, frameId: Int, ringTick: Long, ringType: Int, ringAux: Int, ringSpeed: Int) {
events.add("$kind:$eventType")
}
override fun onBatteryState(headsetBattery: Int, headsetCharging: Int, ringBattery: Int, ringCharging: Int) {
batteries.add("$headsetBattery/$headsetCharging/$ringBattery/$ringCharging")
}
override fun onSilentMode(silent: Boolean) {}
override fun onWearState(wearing: Boolean) {}
override fun onPhoneLockState(locked: Boolean) {}
override fun onEvenAppConflict(message: String?) {}
override fun onFrameMetrics(paintMs: Int, transmitMs: Int, tileCount: Int) {}
override fun onFrameFinished(frameId: Int, outcome: String?) {
finished.add("$frameId:$outcome")
}
override fun onFirmwareInfo(leftVersion: String?, rightVersion: String?, extension: String?) {}
}
private class Session {
val platform = ClockPlatform(testPlatform())
val link = FakeLink(platform)
val host = FakeHost(platform)
val listener = FakeListener()
val frameTimings = FrameTimingsCore(platform)
val core = GlassesSessionCore(link, host, frameTimings, platform, RIGHT, LEFT, "")
init {
link.core = core
core.setListener(listener)
}
fun layoutCreated(): Boolean = core.monitor.withLock { core.fixedLayoutCreated }
fun startAndAwaitLayout() {
assertTrue(core.start())
assertTrue(waitUntil(5_000) { layoutCreated() }, "layout never created; logs=${host.logs().takeLast(20)}")
}
/** A 64x32 surface frame with a marker byte at [marker]; returns the expected composite fingerprint. */
fun submitFrame(marker: Int, value: Byte, frameId: Int): Unit {
val pixels = ByteArray(64 * 32)
pixels[marker] = value
core.submitSurfaceFrame(ArrayByteReader(pixels), "s", 0, 0, 64, 32, "content-$marker-$value", 0, frameId, null)
}
fun displayedMatchesDesired(): Boolean = core.monitor.withLock {
core.desiredTilesLock.locked { core.desiredFingerprint.isNotEmpty() && core.desiredFingerprint == core.displayedFingerprint }
}
}
class GlassesSessionCoreTest {
@Test
fun connectsAuthenticatesPreludesAndCreatesTheLayout() {
val s = Session()
s.startAndAwaitLayout()
val writes = s.link.writes
assertEquals(setOf(RIGHT, LEFT), writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.address }.toSet())
assertTrue(s.host.hasLog("security auth R=ok L=ok"), s.host.logs().toString())
assertEquals(1, writes.count { it.kind == "prelude" })
// The framebuffer lease is acquired on both arms (fire-and-forget settings writes, magic 0).
val fbAcquire = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte())
assertEquals(setOf(RIGHT, LEFT), writes.filter { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(fbAcquire) }.map { it.address }.toSet())
assertEquals(1, writes.count { it.kind == "battery" }, "settings query for firmware info")
assertEquals(1, writes.count { it.kind == "create-layout" })
assertTrue(s.core.isSessionReady())
assertEquals(listOf("disconnected", "connecting", "connected"), s.listener.phases.distinct())
s.core.disconnect()
assertFalse(s.core.isSessionReady())
assertEquals("disconnected", s.listener.phases.last())
assertTrue(s.listener.phases.contains("disconnecting"))
assertTrue(s.link.closed)
assertEquals(listOf(RIGHT, LEFT), s.link.disconnected.takeLast(2))
assertFalse(s.host.wakeLock)
assertTrue(s.host.hasLog("communicator stop"))
}
@Test
fun imagePipelineSendsAKeyframeThenABoundingBoxDelta() {
val s = Session()
s.startAndAwaitLayout()
s.core.configureCompositorScreen(64, 32)
s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE)
s.submitFrame(100, 0x80.toByte(), 1)
assertTrue(waitUntil(5_000) { s.link.cfwMessages().size >= 1 && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString())
assertTrue(s.listener.finished.contains("1:sent"), s.listener.finished.toString())
s.submitFrame(101, 0x80.toByte(), 2)
assertTrue(waitUntil(5_000) { s.link.cfwMessages().size >= 2 && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString())
val messages = s.link.cfwMessages()
assertEquals(CFW_MSG_FULL_FRAME, messages[0][0].toInt() and CFW_MSG_TYPE_MASK, "first frame is a keyframe")
assertEquals(CFW_MSG_BOUNDING_BOX, messages[1][0].toInt() and CFW_MSG_TYPE_MASK, "second frame is a delta")
assertTrue(s.listener.finished.contains("2:sent"))
// Identical content again: nothing is sent and the frame is finished as a no-change.
s.submitFrame(101, 0x80.toByte(), 3)
assertTrue(waitUntil(2_000) { s.listener.finished.any { it.startsWith("3:") } })
assertTrue(s.listener.finished.any { it == "3:discarded: no change from displayed image" }, s.listener.finished.toString())
assertEquals(2, s.link.cfwMessages().size)
s.core.disconnect()
}
@Test
fun heartbeatAndLeaseRenewalFollowTheClock() {
val s = Session()
s.startAndAwaitLayout()
// The first heartbeat follows the layout at once (nothing has been acked yet); the
// next one waits for HEARTBEAT_READY_MS since that ack.
assertTrue(waitUntil(3_000) { s.link.writes.count { it.kind == "heartbeat" } == 1 }, s.host.logs().takeLast(10).toString())
assertTrue(s.host.hasLog("Got ACK for heartbeat"))
sleepMs(300)
assertEquals(1, s.link.writes.count { it.kind == "heartbeat" })
s.platform.offsetMs += ConnectionOptions.HEARTBEAT_READY_MS + 100L
s.core.interruptibleSleep.interrupt()
assertTrue(waitUntil(3_000) { s.link.writes.count { it.kind == "heartbeat" } == 2 }, s.host.logs().takeLast(10).toString())
val leaseWritesBefore = s.link.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte())) }
s.platform.offsetMs += GlassesSessionCore.FACECLAW_WAKE_LEASE_RENEW_MS + 100L
s.core.interruptibleSleep.interrupt()
assertTrue(waitUntil(3_000) {
s.link.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.pb.containsSequence(byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_FB_OP_ACQUIRE.toByte())) } == leaseWritesBefore + 2
}, "framebuffer lease renewed on both arms")
s.core.disconnect()
}
@Test
fun deferredWakeIsClaimedAndReadyIsSentAfterResume() {
val s = Session()
s.startAndAwaitLayout()
s.core.configureCompositorScreen(64, 32)
s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE)
s.submitFrame(7, 0x40, 1)
assertTrue(waitUntil(5_000) { s.displayedMatchesDesired() })
assertTrue(s.core.suspendEvenHubSession())
assertTrue(s.core.monitor.withLock { s.core.shutdownRequested })
assertEquals(1, s.link.writes.count { it.kind == "shutdown" })
// CFW deferred double-tap wake on the right arm: field 102 [F,C,ver,event=1,nonce LE16].
val nonce = 0x1234
val event = byteArrayOf(70, 67, 1, 1, (nonce and 0xff).toByte(), (nonce ushr 8).toByte())
val pb = BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, 0) + BleProtocol.encodeBytesField(BleProtocol.FACECLAW_WAKE_EVENT_FIELD, event)
s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, BleProtocol.framePb(pb, BleProtocol.SID_UI_SETTING, BleProtocol.FLAG_NOTIFY, 9)[0])
assertTrue(s.listener.events.contains("display-wake:${BleProtocol.EVENT_DOUBLE_CLICK}"), s.listener.events.toString())
val claim = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_WAKE_OP_CLAIM.toByte(), (nonce and 0xff).toByte(), (nonce ushr 8).toByte())
assertTrue(waitUntil(3_000) { s.link.writes.count { it.pb.containsSequence(claim) } == 2 }, "CLAIM written to both arms")
assertEquals(RIGHT, s.link.writes.first { it.pb.containsSequence(claim) }.address, "right arm first")
assertTrue(s.core.resumeEvenHubSession())
assertEquals(2, s.link.writes.count { it.kind == "prelude" })
assertTrue(waitUntil(5_000) { s.layoutCreated() && s.displayedMatchesDesired() }, s.host.logs().takeLast(20).toString())
assertTrue(s.core.awaitEvenHubSessionReady(5_000))
val ready = byteArrayOf(70, 67, 1, BleProtocol.FACECLAW_WAKE_OP_READY.toByte(), (nonce and 0xff).toByte(), (nonce ushr 8).toByte())
assertEquals(2, s.link.writes.count { it.pb.containsSequence(ready) }, "READY written to both arms")
s.core.disconnect()
}
@Test
fun unexpectedAndDuplicateAcksAreLoggedNotApplied() {
val s = Session()
s.startAndAwaitLayout()
val unknown = BleProtocol.framePb(BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, 150), BleProtocol.SID_EVENHUB, 0, 5)[0]
s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, unknown)
assertTrue(s.host.hasLog("unexpected ACK sid=${BleProtocol.SID_EVENHUB} magic=150"), s.host.logs().takeLast(5).toString())
val layout = s.link.writes.first { it.kind == "create-layout" }
val duplicate = BleProtocol.framePb(BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, layout.magic), layout.sid, 0, 6)[0]
s.core.onNotification(RIGHT, BleProtocol.NOTIFY_CHAR_UUID, duplicate)
assertTrue(s.host.hasLog("duplicate ACK for already-acked message sid=${layout.sid} magic=${layout.magic}"), s.host.logs().takeLast(5).toString())
assertTrue(s.layoutCreated())
s.core.disconnect()
}
@Test
fun ackTimeoutOnTheLayoutReconnects() {
val s = Session()
s.link.muted.add("create-layout")
assertTrue(s.core.start())
assertTrue(waitUntil(5_000) { s.link.writes.any { it.kind == "create-layout" } })
assertFalse(s.layoutCreated())
s.platform.offsetMs += ConnectionOptions.ACK_TIMEOUT_MS + ConnectionOptions.WRITE_TIMEOUT_MS + 1_000L
s.core.interruptibleSleep.interrupt()
assertTrue(waitUntil(3_000) { s.listener.phases.contains("retrying") }, s.host.logs().takeLast(10).toString())
assertTrue(s.host.hasLog("message timed out: create-layout"))
assertTrue(s.host.hasLog("Transport failure: ack timeout"))
assertTrue(s.link.disconnected.containsAll(listOf(RIGHT, LEFT)))
s.core.disconnect()
assertEquals("disconnected", s.listener.phases.last())
}
@Test
fun cfwNackTriggersReplayWithAFreshStreamId() {
val s = Session()
s.link.muted.add("image")
s.startAndAwaitLayout()
s.core.configureCompositorScreen(64, 32)
s.core.configureSurface("s", 0, 0, 64, 32, 0, SurfaceCompositor.TRANSPARENCY_OPAQUE)
s.submitFrame(3, 0x20, 1)
assertTrue(waitUntil(5_000) { s.link.cfwMessages().size == 1 })
val first = s.link.writes.first { it.sid == CfwTransport.SID && it.kind == "image" }
val message = assertNotNull(s.core.monitor.withLock { s.core.inFlightMessages.firstOrNull { it.magic == first.magic } })
// A NACK from lens 1 marks the message for replay; the window re-sends it under a new id.
val nack = cfwAck(first.magic, 1, 0, 0).also { it[8] = 3; val crc = CfwTransport.crc(it.copyOfRange(8, 17)); it[17] = crc.toByte(); it[18] = (crc ushr 8).toByte() }
s.core.onNotification(LEFT, BleProtocol.NOTIFY_CHAR_UUID, nack)
assertTrue(waitUntil(3_000) { s.link.cfwMessages().size == 2 }, s.host.logs().takeLast(10).toString())
assertTrue(s.host.hasLog("CFW replay id=${first.magic}"))
val second = s.link.writes.filter { it.sid == CfwTransport.SID && it.kind == "image" }[1]
assertTrue(second.magic != first.magic)
assertEquals(1, message.cfwRetries)
assertContentEquals(first.message, second.message)
s.core.disconnect()
}
}
@@ -0,0 +1,128 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
class GlyphAndIconSourceTest {
private fun u16(bytes: ByteArray, at: Int) = (bytes[at].toInt() and 0xff) or ((bytes[at + 1].toInt() and 0xff) shl 8)
private fun s16(bytes: ByteArray, at: Int) = u16(bytes, at).toShort().toInt()
// 6x4 raster: ink in columns 2..4, rows 1..2, one 128 edge pixel.
private val raster = byteArrayOf(
0, 0, 0, 0, 0, 0,
0, 0, -1, -1, 0, 0,
0, 0, 0, -1, 0x7f, 0,
0, 0, 0, 0, 0, 0,
)
@Test
fun gammaLutIsIdentityAtOneAndDarkensAboveOne() {
val linear = GlyphPacket.gammaLut(1.0)
for (i in 0 until 256) assertEquals(i, linear[i].toInt() and 0xff)
assertContentEquals(linear, GlyphPacket.gammaLut(0.0))
val squared = GlyphPacket.gammaLut(2.0)
assertEquals(64, squared[128].toInt() and 0xff)
assertEquals(255, squared[255].toInt() and 0xff)
}
@Test
fun inkScansFindPaintedBoxes() {
assertContentEquals(intArrayOf(2, 4), GlyphPacket.paintedColumnRange(raster, 6, 4))
assertContentEquals(intArrayOf(2, 1, 4, 2), GlyphPacket.inkBounds(raster, 6, 4))
assertNull(GlyphPacket.paintedColumnRange(ByteArray(12), 4, 3))
assertNull(GlyphPacket.inkBounds(ByteArray(12), 4, 3))
}
@Test
fun imagePacketsCarryLittleEndianHeaderAndGammaMappedRegion() {
val whole = GlyphPacket.packImage(raster, 6, 4, 1.0)
assertEquals(4 + 24, whole.size)
assertEquals(6, u16(whole, 0))
assertEquals(4, u16(whole, 2))
assertContentEquals(raster, whole.copyOfRange(4, whole.size))
val region = GlyphPacket.packImageRegion(raster, 6, 2, 1, 3, 2, 2.0)
assertEquals(3, u16(region, 0))
assertEquals(2, u16(region, 2))
assertContentEquals(byteArrayOf(-1, -1, 0, 0, -1, 63), region.copyOfRange(4, region.size))
}
@Test
fun glyphCellPacketsCropToInkAndEncodeSignedOffsets() {
val cell = GlyphPacket.packGlyphCell(GlyphPacket.advanceFixed(5.5), raster, 6, 4, penX = 3, baselineY = 3, ascent = 10, gamma = 1.0)
assertEquals(352, u16(cell, 0))
assertEquals(-1, s16(cell, 2)) // ink starts one pixel left of the pen
assertEquals(8, s16(cell, 4)) // ascent 10, ink top two rows above the baseline
assertEquals(3, u16(cell, 6))
assertEquals(2, u16(cell, 8))
assertContentEquals(byteArrayOf(-1, -1, 0, 0, -1, 0x7f), cell.copyOfRange(10, cell.size))
val empty = GlyphPacket.packGlyphCell(64, ByteArray(12), 4, 3, 0, 0, 10, 1.0)
assertContentEquals(byteArrayOf(64, 0, 0, 0, 0, 0, 0, 0, 0, 0), empty)
assertEquals(0xffff, GlyphPacket.advanceFixed(5000.0))
}
private fun nameTable(vararg records: Triple<Int, Int, String>): ByteArray {
// records: (platformId, nameId, value); platform 1 stored Latin-1, others UTF-16BE
val strings = ArrayList<ByteArray>()
for ((platform, _, value) in records) {
strings.add(if (platform == 1) ByteArray(value.length) { value[it].code.toByte() }
else ByteArray(value.length * 2) { if (it % 2 == 0) (value[it / 2].code shr 8).toByte() else value[it / 2].code.toByte() })
}
val out = ByteSink()
fun u16(v: Int) { out.write(v ushr 8); out.write(v) }
fun u32(v: Int) { u16(v ushr 16); u16(v and 0xffff) }
val nameOffset = 12 + 16
val nameLength = 6 + records.size * 12 + strings.sumOf { it.size }
u32(0x00010000); u16(1); u16(0); u16(0); u16(0) // offset table, 1 table
u32(0x6e616d65); u32(0); u32(nameOffset); u32(nameLength) // 'name' record
u16(0); u16(records.size); u16(6 + records.size * 12)
var offset = 0
for ((i, record) in records.withIndex()) {
u16(record.first); u16(0); u16(0); u16(record.second); u16(strings[i].size); u16(offset)
offset += strings[i].size
}
for (s in strings) out.write(s)
return out.toByteArray()
}
@Test
fun openTypeNamesPreferTypographicFamilyAndStyle() {
assertEquals("Roboto\nLight", OpenTypeNames.parse(nameTable(Triple(3, 1, "Roboto Light"), Triple(3, 2, "Regular"), Triple(3, 16, "Roboto"), Triple(1, 17, "Light"))))
assertEquals("Mono\n", OpenTypeNames.parse(nameTable(Triple(1, 1, " Mono "))))
assertEquals("", OpenTypeNames.parse(nameTable(Triple(3, 2, "Bold"))))
assertEquals("", OpenTypeNames.parse(ByteArray(8)))
}
@Test
fun svgIconsParseShapesStrokeRuleAndPlacement() {
val svg = """<svg viewBox="0 0 24 24" fill="none" stroke="currentColor"><rect x="3" y="3" width="18" height="18" rx="2"/>
<circle cx="12" cy="12" r="0"/><circle cx="12" cy="12" r="3"/><path d="M4 4h16"/><polygon points="1,2 3 4 5"/><line x1="0" y1="0" x2="1" y2="1"/></svg>"""
val icon = SvgIconSource.parse(svg)!!
assertTrue(icon.stroked)
assertEquals(listOf("Rect", "Circle", "PathData", "Polyline", "Line"), icon.shapes.map { it::class.simpleName })
val rect = icon.shapes[0] as SvgShape.Rect
assertEquals(2f, rect.rx); assertEquals(2f, rect.ry)
assertEquals(3f, (icon.shapes[1] as SvgShape.Circle).r)
assertEquals("M4 4h16", (icon.shapes[2] as SvgShape.PathData).d)
val polygon = icon.shapes[3] as SvgShape.Polyline
assertContentEquals(floatArrayOf(1f, 2f, 3f, 4f), polygon.points)
assertTrue(polygon.closed)
assertContentEquals(floatArrayOf(0f, 0f, 2f), icon.placement(48))
val filled = SvgIconSource.parse("""<svg viewBox="-2 0 40 20"><rect width="4" height="2"/></svg>""")!!
assertTrue(!filled.stroked)
assertContentEquals(floatArrayOf(0f, 5f, 0.5f), filled.placement(20))
assertEquals(-2f, filled.minX)
assertNull(SvgIconSource.parse("""<svg><circle r="0"/><rect width="0" height="3"/></svg>"""))
}
@Test
fun coverageQuantizesToThreeLevels() {
assertEquals(0, SvgIconSource.quantizeCoverage(63))
assertEquals(128, SvgIconSource.quantizeCoverage(64))
assertEquals(128, SvgIconSource.quantizeCoverage(191))
assertEquals(255, SvgIconSource.quantizeCoverage(192))
assertContentEquals(byteArrayOf(0, -128, -1), SvgIconSource.quantizeArgb(intArrayOf(0x10ffffff, 0x80000000.toInt(), 0xff000000.toInt())))
}
}
@@ -0,0 +1,99 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertSame
import kotlin.test.assertTrue
class GraphicsCoresTest {
private fun checksum(bytes: ByteArray): Long {
var sum = 7L
for (b in bytes) sum = sum * 31 + (b.toInt() and 0xff)
return sum
}
@Test
fun grayPacketHeaderAndFlatConversion() {
// 3x2: opaque white, opaque mid, transparent white, half-alpha red, opaque black, opaque green
val pixels = intArrayOf(
0xffffffff.toInt(), 0xff808080.toInt(), 0x00ffffff,
0x80ff0000.toInt(), 0xff000000.toInt(), 0xff00ff00.toInt(),
)
val packet = GrayPacket.fromArgb(pixels, 3, 2, 1f, false)
assertEquals(4 + 6, packet.size)
assertContentEquals(byteArrayOf(3, 0, 2, 0), packet.copyOfRange(0, 4))
assertEquals(255, packet[4].toInt() and 0xff)
assertEquals(128, packet[5].toInt() and 0xff)
assertEquals(0, packet[6].toInt() and 0xff) // alpha 0 darkens to black
assertEquals(76 * 128 / 255, packet[7].toInt() and 0xff) // luma 76 at half alpha
assertEquals(0, packet[8].toInt() and 0xff)
assertEquals(149, packet[9].toInt() and 0xff)
val wide = GrayPacket.fromArgb(IntArray(300 * 2), 300, 2, 1f, false)
assertContentEquals(byteArrayOf(0x2c, 1, 2, 0), wide.copyOfRange(0, 4))
}
@Test
fun toneCurveDarkensMidtones() {
val identity = GrayPacket.toneCurve(1f)
assertEquals((0..255).toList(), identity.toList())
val curve = GrayPacket.toneCurve(2.2f)
assertEquals(0, curve[0])
assertEquals(255, curve[255])
assertEquals(56, curve[128])
assertTrue(curve.toList() == curve.sorted())
}
@Test
fun ditherIsDeterministicAndQuantized() {
// A horizontal ramp over 16x4 so the diffusion has somewhere to go.
val pixels = IntArray(16 * 4) { i -> val v = (i % 16) * 17; 0xff000000.toInt() or (v shl 16) or (v shl 8) or v }
val a = GrayPacket.fromArgb(pixels, 16, 4, 1f, true)
val b = GrayPacket.fromArgb(pixels, 16, 4, 1f, true)
assertContentEquals(a, b)
for (i in 4 until a.size) {
val v = a[i].toInt() and 0xff
assertTrue(v == 1 || (v % 16 == 0 && v in 16..240), "byte $i = $v")
}
// Mid-gray flat area dithers to a mix of two adjacent levels, never a solid.
val flat = GrayPacket.fromArgb(IntArray(8 * 8) { 0xff7f7f7f.toInt() }, 8, 8, 1f, true)
val levels = flat.drop(4).map { it.toInt() and 0xff }.toSet()
assertEquals(setOf(112, 128), levels)
assertEquals(2001783152304756847L, checksum(a))
}
@Test
fun previewPaletteValues() {
assertEquals(0xff000000.toInt(), PreviewPalette.argb(0, 0.7, false))
assertEquals(0xffffffff.toInt(), PreviewPalette.argb(255, 0.7, false))
assertEquals(0xff00ff00.toInt(), PreviewPalette.argb(255, 0.7, true))
val gray = PreviewPalette.argb(128, 0.7, false)
val v = gray and 0xff
assertEquals(157, v)
assertEquals(0xff000000.toInt() or (v shl 16) or (v shl 8) or v, gray)
assertEquals(0xff000000.toInt() or (v shl 8), PreviewPalette.argb(128, 0.7, true))
assertEquals(0xff808080.toInt(), PreviewPalette.argb(128, 1.0, false))
val lut = PreviewPalette.lookupTable(0.7, false)
assertSame(lut, PreviewPalette.lookupTable(0.7, false))
assertEquals(256, lut.size)
val expanded = PreviewPalette.expand(byteArrayOf(0, 0x80.toByte(), 0xff.toByte()), 0, 3, 0.7, false)
assertContentEquals(intArrayOf(lut[0], lut[128], lut[255]), expanded)
}
@Test
fun statusIconsAreStable() {
val wifi = StatusIconArt.wifi(4)
val cell = StatusIconArt.cell(2)
val hotspot = StatusIconArt.hotspot()
for (icon in listOf(wifi, cell, hotspot)) {
assertEquals(24 * 24, icon.size)
assertTrue(icon.any { it.toInt() != 0 })
}
assertTrue(StatusIconArt.wifi(0).count { it.toInt() != 0 } < wifi.count { it.toInt() != 0 })
assertEquals(StatusIconArt.wifi(2).toList(), StatusIconArt.wifi(2).toList())
assertEquals(listOf(-7515428168691714401L, -7643893777543336767L, 7675840471157348143L), listOf(checksum(wifi), checksum(cell), checksum(hotspot)))
val scaled = StatusIconArt.scale(wifi, 12)
assertEquals(144, scaled.size)
assertSame(wifi, StatusIconArt.scale(wifi, 24))
}
}
@@ -0,0 +1,67 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFailsWith
class Lc3PacketFramerTest {
private class FakeCodec(var fail: Boolean = false) : Lc3Codec {
var calls = 0
override fun decodeFrames(packet: ByteArray, offset: Int, length: Int, pcmOut: ShortArray): Int {
calls++
assertEquals(0, offset)
assertEquals(Lc3PacketFramer.LC3_BYTES, length)
return if (fail) -1 else Lc3PacketFramer.SAMPLES_PER_PACKET
}
}
private fun packet(counter: Int, ssr: Int = 0, angle: Int = 0): ByteArray {
val p = ByteArray(Lc3PacketFramer.PACKET_BYTES)
p[Lc3PacketFramer.COUNTER_OFFSET] = counter.toByte()
p[Lc3PacketFramer.SSR_OFFSET] = (ssr and 0xff).toByte()
p[Lc3PacketFramer.SSR_OFFSET + 1] = ((ssr shr 8) and 0xff).toByte()
p[Lc3PacketFramer.ANGLE_OFFSET] = (angle and 0xff).toByte()
p[Lc3PacketFramer.ANGLE_OFFSET + 1] = ((angle shr 8) and 0xff).toByte()
return p
}
@Test
fun countsRealDuplicateLateAndMissingPackets() {
val codec = FakeCodec()
val framer = Lc3PacketFramer(codec)
val pcm = ShortArray(Lc3PacketFramer.SAMPLES_PER_PACKET)
assertEquals(800, framer.decodePacket(packet(250), pcm))
assertEquals(800, framer.decodePacket(packet(251), pcm))
assertEquals(0, framer.decodePacket(packet(251), pcm)) // relayed by the other arm
assertEquals(0, framer.decodePacket(packet(250), pcm)) // late copy (gap 255)
assertEquals(800, framer.decodePacket(packet(254), pcm)) // 252, 253 lost
assertEquals(800, framer.decodePacket(packet(1), pcm)) // wraps past 255 -> 0; 255, 0 lost
assertEquals(0, framer.decodePacket(packet(129), pcm)) // gap 128 is late, not 127 losses
assertEquals(4, framer.realPackets)
assertEquals(3, framer.duplicatePackets)
assertEquals(4, framer.missingPackets)
assertEquals(0, framer.decodeErrors)
assertEquals(4, codec.calls)
}
@Test
fun parsesSignedTrailerAndRejectsBadInput() {
val codec = FakeCodec()
val framer = Lc3PacketFramer(codec)
val pcm = ShortArray(Lc3PacketFramer.SAMPLES_PER_PACKET)
framer.decodePacket(packet(7, ssr = -1234, angle = -45), pcm)
assertEquals(-1234, framer.lastSsr)
assertEquals(-45, framer.lastAngleDegrees)
assertEquals(0, framer.decodePacket(ByteArray(10), pcm))
assertEquals(0, framer.decodePacket(null, pcm))
assertEquals(2, framer.decodeErrors)
assertFailsWith<IllegalArgumentException> { framer.decodePacket(packet(8), ShortArray(10)) }
codec.fail = true
assertEquals(0, framer.decodePacket(packet(8), pcm))
assertEquals(3, framer.decodeErrors)
// A failed decode does not advance the counter, so 8 is still accepted afterwards.
codec.fail = false
assertEquals(800, framer.decodePacket(packet(8), pcm))
assertEquals(2, framer.realPackets)
}
}
@@ -0,0 +1,178 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNull
import kotlin.test.assertTrue
private class FakeConnection(input: String, private val bytesOverride: ByteArray? = null) : RemoteInputConnection {
private val bytes = bytesOverride ?: input.encodeToByteArray()
private var index = 0
private val written = ByteSink()
var closed = false
val output: String get() = written.toByteArray().decodeToString()
override fun readByte(timeoutMs: Long): Int = if (index < bytes.size) bytes[index++].toInt() and 255 else -1
override fun write(bytes: ByteArray) = written.write(bytes)
override fun close() { closed = true }
}
private class FakePort(platform: ProtocolPlatform) : LocalServerPort {
val connections = BlockingQueue<FakeConnection>(platform)
var localAddresses = setOf("127.0.0.1", "192.168.1.5")
var bindFails = false
var listener: FakeListener? = null
inner class FakeListener : RemoteInputListener {
override var isClosed = false
override fun accept(): RemoteInputConnection? {
while (!isClosed) {
val next = connections.poll(20)
if (next != null) return next
}
return null
}
override fun close() { isClosed = true }
}
override fun isLocalUnicastAddress(address: String): Boolean = address in localAddresses
override fun bind(address: String, port: Int): RemoteInputListener {
if (bindFails) throw IllegalStateException("in use")
return FakeListener().also { listener = it }
}
}
private fun waitFor(timeoutMs: Long = 3000, condition: () -> Boolean): Boolean {
val platform = testPlatform()
val deadline = platform.elapsedRealtimeMs() + timeoutMs
while (platform.elapsedRealtimeMs() < deadline) {
if (condition()) return true
sleepMs(5)
}
return condition()
}
class RemoteInputSessionTest {
private val platform = testPlatform()
private val random = object : RandomSource { override fun nextBytes(count: Int) = ByteArray(count) { it.toByte() } }
private fun dispatchOnThread(action: () -> Unit) = startThread("remote-input-dispatch", true, action)
private fun session(port: FakePort, requestTimeoutMs: Long = 5000, readTimeoutMs: Long = 5000) =
RemoteInputSession(port, random, ::dispatchOnThread, platform, requestTimeoutMs = requestTimeoutMs, readTimeoutMs = readTimeoutMs)
@Test
fun startValidatesAddressesAndBindFailures() {
val port = FakePort(platform)
val session = session(port)
assertEquals("Invalid local address.", session.start(7000, "localhost"))
assertEquals("Address is not assigned to a local interface.", session.start(7000, "10.0.0.9"))
port.bindFails = true
assertEquals("Could not open local input port 7000.", session.start(7000, "127.0.0.1"))
port.bindFails = false
assertEquals("", session.start(7000, "127.0.0.1"))
assertEquals("", session.start(7000, "127.0.0.1"), "second start is a no-op")
session.stop()
assertTrue(waitFor { port.listener!!.isClosed })
assertTrue(RemoteInputSession.isNumericLocalAddress("fe80::1%en0"))
assertFalse(RemoteInputSession.isNumericLocalAddress("example.com"))
}
@Test
fun requestIsPublishedAnsweredAndWrittenBack() {
val port = FakePort(platform)
val session = session(port)
var notified = 0
session.setRequestListener { notified++ }
assertEquals("", session.start(7000, "127.0.0.1"))
val connection = FakeConnection("{\"token\":\"t\",\"text\":\"hi\"}\n")
port.connections.put(connection)
assertTrue(waitFor { notified == 1 })
val pending = Json.parseObject(session.nextRequest()!!)
assertEquals(1L, pending["id"].asLong())
assertEquals("{\"token\":\"t\",\"text\":\"hi\"}", pending["body"].asString())
assertTrue(pending["expiresAt"].asLong()!! > currentTimeMillis())
assertNull(session.nextRequest(), "consumed once")
session.complete(99L, "{\"ok\":false}")
assertFalse(waitFor(100) { connection.output.isNotEmpty() }, "wrong id is ignored")
session.complete(1L, "{\"ok\":true}")
assertTrue(waitFor { connection.output == "{\"ok\":true}\n" })
assertTrue(waitFor { connection.closed })
session.stop()
}
@Test
fun timeoutRepliesWithTheTimeoutResponse() {
val port = FakePort(platform)
val session = session(port, requestTimeoutMs = 150)
assertEquals("", session.start(7000, "127.0.0.1"))
val connection = FakeConnection("slow\n")
port.connections.put(connection)
assertTrue(waitFor { connection.output == RemoteInputSession.TIMEOUT_RESPONSE + "\n" })
assertNull(session.nextRequest(), "expired request is dropped")
session.stop()
}
@Test
fun oversizedAndIncompleteFramesGetNoReply() {
val port = FakePort(platform)
val session = session(port)
var notified = 0
session.setRequestListener { notified++ }
assertEquals("", session.start(7000, "127.0.0.1"))
val huge = FakeConnection("", ByteArray(RemoteInputSession.MAX_FRAME_BYTES + 2) { 'a'.code.toByte() }.also { it[it.size - 1] = '\n'.code.toByte() })
port.connections.put(huge)
assertTrue(waitFor { huge.closed })
assertEquals("", huge.output)
val partial = FakeConnection("no newline")
port.connections.put(partial)
assertTrue(waitFor { partial.closed })
assertEquals("", partial.output)
val invalidUtf8 = FakeConnection("", byteArrayOf(0xff.toByte(), 0xfe.toByte(), '\n'.code.toByte()))
port.connections.put(invalidUtf8)
assertTrue(waitFor { invalidUtf8.closed })
assertEquals("", invalidUtf8.output)
assertEquals(0, notified)
assertNull(session.nextRequest())
session.stop()
}
@Test
fun stopWhilePendingReleasesTheServingThread() {
val port = FakePort(platform)
val session = session(port)
var notified = 0
session.setRequestListener { notified++ }
assertEquals("", session.start(7000, "127.0.0.1"))
val connection = FakeConnection("pending\n")
port.connections.put(connection)
assertTrue(waitFor { notified == 1 })
session.stop()
assertTrue(waitFor { connection.closed })
assertNull(session.nextRequest())
assertEquals("", session.start(7001, "127.0.0.1"), "restart after stop")
session.stop()
}
@Test
fun helpersFilterInterfacesAndHashTokens() {
val all = listOf(
InterfaceAddress("lo0", "127.0.0.1", up = true, pointToPoint = false, loopback = true, linkLocal = false, anyLocal = false),
InterfaceAddress("wlan0", "192.168.1.5", up = true, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = false),
InterfaceAddress("wlan0", "fe80::1%wlan0", up = true, pointToPoint = false, loopback = false, linkLocal = true, anyLocal = false),
InterfaceAddress("tun0", "100.64.0.2", up = true, pointToPoint = true, loopback = false, linkLocal = false, anyLocal = false),
InterfaceAddress("eth0", "10.0.0.2", up = false, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = false),
InterfaceAddress("any", "0.0.0.0", up = true, pointToPoint = false, loopback = false, linkLocal = false, anyLocal = true),
)
assertEquals(
"[{\"name\":\"wlan0\",\"address\":\"192.168.1.5\",\"pointToPoint\":false},{\"name\":\"tun0\",\"address\":\"100.64.0.2\",\"pointToPoint\":true}]",
RemoteInputSession.interfacesJson(all),
)
assertEquals("ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", RemoteInputSession.tokenHash("abc"))
val session = session(FakePort(platform))
assertEquals("000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", session.randomSecret())
}
}
@@ -0,0 +1,149 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull
import kotlin.test.assertTrue
class ResumableDownloadTest {
private class Storage : DownloadStorage {
val files = HashMap<String, ByteArray>()
override fun length(path: String): Long = files[path]?.size?.toLong() ?: -1L
override fun prepareParent(path: String) {}
override fun openWrite(path: String, append: Boolean): FileSink {
if (!append) files[path] = ByteArray(0)
return object : FileSink {
override fun write(bytes: ByteArray, offset: Int, length: Int) {
files[path] = (files[path] ?: ByteArray(0)) + bytes.copyOfRange(offset, offset + length)
}
override fun close() {}
}
}
override fun readAll(path: String, consumer: (ByteArray, Int) -> Boolean): Long {
val data = files[path] ?: return 0
var at = 0L
for (chunk in data.toList().chunked(7)) {
val bytes = chunk.toByteArray()
if (!consumer(bytes, bytes.size)) break
at += bytes.size
}
return at
}
override fun rename(from: String, to: String): Boolean { files[to] = files.remove(from) ?: return false; return true }
override fun delete(path: String): Boolean = files.remove(path) != null
}
private class Transport(val content: ByteArray, val honorRange: Boolean = true, val truncateAt: Int = -1) : DownloadTransport {
var lastRange = -1L
override fun open(url: String, rangeFrom: Long): DownloadResponse {
lastRange = rangeFrom
val start = if (honorRange) rangeFrom.toInt() else 0
val end = if (truncateAt >= 0) minOf(truncateAt, content.size) else content.size
val body = content.copyOfRange(start, end)
var at = 0
return object : DownloadResponse {
override val code: Int = if (honorRange && rangeFrom > 0) 206 else 200
override val contentLength: Long = body.size.toLong()
override fun read(buffer: ByteArray): Int {
if (at >= body.size) return -1
val n = minOf(5, body.size - at)
body.copyInto(buffer, 0, at, at + n)
at += n
return n
}
override fun close() {}
}
}
}
private class Events : FaceclawModelDownloaderListener {
val progress = ArrayList<Pair<Long, Long>>()
var done: String? = null
var error: String? = null
override fun onProgress(bytesDownloaded: Long, totalBytes: Long) { progress.add(bytesDownloaded to totalBytes) }
override fun onDone(path: String?) { done = path }
override fun onError(message: String?) { error = message }
}
private val content = ByteArray(40) { (it * 3).toByte() }
private val sha = sha256Hex(content)
private fun run(storage: Storage, transport: Transport, expectedSha: String? = sha, total: Long = content.size.toLong()): Events {
val events = Events()
var clock = 0L
ResumableDownload("http://x/m", "/dest/model", expectedSha, total, transport, storage, events) { clock += 600; clock }.run()
return events
}
@Test
fun freshDownloadVerifiesAndRenames() {
val storage = Storage()
val transport = Transport(content)
val events = run(storage, transport)
assertNull(events.error)
assertEquals("/dest/model", events.done)
assertEquals(content.toList(), storage.files["/dest/model"]!!.toList())
assertEquals(0L, transport.lastRange)
assertTrue(events.progress.isNotEmpty() && events.progress.last().second == 40L)
val again = run(storage, transport)
assertEquals("/dest/model", again.done)
}
@Test
fun resumesFromPartWhenServerHonorsRange() {
val storage = Storage()
storage.files["/dest/model.part"] = content.copyOfRange(0, 17)
val transport = Transport(content)
val events = run(storage, transport)
assertEquals(17L, transport.lastRange)
assertNull(events.error)
assertEquals(content.toList(), storage.files["/dest/model"]!!.toList())
}
@Test
fun restartsWhenServerIgnoresRange() {
val storage = Storage()
storage.files["/dest/model.part"] = ByteArray(17) { 1 }
val transport = Transport(content, honorRange = false)
val events = run(storage, transport)
assertEquals(17L, transport.lastRange)
assertNull(events.error)
assertEquals(content.toList(), storage.files["/dest/model"]!!.toList())
}
@Test
fun checksumMismatchDeletesPart() {
val storage = Storage()
val events = run(storage, Transport(content), expectedSha = "00".repeat(32))
assertEquals("Model download was corrupted (checksum mismatch); download it again", events.error)
assertNull(storage.files["/dest/model.part"])
assertNull(storage.files["/dest/model"])
}
@Test
fun shortBodyKeepsPartForResume() {
val storage = Storage()
val events = run(storage, Transport(content, truncateAt = 30))
assertEquals("Model download ended early (30 of 40 bytes); try again to resume", events.error)
assertEquals(30, storage.files["/dest/model.part"]!!.size)
val resumed = run(storage, Transport(content))
assertNull(resumed.error)
assertEquals(content.toList(), storage.files["/dest/model"]!!.toList())
}
@Test
fun httpErrorIsReported() {
val storage = Storage()
val transport = object : DownloadTransport {
override fun open(url: String, rangeFrom: Long): DownloadResponse = object : DownloadResponse {
override val code = 404
override val contentLength = -1L
override fun read(buffer: ByteArray) = -1
override fun close() {}
}
}
val events = Events()
ResumableDownload("u", "/dest/model", null, 0, transport, storage, events).run()
assertEquals("Model download failed: HTTP 404", events.error)
}
}
@@ -0,0 +1,46 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class SettingsChangeHubTest {
private class MemorySettings : SettingsStorage {
val values = HashMap<String?, Any?>()
override fun getString(key: String?, defaultValue: String?): String? = values[key] as? String ?: defaultValue
override fun putString(key: String?, value: String?) { values[key] = value }
override fun getBoolean(key: String?, defaultValue: Boolean): Boolean = values[key] as? Boolean ?: defaultValue
override fun putBoolean(key: String?, value: Boolean) { values[key] = value }
}
@Test
fun writesNotifyEachListenerThroughItsOwnDispatcher() {
val warnings = ArrayList<String>()
val hub = SettingsChangeHub(MemorySettings(), { warnings.add(it) }, testPlatform())
val queued = ArrayList<() -> Unit>()
val seen = ArrayList<String?>()
val listener = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { seen.add(key) } }
hub.register(listener) { queued.add(it) }
val silent = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { seen.add("silent") } }
hub.register(silent, null)
assertEquals(1, warnings.size)
hub.setString("font", "Inter")
hub.setBoolean("dark", true)
assertEquals("Inter", hub.getString("font", null))
assertTrue(hub.getBoolean("dark", false))
assertTrue(seen.isEmpty()) // delivered only when the dispatcher runs
queued.forEach { it() }
assertEquals(listOf<String?>("font", "dark"), seen)
val failing = object : FaceclawSettingsListener { override fun onSettingChanged(key: String?) { throw IllegalStateException("boom") } }
hub.register(failing) { it() }
hub.setString("x", null)
assertTrue(warnings.last().contains("boom"))
hub.unregister(listener)
hub.unregister(failing)
queued.clear()
hub.setBoolean("dark", false)
assertTrue(queued.isEmpty())
}
}
@@ -0,0 +1,87 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertTrue
class SseStreamTest {
private fun feed(stream: SseStream, vararg chunks: String) {
for (chunk in chunks) {
val bytes = chunk.encodeToByteArray()
stream.onChunk(bytes, 0, bytes.size)
}
}
@Test
fun splitsLinesAcrossChunksWithBomAndCrlf() {
val stream = SseStream()
stream.onStatus(200)
feed(stream, "event: a\r", "\ndata: {\"x\":1}\n\nda", "ta: tail")
assertEquals(listOf(SseEvent.Line("event: a"), SseEvent.Line("data: {\"x\":1}"), SseEvent.Line("")), stream.takeEvents())
stream.onComplete()
assertEquals(listOf(SseEvent.Line("data: tail"), SseEvent.Complete), stream.takeEvents())
assertTrue(stream.isFinished)
feed(stream, "late\n")
assertEquals(emptyList(), stream.takeEvents())
}
@Test
fun errorBodyIsBufferedAndCapped() {
val stream = SseStream()
stream.onStatus(429)
feed(stream, "{\"error\":", "\"slow down\"}\n")
assertEquals(emptyList(), stream.takeEvents())
stream.onComplete()
assertEquals(listOf(SseEvent.HttpError(429, "{\"error\":\"slow down\"}\n")), stream.takeEvents())
val capped = SseStream()
capped.onStatus(500)
val big = ByteArray(SseStream.ERROR_BODY_LIMIT + 100) { 'x'.code.toByte() }
capped.onChunk(big, 0, big.size)
capped.onComplete()
val event = capped.takeEvents().single() as SseEvent.HttpError
assertEquals(SseStream.ERROR_BODY_LIMIT, event.body.length)
}
@Test
fun queueOverflowAndOversizedLinesFail() {
val stream = SseStream()
stream.onStatus(200)
val line = "x".repeat(1023) + "\n"
val block = line.repeat(64).encodeToByteArray()
var fed = 0L
while (fed <= SseStream.QUEUE_BYTE_LIMIT + block.size) {
stream.onChunk(block, 0, block.size)
fed += block.size
}
val events = stream.takeEvents()
assertEquals(SseEvent.Failure("Streaming response queue overflow"), events.single())
val long = SseStream()
long.onStatus(200)
val huge = ByteArray(SseStream.LINE_LIMIT + 1) { 'y'.code.toByte() }
long.onChunk(huge, 0, huge.size)
assertEquals(SseEvent.Failure("Streaming response line too large"), long.takeEvents().single())
}
@Test
fun failureAndCancelAreTerminal() {
val stream = SseStream()
stream.onStatus(200)
feed(stream, "a\n")
stream.onFailure("boom")
assertEquals(listOf(SseEvent.Line("a"), SseEvent.Failure("boom")), stream.takeEvents())
val cancelled = SseStream()
cancelled.onStatus(200)
feed(cancelled, "a\n")
cancelled.cancel()
cancelled.onComplete()
assertEquals(emptyList(), cancelled.takeEvents())
}
@Test
fun unpacksFlatHeaderArrays() {
assertEquals(listOf("A" to "1", "C" to "3"), SseStream.unpackHeaders(arrayOf("A", "1", "", "2", "C", "3", "D")))
assertEquals(emptyList(), SseStream.unpackHeaders(null))
}
}
@@ -0,0 +1,382 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertContains
import kotlin.test.assertContentEquals
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNotNull
import kotlin.test.assertTrue
/** A stock-firmware lens pair behind the StockLink port; replies are scripted per written message. */
private class FakeStockLink : StockLink {
class Written(val address: String, val uuid: String, val sid: Int, val flag: Int, val seq: Int, val pb: ByteArray) {
/** Protobuf magic (field 2) of a control message; -1 for OTA payloads. */
val magic: Int get() = if (sid == 0xc0 || sid == 0xc1) -1 else BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(pb), 2, -1)
}
var inbound: StockLinkListener? = null
val connected = HashSet<String>()
val bond = HashMap<String, BondState>()
val writes = ArrayList<Written>()
var connectResult: (String) -> Boolean = { true }
var responder: (Written) -> Unit = {}
var closed = false
override fun setListener(listener: StockLinkListener?) { inbound = listener }
override fun connect(address: String, timeoutMs: Int): Boolean = connectResult(address).also { if (it) connected.add(address) }
override fun prepareLink(address: String, desiredMtu: Int, timeoutMs: Int) {}
override fun discoverServices(address: String, timeoutMs: Int): Boolean = address in connected
override fun enableNotifications(address: String, characteristicUuid: String, timeoutMs: Int): Boolean = address in connected
override fun writeFrames(address: String, characteristicUuid: String, frames: List<ByteArray>, mode: GattWriteMode, timeoutMs: Int): Boolean {
if (address !in connected) throw IllegalStateException("Not connected: $address")
val head = frames.first()
val body = ByteArray(frames.sumOf { it.size - 8 })
var at = 0
for (f in frames) { f.copyInto(body, at, 8); at += f.size - 8 }
val written = Written(address, characteristicUuid, head[6].toInt() and 0xff, head[7].toInt() and 0xff, head[2].toInt() and 0xff, body)
writes.add(written)
responder(written)
return true
}
override fun isConnected(address: String): Boolean = address in connected
override fun bondState(address: String): BondState = bond[address] ?: BondState.BONDED
override fun disconnect(address: String) { connected.remove(address) }
override fun close() { closed = true; connected.clear() }
/** Deliver a lens frame on the control notify characteristic. */
fun notify(address: String, pb: ByteArray, sid: Int, flag: Int = 0) {
for (frame in BleProtocol.framePb(pb, sid, flag, 7)) inbound!!.onNotification(address, BleProtocol.NOTIFY_CHAR_UUID, frame)
}
fun otaAck(address: String, op: Int, status: Int) {
inbound!!.onNotification(address, BleProtocol.OTA_DATA_NOTIFY_UUID, BleProtocol.framePb(byteArrayOf(op.toByte(), status.toByte()), 0xc1, 0, 1).single())
}
fun drop(address: String) { connected.remove(address); inbound!!.onConnectionStateChange(address, false) }
}
private fun varint(value: Int): ByteArray { var v = value; val out = ArrayList<Byte>(); do { var b = v and 0x7f; v = v ushr 7; if (v != 0) b = b or 0x80; out.add(b.toByte()) } while (v != 0); return out.toByteArray() }
private fun field(number: Int, bytes: ByteArray): ByteArray = varint((number shl 3) or 2) + varint(bytes.size) + bytes
private fun field(number: Int, text: String): ByteArray = field(number, text.encodeToByteArray())
private fun authSuccess(magic: Int) = BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + field(3, ByteArray(0))
private fun authPending(magic: Int) = BleProtocol.encodeVarintField(1, 4) + BleProtocol.encodeVarintField(2, magic) + field(3, BleProtocol.encodeVarintField(1, 1))
private fun ack(magic: Int) = BleProtocol.encodeVarintField(1, 1) + BleProtocol.encodeVarintField(2, magic)
private fun settingsAck(magic: Int, battery: Int = -1, left: String = "", right: String = "", extension: String = ""): ByteArray {
var request = ByteArray(0)
if (left.isNotEmpty()) request += field(5, left)
if (right.isNotEmpty()) request += field(6, right)
if (battery >= 0) request += BleProtocol.encodeVarintField(12, battery)
var pb = BleProtocol.encodeVarintField(1, 2) + BleProtocol.encodeVarintField(2, magic) + field(4, request)
if (extension.isNotEmpty()) pb += field(100, extension)
return pb
}
private fun listSelection(container: String, item: String, index: Int, eventType: Int): ByteArray =
field(13, field(1, field(2, container) + field(3, item) + BleProtocol.encodeVarintField(4, index) + BleProtocol.encodeVarintField(5, eventType)))
private val fast = StockFlowTimings(
securityAuthTimeoutMs = 300, securityAuthSoftTimeoutMs = 100, pairingWaitCapMs = 2_000, postBondGraceMs = 20, pollMs = 5,
armRetryDelayMs = 5, queryTimeoutMs = 40, preludeTimeoutMs = 40, heartbeatIntervalMs = 15, selectionTimeoutMs = 80, createAckTimeoutMs = 40,
batteryAckTimeoutMs = 40, blockAckTimeoutMs = 60, ctrlAckTimeoutMs = 60, firstLensConnectWindowMs = 200, secondLensConnectWindowMs = 200,
rebootSettleMs = 5, notifySettleMs = 5, otaRetryDelayMs = 5, componentRetryDelayMs = 5,
)
private const val R = "AA:00:00:00:00:01"
private const val L = "AA:00:00:00:00:02"
private class ProbeEvents : FaceclawDeviceInfoProbeListener {
val logs = ArrayList<String>(); val states = ArrayList<String>(); var result: List<String>? = null; var error: String? = null
override fun onLog(line: String?) { logs.add(line ?: "") }
override fun onState(state: String?, detail: String?) { states.add("$state:$detail") }
override fun onResult(leftVersion: String?, rightVersion: String?, extension: String?) { result = listOf(leftVersion ?: "", rightVersion ?: "", extension ?: "") }
override fun onError(message: String?) { error = message }
}
/** Standard lens behaviour: auth succeeds, prelude and settings are acked with the given versions/battery. */
private fun stockResponder(link: FakeStockLink, left: String = "2.2.9", right: String = "2.2.9", extension: String = "", battery: Int = 85): (FakeStockLink.Written) -> Unit = { w ->
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid)
BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid)
BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, battery, left, right, extension), w.sid)
BleProtocol.SID_EVENHUB -> link.notify(w.address, ack(w.magic), w.sid)
}
}
class StockFlowsTest {
@Test
fun probeAuthenticatesBothArmsThenReadsVersions() {
val link = FakeStockLink()
link.responder = stockResponder(link, "L1", "R1", "Faceclaw/3")
val events = ProbeEvents()
DeviceInfoProbeFlow(link, R, L, events, fast, testPlatform()).run()
assertEquals(listOf("L1", "R1", "Faceclaw/3"), events.result)
assertEquals(null, events.error)
assertEquals(listOf("connecting:right", "authenticating:right", "connecting:left", "authenticating:left", "querying:right"), events.states)
// Right arm then left arm auth, then prelude + one settings read on the right arm only.
assertEquals(listOf(R, L), link.writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.address })
assertEquals(listOf(R), link.writes.filter { it.sid == BleProtocol.SID_UI_SETTING }.map { it.address })
assertTrue(link.closed)
}
@Test
fun probeUsesUnsolicitedSettingsPushAndFallsBackToLeftLens() {
// Right lens: auth ok, but the settings read is never acked; instead a push on a
// device-chosen magic carries the versions.
val pushed = FakeStockLink()
pushed.responder = { w ->
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> pushed.notify(w.address, authSuccess(w.magic), w.sid)
BleProtocol.PRELUDE_ACK_SID -> pushed.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid)
BleProtocol.SID_UI_SETTING -> pushed.notify(w.address, settingsAck(250, -1, "P1", "P2", ""), w.sid)
}
}
var events = ProbeEvents()
DeviceInfoProbeFlow(pushed, R, "", events, fast, testPlatform()).run()
assertEquals(listOf("P1", "P2", ""), events.result)
assertTrue(events.logs.any { it.contains("unsolicited settings push") })
// Right lens silent on settings (both attempts); the left lens answers.
val silentRight = FakeStockLink()
silentRight.responder = { w ->
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> silentRight.notify(w.address, authSuccess(w.magic), w.sid)
BleProtocol.PRELUDE_ACK_SID -> silentRight.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid)
BleProtocol.SID_UI_SETTING -> if (w.address == L) silentRight.notify(w.address, settingsAck(w.magic, -1, "LL", "RR", ""), w.sid)
}
}
events = ProbeEvents()
DeviceInfoProbeFlow(silentRight, R, L, events, fast, testPlatform()).run()
assertEquals(listOf("LL", "RR", ""), events.result)
assertTrue(events.logs.any { it.contains("probing the left lens") })
assertEquals(2, silentRight.writes.count { it.sid == BleProtocol.SID_UI_SETTING && it.address == R })
assertContains(events.states, "querying:left")
}
@Test
fun probeWaitsThroughPairingAndResendsAfterBond() {
// Unbonded phone: the first auth request is answered with the pre-encryption
// non-success result, the OS then pairs (no writes while BONDING), and once bonded the
// request is re-sent after the grace period and finally succeeds.
val link = FakeStockLink()
link.bond[R] = BondState.NONE
val authReplies = ArrayList<Int>()
link.responder = { w ->
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> { authReplies.add(w.magic); link.notify(w.address, if (authReplies.size == 1) authPending(w.magic) else authSuccess(w.magic), w.sid) }
BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid)
BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, -1, "a", "b", ""), w.sid)
}
}
startThread("bond-flip", true) { sleepMs(20); link.bond[R] = BondState.BONDING; sleepMs(40); link.bond[R] = BondState.BONDED }
val events = ProbeEvents()
DeviceInfoProbeFlow(link, R, "", events, fast, testPlatform()).run()
assertEquals(listOf("a", "b", ""), events.result, events.logs.joinToString("\n"))
assertTrue(events.logs.any { it.contains("is not the success result") })
assertTrue(events.logs.any { it.contains("is pairing with the right lens") })
assertTrue(events.logs.any { it.contains("bond established") })
assertTrue(events.logs.any { it.contains("re-sending the auth request after pairing") })
assertEquals(2, authReplies.size)
assertTrue(events.logs.any { it.contains("security auth complete: right lens") })
}
@Test
fun probeReportsUnreachableArmAndCancel() {
val link = FakeStockLink()
link.connectResult = { false }
val events = ProbeEvents()
DeviceInfoProbeFlow(link, R, L, events, fast, testPlatform()).run()
assertNotNull(events.error).let { assertTrue(it.contains("could not connect to the right lens"), it) }
assertEquals(3, events.states.count { it == "connecting:right" })
val cancelled = FakeStockLink()
cancelled.responder = { w -> if (w.sid == BleProtocol.SID_SECURITY_AUTH) cancelled.notify(w.address, authPending(w.magic), w.sid) }
val slow = StockFlowTimings(securityAuthTimeoutMs = 5_000, pollMs = 5)
val flow = DeviceInfoProbeFlow(cancelled, R, "", ProbeEvents().also { events.logs.clear() }, slow, testPlatform())
startThread("cancel", true) { sleepMs(30); flow.cancel() }
val start = testPlatform().elapsedRealtimeMs()
flow.run()
assertTrue(testPlatform().elapsedRealtimeMs() - start < 2_000)
assertTrue(flow.cancelled)
}
private class PromptEvents : FaceclawFlashPromptListener {
val logs = ArrayList<String>(); val states = ArrayList<String>(); var battery: Pair<Int, Int>? = null; var result: Boolean? = null
override fun onLog(line: String?) { logs.add(line ?: "") }
override fun onState(state: String?, detail: String?) { states.add("$state:$detail") }
override fun onBattery(rightPercent: Int, leftPercent: Int) { battery = rightPercent to leftPercent }
override fun onResult(approved: Boolean) { result = approved }
}
private fun promptResponder(link: FakeStockLink, selection: Int?, batteries: Map<String, Int>): (FakeStockLink.Written) -> Unit = { w ->
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid)
BleProtocol.PRELUDE_ACK_SID -> link.notify(w.address, ack(BleProtocol.PRELUDE_ACK_MAGIC), w.sid)
BleProtocol.SID_UI_SETTING -> link.notify(w.address, settingsAck(w.magic, batteries[w.address] ?: -1), w.sid)
BleProtocol.SID_EVENHUB -> {
val cmd = BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(w.pb), 1, -1)
link.notify(w.address, ack(w.magic), w.sid)
// Cmd 0 = create page: the user taps a row right away (a scroll first, which must be ignored).
if (cmd == 0 && selection != null) {
link.notify(w.address, listSelection(FlashPromptFlow.LIST_NAME, "other", 0, BleProtocol.EVENT_SCROLL_TOP), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_NOTIFY)
link.notify(w.address, listSelection(FlashPromptFlow.LIST_NAME, FlashPromptFlow.ITEMS[selection], selection, BleProtocol.EVENT_CLICK), BleProtocol.SID_EVENHUB, BleProtocol.FLAG_NOTIFY)
}
}
}
}
@Test
fun promptApprovalReadsBothBatteriesAndShutsDown() {
val link = FakeStockLink()
link.responder = promptResponder(link, 1, mapOf(R to 85, L to 70))
val events = PromptEvents()
FlashPromptFlow(link, R, L, "warning", false, events, fast, testPlatform()).run()
assertEquals(true, events.result, events.logs.joinToString("\n"))
assertEquals(85 to 70, events.battery)
assertEquals(listOf("connecting:", "connected:", "prompting:", "battery:", "result:approved"), events.states)
val hub = link.writes.filter { it.sid == BleProtocol.SID_EVENHUB }.map { BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(it.pb), 1, -1) }
assertEquals(0, hub.first()) // create page
assertEquals(9, hub.last()) // shutdown
assertTrue(link.writes.all { it.sid != BleProtocol.SID_EVENHUB || it.address == R })
assertTrue(link.closed)
}
@Test
fun promptDeclineTimeoutAndSkip() {
val declined = FakeStockLink()
declined.responder = promptResponder(declined, 0, mapOf())
var events = PromptEvents()
FlashPromptFlow(declined, R, "", "w", false, events, fast, testPlatform()).run()
assertEquals(false, events.result)
assertEquals("result:declined", events.states.last())
assertEquals(null, events.battery)
val silent = FakeStockLink()
silent.responder = promptResponder(silent, null, mapOf())
events = PromptEvents()
FlashPromptFlow(silent, R, "", "w", false, events, fast, testPlatform()).run()
assertEquals(null, events.result)
assertEquals("timeout:No response from the glasses.", events.states.last())
// Heartbeats (EvenHub cmd 12) kept the page alive while waiting (interval 15 ms, wait 80 ms).
assertTrue(silent.writes.count { it.sid == BleProtocol.SID_EVENHUB && BleProtocol.readVarintFieldValue(BleProtocol.stripTrailingCrc(it.pb), 1, -1) == 12 } >= 2)
val skipped = FakeStockLink()
skipped.responder = promptResponder(skipped, null, mapOf(R to 50))
events = PromptEvents()
FlashPromptFlow(skipped, R, L, "w", true, events, fast, testPlatform()).run()
assertEquals(true, events.result)
assertEquals(50 to -1, events.battery)
assertFalse(skipped.writes.any { it.sid == BleProtocol.SID_EVENHUB })
val lost = FakeStockLink()
lost.responder = { w ->
promptResponder(lost, null, mapOf())(w)
if (w.sid == BleProtocol.SID_EVENHUB) lost.drop(R)
}
events = PromptEvents()
FlashPromptFlow(lost, R, "", "w", false, events, fast, testPlatform()).run()
assertEquals("disconnected:Lost connection to the glasses.", events.states.last())
}
// ---- OTA ---------------------------------------------------------------
private fun putU32(buf: ByteArray, offset: Int, value: Long) { for (i in 0 until 4) buf[offset + i] = ((value ushr (8 * i)) and 0xff).toByte() }
private fun container(): ByteArray {
val mainApp = ByteArray(6000) { (it * 7).toByte() }.also { putU32(it, 0, 6000L); putU32(it, 0x14, FirmwareImage.APP_LOAD_ADDR) }
val components = listOf("ota/bootloader.bin" to ByteArray(300) { it.toByte() }, "ota/a.bin" to ByteArray(64) { 1 }, "ota/b.bin" to ByteArray(128) { 2 }, "ota/c.bin" to ByteArray(32) { 3 }, FirmwareImage.REQUIRED_SEGMENT to mainApp)
val toc = FirmwareImage.HEADER_SIZE + components.size * FirmwareImage.TOC_ENTRY_SIZE
var offset = toc
val offsets = components.map { c -> offset.also { offset += FirmwareImage.SUBHEADER_SIZE + c.second.size } }
val img = ByteArray(offset)
putU32(img, 8, components.size.toLong())
components.forEachIndexed { i, c ->
val off = offsets[i]
putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 4, off.toLong())
putU32(img, off + 8, c.second.size.toLong())
c.first.encodeToByteArray().copyInto(img, off + FirmwareImage.NAME_OFFSET)
c.second.copyInto(img, off + FirmwareImage.SUBHEADER_SIZE)
val crc = FirmwareImage.crc32cUnsigned(c.second)
putU32(img, FirmwareImage.HEADER_SIZE + i * 16 + 12, crc)
putU32(img, off + 12, crc)
}
return img
}
private class FlashEvents : FaceclawFirmwareFlasherListener {
val logs = ArrayList<String>(); val states = ArrayList<String>(); val progress = ArrayList<String>(); var complete: Pair<Boolean, String>? = null
override fun onLog(line: String?) { logs.add(line ?: "") }
override fun onProgress(lens: String?, componentIndex: Int, componentCount: Int, blockIndex: Int, blockCount: Int, bytesSent: Long, bytesTotal: Long) { progress.add("$lens $componentIndex/$componentCount $blockIndex/$blockCount $bytesSent/$bytesTotal") }
override fun onState(state: String?, detail: String?) { states.add("$state:$detail") }
override fun onComplete(success: Boolean, detail: String?) { complete = success to (detail ?: "") }
}
/** OTA lens: NAKs one block once and fails one END once, exercising both retry paths. */
private class OtaLens(val link: FakeStockLink) {
val received = HashMap<String, ByteArray>()
var nakOnce = true
var failEndOnce = true
var component = ByteArray(0)
fun respond(w: FakeStockLink.Written) {
when (w.sid) {
BleProtocol.SID_SECURITY_AUTH -> link.notify(w.address, authSuccess(w.magic), w.sid)
OtaFlashFlow.SID_CTRL -> {
val pb = BleProtocol.stripTrailingCrc(w.pb)
when (pb[0].toInt()) {
OtaFlashFlow.OP_BEGIN -> link.otaAck(w.address, OtaFlashFlow.OP_BEGIN, 0)
OtaFlashFlow.OP_FILE_CHECK -> { component = ByteArray(0); link.otaAck(w.address, OtaFlashFlow.OP_FILE_CHECK, 0) }
OtaFlashFlow.OP_END -> {
val status = if (failEndOnce) { failEndOnce = false; 5 } else { received[w.address] = (received[w.address] ?: ByteArray(0)) + component; 9 }
link.otaAck(w.address, OtaFlashFlow.OP_END, status)
}
OtaFlashFlow.OP_BLOCK -> {}
}
}
OtaFlashFlow.SID_DATA -> {
val block = BleProtocol.stripTrailingCrc(w.pb)
if (nakOnce) { nakOnce = false; link.otaAck(w.address, OtaFlashFlow.OP_BLOCK, 1) } else { component += block; link.otaAck(w.address, OtaFlashFlow.OP_BLOCK, 0) }
}
}
}
}
@Test
fun otaFlashesBothLensesWithBlockNakAndEndRetry() {
val img = container()
val segs = FirmwareImage.validate(img)
val link = FakeStockLink()
val lens = OtaLens(link)
link.responder = lens::respond
val events = FlashEvents()
OtaFlashFlow(link, R, L, "fw.bin", events, fast, testPlatform()) { img }.run()
assertEquals(true to "Both lenses flashed. The glasses are rebooting into the custom firmware.", events.complete, events.logs.joinToString("\n"))
assertEquals(listOf("validating:", "connecting:left", "flashing:left", "rebooting:Left lens done. Both lenses reboot briefly; reconnecting for the right lens.", "connecting:right", "flashing:right", "done:"), events.states)
// Every component payload arrived intact on both lenses, in order.
val expected = segs.fold(ByteArray(0)) { acc, s -> acc + img.copyOfRange(s.payloadStart, s.payloadStart + s.ps) }
assertContentEquals(expected, lens.received[L]!!)
assertContentEquals(expected, lens.received[R]!!)
assertTrue(events.logs.any { it.contains("NAK=1 resend 1/3") })
assertTrue(events.logs.any { it.contains("END verify FAILED (status 5)") })
assertTrue(events.logs.any { it.contains("re-flash attempt 2/3") })
assertEquals("left 5/5 2/2 6524/6524", events.progress.first { it.startsWith("left 5/5 2") })
// Marker + data share one envelope seq; the OTA stream restarts at 1 per lens.
val leftOta = link.writes.filter { it.address == L && it.uuid == BleProtocol.OTA_DATA_WRITE_UUID }
assertEquals(1, leftOta.first().seq)
val firstBlock = leftOta.indexOfFirst { it.sid == OtaFlashFlow.SID_DATA }
assertEquals(leftOta[firstBlock - 1].seq, leftOta[firstBlock].seq)
assertEquals(setOf(BleProtocol.WRITE_CHAR_UUID), link.writes.filter { it.sid == BleProtocol.SID_SECURITY_AUTH }.map { it.uuid }.toSet())
}
@Test
fun otaGivesUpWhenLensConnectWindowExpires() {
val link = FakeStockLink()
link.connectResult = { false }
val events = FlashEvents()
OtaFlashFlow(link, R, L, "fw.bin", events, fast, testPlatform()) { container() }.run()
assertEquals(false to "could not reach left lens: connect/discover incomplete", events.complete)
assertEquals("error:could not reach left lens: connect/discover incomplete", events.states.last())
assertTrue(events.logs.count { it.contains("left connect attempt") } >= 2)
assertTrue(link.closed)
val unreadable = FlashEvents()
OtaFlashFlow(FakeStockLink(), R, L, "/nonexistent/fw.bin", unreadable, fast, testPlatform()).run()
assertTrue(unreadable.complete!!.second.startsWith("could not read firmware"), unreadable.complete!!.second)
}
}
@@ -0,0 +1,273 @@
package com.faceclaw.app
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertFalse
import kotlin.test.assertNull
import kotlin.test.assertTrue
class VoiceCaptureSessionTest {
/** Real clock plus an offset the test can advance to force decode cadences. */
private class ShiftedPlatform(private val base: ProtocolPlatform) : ProtocolPlatform by base {
var offsetMs = 0L
override fun elapsedRealtimeMs(): Long = base.elapsedRealtimeMs() + offsetMs
}
private class Events : FaceclawVoiceControllerListener {
val statuses = mutableListOf<String?>()
val transcripts = mutableListOf<Pair<String?, Boolean>>()
val pcmChunks = mutableListOf<ByteArray>()
val frameMeta = mutableListOf<Pair<Int, Int>>()
var speechEnds = 0
var verified: Pair<Boolean, Float>? = null
val stopped = mutableListOf<Int>()
val stoppedLatch = Latch(1, testPlatform())
override fun onStatus(status: String?) { statuses.add(status) }
override fun onTranscript(text: String?, isFinal: Boolean) { transcripts.add(text to isFinal) }
override fun onStopped(captureId: Int) { stopped.add(captureId); stoppedLatch.countDown() }
override fun onPcm(pcm16le: ByteArray?) { pcmChunks.add(pcm16le!!) }
override fun onFrameMeta(angleDegrees: Int, ssr: Int) { frameMeta.add(angleDegrees to ssr) }
override fun onSpeechEnd() { speechEnds++ }
override fun onSpeakerVerified(isWearer: Boolean, similarity: Float) { verified = isWearer to similarity }
}
/** Scripted phone mic: each entry is a chunk amplitude; after the script the read fails (-1). */
private class ScriptedMic(private val amplitudes: List<Int>, private val onRead: () -> Unit = {}) : PcmAudioSource {
var index = 0
var closed = false
override fun read(pcm: ShortArray): Int {
onRead()
if (index >= amplitudes.size) return -1
val amplitude = amplitudes[index++]
for (i in pcm.indices) pcm[i] = (if (i % 2 == 0) amplitude else -amplitude).toShort()
return pcm.size
}
override fun close() { closed = true }
}
private class FakeTranscriber(private val text: (Int) -> String) : OfflineTranscriber {
val calls = mutableListOf<Int>()
var released = false
override fun recognize(samples: FloatArray, count: Int): String { calls.add(count); return text(count) }
override fun release() { released = true }
}
private class FakeHost(
val platform: ProtocolPlatform,
var mic: PcmAudioSource? = null,
var transcriber: OfflineTranscriber? = null,
var hasModel: Boolean = true,
var embedding: FloatArray? = null,
var g2Ready: Boolean = false,
) : VoiceCaptureHost {
override val dispatcher = CallbackDispatcher { it() }
var packetListener: FaceclawAudioPacketListener? = null
var g2Stops = 0
val recordings = mutableListOf<ByteArray>()
override fun onSessionStarting() {}
override fun isG2SessionReady(): Boolean = g2Ready
override fun isG2AudioCaptureActive(): Boolean = packetListener != null
override fun g2AudioSource(): PacketAudioSource? = if (!g2Ready) null else object : PacketAudioSource {
override fun start(listener: FaceclawAudioPacketListener): Boolean { packetListener = listener; return true }
override fun stop() { g2Stops++ }
override fun isActive(): Boolean = packetListener != null
}
override fun createPacketDecoder(): AudioPacketDecoder = object : AudioPacketDecoder {
override fun decodePacket(packet: ByteArray?, pcmOut: ShortArray?): Int {
if (packet == null || packet.size != Lc3PacketFramer.PACKET_BYTES) return 0
pcmOut!!.fill(packet[0].toShort())
lastAngleDegrees = Lc3PacketFramer.readTrailerS16(packet, Lc3PacketFramer.ANGLE_OFFSET)
lastSsr = Lc3PacketFramer.readTrailerS16(packet, Lc3PacketFramer.SSR_OFFSET)
realPackets++
return Lc3PacketFramer.SAMPLES_PER_PACKET
}
override var lastAngleDegrees = 0
override var lastSsr = 0
override var realPackets = 0L
override val duplicatePackets = 0L
override val missingPackets = 0L
override val decodeErrors = 0L
override fun close() {}
}
override fun openPhoneMic(): PcmAudioSource? = mic
override fun hasTranscriberModel(kind: VoiceModelKind): Boolean = hasModel
override fun loadTranscriber(kind: VoiceModelKind): OfflineTranscriber = transcriber ?: error("no transcriber")
override fun speakerEmbedder(modelPath: String): SpeakerEmbedder = object : SpeakerEmbedder {
override fun embed(pcm16le: ByteArray, sampleRate: Int): FloatArray? = embedding
override fun release() {}
}
override fun recordingSink(): RecordingSink = RecordingSink { bytes, _ -> recordings.add(bytes) }
}
private val logs = mutableListOf<String>()
private fun session(host: VoiceCaptureHost, platform: ProtocolPlatform, name: String) =
VoiceCaptureSession(host, platform, name, { logs.add("I $it") }, { logs.add("W $it") }, { m, e -> logs.add("E $m ${e.message}") })
private fun packet(value: Int, angle: Int, ssr: Int): ByteArray {
val bytes = ByteArray(Lc3PacketFramer.PACKET_BYTES)
bytes[0] = value.toByte()
bytes[Lc3PacketFramer.SSR_OFFSET] = (ssr and 0xff).toByte()
bytes[Lc3PacketFramer.SSR_OFFSET + 1] = (ssr shr 8).toByte()
bytes[Lc3PacketFramer.ANGLE_OFFSET] = (angle and 0xff).toByte()
bytes[Lc3PacketFramer.ANGLE_OFFSET + 1] = (angle shr 8).toByte()
return bytes
}
@Test
fun beamGateWrapsAroundAndClampsHalfWidth() {
val session = session(FakeHost(testPlatform()), testPlatform(), "voice-test-beam")
session.setBeamFilter(true, 170, 30)
assertTrue(session.withinBeam(170))
assertTrue(session.withinBeam(-170)) // 20 degrees away across the wrap
assertFalse(session.withinBeam(120))
session.setBeamFilter(true, 0, 1) // clamps to 5
assertTrue(session.withinBeam(5))
assertFalse(session.withinBeam(6))
}
@Test
fun droppingQueueKeepsNewestAndCountsDrops() {
val queue = DroppingPacketQueue<Int>(3, testPlatform())
for (i in 1..5) queue.put(i)
assertEquals(2L, queue.droppedCount)
assertEquals(3, queue.take(10) { true })
assertEquals(4, queue.take(10) { true })
assertEquals(5, queue.take(10) { true })
assertNull(queue.take(10, { true }))
assertNull(queue.take(10, { false }))
}
@Test
fun g2PacketsAreCountedGatedAndForwarded() {
val platform = ShiftedPlatform(testPlatform())
val host = FakeHost(platform, g2Ready = true)
val events = Events()
val session = session(host, platform, "voice-test-g2")
session.setListener(events)
session.setBeamFilter(true, 0, 30)
session.start("cloud", 7)
val deadline = platform.elapsedRealtimeMs() + 3000
while (host.packetListener == null && platform.elapsedRealtimeMs() < deadline) sleepMs(5)
val listener = host.packetListener!!
listener.onAudioPacket(packet(1, 10, 5), "L", 1000)
listener.onAudioPacket(packet(2, 90, 5), "L", 1050) // outside the beam: meta only
listener.onAudioPacket(packet(3, 0, 0), "R", 1200) // wrong arm, late (150 ms gap), ssr 0 passes the gate
listener.onAudioPacket(ByteArray(3), "L", 1230) // bad length: dropped by the decoder
val waitUntil = platform.elapsedRealtimeMs() + 3000
while (events.pcmChunks.size < 2 && platform.elapsedRealtimeMs() < waitUntil) sleepMs(5)
session.stop()
assertTrue(events.stoppedLatch.await(3000))
val stats = session.audioStats()
assertEquals(4L, stats.queuedPackets)
assertEquals(1L, stats.wrongArmPackets)
assertEquals(1L, stats.latePackets)
assertEquals(150L, stats.maxInterPacketMs)
assertEquals(2, events.pcmChunks.size)
assertEquals(Lc3PacketFramer.SAMPLES_PER_PACKET * 2, events.pcmChunks[0].size)
assertEquals(listOf(10 to 5, 90 to 5, 0 to 0), events.frameMeta)
assertEquals(listOf(7), events.stopped)
assertTrue(host.g2Stops >= 1)
assertTrue(events.statuses.contains("Listening (cloud)..."))
assertTrue(events.transcripts.isEmpty())
assertTrue(logs.any { it.startsWith("I G2 mic packets=4 decoded=3") && it.contains("wrongArm=1") }, logs.toString())
}
@Test
fun startRefusesWithoutGlassesAndWhenAlreadyRunning() {
val host = FakeHost(testPlatform(), g2Ready = false)
val events = Events()
val session = session(host, testPlatform(), "voice-test-refuse")
session.setListener(events)
session.start("onboard", 3)
assertEquals<List<String?>>(listOf("Voice control needs an active G2 connection."), events.statuses)
assertEquals(listOf(3), events.stopped)
assertFalse(session.isCapturing())
}
@Test
fun onboardTranscriptCommitsSegmentsAndFinalizesWithReplaceSemantics() {
val platform = ShiftedPlatform(testPlatform())
// 170 chunks of 800 samples = 8.5 s: one 8 s segment commit plus a tail.
val mic = ScriptedMic(List(170) { 3000 }) { platform.offsetMs += 1000 }
val transcriber = FakeTranscriber { "x" }
val host = FakeHost(platform, mic = mic, transcriber = transcriber)
val events = Events()
val session = session(host, platform, "voice-test-onboard")
session.setListener(events)
session.setUsePhoneMic(true)
session.setSaveRecordings(true)
session.start("onboard", 1)
assertTrue(events.stoppedLatch.await(10000))
assertTrue(mic.closed)
assertTrue(transcriber.released)
assertEquals("Loading transcription model...", events.statuses[0])
assertEquals("Listening...", events.statuses[1])
val finals = events.transcripts.filter { it.second }
assertEquals(1, finals.size)
assertEquals("x x", finals.single().first)
assertEquals("x" to false, events.transcripts.first())
assertTrue(events.transcripts.any { it == ("x x" to false) })
// Every segment handed to the model is at most 8 s and was peak-normalized.
assertTrue(transcriber.calls.all { it in 1..VoiceCaptureSession.TRANSCRIPT_SEGMENT_MAX_SAMPLES })
assertTrue(logs.any { it.startsWith("I Moonshine decode final=true") }, logs.toString())
assertEquals(1, host.recordings.size)
assertEquals(170 * 800 * 2, host.recordings[0].size)
assertEquals(170, events.pcmChunks.size)
assertEquals("x x", VoiceCaptureSession.joinTranscript("x", "x"))
assertEquals("x, y", VoiceCaptureSession.joinTranscript("x", ", y"))
}
@Test
fun endpointingFiresOnceAfterSpeechThenSilence() {
val platform = ShiftedPlatform(testPlatform())
val script = List(8) { 0 } + List(10) { 3000 } + List(24) { 0 }
val host = FakeHost(platform, mic = ScriptedMic(script))
val events = Events()
val session = session(host, platform, "voice-test-endpoint")
session.setListener(events)
session.setUsePhoneMic(true)
session.setEndpointing(true)
session.start("cloud", 2)
assertTrue(events.stoppedLatch.await(10000))
assertEquals(1, events.speechEnds)
assertNull(events.verified)
}
@Test
fun speakerVerificationFailsOpenAndComparesEmbeddings() {
fun run(embedding: FloatArray?, chunks: Int): Pair<Boolean, Float>? {
val platform = ShiftedPlatform(testPlatform())
val host = FakeHost(platform, mic = ScriptedMic(List(chunks) { 1000 }), embedding = embedding)
val events = Events()
val session = session(host, platform, "voice-test-verify")
session.setListener(events)
session.setUsePhoneMic(true)
session.setSpeakerVerification("model", floatArrayOf(1f, 0f), 0.8f)
session.start("cloud", 4)
assertTrue(events.stoppedLatch.await(10000))
return events.verified
}
assertEquals<Pair<Boolean, Float>?>(true to 0f, run(floatArrayOf(1f, 0f), 10)) // under 1 s: too short, fails open
assertEquals<Pair<Boolean, Float>?>(true to 0f, run(null, 30)) // no embedding: fails open
assertEquals<Pair<Boolean, Float>?>(true to 0f, run(floatArrayOf(1f), 30)) // dimension mismatch: fails open
assertEquals<Pair<Boolean, Float>?>(true to 1f, run(floatArrayOf(1f, 0f), 30))
assertEquals<Pair<Boolean, Float>?>(false to 0f, run(floatArrayOf(0f, 1f), 30))
}
@Test
fun missingModelReportsAndStops() {
val host = FakeHost(testPlatform(), mic = ScriptedMic(emptyList()), hasModel = false)
val events = Events()
val session = session(host, testPlatform(), "voice-test-nomodel")
session.setListener(events)
session.setUsePhoneMic(true)
session.start("onboard", 9)
assertTrue(events.stoppedLatch.await(5000))
assertEquals<List<String?>>(listOf("Voice model not downloaded (see Settings > Voice)."), events.statuses)
assertEquals(listOf(9), events.stopped)
assertEquals("1.50", formatFixed(1.5, 2))
assertEquals("-0.333", formatFixed(-1.0 / 3, 3))
assertEquals("0", formatFixed(0.4, 0))
}
}