Merge branch 'main' into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-10-25 01:06:38 -07:00
commit 4f8c082b52
368 files changed
+26238 -3401

No files matched your search

+2 -2
View File
@@ -12,8 +12,8 @@ local `@faceclaw/kotlin` plugin. Generated binaries and Gradle caches are ignore
the Android adapters: `util/` (Json codec, FrameTimingsCore, concurrency
primitives, sha256/clock/log/file expects), `g2protocol/FirmwareImage`,
`audio/` (Lc3PacketFramer, AudioSegmentation, VoicePrint, WavPcmReader),
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, GlyphPacket,
OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
`graphics/` (GrayPacket, PreviewPalette, StatusIconArt, NotificationIconArt,
GlyphPacket, OpenTypeNames, SvgIconSource), `net/` (HttpIdentity, SseStream,
ResumableDownload, RemoteInputSession), `evenhub/EvenHubAssetServer`,
`alarms/` (AlarmSchedule, AlarmRingingPolicy), `settings/SettingsChangeHub`,
the stock-firmware flows over a `StockLink` GATT port (`g2protocol/StockLinkSession`,
@@ -13,6 +13,30 @@ constructor(
private val sampleRate: Int,
private val channels: Int,
) {
companion object {
/**
* Rewrite the header of a recording whose writer never finished (the process died
* mid-recording, leaving the empty header written at start) to cover the PCM actually on
* disk, trimming any partial sample frame. Returns the PCM byte count, or -1 when the
* file doesn't exist.
*/
@JvmStatic
@Throws(IOException::class)
fun repairHeader(path: String, sampleRate: Int, channels: Int): Long {
val target = File(path)
if (!target.exists()) return -1
val headerBytes = BinaryEncoding.wavHeader(0, sampleRate, channels).size.toLong()
val frameBytes = channels * 2L
val pcmBytes = maxOf(0L, target.length() - headerBytes) / frameBytes * frameBytes
RandomAccessFile(target, "rw").use {
it.setLength(headerBytes + pcmBytes)
it.seek(0)
it.write(BinaryEncoding.wavHeader(minOf(pcmBytes, Int.MAX_VALUE.toLong()).toInt(), sampleRate, channels))
}
return pcmBytes
}
}
private var file: RandomAccessFile?
private var dataBytes = 0L
@@ -312,6 +312,21 @@ class CaptionSession(
}
}
/**
* Stops without waiting. The worker still flushes the trailing utterance and releases the
* models, then reports onStopped, so a caller can close out after the last line arrives
* instead of blocking its thread on the decode.
*/
fun requestStop() {
lock.withLock {
if (!started) {
return
}
started = false
}
queue.wake()
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
fun acceptPcm(pcm16le: ByteArray?) {
if (!started || pcm16le == null || pcm16le.size < 2) {
@@ -322,6 +337,7 @@ class CaptionSession(
private fun runLoop(finished: Latch) {
workerThreadName = currentThreadName()
var failure: String? = null
try {
loadModels()
segmenter.reset()
@@ -329,18 +345,35 @@ class CaptionSession(
val chunk = queue.take(250) { started } ?: continue
segmenter.processChunk(chunk)
}
// Flush a trailing utterance so its text isn't lost on stop.
// Finish the audio already queued when the stop landed, then flush the trailing
// utterance so its text isn't lost on stop.
while (true) {
val chunk = queue.take(0) { false } ?: break
segmenter.processChunk(chunk)
}
segmenter.flush()
} catch (t: Throwable) {
logError("caption engine failed", t)
emitStatus("Captions failed: " + t.message)
failure = t.message ?: t.toString()
emitStatus("Captions failed: $failure")
} finally {
decoder?.release()
try {
decoder?.release()
} catch (t: Throwable) {
logError("caption model release failed", t)
}
decoder = null
lock.withLock {
workerFinished = null
workerThreadName = null
// A worker that died on its own leaves the session stopped, so the owner can
// tell (and acceptPcm stops queueing for nobody).
if (workerFinished === finished) {
started = false
workerFinished = null
workerThreadName = null
}
}
// Posted before the latch opens, so a blocking stop() returns after it.
emitStopped(failure)
finished.countDown()
}
}
@@ -366,8 +399,20 @@ class CaptionSession(
override fun onUtterance(samples: ShortArray, length: Int, startMs: Long, endMs: Long, peakRms: Double) {
val currentDecoder = decoder ?: return
val text = currentDecoder.recognize(samples, length)
val embedding = currentDecoder.embed(samples, length)
// One utterance the models choke on must not end captioning for the session: deliver
// what did decode and keep listening.
val text = try {
currentDecoder.recognize(samples, length)
} catch (t: Throwable) {
logError("caption recognition failed", t)
""
}
val embedding = try {
currentDecoder.embed(samples, length)
} catch (t: Throwable) {
logError("caption voice-print failed", t)
null
}
val currentListener = listener ?: return
val embeddingCopy = embedding?.copyOf()
host.dispatcher.post { currentListener.onUtterance(text, embeddingCopy, startMs, endMs, peakRms) }
@@ -377,4 +422,9 @@ class CaptionSession(
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onStatus(status) }
}
private fun emitStopped(error: String?) {
val currentListener = listener ?: return
host.dispatcher.post { currentListener.onStopped(error) }
}
}
@@ -0,0 +1,10 @@
package com.faceclaw.app
/** Callbacks for FaceclawBinaryPost, delivered on the constructing thread's Looper. */
interface FaceclawBinaryPostListener {
/** The server answered, with any status; body is the response text. */
fun onResponse(status: Int, body: String?): Unit
/** Network-level failure (connect, TLS, timeout, mid-body drop); no response arrived. */
fun onFailure(message: String?): Unit
}
@@ -19,4 +19,10 @@ interface FaceclawCaptionEngineListener {
fun onSpeechStart(startMs: Long): Unit
fun onStatus(status: String?): Unit
/**
* The worker has exited, after delivering any trailing utterance. error is null after
* stop()/requestStop(), or the failure that ended captioning on its own.
*/
fun onStopped(error: String?): Unit
}
@@ -15,4 +15,10 @@ interface FaceclawEvenHubListener {
/** The WebView finished loading the app's page. */
fun onPageFinished(url: String?): Unit
/**
* The WebView's renderer process is gone: it [crashed], or the system killed it to reclaim
* memory. The WebView has already been detached and destroyed, so the app's page is lost.
*/
fun onRenderProcessGone(crashed: Boolean): Unit
}
@@ -4,6 +4,7 @@ interface FaceclawMediaControllerListener {
/** All observed session owners, before filtering or choosing the active player. */
fun onSessionAppsChanged(appsJson: String?): Unit
/** artVersion changes whenever the current item's album art may have changed. */
fun onStateChange(
playbackState: String?,
packageName: String?,
@@ -19,5 +20,6 @@ interface FaceclawMediaControllerListener {
canSkipPrevious: Boolean,
accessEnabled: Boolean,
status: String?,
artVersion: Int,
): Unit
}
@@ -2,4 +2,5 @@ package com.faceclaw.app
interface FaceclawNotificationListener {
fun onNotificationPosted(key: String?): Unit
fun onNotificationRemoved(key: String?): Unit
}
@@ -0,0 +1,44 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
/**
* Process-wide counts of connection-reliability events: connect attempts,
* transport failures by reason, ack timeouts, CFW replays, arm drops, how long
* sessions lasted, and the like. The phone app's analytics drains them
* (app/native/connection-counters.ts, app/analytics/). Only counts under fixed
* event names are kept, never addresses or message contents. They pile up in
* memory whether or not analytics is on; the app throws a drain away when it
* is off.
*/
object ConnectionCounters {
/** Distinct names kept between drains; new names past this are dropped. */
private const val MAX_NAMES = 200
private val lock = protocolPlatform().createLock()
private var counts = LinkedHashMap<String, Long>()
@JvmStatic
fun increment(name: String) {
lock.withLock {
val current = counts[name]
if (current != null) counts[name] = current + 1
else if (counts.size < MAX_NAMES) counts[name] = 1
}
}
/** The counts since the last drain, as a JSON object of name to count, and starts over. */
@JvmStatic
fun drain(): String {
val drained = lock.withLock {
val out = counts
counts = LinkedHashMap()
out
}
return Json.write(drained)
}
/** An event name from fixed text: "CFW recovery retry limit" -> "cfw-recovery-retry-limit". */
@JvmStatic
fun slug(text: String): String = text.lowercase().replace(Regex("[^a-z0-9]+"), "-").trim('-').take(60)
}
@@ -359,6 +359,7 @@ internal fun GlassesSessionCore.firstUnpairedArm(): String? {
*/
internal fun GlassesSessionCore.handleUnpairedFailure(address: String) {
logError("Connect failed and $address is not paired; suspending reconnect")
ConnectionCounters.increment("g2.unpaired")
monitor.withLock { haltReconnectLocked("arm not paired: $address") }
if (!userDisconnectRequested) {
setStateDisplay(
@@ -387,6 +388,7 @@ internal fun GlassesSessionCore.handleIncompatibleFirmware() {
incompatibleFirmware = null
pending
} ?: return
ConnectionCounters.increment("g2.incompatible-firmware")
logError("Glasses report incompatible firmware L=" + info.leftVersion + " R=" + info.rightVersion
+ " ext=\"" + info.extension + "\" (need Faceclaw/" + requiredFirmwareRevision + "); suspending reconnect")
monitor.withLock { haltReconnectLocked("incompatible firmware") }
@@ -429,11 +431,13 @@ internal fun GlassesSessionCore.haltReconnectLocked(reason: String) {
*/
internal fun GlassesSessionCore.handleTransportFailure(reason: String?, displayStatus: String? = null) {
logError("Transport failure: $reason")
ConnectionCounters.increment("g2.failure." + ConnectionCounters.slug(reason ?: "unknown"))
val retryDelayMs: Long
monitor.withLock {
maybeEmitEvenAppConflictLocked(reason)
finishBenchmarkLocked(true, "transport failure")
val sessionWasReady = sessionReady
if (sessionWasReady) countSessionEnded("failure")
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -460,6 +464,23 @@ internal fun GlassesSessionCore.handleTransportFailure(reason: String?, displayS
interruptibleSleep.interrupt()
}
/**
* Counts a ready session ending, by [cause] and by how long it had been up
* (for the analytics in app/analytics/).
*/
internal fun GlassesSessionCore.countSessionEnded(cause: String) {
ConnectionCounters.increment("g2.session-end.$cause")
val minutes = (now() - lastSessionReadyAtMs) / 60_000
ConnectionCounters.increment(
"g2.session-length." + when {
minutes < 1 -> "under-1m"
minutes < 10 -> "1m-10m"
minutes < 60 -> "10m-1h"
else -> "over-1h"
}
)
}
/**
* Set [reconnectAfterMs] for the next attempt after a failure and return the delay. Repeated
* failures back off along RECONNECT_BACKOFF_MS; a session that stayed up for
@@ -742,6 +763,7 @@ internal fun GlassesSessionCore.maybeEmitEvenAppConflictLocked(reason: String?)
return
}
lastEvenAppConflictAtMs = now
ConnectionCounters.increment("g2.even-app-conflict")
emitEvenAppConflict("The Even Realities app still appears to be running. It can hold the glasses BLE link and cause Faceclaw write failures. Open its app settings and force stop it, then reconnect Faceclaw.")
}
@@ -567,6 +567,8 @@ class GlassesSessionCore(
releaseFaceclawFramebufferLease()
}
monitor.withLock {
ConnectionCounters.increment("g2.disconnect-requested")
if (sessionReady) countSessionEnded("requested")
userDisconnectRequested = true
running = false
audioCaptureActive = false
@@ -1254,16 +1256,17 @@ class GlassesSessionCore(
* Queue a CFW mic_control record (['M','C',ver,op,...]) as a settings
* field-103 write to both temples, or to a single one. Fire-and-forget:
* the firmware answers with a field-104 status notify per temple, which
* arrives through addMicStatusListener.
* arrives through addMicStatusListener. Returns false when nothing was
* queued (no session, or a malformed record).
*/
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) {
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean {
if (record == null || record.size < 4) {
return
return false
}
monitor.withLock {
if (!running || !sessionReady) {
logLine("skip mic control ($label); session not ready")
return
return false
}
if (rightTemple) {
pendingMessages.addLast(messageBuilder.faceclawMicControl(record, label, false))
@@ -1274,6 +1277,7 @@ class GlassesSessionCore(
logLine("queue mic control $label")
}
interruptibleSleep.interrupt()
return true
}
/**
@@ -1750,6 +1754,7 @@ class GlassesSessionCore(
val ingress = if (message.isLeftArmMessage) leftAddress else rightAddress
if (message.sid == CfwTransport.SID && address.equals(ingress, ignoreCase = true) && message.magic == ack.streamId) {
message.acceptCfwAck(ack)
if (ack.nack) ConnectionCounters.increment("g2.cfw-nack")
logLine("CFW " + (if (ack.nack) "NACK" else "ACK") + " id=" + ack.streamId
+ " ordinal=" + ack.messageId + " lens=" + ack.lens + " txseq=" + (data[2].toInt() and 255)
+ " redundant=" + (ack !== acks[0])
@@ -1960,6 +1965,11 @@ class GlassesSessionCore(
if (decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT || decoded.eventType == BleProtocol.EVENT_SYSTEM_EXIT) {
if (shutdownRequested) {
lastShutdownExitAtMs = now()
} else {
ConnectionCounters.increment(
if (decoded.eventType == BleProtocol.EVENT_ABNORMAL_EXIT) "g2.firmware-abnormal-exit"
else "g2.firmware-system-exit"
)
}
fixedLayoutCreated = false
displayedFingerprint = ""
@@ -2093,6 +2103,7 @@ class GlassesSessionCore(
// Wakes the ring worker, which owns ring connect/setup.
monitor.signalAll()
}
ConnectionCounters.increment(if (connected) "ring.direct.connect" else "ring.direct.disconnect")
logLine(if (connected) "direct ring BLE connected" else "direct ring BLE disconnected")
return
}
@@ -2110,6 +2121,12 @@ class GlassesSessionCore(
// dropping mid-setup): the connect sequence notices on its own and
// schedules the retry, so leave the backoff alone.
lostSession = sessionReady
if (lostSession) {
ConnectionCounters.increment(
if (address.equals(rightAddress, ignoreCase = true)) "g2.arm-drop.right" else "g2.arm-drop.left"
)
countSessionEnded("arm-drop")
}
sessionReady = false
fixedLayoutCreated = false
startupProbePending = false
@@ -2141,6 +2158,7 @@ class GlassesSessionCore(
wearState = -1
}
setStateDisplay("connecting", "Connecting to the glasses...")
ConnectionCounters.increment("g2.connect-attempt")
try {
// Dial both arms at once, so the left arm's attempt is already pending (and
// can land) while the right arm is being set up; they share one window.
@@ -2194,6 +2212,7 @@ class GlassesSessionCore(
}
}
setStateDisplay("connected", "Connected.")
ConnectionCounters.increment("g2.session-ready")
logLine("session ready")
monitor.withLock {
// Query settings at the start of every session so firmware
@@ -2464,6 +2483,7 @@ class GlassesSessionCore(
monitor.withLock {
if (!inFlightMessages.isEmpty()) {
clearInFlightMessagesLocked("security auth timeout")
ConnectionCounters.increment("g2.auth-unacknowledged")
logLine("security auth not acknowledged; continuing (2.2.9 stock requires it; older/custom firmware may not answer)")
return
}
@@ -2490,7 +2510,7 @@ class GlassesSessionCore(
prelude.onAck = MessageCallback {
}
prelude.onTimeout = MessageCallback {
handleTransportFailure("ack timeout")
handleTransportFailure("prelude ack timeout")
}
prelude.sentAtMs = now
writeMessage(prelude)
@@ -87,6 +87,7 @@ internal fun GlassesSessionCore.driveSession(): Long {
val oldest = inFlightMessages.firstOrNull()
val replay = CfwMessageWindow.replayWindow(inFlightMessages, now)
if (!replay.isEmpty()) {
ConnectionCounters.increment("g2.cfw-replay")
logLine("CFW recovery: replay " + replay.size
+ " unresolved message(s) from id=" + replay[0].magic)
for (candidate in replay) {
@@ -665,7 +666,7 @@ internal fun GlassesSessionCore.enqueueCreateLayoutLocked() {
}
startupProbePending = false
}
handleTransportFailure("ack timeout")
handleTransportFailure("create layout ack timeout")
}
pendingMessages.addLast(message)
logLine("queue create layout")
@@ -706,7 +707,7 @@ internal fun GlassesSessionCore.enqueueStartupProbeLocked() {
if (hasPendingOrInflightKindLocked("create-layout")) {
return@MessageCallback
}
handleTransportFailure("ack timeout")
handleTransportFailure("startup probe ack timeout")
}
pendingMessages.addLast(message)
startupProbePending = true
@@ -1280,6 +1281,7 @@ internal fun GlassesSessionCore.hasPendingMagicLocked(sid: Int, magic: Int): Boo
internal fun GlassesSessionCore.handleAckTimeoutLocked(message: OutboundMessage) {
consecutiveAckTimeouts += 1
ConnectionCounters.increment("g2.ack-timeout")
val onTimeout = message.onTimeout
if (onTimeout != null) {
@@ -1,5 +1,6 @@
package com.faceclaw.app
import kotlin.jvm.JvmOverloads
import kotlin.jvm.JvmStatic
import kotlin.math.pow
import kotlin.math.roundToInt
@@ -23,20 +24,28 @@ object GrayPacket {
* (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to round on its own, which turns
* smooth gradients into visible bands. Dithered output is already quantized: every byte is
* a level times 16, which BmpUtil re-quantizes back to exactly that level.
*
* invertLightBackground converts a picture drawn on a light background (hasLightBackground)
* as its negative, so a chart or screenshot made for a light theme shows light-on-dark: the
* display lights what is light, so a light background is a glaring panel, while black is
* see-through. The colors are inverted, not their alpha, before the tone curve, where the
* sRGB-encoded values are roughly perceptually even; inverting the curve's output instead
* would turn dim strokes into ones barely darker than the background.
*/
@JvmStatic
fun fromArgb(pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean): ByteArray {
@JvmOverloads
fun fromArgb(
pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean = false,
): ByteArray {
require(width > 0 && height > 0 && pixels.size >= width * height) { "bad pixel plane" }
val out = ByteArray(HEADER_SIZE + width * height)
writeHeader(out, width, height)
val tone = toneCurve(gamma)
val invert = invertLightBackground && hasLightBackground(pixels, width, height)
for (i in 0 until width * height) {
val p = pixels[i]
val a = (p ushr 24) and 0xff
val r = (p shr 16) and 0xff
val g = (p shr 8) and 0xff
val b = p and 0xff
out[HEADER_SIZE + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte()
val luma = luma(p)
out[HEADER_SIZE + i] = tone[(if (invert) 255 - luma else luma) * alpha(p) / 255].toByte()
}
if (dither) {
ditherToDisplayLevels(out, HEADER_SIZE, width, height)
@@ -44,6 +53,53 @@ object GrayPacket {
return out
}
/** Grays within this span of each other count as one background (compression noise, faint shading). */
private const val BACKGROUND_SPAN = 16
/** The share of the pixels a background covers; a photo's tones mostly spread wider than this. */
private const val BACKGROUND_MIN_SHARE = 0.4f
/** A background at least this light (source gray, before the tone curve) counts as light. */
private const val LIGHT_BACKGROUND_MIN_GRAY = 160
/**
* Whether a picture is drawn on a light background: the most populous span of
* BACKGROUND_SPAN adjacent grays (as displayed: composited over black, before the tone
* curve) covers BACKGROUND_MIN_SHARE of the picture and is light. The flat backgrounds of
* charts, screenshots and documents qualify; photographs mostly don't, though a large
* blown-out sky can.
*/
@JvmStatic
fun hasLightBackground(pixels: IntArray, width: Int, height: Int): Boolean {
val count = width * height
val histogram = IntArray(256)
for (i in 0 until count) {
val p = pixels[i]
histogram[luma(p) * alpha(p) / 255] += 1
}
var window = 0
var best = 0
var bestStart = 0
for (v in 0 until 256) {
window += histogram[v]
if (v >= BACKGROUND_SPAN) {
window -= histogram[v - BACKGROUND_SPAN]
}
if (window > best) {
best = window
bestStart = maxOf(0, v - BACKGROUND_SPAN + 1)
}
}
return best >= BACKGROUND_MIN_SHARE * count && bestStart + BACKGROUND_SPAN / 2 >= LIGHT_BACKGROUND_MIN_GRAY
}
private fun luma(p: Int): Int {
val r = (p shr 16) and 0xff
val g = (p shr 8) and 0xff
val b = p and 0xff
return (r * 299 + g * 587 + b * 114) / 1000
}
private fun alpha(p: Int): Int = (p ushr 24) and 0xff
@JvmStatic
fun writeHeader(out: ByteArray, width: Int, height: Int) {
out[0] = (width and 0xff).toByte()
@@ -0,0 +1,378 @@
package com.faceclaw.app
import kotlin.jvm.JvmStatic
import kotlin.math.abs
import kotlin.math.cbrt
import kotlin.math.max
import kotlin.math.min
import kotlin.math.pow
import kotlin.math.roundToInt
import kotlin.math.sqrt
/**
* Turns a rendered notification icon into the lit-on-black gray silhouette the glasses' status
* bar shows: outputSize x outputSize bytes, 0 = off. Platform code picks the source drawables
* (small icon, launcher icon layers, large icon) and renders them to ARGB at [ANALYSIS_SCALE]
* times the output size; this object owns the pixel analysis, so the classification can be
* tested off-device.
*
* Small icons are supposed to be alpha templates: the platform tints them and ignores their
* color channels, which apps fill with anything (a brand color, Discord's noise, a theme tint).
* Some apps instead ship a filled color tile whose alpha is just a square or circle, with the
* shape carried by the colors; tinted, those become a blank block. The alpha's own shape tells
* the two apart, never the colors: alpha with holes or concavities is a template and is read by
* alpha alone, while a plain convex tile has its glyph pulled out of the tile's background color.
*/
object NotificationIconArt {
/** Sources are rendered at this multiple of the output size, analyzed, then box-filtered down. */
const val ANALYSIS_SCALE = 2
/** Output gray = lit^GAMMA, for lit (coverage or glyph strength) in 0..1. */
const val GAMMA = 1.6
/** Below this peak alpha an icon counts as invisible. */
private const val MIN_VISIBLE_ALPHA = 16
/**
* A plain tile fills at least this share of the pixel centers inside its own convex hull; a
* digitized convex shape fills all of them. Templates have notches, gaps and separate parts.
*/
private const val MIN_TILE_SOLIDITY = 0.92
/** A plain tile has alpha cut-outs (holes) covering at most this share of its area. */
private const val MAX_TILE_HOLE_SHARE = 0.005
/** OKLab distance from the tile's background color that still counts as background. */
private const val BACKGROUND_TOLERANCE = 0.1
/** Share of the tile's rim that must be background-colored for the tile to have one background. */
private const val MIN_BACKGROUND_RIM_SHARE = 0.8
/** Weakest OKLab contrast between glyph and background worth extracting; below it the tile is solid. */
private const val MIN_GLYPH_CONTRAST = 0.12
/** The quantile of a tile's distances from its background taken as the glyph's color contrast. */
private const val GLYPH_CONTRAST_QUANTILE = 0.98
/** A picture whose middle 90% of brightness spans less than this is left unstretched. */
private const val MIN_PICTURE_RANGE = 0.1
enum class Kind {
/** Nothing visible. */
EMPTY,
/** Alpha carries the shape: lit by alpha alone, colors ignored. */
TEMPLATE,
/** A plain tile with no glyph on it (a filled shape): lit by alpha like a template. */
SOLID,
/** A plain tile with a glyph on a one-color background: lit by color distance from the background. */
GLYPH,
/** A plain tile with no one background color (a photo, a gradient): lit by brightness. */
PICTURE,
}
class Art(val kind: Kind, val gray: ByteArray)
/** sRGB byte value to linear light. */
private val SRGB_TO_LINEAR = DoubleArray(256) { value ->
val c = value / 255.0
if (c <= 0.04045) c / 12.92 else ((c + 0.055) / 1.055).pow(2.4)
}
/**
* Classify and convert an icon rendered at (outputSize * ANALYSIS_SCALE) square, row-major
* ARGB (not premultiplied).
*/
@JvmStatic
fun convert(argb: IntArray, outputSize: Int): Art {
val size = analysisSize(argb, outputSize)
val maxAlpha = peakAlpha(argb)
if (maxAlpha < MIN_VISIBLE_ALPHA) {
return Art(Kind.EMPTY, ByteArray(outputSize * outputSize))
}
val mask = BooleanArray(argb.size) { alpha(argb[it]) * 2 >= maxAlpha }
if (!isPlainTile(mask, size)) {
return Art(Kind.TEMPLATE, toGray(alphaLit(argb, maxAlpha), size, outputSize))
}
return readTile(argb, mask, size, maxAlpha, outputSize)
}
/**
* Convert an icon known to be an alpha template (a themed icon's monochrome layer, a drawn
* initial) without classifying it: lit by alpha alone.
*/
@JvmStatic
fun template(argb: IntArray, outputSize: Int): Art {
val size = analysisSize(argb, outputSize)
val maxAlpha = peakAlpha(argb)
if (maxAlpha < MIN_VISIBLE_ALPHA) {
return Art(Kind.EMPTY, ByteArray(outputSize * outputSize))
}
return Art(Kind.TEMPLATE, toGray(alphaLit(argb, maxAlpha), size, outputSize))
}
private fun analysisSize(argb: IntArray, outputSize: Int): Int {
val size = outputSize * ANALYSIS_SCALE
require(outputSize > 0 && argb.size == size * size) { "expected ${size}x$size pixels, got ${argb.size}" }
return size
}
/**
* Whether the alpha is a plain tile (a square, circle or squircle, at any size) rather than
* a shape: no holes worth the name, and nearly convex.
*/
private fun isPlainTile(mask: BooleanArray, size: Int): Boolean {
val opaque = mask.count { it }
if (holeCount(mask, size) > opaque * MAX_TILE_HOLE_SHARE) {
return false
}
return solidity(mask, size, opaque) >= MIN_TILE_SOLIDITY
}
/** Transparent pixels the outside can't reach through other transparent pixels (4-connected). */
private fun holeCount(mask: BooleanArray, size: Int): Int {
val outside = BooleanArray(mask.size)
val stack = IntArray(mask.size)
var top = 0
fun visit(i: Int) {
if (!mask[i] && !outside[i]) {
outside[i] = true
stack[top++] = i
}
}
for (k in 0 until size) {
visit(k)
visit((size - 1) * size + k)
visit(k * size)
visit(k * size + size - 1)
}
while (top > 0) {
val i = stack[--top]
val x = i % size
val y = i / size
if (x > 0) visit(i - 1)
if (x < size - 1) visit(i + 1)
if (y > 0) visit(i - size)
if (y < size - 1) visit(i + size)
}
var holes = 0
for (i in mask.indices) {
if (!mask[i] && !outside[i]) holes += 1
}
return holes
}
/**
* The mask's pixel count over the number of pixel centers inside its convex hull, which
* Pick's theorem gives from the hull's area and the lattice points on its edges.
*/
private fun solidity(mask: BooleanArray, size: Int, opaque: Int): Double {
// Each row's leftmost and rightmost pixels are the only hull candidates. Generated row by
// row, they are already sorted by (y, x), the order the monotone chain below works in.
val xs = IntArray(size * 2)
val ys = IntArray(size * 2)
var count = 0
for (y in 0 until size) {
var left = -1
var right = -1
for (x in 0 until size) {
if (mask[y * size + x]) {
if (left < 0) left = x
right = x
}
}
if (left < 0) continue
xs[count] = left
ys[count] = y
count += 1
if (right != left) {
xs[count] = right
ys[count] = y
count += 1
}
}
// Cross product in (y, x) coordinates, matching the sort order.
fun cross(o: Int, a: Int, b: Int): Long =
(ys[a] - ys[o]).toLong() * (xs[b] - xs[o]) - (xs[a] - xs[o]).toLong() * (ys[b] - ys[o])
val hull = IntArray(count * 2 + 1)
var k = 0
for (i in 0 until count) {
while (k >= 2 && cross(hull[k - 2], hull[k - 1], i) <= 0) k -= 1
hull[k++] = i
}
val lowerSize = k + 1
for (i in count - 2 downTo 0) {
while (k >= lowerSize && cross(hull[k - 2], hull[k - 1], i) <= 0) k -= 1
hull[k++] = i
}
// The chain ends back at its first point; with one point there is no closing repeat.
val vertices = if (count == 1) 1 else k - 1
var twiceArea = 0L
var boundary = 0L
for (v in 0 until vertices) {
val a = hull[v]
val b = hull[(v + 1) % vertices]
twiceArea += xs[a].toLong() * ys[b] - xs[b].toLong() * ys[a]
boundary += gcd(abs(xs[b] - xs[a]), abs(ys[b] - ys[a]))
}
val latticePoints = abs(twiceArea) / 2.0 + boundary / 2.0 + 1.0
return opaque / latticePoints
}
private fun gcd(a: Int, b: Int): Int {
var x = a
var y = b
while (y != 0) {
val t = x % y
x = y
y = t
}
return x
}
private fun readTile(argb: IntArray, mask: BooleanArray, size: Int, maxAlpha: Int, outputSize: Int): Art {
val lab = DoubleArray(argb.size * 3)
for (i in argb.indices) {
if (mask[i]) toOklab(argb[i], lab, i * 3)
}
// The background is the color of the tile's rim, about one output pixel deep.
val rim = rimIndices(mask, size)
val background = DoubleArray(3) { channel ->
val values = DoubleArray(rim.size) { lab[rim[it] * 3 + channel] }
values.sort()
values[values.size / 2]
}
val rimDistances = DoubleArray(rim.size) { distance(lab, rim[it], background) }
rimDistances.sort()
val uniformRim = rimDistances.count { it <= BACKGROUND_TOLERANCE }
if (uniformRim < rim.size * MIN_BACKGROUND_RIM_SHARE) {
return Art(Kind.PICTURE, toGray(pictureLit(argb, mask, maxAlpha), size, outputSize))
}
// How far the background itself wanders, and how far the glyph stands from it.
val noise = quantile(rimDistances, MIN_BACKGROUND_RIM_SHARE)
val distances = DoubleArray(argb.size)
var tileCount = 0
for (i in argb.indices) {
if (mask[i]) {
distances[i] = distance(lab, i, background)
tileCount += 1
}
}
val tileDistances = DoubleArray(tileCount)
var next = 0
for (i in argb.indices) {
if (mask[i]) tileDistances[next++] = distances[i]
}
tileDistances.sort()
val contrast = quantile(tileDistances, GLYPH_CONTRAST_QUANTILE)
if (contrast < max(MIN_GLYPH_CONTRAST, 2 * noise)) {
return Art(Kind.SOLID, toGray(alphaLit(argb, maxAlpha), size, outputSize))
}
// Ramp from background to glyph; the glyph is lit whether it is lighter or darker than the tile.
val low = max(noise, 0.25 * contrast)
val high = 0.75 * contrast
val lit = DoubleArray(argb.size) { i ->
if (!mask[i]) {
0.0
} else {
val strength = ((distances[i] - low) / (high - low)).coerceIn(0.0, 1.0)
strength * min(1.0, alpha(argb[i]).toDouble() / maxAlpha)
}
}
return Art(Kind.GLYPH, toGray(lit, size, outputSize))
}
/** Tile pixels within one output pixel of its edge (or the image's edge). */
private fun rimIndices(mask: BooleanArray, size: Int): IntArray {
var inner = mask
repeat(ANALYSIS_SCALE) { inner = erode(inner, size) }
val rim = ArrayList<Int>()
for (i in mask.indices) {
if (mask[i] && !inner[i]) rim.add(i)
}
return rim.toIntArray()
}
private fun erode(mask: BooleanArray, size: Int): BooleanArray = BooleanArray(mask.size) { i ->
val x = i % size
val y = i / size
mask[i] && x > 0 && mask[i - 1] && x < size - 1 && mask[i + 1] &&
y > 0 && mask[i - size] && y < size - 1 && mask[i + size]
}
/** Lit by brightness, stretched so the picture's middle 90% spans the full range. */
private fun pictureLit(argb: IntArray, mask: BooleanArray, maxAlpha: Int): DoubleArray {
val brightness = DoubleArray(argb.size) { i ->
val p = argb[i]
(0.2126 * ((p shr 16) and 0xff) + 0.7152 * ((p shr 8) and 0xff) + 0.0722 * (p and 0xff)) / 255.0
}
val tile = ArrayList<Double>()
for (i in argb.indices) {
if (mask[i]) tile.add(brightness[i])
}
val sorted = tile.toDoubleArray()
sorted.sort()
var low = quantile(sorted, 0.05)
var high = quantile(sorted, 0.95)
if (high - low < MIN_PICTURE_RANGE) {
low = 0.0
high = 1.0
}
return DoubleArray(argb.size) { i ->
((brightness[i] - low) / (high - low)).coerceIn(0.0, 1.0) * min(1.0, alpha(argb[i]).toDouble() / maxAlpha)
}
}
/** Lit by alpha alone, normalized so a faint template (a theme's 54% tint) still lights fully. */
private fun alphaLit(argb: IntArray, maxAlpha: Int): DoubleArray =
DoubleArray(argb.size) { min(1.0, alpha(argb[it]).toDouble() / maxAlpha) }
/** Box-filter lit values down to the output size, then apply the tone curve. */
private fun toGray(lit: DoubleArray, size: Int, outputSize: Int): ByteArray {
val scale = size / outputSize
val gray = ByteArray(outputSize * outputSize)
for (oy in 0 until outputSize) {
for (ox in 0 until outputSize) {
var sum = 0.0
for (dy in 0 until scale) {
val row = (oy * scale + dy) * size + ox * scale
for (dx in 0 until scale) sum += lit[row + dx]
}
val mean = sum / (scale * scale)
gray[oy * outputSize + ox] = (255.0 * mean.coerceIn(0.0, 1.0).pow(GAMMA)).roundToInt().toByte()
}
}
return gray
}
private fun toOklab(pixel: Int, out: DoubleArray, offset: Int) {
val r = SRGB_TO_LINEAR[(pixel shr 16) and 0xff]
val g = SRGB_TO_LINEAR[(pixel shr 8) and 0xff]
val b = SRGB_TO_LINEAR[pixel and 0xff]
val l = cbrt(0.4122214708 * r + 0.5363325363 * g + 0.0514459929 * b)
val m = cbrt(0.2119034982 * r + 0.6806995451 * g + 0.1073969566 * b)
val s = cbrt(0.0883024619 * r + 0.2817188376 * g + 0.6299787005 * b)
out[offset] = 0.2104542553 * l + 0.7936177850 * m - 0.0040720468 * s
out[offset + 1] = 1.9779984951 * l - 2.4285922050 * m + 0.4505937099 * s
out[offset + 2] = 0.0259040371 * l + 0.7827717662 * m - 0.8086757660 * s
}
private fun distance(lab: DoubleArray, index: Int, background: DoubleArray): Double {
val dl = lab[index * 3] - background[0]
val da = lab[index * 3 + 1] - background[1]
val db = lab[index * 3 + 2] - background[2]
return sqrt(dl * dl + da * da + db * db)
}
private fun quantile(sorted: DoubleArray, q: Double): Double = sorted[((sorted.size - 1) * q).roundToInt()]
private fun peakAlpha(argb: IntArray): Int {
var peak = 0
for (p in argb) peak = max(peak, alpha(p))
return peak
}
private fun alpha(p: Int): Int = (p ushr 24) and 0xff
}
@@ -195,6 +195,28 @@ class IosProtocol {
fun png(data: NSData, width: Int, height: Int): NSData =
SharedScreenshots.encode4BitGrayPng(data.byteArray(), width, height).data()
/**
* A [widthLo, widthHi, heightLo, heightHi, B, G, R, A...] packet (little-endian ARGB words,
* as CoreGraphics draws them) to the gray packet Android's ImageFileLoader returns, with the
* same tone curve and dither. Malformed packets give empty data (no image) rather than an
* exception, which would terminate the app on its way into Objective-C.
*/
fun grayPacket(bgra: NSData, gamma: Float, dither: Boolean): NSData {
val bytes = bgra.byteArray()
if (bytes.size < GrayPacket.HEADER_SIZE) return NSData()
val width = (bytes[0].toInt() and 0xff) or ((bytes[1].toInt() and 0xff) shl 8)
val height = (bytes[2].toInt() and 0xff) or ((bytes[3].toInt() and 0xff) shl 8)
if (width <= 0 || height <= 0 || bytes.size != GrayPacket.HEADER_SIZE + width * height * 4) return NSData()
val pixels = IntArray(width * height) { i ->
val at = GrayPacket.HEADER_SIZE + i * 4
(bytes[at].toInt() and 0xff) or
((bytes[at + 1].toInt() and 0xff) shl 8) or
((bytes[at + 2].toInt() and 0xff) shl 16) or
((bytes[at + 3].toInt() and 0xff) shl 24)
}
return GrayPacket.fromArgb(pixels, width, height, gamma, dither).data()
}
}
/** iOS retains/coalesces surface updates in the same compositor Android uses. */
@@ -584,6 +584,7 @@ class IosBleCentral(private val platform: ProtocolPlatform = IosProtocolPlatform
private fun onDisconnected(peripheral: CBPeripheral, error: NSError?) {
val peer = remember(peripheral)
if (error != null) ConnectionCounters.increment("ble.ios-disconnect-error." + error.code)
failPeer(peer, error?.localizedDescription ?: "Disconnected", notifyListeners = true)
}
@@ -15,9 +15,13 @@ interface IosAncsListener {
fun onRelayFrame(data: NSData)
}
/** Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue. */
/**
* Audio packets for the NativeScript side as NSData (a Kotlin ByteArray crosses per byte). Main queue.
* Not named onAudioPacket: that selector collides with FaceclawAudioPacketListener's, and the
* mangled name Kotlin/Native picks broke the TypeScript proxy ("unrecognized selector").
*/
interface IosAudioPacketListener {
fun onAudioPacket(data: NSData, arm: String, arrivalMs: Long)
fun onNsDataAudioPacket(data: NSData, arm: String, arrivalMs: Long)
}
/**
@@ -187,7 +191,7 @@ class IosGlassesSession internal constructor(
fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener)
fun sendFaceclawMicControl(record: NSData?, label: String, rightTemple: Boolean, leftTemple: Boolean) =
fun sendFaceclawMicControl(record: NSData?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
core.sendFaceclawMicControl(record?.byteArray(), label, rightTemple, leftTemple)
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean =
@@ -215,14 +219,21 @@ class IosGlassesSession internal constructor(
/** Set the compositor's output frame size. Call before configuring surfaces. */
fun configureCompositorScreen(width: Int, height: Int) = core.configureCompositorScreen(width, height)
fun setSurfaceVisible(id: String, visible: Boolean) = core.setSurfaceVisible(id, visible)
// The compositor throws for an unknown surface or a bad rect, and without @Throws (which
// would rename the selectors) that aborts the process. Log and drop instead; false then.
fun setSurfaceDepth(id: String, depth: Int) = core.setSurfaceDepth(id, depth)
fun setSurfaceVisible(id: String, visible: Boolean): Boolean = surfaceCall("setSurfaceVisible $id") {
core.setSurfaceVisible(id, visible)
}
fun setSurfaceDepth(id: String, depth: Int): Boolean = surfaceCall("setSurfaceDepth $id") {
core.setSurfaceDepth(id, depth)
}
fun setScreenBlanked(blanked: Boolean) = core.setScreenBlanked(blanked)
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int) =
core.configureSurface(id, x, y, width, height, zOrder, transparency)
fun configureSurface(id: String, x: Int, y: Int, width: Int, height: Int, zOrder: Int, transparency: Int): Boolean =
surfaceCall("configureSurface $id") { core.configureSurface(id, x, y, width, height, zOrder, transparency) }
fun removeSurface(id: String) = core.removeSurface(id)
@@ -241,10 +252,28 @@ class IosGlassesSession internal constructor(
paintMs: Int,
frameId: Int,
glyphs: NSData?,
) = core.submitSurfaceFrame(
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
glyphs?.let { IosByteReader(it) },
)
): Boolean {
val ok = surfaceCall("submitSurfaceFrame $surfaceId") {
core.submitSurfaceFrame(
IosByteReader(pixels8bpp), surfaceId, rectX, rectY, rectWidth, rectHeight, contentFingerprint, paintMs, frameId,
glyphs?.let { IosByteReader(it) },
)
}
if (!ok) frameTimings.finishFrame(frameId, "discarded: surface $surfaceId rejected")
return ok
}
private inline fun surfaceCall(what: String, block: () -> Unit): Boolean =
try {
block()
true
} catch (e: IllegalArgumentException) {
core.logLine("$what dropped: ${e.message}")
false
} catch (e: IllegalStateException) {
core.logLine("$what dropped: ${e.message}")
false
}
/** The current composited screen (gray, 8bpp) or null before any surface was configured. */
fun previewComposite(): IosTextureFrame? = core.previewComposite()?.let { IosTextureFrame(it) }
@@ -271,7 +300,7 @@ class IosGlassesSession internal constructor(
override fun onAudioPacket(data: ByteArray?, arm: String?, arrivalMs: Long) {
val packet = (data ?: ByteArray(0)).data()
val label = arm ?: ""
dispatch_async(dispatch_get_main_queue()) { delegate.onAudioPacket(packet, label, arrivalMs) }
dispatch_async(dispatch_get_main_queue()) { delegate.onNsDataAudioPacket(packet, label, arrivalMs) }
}
}
@@ -57,8 +57,9 @@ internal object IosStreamingHttp : StreamingHttp {
}
val request = buildRequest(target, "POST", body, listOf("Content-Type" to "application/json; charset=utf-8", "Accept" to "text/event-stream") + headers)
val configuration = NSURLSessionConfiguration.ephemeralSessionConfiguration
// An idle limit only, like Android's OkHttp read timeout: callers cap a stream's
// total time, and a Hermes agent turn can stream for many minutes.
configuration.timeoutIntervalForRequest = 180.0
configuration.timeoutIntervalForResource = 180.0
val delegate = StreamingDelegate(sink)
val session = NSURLSession.sessionWithConfiguration(configuration, delegate, delegateQueue = null)
val task = session.dataTaskWithRequest(request)