mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' into ring-health-fixes
This commit is contained in:
commit
4f8c082b52
368 files changed
+26238
-3401
No files matched your search
@@ -0,0 +1,6 @@
|
||||
# Check text out with LF everywhere, so Windows checkouts keep runnable bash
|
||||
# scripts and the LF-anchored source regexes in tests. Batch files are the
|
||||
# exception: cmd.exe can miss goto/call labels in LF-only batch files.
|
||||
* text=auto eol=lf
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
@@ -26,3 +26,48 @@ jobs:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npm run test:kotlin
|
||||
|
||||
# The stats.faceclaw.org server: typecheck, and the tests including the
|
||||
# database one against a throwaway Postgres.
|
||||
stats-server:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: stats-server
|
||||
services:
|
||||
postgres:
|
||||
image: postgres:16
|
||||
env:
|
||||
POSTGRES_PASSWORD: postgres
|
||||
ports: ["5432:5432"]
|
||||
options: >-
|
||||
--health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10
|
||||
env:
|
||||
PGHOST: localhost
|
||||
PGUSER: postgres
|
||||
PGPASSWORD: postgres
|
||||
STATS_TEST_DATABASE: postgres
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: 22
|
||||
- run: npm ci
|
||||
- run: npm run check
|
||||
- run: npm test
|
||||
|
||||
# The @faceclaw/evenhub-extensions npm package: typecheck (including
|
||||
# compatibility with the stock SDK's container classes), build, and check
|
||||
# that the committed API reference matches the source.
|
||||
evenhub-extensions:
|
||||
runs-on: ubuntu-latest
|
||||
defaults:
|
||||
run:
|
||||
working-directory: evenhub-extensions
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- run: npm ci
|
||||
- run: npm run check
|
||||
- run: npm run build
|
||||
- run: npm run docs
|
||||
- run: git diff --exit-code -- docs
|
||||
@@ -35,6 +35,7 @@ typings/
|
||||
App_Resources/Android/src/main/jniLibs/**/*.so
|
||||
App_Resources/Android/src/main/assets/faceclaw-voice-asr/
|
||||
old
|
||||
__pycache__
|
||||
|
||||
# Visual Studio Code
|
||||
.vscode/*
|
||||
|
||||
+1
-1
@@ -78,7 +78,7 @@
|
||||
"node_modules/**",
|
||||
"platforms/**",
|
||||
"dist/**",
|
||||
"faceclaw-extensions/dist/**",
|
||||
"evenhub-extensions/dist/**",
|
||||
"old/**",
|
||||
"tmp/**"
|
||||
]
|
||||
|
||||
@@ -13,6 +13,9 @@ and others who contributed to documenting the G2's bluetooth protocol and firmwa
|
||||
|
||||
G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3
|
||||
|
||||
Weather data outside the US is by Open-Meteo.com: https://open-meteo.com/. It is
|
||||
licensed under CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/).
|
||||
|
||||
The Terminus font is by Dimitar Zhekov's Terminus: https://framagit.org/ohnonot/terv-terc
|
||||
and is distributed under the SIL Open Font License; see app/fonts/terminus/LICENSE.
|
||||
|
||||
|
||||
@@ -46,6 +46,17 @@ def faceclawNdkHostTag = {
|
||||
return "linux-x86_64"
|
||||
}
|
||||
|
||||
// Host tools carry .exe on Windows. Call the binaries themselves: the NDK's
|
||||
// target-prefixed clang wrappers are bash scripts (or .cmd files) that only add
|
||||
// --target.
|
||||
def faceclawHostExecutable = { File dir, String name ->
|
||||
return file("$dir/$name${Os.isFamily(Os.FAMILY_WINDOWS) ? ".exe" : ""}")
|
||||
}
|
||||
|
||||
def faceclawNdkTool = { File ndkRoot, String name ->
|
||||
return faceclawHostExecutable(file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin"), name)
|
||||
}
|
||||
|
||||
def faceclawFindNdkRoot = {
|
||||
def explicitNdk = System.getenv("ANDROID_NDK_HOME")
|
||||
if (explicitNdk != null && file(explicitNdk).exists()) {
|
||||
@@ -111,7 +122,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
if (ndkRoot == null || !ndkRoot.exists()) {
|
||||
throw new GradleException("Android NDK is required to build libfaceclaw_lc3.so")
|
||||
}
|
||||
def clang = file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/aarch64-linux-android24-clang")
|
||||
def clang = faceclawNdkTool(ndkRoot, "clang")
|
||||
if (!clang.exists()) {
|
||||
throw new GradleException("Could not find Android clang at $clang")
|
||||
}
|
||||
@@ -125,6 +136,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
exec {
|
||||
commandLine([
|
||||
clang.absolutePath,
|
||||
"--target=aarch64-linux-android24",
|
||||
"-shared",
|
||||
"-fPIC",
|
||||
"-O3",
|
||||
@@ -192,8 +204,8 @@ tasks.register("prepareFaceclawLlama") {
|
||||
if (cmakeDir == null) {
|
||||
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
|
||||
}
|
||||
def cmake = file("$cmakeDir/bin/cmake")
|
||||
def ninja = file("$cmakeDir/bin/ninja")
|
||||
def cmake = faceclawHostExecutable(file("$cmakeDir/bin"), "cmake")
|
||||
def ninja = faceclawHostExecutable(file("$cmakeDir/bin"), "ninja")
|
||||
def march = "-march=armv8.2-a+dotprod+fp16"
|
||||
|
||||
// Configure once; afterwards the ninja no-op build keeps this cheap while
|
||||
@@ -227,14 +239,13 @@ tasks.register("prepareFaceclawLlama") {
|
||||
}
|
||||
// The static-linked lib carries llama.cpp's full debug info (~50MB);
|
||||
// strip it so the APK only pays for the code (~8MB).
|
||||
def ndkRootForStrip = faceclawFindNdkRoot()
|
||||
if (ndkRootForStrip != null) {
|
||||
def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
def llvmStrip = faceclawNdkTool(ndkRoot, "llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
} else {
|
||||
logger.warn("Could not find llvm-strip at $llvmStrip; packaging an unstripped libfaceclaw_llama.so (~50MB)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,7 +262,8 @@ if (faceclawVersionFile.exists()) {
|
||||
}
|
||||
}
|
||||
def faceclawVersionCode = {
|
||||
def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
// Prerelease labels belong in versionName, not the numeric versionCode.
|
||||
def parts = (faceclawVersionName.split('-')[0].tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
return parts[0] * 10000 + parts[1] * 100 + parts[2]
|
||||
}()
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ import java.io.IOException
|
||||
/**
|
||||
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
|
||||
* container, cutting speech recordings to roughly a tenth of their WAV size.
|
||||
* Runs on its own thread; the source WAV is deleted on success when requested.
|
||||
* Runs on its own thread; when requested, the source WAV is deleted once the
|
||||
* listener's onDone has returned normally.
|
||||
*/
|
||||
class FaceclawAudioTranscoder(
|
||||
private val wavPath: String?,
|
||||
@@ -66,19 +67,38 @@ class FaceclawAudioTranscoder(
|
||||
private fun run() {
|
||||
try {
|
||||
transcode()
|
||||
val size = File(outputPath).length()
|
||||
if (deleteSourceOnSuccess) {
|
||||
File(wavPath).delete()
|
||||
}
|
||||
if (listener != null) {
|
||||
listener.onDone(outputPath, size)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "transcode failed for $wavPath", t)
|
||||
File(outputPath).delete()
|
||||
if (listener != null) {
|
||||
listener.onError(if (t.message == null) t.toString() else t.message)
|
||||
try {
|
||||
listener?.onError(if (t.message == null) t.toString() else t.message)
|
||||
} catch (listenerError: Throwable) {
|
||||
Log.w(TAG, "transcode error listener failed", listenerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
val size = File(outputPath).length()
|
||||
if (size <= 0) {
|
||||
Log.w(TAG, "transcode produced no output for $wavPath")
|
||||
File(outputPath).delete()
|
||||
try {
|
||||
listener?.onError("transcode produced no output")
|
||||
} catch (listenerError: Throwable) {
|
||||
Log.w(TAG, "transcode error listener failed", listenerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
try {
|
||||
listener?.onDone(outputPath, size)
|
||||
} catch (t: Throwable) {
|
||||
// The caller never took up the new file, so whatever it recorded still names the
|
||||
// source: keep that and drop the output, rather than lose both.
|
||||
Log.w(TAG, "transcode listener failed for $wavPath", t)
|
||||
File(outputPath).delete()
|
||||
return
|
||||
}
|
||||
if (deleteSourceOnSuccess) {
|
||||
File(wavPath).delete()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +113,7 @@ class FaceclawAudioTranscoder(
|
||||
|
||||
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
|
||||
var muxer: MediaMuxer? = null
|
||||
var muxerStopped = false
|
||||
var input: FileInputStream? = null
|
||||
try {
|
||||
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
||||
@@ -159,6 +180,11 @@ class FaceclawAudioTranscoder(
|
||||
}
|
||||
}
|
||||
}
|
||||
// stop() writes the index (moov); without it the file won't play, so its failure
|
||||
// (storage full, I/O error, a muxer that never started) fails the transcode rather
|
||||
// than reporting a broken output as done.
|
||||
muxerStopped = true
|
||||
muxer.stop()
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
@@ -172,9 +198,12 @@ class FaceclawAudioTranscoder(
|
||||
}
|
||||
codec.release()
|
||||
if (muxer != null) {
|
||||
try {
|
||||
muxer.stop()
|
||||
} catch (ignored: Throwable) {
|
||||
if (!muxerStopped) {
|
||||
// Already failing; the output is deleted either way.
|
||||
try {
|
||||
muxer.stop()
|
||||
} catch (ignored: Throwable) {
|
||||
}
|
||||
}
|
||||
muxer.release()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
|
||||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import okhttp3.Call
|
||||
import okhttp3.Callback
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import okhttp3.Response
|
||||
|
||||
/**
|
||||
* One POST of a binary body (a Whisper server WAV upload), sent by start().
|
||||
* NativeScript's http module hands an ArrayBuffer body to Java one bridge call
|
||||
* per byte, which for a ~1 MB upload stalls the JS thread; here the body
|
||||
* arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk.
|
||||
* headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and
|
||||
* may carry the Content-Type. The response text and status reach the listener
|
||||
* on the constructing thread's Looper.
|
||||
*/
|
||||
class FaceclawBinaryPost(
|
||||
url: String?,
|
||||
body: ByteBuffer?,
|
||||
headers: Array<String?>?,
|
||||
timeoutMs: Long,
|
||||
listener: FaceclawBinaryPostListener?
|
||||
) {
|
||||
private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
|
||||
private val listener: FaceclawBinaryPostListener =
|
||||
listener ?: throw IllegalArgumentException("listener is required")
|
||||
private val call: Call
|
||||
|
||||
@Volatile
|
||||
private var cancelled = false
|
||||
|
||||
init {
|
||||
if (url == null || url.trim().isEmpty()) {
|
||||
throw IllegalArgumentException("url is required")
|
||||
}
|
||||
val bytes = ByteArray(body?.remaining() ?: 0)
|
||||
body?.duplicate()?.get(bytes)
|
||||
// No media type on the body, so okhttp keeps the caller's Content-Type header.
|
||||
val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody())
|
||||
for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value)
|
||||
call = getClient().newCall(builder.build())
|
||||
call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
|
||||
fun start() {
|
||||
call.enqueue(object : Callback {
|
||||
override fun onFailure(call: Call, e: IOException) {
|
||||
post { listener.onFailure(e.message ?: e.toString()) }
|
||||
}
|
||||
|
||||
override fun onResponse(call: Call, response: Response) {
|
||||
val text = try {
|
||||
response.use { it.body?.string() ?: "" }
|
||||
} catch (e: IOException) {
|
||||
post { listener.onFailure(e.message ?: e.toString()) }
|
||||
return
|
||||
}
|
||||
post { listener.onResponse(response.code, text) }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** Aborts the request; no callback follows. */
|
||||
fun cancel() {
|
||||
cancelled = true
|
||||
call.cancel()
|
||||
}
|
||||
|
||||
private fun post(runnable: () -> Unit) {
|
||||
callbackHandler.post {
|
||||
if (!cancelled) runnable()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
@Volatile
|
||||
private var sharedClient: OkHttpClient? = null
|
||||
|
||||
private fun getClient(): OkHttpClient {
|
||||
if (sharedClient == null) {
|
||||
synchronized(FaceclawBinaryPost::class.java) {
|
||||
if (sharedClient == null) {
|
||||
sharedClient = OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.addInterceptor(FaceclawHttp.userAgentInterceptor())
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
return sharedClient!!
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,7 +294,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener)
|
||||
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) =
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
|
||||
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
|
||||
@@ -329,6 +329,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||
}
|
||||
|
||||
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||
val composite = core.previewComposite() ?: return null
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||
cropX, cropY, cropWidth, cropHeight)
|
||||
}
|
||||
|
||||
/**
|
||||
* Posted to the main looper after each step of an animation the preview is replaying (menu
|
||||
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
|
||||
|
||||
@@ -550,6 +550,7 @@ class FaceclawBleManager(context: Context) {
|
||||
override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) {
|
||||
val address = gatt.device.address
|
||||
Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState")
|
||||
if (status != BluetoothGatt.GATT_SUCCESS) ConnectionCounters.increment("ble.gatt-status.$status")
|
||||
val link = links[address]
|
||||
val current = link?.gatt
|
||||
if (link == null || (current != null && current !== gatt)) {
|
||||
|
||||
@@ -68,6 +68,11 @@ class FaceclawCaptionEngine {
|
||||
session.stop()
|
||||
}
|
||||
|
||||
/** Stop without blocking; the listener's onStopped follows the trailing utterance. */
|
||||
fun requestStop() {
|
||||
session.requestStop()
|
||||
}
|
||||
|
||||
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
|
||||
fun acceptPcm(pcm16le: ByteArray?) {
|
||||
session.acceptPcm(pcm16le)
|
||||
|
||||
@@ -718,6 +718,94 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
|
||||
}
|
||||
}
|
||||
|
||||
// ---- crash recovery ----
|
||||
|
||||
/**
|
||||
* Close out sessions a dead process left open (ended_at NULL): the end becomes the last
|
||||
* stored segment's end (or the start), and the tone average comes from the stored segments.
|
||||
* excludeIdsJson is a JSON array of session ids that are legitimately still open. Returns
|
||||
* the sessions it closed as a JSON array of {id, audioPath, audioCodec}.
|
||||
*/
|
||||
fun closeInterruptedSessions(excludeIdsJson: String?): String {
|
||||
val exclude = HashSet<Long>()
|
||||
if (excludeIdsJson != null && !excludeIdsJson.isEmpty()) {
|
||||
val ids = JSONArray(excludeIdsJson)
|
||||
for (i in 0 until ids.length()) {
|
||||
exclude.add(ids.getLong(i))
|
||||
}
|
||||
}
|
||||
val db = writableDatabase
|
||||
val closed = JSONArray()
|
||||
val closedIds = ArrayList<Long>()
|
||||
db.rawQuery("SELECT id, audio_path, audio_codec FROM sessions WHERE ended_at IS NULL", null).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
val id = c.getLong(0)
|
||||
if (id in exclude) continue
|
||||
closedIds.add(id)
|
||||
val row = JSONObject()
|
||||
row.put("id", id)
|
||||
row.put("audioPath", if (c.isNull(1)) JSONObject.NULL else c.getString(1))
|
||||
row.put("audioCodec", if (c.isNull(2)) JSONObject.NULL else c.getString(2))
|
||||
closed.put(row)
|
||||
}
|
||||
}
|
||||
for (id in closedIds) {
|
||||
db.execSQL(
|
||||
"UPDATE sessions SET"
|
||||
+ " ended_at = COALESCE((SELECT MAX(ended_at) FROM segments WHERE session_id = ?), started_at),"
|
||||
+ " avg_sentiment = COALESCE((SELECT AVG(sentiment) FROM segments WHERE session_id = ?), 0)"
|
||||
+ " WHERE id = ?",
|
||||
arrayOf<Any?>(id, id, id))
|
||||
}
|
||||
return closed.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Closed sessions whose audio is still the WAV: a transcode that never finished (the process
|
||||
* died mid-transcode, or recovery failed partway on an earlier boot). Returns a JSON array of
|
||||
* {id, audioPath}.
|
||||
*/
|
||||
fun listUntranscodedSessions(): String {
|
||||
val sessions = JSONArray()
|
||||
readableDatabase.rawQuery(
|
||||
"SELECT id, audio_path FROM sessions"
|
||||
+ " WHERE ended_at IS NOT NULL AND audio_codec = 'wav' AND audio_path IS NOT NULL",
|
||||
null,
|
||||
).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
val row = JSONObject()
|
||||
row.put("id", c.getLong(0))
|
||||
row.put("audioPath", c.getString(1))
|
||||
sessions.put(row)
|
||||
}
|
||||
}
|
||||
return sessions.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete recordings in dirPath that no session's audio_path names: files a session row never
|
||||
* learned about, or partial transcoder output. Matched by file name, so a storage move that
|
||||
* changed the directory's absolute path can't make live recordings look orphaned. Only call
|
||||
* while no recording or transcode is in flight. Returns how many files were deleted.
|
||||
*/
|
||||
fun deleteUnreferencedRecordings(dirPath: String?): Int {
|
||||
if (dirPath == null) return 0
|
||||
val files = File(dirPath).listFiles() ?: return 0
|
||||
val referenced = HashSet<String>()
|
||||
readableDatabase.rawQuery("SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL", null).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
referenced.add(File(c.getString(0)).name)
|
||||
}
|
||||
}
|
||||
var deleted = 0
|
||||
for (file in files) {
|
||||
if (file.isFile && file.name !in referenced && file.delete()) {
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
fun deleteSession(sessionId: Long) {
|
||||
val db = writableDatabase
|
||||
val path = getSessionAudioPath(sessionId)
|
||||
|
||||
@@ -1,32 +1,100 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.net.Uri
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.webkit.JavascriptInterface
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import androidx.webkit.JavaScriptReplyProxy
|
||||
import androidx.webkit.WebMessageCompat
|
||||
import androidx.webkit.WebViewCompat
|
||||
import androidx.webkit.WebViewFeature
|
||||
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONException
|
||||
|
||||
/**
|
||||
* The object injected into a hosted EvenHub app's WebView as
|
||||
* window.__faceclawEvenHub. The document-start shim (served inline by
|
||||
* FaceclawEvenHubWebViewClient) wraps it in a promise-returning
|
||||
* window.flutter_inappwebview.callHandler, which is what the EvenHub SDK
|
||||
* actually talks to.
|
||||
* The host end of a hosted EvenHub app's bridge: window.__faceclawEvenHubPort,
|
||||
* a WebMessageListener object that WebView injects only into frames from the
|
||||
* app's own origin. Sites the app navigates to, and other-origin iframes it
|
||||
* embeds, never see it, and messages from the app's own subframes are dropped,
|
||||
* so only the app's main document reaches the host (its SDK storage, sensors,
|
||||
* glasses display and the Faceclaw extensions, API keys included). This
|
||||
* replaced an addJavascriptInterface object, which WebView exposes to every
|
||||
* frame of every page.
|
||||
*
|
||||
* JavascriptInterface methods run on a WebView-internal thread; everything
|
||||
* is bounced to the main thread before touching the NativeScript listener.
|
||||
* The transport script (webview.ts) wraps the port as window.__faceclawEvenHub,
|
||||
* whose postMessage(name, argsJson, id) and wakeTimers() the stock bridge shim
|
||||
* and the extensions call. Messages are JSON arrays: ["call", name, argsJson,
|
||||
* id] or ["wake"].
|
||||
*/
|
||||
class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) {
|
||||
class FaceclawEvenHubJsBridge private constructor(
|
||||
private val listener: FaceclawEvenHubListener
|
||||
) : WebViewCompat.WebMessageListener {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
@JavascriptInterface
|
||||
fun postMessage(handlerName: String?, argsJson: String?, callId: Int) {
|
||||
mainHandler.post {
|
||||
listener.onEvenAppMessage(handlerName, argsJson, callId)
|
||||
override fun onPostMessage(
|
||||
view: WebView,
|
||||
message: WebMessageCompat,
|
||||
sourceOrigin: Uri,
|
||||
isMainFrame: Boolean,
|
||||
replyProxy: JavaScriptReplyProxy
|
||||
) {
|
||||
if (!isMainFrame) return
|
||||
val parts = try {
|
||||
JSONArray(message.data ?: return)
|
||||
} catch (e: JSONException) {
|
||||
return
|
||||
}
|
||||
when (parts.optString(0)) {
|
||||
"call" -> {
|
||||
val name = parts.optString(1)
|
||||
val argsJson = parts.optString(2)
|
||||
val callId = parts.optInt(3)
|
||||
mainHandler.post { listener.onEvenAppMessage(name, argsJson, callId) }
|
||||
}
|
||||
// The page's timer shim has work due before the host's next tick.
|
||||
"wake" -> FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
|
||||
}
|
||||
}
|
||||
|
||||
/** The page's timer shim has work due before the host's next tick. */
|
||||
@JavascriptInterface
|
||||
fun wakeTimers() {
|
||||
FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
private const val OBJECT_NAME = "__faceclawEvenHubPort"
|
||||
|
||||
/**
|
||||
* The origin rule matching exactly [url]'s origin (default port elided),
|
||||
* or null when it isn't an http(s) URL with a host.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun originRule(url: String): String? {
|
||||
val uri = Uri.parse(url)
|
||||
val scheme = uri.scheme?.lowercase()
|
||||
val host = uri.host?.lowercase()
|
||||
if ((scheme != "http" && scheme != "https") || host.isNullOrEmpty()) return null
|
||||
val defaultPort = if (scheme == "https") 443 else 80
|
||||
return if (uri.port == -1 || uri.port == defaultPort) "$scheme://$host" else "$scheme://$host:${uri.port}"
|
||||
}
|
||||
|
||||
/**
|
||||
* Expose the bridge to [webView]'s frames from [originRule]. Must run
|
||||
* before the WebView loads anything.
|
||||
*
|
||||
* @return null on success, else why the bridge could not be installed.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun install(webView: WebView, originRule: String, listener: FaceclawEvenHubListener): String? {
|
||||
if (!WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
|
||||
Log.w(TAG, "WebMessageListener unsupported; not exposing the EvenHub bridge")
|
||||
return "This Android System WebView is too old to run EvenHub apps safely. Update it, then reopen the app."
|
||||
}
|
||||
return try {
|
||||
WebViewCompat.addWebMessageListener(webView, OBJECT_NAME, setOf(originRule), FaceclawEvenHubJsBridge(listener))
|
||||
null
|
||||
} catch (e: IllegalArgumentException) {
|
||||
Log.w(TAG, "bridge origin rule rejected: $originRule", e)
|
||||
"The app's address ($originRule) can't be used as a web origin."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import androidx.webkit.ProfileStore
|
||||
import androidx.webkit.WebViewCompat
|
||||
import androidx.webkit.WebViewFeature
|
||||
|
||||
import java.security.MessageDigest
|
||||
|
||||
/**
|
||||
* Gives each EvenHub package its own WebView profile, so one app's browser
|
||||
* storage (cookies, localStorage, IndexedDB, caches, service workers) is out of
|
||||
* reach of every other app. That includes the storage of third-party sites an
|
||||
* app navigates to, such as a sign-in provider's session cookies, which in the
|
||||
* shared default profile any other app's WebView could reuse. The per-app fake
|
||||
* origins alone don't cover that, and `*.evenhub.invalid` hosts share a site,
|
||||
* so domain cookies would cross between packaged apps too.
|
||||
*
|
||||
* The profile persists across launches, and is shared by every instance of the
|
||||
* same package, like the iOS per-package WKWebsiteDataStore.
|
||||
*/
|
||||
class FaceclawEvenHubProfile private constructor() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
|
||||
/** Stable per package; the hash input matches the iOS data-store identifier's. */
|
||||
@JvmStatic
|
||||
fun nameFor(packageId: String): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest("com.faceclaw.ehpk:$packageId".toByteArray(Charsets.UTF_8))
|
||||
return "evenhub-" + digest.take(16).joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a new WebView in [packageId]'s profile. Must run on the main thread
|
||||
* before anything else is done with the WebView.
|
||||
*
|
||||
* @return false when the WebView provider lacks multi-profile support,
|
||||
* leaving the WebView in the shared default profile.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun apply(webView: WebView, packageId: String): Boolean {
|
||||
if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) {
|
||||
Log.w(TAG, "multi-profile WebView unsupported; $packageId shares the default profile")
|
||||
return false
|
||||
}
|
||||
val name = nameFor(packageId)
|
||||
ProfileStore.getInstance().getOrCreateProfile(name)
|
||||
WebViewCompat.setProfile(webView, name)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.faceclaw.app
|
||||
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.webkit.RenderProcessGoneDetail
|
||||
import android.webkit.WebResourceRequest
|
||||
import android.webkit.WebResourceResponse
|
||||
import android.webkit.WebView
|
||||
@@ -33,6 +34,11 @@ import java.nio.charset.StandardCharsets
|
||||
* The shim is registered as a document-start script instead
|
||||
* (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for
|
||||
* WebView versions that lack that API.
|
||||
*
|
||||
* An app can crash its renderer, or the system can kill it for memory. Unless
|
||||
* every WebView that renderer served handles that, WebView aborts the whole
|
||||
* process, taking Faceclaw down with it; so onRenderProcessGone retires the
|
||||
* affected WebView and reports it instead.
|
||||
*/
|
||||
class FaceclawEvenHubWebViewClient(
|
||||
private val rootDir: String?,
|
||||
@@ -93,6 +99,15 @@ class FaceclawEvenHubWebViewClient(
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean {
|
||||
// Called on the main thread for each WebView the renderer served. This
|
||||
// one can't be used again: drop it, and the app shows as failed.
|
||||
Log.w(TAG, "renderer gone (crashed=${detail.didCrash()}) for $host")
|
||||
FaceclawEvenHubWebViewHost.getInstance().detach(view)
|
||||
listener.onRenderProcessGone(detail.didCrash())
|
||||
return true
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
|
||||
|
||||
@@ -59,8 +59,8 @@ import java.util.HashMap
|
||||
* requestAnimationFrame) with JS queues fired by window.__fcTick(), and this
|
||||
* host ticks those on the main thread regardless of screen state. The tick
|
||||
* returns when the page next has work, so an idle page isn't evaluated at
|
||||
* 60Hz; work scheduled between ticks wakes the host through
|
||||
* FaceclawEvenHubJsBridge.wakeTimers. The main Looper keeps running under the
|
||||
* 60Hz; work scheduled between ticks wakes the host through the bridge's
|
||||
* "wake" message (FaceclawEvenHubJsBridge). The main Looper keeps running under the
|
||||
* foreground service, so the tick survives the screen turning off. (Matches the
|
||||
* official Even app's approach.)
|
||||
*
|
||||
@@ -105,8 +105,12 @@ class FaceclawEvenHubWebViewHost {
|
||||
* the page wakes it. It also re-ticks after IDLE_POLL_MS regardless, so a
|
||||
* lost evaluateJavascript callback (e.g. mid-navigation) or a page whose
|
||||
* wake didn't reach us stalls its timers for at most that long.
|
||||
*
|
||||
* The tick only runs in the app's own documents ([appCheck], a JS
|
||||
* expression): after a navigation the WebView may show another site.
|
||||
*/
|
||||
private inner class Ticker(private val web: WebView) : Runnable {
|
||||
private inner class Ticker(private val web: WebView, appCheck: String) : Runnable {
|
||||
private val tickJs = "if ($appCheck) { $TICK_JS }"
|
||||
private var inFlight = false
|
||||
private var wakePending = false
|
||||
private var stopped = false
|
||||
@@ -114,7 +118,7 @@ class FaceclawEvenHubWebViewHost {
|
||||
override fun run() {
|
||||
if (stopped) return
|
||||
inFlight = true
|
||||
web.evaluateJavascript(TICK_JS) { result -> onTicked(result) }
|
||||
web.evaluateJavascript(tickJs) { result -> onTicked(result) }
|
||||
schedule(IDLE_POLL_MS)
|
||||
}
|
||||
|
||||
@@ -278,8 +282,11 @@ class FaceclawEvenHubWebViewHost {
|
||||
(web.context as? MutableContextWrapper)?.baseContext = activity
|
||||
}
|
||||
|
||||
/** Add a WebView to the host, full-size and rendering, hidden behind the UI. */
|
||||
fun attach(activity: Activity, web: WebView) {
|
||||
/**
|
||||
* Add a WebView to the host, full-size and rendering, hidden behind the UI.
|
||||
* [appCheck] is a JS expression that is true only in the app's own documents.
|
||||
*/
|
||||
fun attach(activity: Activity, web: WebView, appCheck: String) {
|
||||
ensureOverlay(activity)
|
||||
rebaseContext(web, activity)
|
||||
web.visibility = View.VISIBLE
|
||||
@@ -301,7 +308,7 @@ class FaceclawEvenHubWebViewHost {
|
||||
}
|
||||
if (!webViews.contains(web)) webViews.add(web)
|
||||
if (!tickers.containsKey(web)) {
|
||||
val ticker = Ticker(web)
|
||||
val ticker = Ticker(web, appCheck)
|
||||
tickers[web] = ticker
|
||||
mainHandler.postDelayed(ticker, TICK_MS)
|
||||
}
|
||||
@@ -348,9 +355,9 @@ class FaceclawEvenHubWebViewHost {
|
||||
return shown
|
||||
}
|
||||
|
||||
/** Remove and destroy a WebView (its app is closing). */
|
||||
/** Remove and destroy a WebView (its app is closing, or its renderer is gone). Idempotent. */
|
||||
fun detach(web: WebView) {
|
||||
webViews.remove(web)
|
||||
if (!webViews.remove(web)) return
|
||||
tickers.remove(web)?.stop()
|
||||
stack?.removeView(web)
|
||||
web.destroy()
|
||||
|
||||
@@ -46,6 +46,7 @@ class FaceclawMediaController(context: Context) {
|
||||
|
||||
override fun onMetadataChanged(metadata: MediaMetadata?) {
|
||||
synchronized(lock) {
|
||||
artVersion++
|
||||
emitStateLocked()
|
||||
}
|
||||
}
|
||||
@@ -92,6 +93,15 @@ class FaceclawMediaController(context: Context) {
|
||||
private var sessionsListenerRegistered = false
|
||||
private var ignoredPackages: Set<String> = HashSet()
|
||||
|
||||
/**
|
||||
* Bumped whenever the art getAlbumArtGray would return may have changed:
|
||||
* every metadata update and every switch of active session. Players
|
||||
* re-publish the same title/album with a different bitmap (art attached
|
||||
* after download, or a stale bitmap from the previous track replaced
|
||||
* after a quick double skip), so the strings can't key an art cache.
|
||||
*/
|
||||
private var artVersion = 0
|
||||
|
||||
init {
|
||||
this.listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java)
|
||||
this.sessionManager = appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as MediaSessionManager?
|
||||
@@ -404,6 +414,7 @@ class FaceclawMediaController(context: Context) {
|
||||
previous.unregisterCallback(controllerCallback)
|
||||
}
|
||||
activeController = controller
|
||||
artVersion++
|
||||
if (controller != null) {
|
||||
controller.registerCallback(controllerCallback, mainHandler)
|
||||
}
|
||||
@@ -503,6 +514,7 @@ class FaceclawMediaController(context: Context) {
|
||||
val finalCanSkipPrevious = canSkipPrevious
|
||||
val finalAccessEnabled = accessEnabled
|
||||
val finalStatus = status
|
||||
val finalArtVersion = artVersion
|
||||
|
||||
mainHandler.post {
|
||||
currentListener.onStateChange(
|
||||
@@ -519,7 +531,8 @@ class FaceclawMediaController(context: Context) {
|
||||
finalCanSkipNext,
|
||||
finalCanSkipPrevious,
|
||||
finalAccessEnabled,
|
||||
finalStatus
|
||||
finalStatus,
|
||||
finalArtVersion
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+330
-123
@@ -3,15 +3,20 @@ package com.faceclaw.app
|
||||
import android.app.Notification
|
||||
import android.app.NotificationManager
|
||||
import android.app.PendingIntent
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.database.ContentObserver
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.drawable.BitmapDrawable
|
||||
import android.graphics.drawable.Drawable
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.os.SystemClock
|
||||
import android.provider.Settings
|
||||
import android.service.notification.NotificationListenerService
|
||||
import android.service.notification.StatusBarNotification
|
||||
import android.util.Log
|
||||
@@ -29,15 +34,177 @@ import org.json.JSONObject
|
||||
class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawNotify"
|
||||
private const val NOTIFICATION_ICON_GAMMA = 1.6
|
||||
private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName"
|
||||
/**
|
||||
* How long notification access can be granted with no listener
|
||||
* connected before the watchdog forces a rebind: longer than a
|
||||
* normal bind at process start, and than the system's own single
|
||||
* retry 10 s after a binding dies. Later attempts double from here.
|
||||
*/
|
||||
private const val REBIND_GRACE_MS = 15_000L
|
||||
private const val REBIND_MAX_DELAY_MS = 15 * 60_000L
|
||||
/** Missing this long, the listener needs the user to toggle notification access. */
|
||||
private const val LISTENER_STUCK_MS = 30_000L
|
||||
/** Notifications posted while disconnected alert on reconnect only if this recent. */
|
||||
private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L
|
||||
|
||||
/** The connected listener; null until onListenerConnected and after onListenerDisconnected. */
|
||||
@Volatile
|
||||
private var activeService: FaceclawMediaNotificationListenerService? = null
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val notificationListeners: MutableSet<FaceclawNotificationListener> = CopyOnWriteArraySet()
|
||||
private val listenerStateObservers: MutableSet<Runnable> = CopyOnWriteArraySet()
|
||||
private val activeNotificationWakeKeys: MutableSet<String> = HashSet()
|
||||
|
||||
// Binding watchdog (see watchListenerBinding); written on the main thread only.
|
||||
private var watchdogContext: Context? = null
|
||||
private var rebindScheduled = false
|
||||
private var rebindDelayMs = REBIND_GRACE_MS
|
||||
/** elapsedRealtime when the watchdog first saw access granted with no listener; 0 when not missing. */
|
||||
@Volatile
|
||||
private var listenerMissingSinceMs = 0L
|
||||
/** Wall-clock time of the last disconnect in this process (compared with postTime); 0 if none. */
|
||||
private var lastListenerDisconnectAtMs = 0L
|
||||
private val forceRebindTask = Runnable { forceRebindIfStillMissing() }
|
||||
private val accessObserver = object : ContentObserver(mainHandler) {
|
||||
override fun onChange(selfChange: Boolean) {
|
||||
checkListenerBinding()
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for the listener going missing while notification
|
||||
* access is granted. Android can drop a listener's binding, or give
|
||||
* up rebinding it after the process dies, and not bring it back;
|
||||
* every read then comes back empty for the life of the process.
|
||||
* Called at app boot; disconnects and access changes re-check.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun watchListenerBinding(context: Context?) {
|
||||
if (context == null) {
|
||||
return
|
||||
}
|
||||
val appContext = context.applicationContext
|
||||
mainHandler.post {
|
||||
if (watchdogContext == null) {
|
||||
watchdogContext = appContext
|
||||
try {
|
||||
appContext.contentResolver.registerContentObserver(
|
||||
Settings.Secure.getUriFor("enabled_notification_listeners"), false, accessObserver)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "notification access observer registration failed", t)
|
||||
}
|
||||
}
|
||||
checkListenerBinding()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "connected"; "no-access" (notification access not granted);
|
||||
* "connecting" (granted, listener not connected yet); or "stuck"
|
||||
* (granted, but still missing past LISTENER_STUCK_MS, which forced
|
||||
* rebinds can't always fix: the user has to toggle access).
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getListenerState(context: Context?): String {
|
||||
if (activeService != null) {
|
||||
return "connected"
|
||||
}
|
||||
if (!FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
return "no-access"
|
||||
}
|
||||
val missingSince = listenerMissingSinceMs
|
||||
if (missingSince > 0 && SystemClock.elapsedRealtime() - missingSince >= LISTENER_STUCK_MS) {
|
||||
return "stuck"
|
||||
}
|
||||
return "connecting"
|
||||
}
|
||||
|
||||
/** Run on the main thread whenever getListenerState may have changed. */
|
||||
@JvmStatic
|
||||
fun addListenerStateObserver(observer: Runnable?) {
|
||||
if (observer != null) {
|
||||
listenerStateObservers.add(observer)
|
||||
}
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun removeListenerStateObserver(observer: Runnable?) {
|
||||
if (observer != null) {
|
||||
listenerStateObservers.remove(observer)
|
||||
}
|
||||
}
|
||||
|
||||
private fun notifyListenerStateChanged() {
|
||||
for (observer in listenerStateObservers) {
|
||||
mainHandler.post {
|
||||
try {
|
||||
observer.run()
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "listener state observer failed", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Main thread. Starts the forced-rebind countdown when access is granted and no listener is connected. */
|
||||
private fun checkListenerBinding() {
|
||||
val context = watchdogContext ?: return
|
||||
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
resetRebindWatchdog()
|
||||
return
|
||||
}
|
||||
if (listenerMissingSinceMs == 0L) {
|
||||
listenerMissingSinceMs = SystemClock.elapsedRealtime()
|
||||
}
|
||||
scheduleForcedRebind()
|
||||
}
|
||||
|
||||
private fun scheduleForcedRebind() {
|
||||
if (!rebindScheduled) {
|
||||
rebindScheduled = true
|
||||
mainHandler.postDelayed(forceRebindTask, rebindDelayMs)
|
||||
}
|
||||
}
|
||||
|
||||
private fun resetRebindWatchdog() {
|
||||
mainHandler.removeCallbacks(forceRebindTask)
|
||||
rebindScheduled = false
|
||||
rebindDelayMs = REBIND_GRACE_MS
|
||||
listenerMissingSinceMs = 0L
|
||||
}
|
||||
|
||||
private fun forceRebindIfStillMissing() {
|
||||
rebindScheduled = false
|
||||
val context = watchdogContext ?: return
|
||||
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
resetRebindWatchdog()
|
||||
return
|
||||
}
|
||||
val missingSeconds = (SystemClock.elapsedRealtime() - listenerMissingSinceMs) / 1000
|
||||
Log.w(TAG, "notification access granted but no listener connected for ${missingSeconds}s; forcing a rebind")
|
||||
val component = ComponentName(context, FaceclawMediaNotificationListenerService::class.java)
|
||||
try {
|
||||
// requestRebind only undoes a requestUnbind, so alone it can't
|
||||
// bring back a binding the system dropped. Unbinding by
|
||||
// component first marks the listener snoozed and drops any
|
||||
// record of it; the rebind then un-snoozes it and binds afresh.
|
||||
// Below API 34 there is no public equivalent (and flipping the
|
||||
// component's enabled state can make the system revoke
|
||||
// access), so there the state goes "stuck" for the user.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||
NotificationListenerService.requestUnbind(component)
|
||||
}
|
||||
NotificationListenerService.requestRebind(component)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "forced notification listener rebind failed", t)
|
||||
}
|
||||
rebindDelayMs = Math.min(rebindDelayMs * 2, REBIND_MAX_DELAY_MS)
|
||||
scheduleForcedRebind()
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun addNotificationListener(listener: FaceclawNotificationListener?) {
|
||||
if (listener != null) {
|
||||
@@ -127,15 +294,16 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) {
|
||||
continue
|
||||
}
|
||||
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
|
||||
if (drawable == null) {
|
||||
Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.packageName)
|
||||
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
|
||||
getNotificationAppName(service, statusBarNotification)
|
||||
}
|
||||
if (icon == null) {
|
||||
Log.i(TAG, "icon skipped (nothing to draw): " + statusBarNotification.packageName)
|
||||
continue
|
||||
}
|
||||
Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.packageName
|
||||
+ " drawable=" + drawable.javaClass.simpleName
|
||||
+ " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight)
|
||||
appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName)
|
||||
+ " source=" + icon.source + " kind=" + icon.art.kind)
|
||||
out.write(icon.art.gray)
|
||||
keysOut?.add(statusBarNotification.key ?: "")
|
||||
if (dedupeGroupKey != null) {
|
||||
emittedGroupKeys.add(dedupeGroupKey)
|
||||
@@ -165,13 +333,37 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
if (statusBarNotification == null || statusBarNotification.notification == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
|
||||
if (drawable == null) {
|
||||
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
|
||||
getNotificationAppName(service, statusBarNotification)
|
||||
}
|
||||
return icon?.art?.gray ?: ByteArray(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* The picture attached to one active notification (BigPictureStyle: a
|
||||
* photo, a screenshot, Dexcom's glucose graph), scaled to fit
|
||||
* maxWidth x maxHeight as a gray packet with the tone handling (and
|
||||
* light-background inversion) of ImageFileLoader.bitmapToGrayPacket;
|
||||
* empty when the notification or its picture is unavailable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getNotificationPictureGray(
|
||||
key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean,
|
||||
): ByteArray {
|
||||
val service = activeService
|
||||
if (service == null || maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val out = ByteArrayOutputStream(size * size)
|
||||
appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.packageName)
|
||||
return out.toByteArray()
|
||||
val statusBarNotification = findActiveNotificationByKey(service, key)
|
||||
if (statusBarNotification == null || statusBarNotification.notification == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val picture = loadNotificationPicture(service, statusBarNotification.notification)
|
||||
if (picture == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
return ImageFileLoader.bitmapToGrayPacket(
|
||||
halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
@@ -267,13 +459,21 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
|
||||
private fun emitNotificationPosted(key: String?) {
|
||||
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
|
||||
}
|
||||
|
||||
private fun emitNotificationRemoved(key: String?) {
|
||||
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationRemoved(eventKey) }
|
||||
}
|
||||
|
||||
private fun emitNotificationEvent(key: String?, deliver: (FaceclawNotificationListener, String) -> Unit) {
|
||||
if (key == null || key.isEmpty() || notificationListeners.isEmpty()) {
|
||||
return
|
||||
}
|
||||
for (listener in notificationListeners) {
|
||||
mainHandler.post {
|
||||
try {
|
||||
listener.onNotificationPosted(key)
|
||||
deliver(listener, key)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "notification listener failed", t)
|
||||
}
|
||||
@@ -281,7 +481,14 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
|
||||
private fun refreshActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
|
||||
/**
|
||||
* Re-seed the wake keys from what is active now. After a reconnect
|
||||
* this also catches the glasses up on the gap: keys that went away
|
||||
* are reported removed (closing a popup for a notification
|
||||
* dismissed meanwhile), and recent notifications posted since the
|
||||
* disconnect are reported posted. A first connect only seeds.
|
||||
*/
|
||||
private fun reconcileActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
|
||||
val notifications: Array<StatusBarNotification?>?
|
||||
try {
|
||||
notifications = service.activeNotifications
|
||||
@@ -289,19 +496,40 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
Log.w(TAG, "failed to refresh active notification wake keys", t)
|
||||
return
|
||||
}
|
||||
if (notifications == null) {
|
||||
return
|
||||
}
|
||||
val disconnectedAtMs = lastListenerDisconnectAtMs
|
||||
val alertAfterMs = Math.max(disconnectedAtMs, System.currentTimeMillis() - RECONNECT_ALERT_WINDOW_MS)
|
||||
val posted = ArrayList<String>()
|
||||
val removed = ArrayList<String>()
|
||||
synchronized(activeNotificationWakeKeys) {
|
||||
val previous = HashSet(activeNotificationWakeKeys)
|
||||
activeNotificationWakeKeys.clear()
|
||||
if (notifications == null) {
|
||||
return
|
||||
}
|
||||
for (statusBarNotification in notifications) {
|
||||
if (shouldShowNotificationInList(service, statusBarNotification)) {
|
||||
val key: String? = statusBarNotification!!.key
|
||||
if (key != null && !key.isEmpty()) {
|
||||
activeNotificationWakeKeys.add(key)
|
||||
}
|
||||
if (!shouldShowNotificationInList(service, statusBarNotification)) {
|
||||
continue
|
||||
}
|
||||
val key: String? = statusBarNotification!!.key
|
||||
if (key == null || key.isEmpty()) {
|
||||
continue
|
||||
}
|
||||
activeNotificationWakeKeys.add(key)
|
||||
if (disconnectedAtMs > 0 && !previous.contains(key) && statusBarNotification.postTime > alertAfterMs) {
|
||||
posted.add(key)
|
||||
}
|
||||
}
|
||||
for (key in previous) {
|
||||
if (!activeNotificationWakeKeys.contains(key)) {
|
||||
removed.add(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (key in removed) {
|
||||
emitNotificationRemoved(key)
|
||||
}
|
||||
for (key in posted) {
|
||||
emitNotificationPosted(key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,16 +547,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return !alreadyActive || !isPersistentNotification(statusBarNotification)
|
||||
}
|
||||
|
||||
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?) {
|
||||
/** Stop tracking a notification; returns whether it was tracked (i.e. listed on the glasses). */
|
||||
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?): Boolean {
|
||||
if (statusBarNotification == null) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
val key: String? = statusBarNotification.key
|
||||
if (key == null || key.isEmpty()) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
synchronized(activeNotificationWakeKeys) {
|
||||
activeNotificationWakeKeys.remove(key)
|
||||
return activeNotificationWakeKeys.remove(key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -397,106 +626,77 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return groupKey
|
||||
}
|
||||
|
||||
private fun loadNotificationIcon(service: FaceclawMediaNotificationListenerService, notification: Notification): Drawable? {
|
||||
/** Whether a notification carries a picture for loadNotificationPicture, without loading it. */
|
||||
@Suppress("DEPRECATION")
|
||||
private fun hasNotificationPicture(extras: Bundle): Boolean {
|
||||
return extras.get(Notification.EXTRA_PICTURE) != null
|
||||
|| (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && extras.get(Notification.EXTRA_PICTURE_ICON) != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's attached picture: BigPictureStyle's bitmap
|
||||
* (EXTRA_PICTURE) or, from Android 12, its Icon (EXTRA_PICTURE_ICON).
|
||||
* Newer releases swap large pictures for a content URI to a bitmap
|
||||
* the system has offloaded (Dexcom's graph arrives that way); the
|
||||
* offload provider lets enabled notification listeners open those.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
private fun loadNotificationPicture(service: FaceclawMediaNotificationListenerService, notification: Notification): Bitmap? {
|
||||
val extras: Bundle = notification.extras ?: return null
|
||||
try {
|
||||
val smallIcon: Icon? = notification.smallIcon
|
||||
if (smallIcon != null) {
|
||||
val drawable: Drawable? = smallIcon.loadDrawable(service)
|
||||
if (drawable != null) {
|
||||
// Status-bar small icons are alpha templates: the platform
|
||||
// draws them tinted and ignores their color channels, which
|
||||
// apps may fill with garbage (Discord ships noise there).
|
||||
// Tint white so the shape comes from alpha alone, matching
|
||||
// how the status bar renders them.
|
||||
drawable.mutate()
|
||||
drawable.setTint(Color.WHITE)
|
||||
if (drawable is BitmapDrawable) {
|
||||
drawable.setFilterBitmap(true)
|
||||
}
|
||||
return drawable
|
||||
}
|
||||
val picture: Bitmap? = extras.getParcelable(Notification.EXTRA_PICTURE)
|
||||
if (picture != null) {
|
||||
return softwareBitmap(picture)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load small notification icon", t)
|
||||
Log.w(TAG, "failed to read notification picture", t)
|
||||
}
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
// Large icons are real color images (avatars, album art); keep color.
|
||||
val largeIcon: Icon? = notification.getLargeIcon()
|
||||
if (largeIcon != null) {
|
||||
return largeIcon.loadDrawable(service)
|
||||
val icon: Icon? = extras.getParcelable(Notification.EXTRA_PICTURE_ICON)
|
||||
val drawable: Drawable? = icon?.loadDrawable(service)
|
||||
if (drawable is BitmapDrawable && drawable.bitmap != null) {
|
||||
return softwareBitmap(drawable.bitmap)
|
||||
}
|
||||
if (drawable != null && drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) {
|
||||
val bitmap = Bitmap.createBitmap(drawable.intrinsicWidth, drawable.intrinsicHeight, Bitmap.Config.ARGB_8888)
|
||||
drawable.setBounds(0, 0, bitmap.width, bitmap.height)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
return bitmap
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load large notification icon", t)
|
||||
Log.w(TAG, "failed to load notification picture icon", t)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun appendIconGrayBytes(drawable: Drawable, size: Int, out: ByteArrayOutputStream,
|
||||
service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?) {
|
||||
val bitmap = renderIconScaled(drawable, size)
|
||||
dumpIconDebugPng(service, index, packageName, bitmap)
|
||||
for (y in 0 until size) {
|
||||
for (x in 0 until size) {
|
||||
val color = bitmap.getPixel(x, y)
|
||||
val alpha = Color.alpha(color)
|
||||
val grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0)
|
||||
val gray = Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)).toInt()
|
||||
out.write(gray and 0xff)
|
||||
}
|
||||
}
|
||||
bitmap.recycle()
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a drawable at the target size with proper downscaling. Detailed
|
||||
* sources (e.g. avatar bitmaps used as notification icons) are rendered at
|
||||
* native resolution and reduced by repeated halving: a single filtered pass
|
||||
* from, say, 126px to 24px samples too sparsely and turns fine detail into
|
||||
* speckle that reads as a garbled icon.
|
||||
*/
|
||||
private fun renderIconScaled(drawable: Drawable, size: Int): Bitmap {
|
||||
val renderW = Math.max(size, drawable.intrinsicWidth)
|
||||
val renderH = Math.max(size, drawable.intrinsicHeight)
|
||||
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
drawable.setBounds(0, 0, renderW, renderH)
|
||||
drawable.draw(canvas)
|
||||
while (bitmap.width >= size * 2 && bitmap.height >= size * 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
bitmap.recycle()
|
||||
bitmap = halved
|
||||
}
|
||||
if (bitmap.width != size || bitmap.height != size) {
|
||||
val scaled = Bitmap.createScaledBitmap(bitmap, size, size, true)
|
||||
bitmap.recycle()
|
||||
bitmap = scaled
|
||||
/** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */
|
||||
private fun softwareBitmap(bitmap: Bitmap): Bitmap {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) {
|
||||
return bitmap.copy(Bitmap.Config.ARGB_8888, false)
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug aid for garbled-icon reports: saves each rendered icon to
|
||||
* <externalFilesDir>/debug-icons/ (adb-pullable) so extraction problems can
|
||||
* be told apart from downstream compositing/transmission problems. Cheap:
|
||||
* runs at most once per icon-cache refresh on tiny bitmaps.
|
||||
* Halve a picture while it is still at least twice the size it will
|
||||
* be shown at, leaving the last (under 2x) step of fitting it into
|
||||
* maxWidth x maxHeight to bitmapToGrayPacket. As with icons
|
||||
* (NotificationIconLoader.renderArgb), one filtered pass from a much larger source
|
||||
* samples too sparsely: a graph's thin lines break up into dots.
|
||||
*/
|
||||
private fun dumpIconDebugPng(service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?, bitmap: Bitmap) {
|
||||
if (index < 0) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
val dir = java.io.File(service.getExternalFilesDir(null), "debug-icons")
|
||||
if (!dir.exists() && !dir.mkdirs()) {
|
||||
return
|
||||
private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
|
||||
var bitmap = source
|
||||
while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
if (bitmap !== source) {
|
||||
bitmap.recycle()
|
||||
}
|
||||
val safeName = if (packageName == null) "unknown" else packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
|
||||
val file = java.io.File(dir, "icon-$index-$safeName.png")
|
||||
java.io.FileOutputStream(file).use { stream ->
|
||||
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to dump debug icon", t)
|
||||
bitmap = halved
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
private fun findActiveNotificationByKey(service: FaceclawMediaNotificationListenerService?, key: String?): StatusBarNotification? {
|
||||
@@ -531,6 +731,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
out.put("appName", getNotificationAppName(service, statusBarNotification))
|
||||
out.put("postTime", statusBarNotification.postTime)
|
||||
out.put("when", notification.`when`)
|
||||
out.put("foregroundService", (notification.flags and Notification.FLAG_FOREGROUND_SERVICE) != 0)
|
||||
putString(out, "category", notification.category)
|
||||
if (extras != null) {
|
||||
putCharSequence(out, "title", firstNonEmpty(
|
||||
@@ -553,8 +754,10 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
out.put("lines", lines)
|
||||
out.put("hasPicture", hasNotificationPicture(extras))
|
||||
} else {
|
||||
out.put("lines", JSONArray())
|
||||
out.put("hasPicture", false)
|
||||
}
|
||||
|
||||
val actionsJson = JSONArray()
|
||||
@@ -626,35 +829,34 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
activeService = this
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
}
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onListenerConnected() {
|
||||
activeService = this
|
||||
super.onListenerConnected()
|
||||
refreshActiveNotificationWakeKeys(this)
|
||||
Log.i(TAG, "notification listener connected")
|
||||
resetRebindWatchdog()
|
||||
reconcileActiveNotificationWakeKeys(this)
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
|
||||
// Also called from onDestroy, which is how an unbind usually arrives.
|
||||
override fun onListenerDisconnected() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
lastListenerDisconnectAtMs = System.currentTimeMillis()
|
||||
Log.w(TAG, "notification listener disconnected")
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
super.onListenerDisconnected()
|
||||
watchListenerBinding(applicationContext)
|
||||
}
|
||||
|
||||
override fun onNotificationPosted(statusBarNotification: StatusBarNotification?) {
|
||||
super.onNotificationPosted(statusBarNotification)
|
||||
if (!shouldShowNotificationInList(this, statusBarNotification)) {
|
||||
forgetActiveNotificationWakeKey(statusBarNotification)
|
||||
// An update that takes a listed notification off the list removes it as far as the glasses go.
|
||||
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
|
||||
emitNotificationRemoved(statusBarNotification!!.key)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (shouldEmitNotificationPosted(statusBarNotification!!)) {
|
||||
@@ -663,7 +865,12 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
|
||||
override fun onNotificationRemoved(statusBarNotification: StatusBarNotification?) {
|
||||
forgetActiveNotificationWakeKey(statusBarNotification)
|
||||
// Includes notifications their source cancels on its own, e.g. a
|
||||
// short-lived foreground-service notification that was standing in
|
||||
// for one posted moments later under another key.
|
||||
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
|
||||
emitNotificationRemoved(statusBarNotification!!.key)
|
||||
}
|
||||
super.onNotificationRemoved(statusBarNotification)
|
||||
}
|
||||
}
|
||||
@@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) {
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||
}
|
||||
|
||||
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||
val composite = compositor.previewComposite() ?: return null
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||
cropX, cropY, cropWidth, cropHeight)
|
||||
}
|
||||
|
||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||
@Throws(java.io.IOException::class)
|
||||
fun saveCompositePngScreenshot(): String {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.Context
|
||||
import android.location.Address
|
||||
import android.location.Geocoder
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import java.util.Locale
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
/**
|
||||
* Place name for the Weather header where the weather source doesn't supply
|
||||
* one (outside National Weather Service coverage), from Android's own
|
||||
* Geocoder. Its lookup blocks on the network, so it runs on a thread of its
|
||||
* own; the listener hears back on the main thread, with nulls when the phone
|
||||
* has no geocoder or the lookup fails.
|
||||
*/
|
||||
class FaceclawReverseGeocoder(context: Context) {
|
||||
interface Listener {
|
||||
fun onResult(locality: String?, region: String?, country: String?)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "FaceclawGeocoder"
|
||||
}
|
||||
|
||||
private val context: Context = context.applicationContext
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
fun lookup(latitude: Double, longitude: Double, listener: Listener) {
|
||||
thread(name = "FaceclawReverseGeocoder", isDaemon = true) {
|
||||
val address = lookupBlocking(latitude, longitude)
|
||||
mainHandler.post { listener.onResult(address?.locality, address?.adminArea, address?.countryName) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun lookupBlocking(latitude: Double, longitude: Double): Address? {
|
||||
if (!Geocoder.isPresent()) {
|
||||
return null
|
||||
}
|
||||
return try {
|
||||
// The listener form needs API 33; this one is fine off the main thread.
|
||||
@Suppress("DEPRECATION")
|
||||
Geocoder(context, Locale.getDefault()).getFromLocation(latitude, longitude, 1)?.firstOrNull()
|
||||
} catch (error: Throwable) {
|
||||
Log.w(TAG, "reverse geocode failed: $error")
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,10 +132,15 @@ class ImageFileLoader private constructor() {
|
||||
* round on its own, which turns smooth gradients into visible bands.
|
||||
* Dithered output is already quantized: every byte is a level times 16,
|
||||
* which BmpUtil re-quantizes back to exactly that level.
|
||||
*
|
||||
* invertLightBackground shows a picture drawn on a light background
|
||||
* (a chart, a screenshot) as its negative; see GrayPacket.fromArgb.
|
||||
*/
|
||||
@JvmStatic
|
||||
@JvmOverloads
|
||||
fun bitmapToGrayPacket(
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean,
|
||||
invertLightBackground: Boolean = false): ByteArray {
|
||||
if (source == null || source.width <= 0 || source.height <= 0
|
||||
|| maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
@@ -152,7 +157,7 @@ class ImageFileLoader private constructor() {
|
||||
}
|
||||
val pixels = IntArray(width * height)
|
||||
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither, invertLightBackground)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "bitmap conversion failed", e)
|
||||
return ByteArray(0)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.app.Notification
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.graphics.Rect
|
||||
import android.graphics.Typeface
|
||||
import android.graphics.drawable.AdaptiveIconDrawable
|
||||
import android.graphics.drawable.BitmapDrawable
|
||||
import android.graphics.drawable.Drawable
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.SystemClock
|
||||
import android.service.notification.StatusBarNotification
|
||||
import android.util.Log
|
||||
|
||||
import com.faceclaw.app.NotificationIconArt.Kind
|
||||
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
|
||||
import kotlin.math.max
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
* Picks and renders the glasses' status-bar icon for a notification (see iconFor).
|
||||
* NotificationIconArt does the pixel analysis; this object owns the drawables, package lookups,
|
||||
* caching and the debug dump.
|
||||
*/
|
||||
internal object NotificationIconLoader {
|
||||
private const val TAG = "FaceclawNotifyIcon"
|
||||
|
||||
/** AdaptiveIconDrawable draws its layers this share of the icon's size past each edge. */
|
||||
private const val ADAPTIVE_LAYER_OUTSET = 0.25f
|
||||
|
||||
private const val CACHE_MAX_ENTRIES = 128
|
||||
/** Cached art is redone after this long, so an updated app's new icon shows up. */
|
||||
private const val CACHE_MAX_AGE_MS = 30 * 60 * 1000L
|
||||
|
||||
/** The debug dump keeps at most this many files, deleting the oldest. */
|
||||
private const val DEBUG_DUMP_MAX_FILES = 96
|
||||
|
||||
class Rendered(
|
||||
/** The drawable it came from: small, monochrome, launcher, foreground, large or letter. */
|
||||
val source: String,
|
||||
val art: NotificationIconArt.Art,
|
||||
)
|
||||
|
||||
/**
|
||||
* An app's launcher icon art: its themed monochrome layer, else a glyph or shape read from
|
||||
* the icon (or its adaptive foreground layer), else the icon read by brightness.
|
||||
*/
|
||||
private class LauncherArt(val monochrome: Rendered?, val shape: Rendered?, val picture: Rendered?)
|
||||
|
||||
private val smallIconCache = ArtCache<Rendered?>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
|
||||
private val launcherCache = ArtCache<LauncherArt>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
|
||||
|
||||
/**
|
||||
* The icon for a notification, size x size gray, from the first of these that works:
|
||||
* 1. the small icon, when its alpha carries a shape (nearly every app);
|
||||
* 2. the launcher icon's themed monochrome layer (Android 13+), which is made to be tinted;
|
||||
* 3. the small icon's glyph, when it is a color tile with one background color;
|
||||
* 4. the launcher icon's glyph or shape, or its adaptive foreground layer's shape;
|
||||
* 5. the small icon, then the launcher icon, read by brightness;
|
||||
* 6. the large icon (usually the sender's avatar: the notification's content, not its app);
|
||||
* 7. the app name's initial in a rounded box.
|
||||
* Null only when even the initial can't be drawn.
|
||||
*/
|
||||
fun iconFor(context: Context, statusBarNotification: StatusBarNotification, size: Int, appName: () -> String): Rendered? {
|
||||
val packageName = statusBarNotification.packageName ?: ""
|
||||
val notification = statusBarNotification.notification
|
||||
val small = smallIconArt(context, packageName, notification.smallIcon, size)
|
||||
val smallKind = small?.art?.kind
|
||||
if (smallKind == Kind.TEMPLATE || smallKind == Kind.SOLID) return small
|
||||
val launcher = launcherArt(context, packageName, size)
|
||||
launcher.monochrome?.let { return it }
|
||||
if (smallKind == Kind.GLYPH) return small
|
||||
launcher.shape?.let { return it }
|
||||
if (smallKind == Kind.PICTURE) return small
|
||||
launcher.picture?.let { return it }
|
||||
largeIconArt(context, packageName, notification, size)?.let { return it }
|
||||
return letterArt(appName(), size)
|
||||
}
|
||||
|
||||
private fun smallIconArt(context: Context, packageName: String, icon: Icon?, size: Int): Rendered? {
|
||||
if (icon == null) return null
|
||||
val key = resourceKey(icon) ?: return loadSmallIcon(context, packageName, icon, size)
|
||||
return smallIconCache.getOrPut("$packageName|$key|$size") { loadSmallIcon(context, packageName, icon, size) }
|
||||
}
|
||||
|
||||
/** Resource icons are cached by resource; bitmap and URI icons are converted every time. */
|
||||
private fun resourceKey(icon: Icon): String? {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P || icon.type != Icon.TYPE_RESOURCE) return null
|
||||
return icon.resPackage + "/" + icon.resId
|
||||
}
|
||||
|
||||
private fun loadSmallIcon(context: Context, packageName: String, icon: Icon, size: Int): Rendered? =
|
||||
tryRender("small", packageName) {
|
||||
val drawable = icon.loadDrawable(context) ?: return@tryRender null
|
||||
// A tint (the Icon's own, or a vector's theme tint) recolors the very pixels that tell
|
||||
// a template from a color tile, and can fade a template's alpha. Templates are read
|
||||
// by alpha alone, so dropping it costs nothing.
|
||||
drawable.mutate()
|
||||
drawable.setTintList(null)
|
||||
renderSource(context, packageName, "small", drawable, size)
|
||||
}
|
||||
|
||||
private fun launcherArt(context: Context, packageName: String, size: Int): LauncherArt =
|
||||
launcherCache.getOrPut("$packageName|$size") { loadLauncherArt(context, packageName, size) }
|
||||
|
||||
private fun loadLauncherArt(context: Context, packageName: String, size: Int): LauncherArt {
|
||||
val icon: Drawable = try {
|
||||
context.packageManager.getApplicationIcon(packageName)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "no launcher icon for $packageName", t)
|
||||
return LauncherArt(null, null, null)
|
||||
}
|
||||
val adaptive = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) icon as? AdaptiveIconDrawable else null
|
||||
if (adaptive != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
val layer = adaptive.monochrome
|
||||
val monochrome = if (layer == null) null else tryRender("monochrome", packageName) {
|
||||
renderSource(context, packageName, "monochrome", layer, size, ADAPTIVE_LAYER_OUTSET, template = true)
|
||||
}
|
||||
if (monochrome != null) return LauncherArt(monochrome, null, null)
|
||||
}
|
||||
val whole = tryRender("launcher", packageName) { renderSource(context, packageName, "launcher", icon, size) }
|
||||
var shape = whole?.takeIf { it.art.kind == Kind.TEMPLATE || it.art.kind == Kind.GLYPH }
|
||||
val foreground = adaptive?.foreground
|
||||
if (shape == null && foreground != null) {
|
||||
// A gradient or picture background hides the glyph; the foreground layer alone may carry it as alpha.
|
||||
shape = tryRender("foreground", packageName) {
|
||||
renderSource(context, packageName, "foreground", foreground, size, ADAPTIVE_LAYER_OUTSET)
|
||||
}?.takeIf { it.art.kind == Kind.TEMPLATE }
|
||||
}
|
||||
return LauncherArt(null, shape, whole?.takeIf { it.art.kind == Kind.PICTURE })
|
||||
}
|
||||
|
||||
private fun largeIconArt(context: Context, packageName: String, notification: Notification, size: Int): Rendered? =
|
||||
tryRender("large", packageName) {
|
||||
val drawable = notification.getLargeIcon()?.loadDrawable(context) ?: return@tryRender null
|
||||
renderSource(context, packageName, "large", drawable, size)
|
||||
}
|
||||
|
||||
/** The app name's first letter or digit in a rounded box, like the shell's letter window icons. */
|
||||
private fun letterArt(appName: String, size: Int): Rendered? = tryRender("letter", appName) {
|
||||
val letter = appName.firstOrNull { it.isLetterOrDigit() }?.uppercaseChar()?.toString() ?: "?"
|
||||
val canvasSize = size * NotificationIconArt.ANALYSIS_SCALE
|
||||
val bitmap = Bitmap.createBitmap(canvasSize, canvasSize, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
val stroke = NotificationIconArt.ANALYSIS_SCALE.toFloat()
|
||||
val box = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = stroke
|
||||
color = Color.argb(160, 255, 255, 255)
|
||||
}
|
||||
val radius = canvasSize / 4f
|
||||
canvas.drawRoundRect(stroke / 2, stroke / 2, canvasSize - stroke / 2, canvasSize - stroke / 2, radius, radius, box)
|
||||
val text = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = Color.WHITE
|
||||
typeface = Typeface.DEFAULT_BOLD
|
||||
textSize = canvasSize * 0.6f
|
||||
textAlign = Paint.Align.CENTER
|
||||
}
|
||||
val bounds = Rect()
|
||||
text.getTextBounds(letter, 0, letter.length, bounds)
|
||||
canvas.drawText(letter, canvasSize / 2f, canvasSize / 2f - bounds.exactCenterY(), text)
|
||||
val argb = IntArray(canvasSize * canvasSize)
|
||||
bitmap.getPixels(argb, 0, canvasSize, 0, 0, canvasSize, canvasSize)
|
||||
bitmap.recycle()
|
||||
val art = NotificationIconArt.template(argb, size)
|
||||
if (art.kind == Kind.EMPTY) null else Rendered("letter", art)
|
||||
}
|
||||
|
||||
private inline fun tryRender(source: String, owner: String, render: () -> Rendered?): Rendered? {
|
||||
return try {
|
||||
render()
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to render $source icon for $owner", t)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
/** Render, dump and convert one source drawable; null when nothing visible came out. */
|
||||
private fun renderSource(
|
||||
context: Context, packageName: String, source: String, drawable: Drawable, size: Int,
|
||||
outset: Float = 0f, template: Boolean = false,
|
||||
): Rendered? {
|
||||
val analysisSize = size * NotificationIconArt.ANALYSIS_SCALE
|
||||
val argb = renderArgb(drawable, analysisSize, outset)
|
||||
dumpDebugPng(context, packageName, source, argb, analysisSize)
|
||||
val art = if (template) NotificationIconArt.template(argb, size) else NotificationIconArt.convert(argb, size)
|
||||
return if (art.kind == Kind.EMPTY) null else Rendered(source, art)
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a drawable to target x target ARGB with proper downscaling. Detailed sources (e.g.
|
||||
* avatar bitmaps used as notification icons) are rendered at native resolution and reduced
|
||||
* by repeated halving: a single filtered pass from, say, 126px to 24px samples too sparsely
|
||||
* and turns fine detail into speckle that reads as a garbled icon. An outset draws the
|
||||
* drawable that share of the canvas past each edge, as AdaptiveIconDrawable draws its layers.
|
||||
*/
|
||||
private fun renderArgb(drawable: Drawable, target: Int, outset: Float): IntArray {
|
||||
drawable.mutate()
|
||||
if (drawable is BitmapDrawable) {
|
||||
drawable.setFilterBitmap(true)
|
||||
}
|
||||
val scale = 1f + 2f * outset
|
||||
val renderW = max(target, (drawable.intrinsicWidth / scale).roundToInt())
|
||||
val renderH = max(target, (drawable.intrinsicHeight / scale).roundToInt())
|
||||
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
|
||||
val dx = (renderW * outset).roundToInt()
|
||||
val dy = (renderH * outset).roundToInt()
|
||||
drawable.setBounds(-dx, -dy, renderW + dx, renderH + dy)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
while (bitmap.width >= target * 2 && bitmap.height >= target * 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
bitmap.recycle()
|
||||
bitmap = halved
|
||||
}
|
||||
if (bitmap.width != target || bitmap.height != target) {
|
||||
val scaled = Bitmap.createScaledBitmap(bitmap, target, target, true)
|
||||
bitmap.recycle()
|
||||
bitmap = scaled
|
||||
}
|
||||
val pixels = IntArray(target * target)
|
||||
bitmap.getPixels(pixels, 0, target, 0, 0, target, target)
|
||||
bitmap.recycle()
|
||||
return pixels
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug aid for icon reports: saves each source as rendered for analysis, before any
|
||||
* conversion, to <externalFilesDir>/debug-icons/<package>-<source>.png (adb-pullable), so a
|
||||
* bad icon can be rerun through NotificationIconArt off-device. Runs only when a source is
|
||||
* converted, not on cached icons; the oldest files past DEBUG_DUMP_MAX_FILES are deleted.
|
||||
*/
|
||||
private fun dumpDebugPng(context: Context, packageName: String, source: String, argb: IntArray, size: Int) {
|
||||
try {
|
||||
val base = context.getExternalFilesDir(null) ?: return
|
||||
val dir = File(base, "debug-icons")
|
||||
if (!dir.exists() && !dir.mkdirs()) {
|
||||
return
|
||||
}
|
||||
val safeName = packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
|
||||
val bitmap = Bitmap.createBitmap(argb, size, size, Bitmap.Config.ARGB_8888)
|
||||
try {
|
||||
FileOutputStream(File(dir, "$safeName-$source.png")).use { stream ->
|
||||
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
|
||||
}
|
||||
} finally {
|
||||
bitmap.recycle()
|
||||
}
|
||||
val files = dir.listFiles() ?: return
|
||||
if (files.size > DEBUG_DUMP_MAX_FILES) {
|
||||
files.sortBy { it.lastModified() }
|
||||
for (index in 0 until files.size - DEBUG_DUMP_MAX_FILES) {
|
||||
files[index].delete()
|
||||
}
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to dump debug icon", t)
|
||||
}
|
||||
}
|
||||
|
||||
/** A small LRU of converted art with an age limit. Conversions run under its lock. */
|
||||
private class ArtCache<V>(private val maxEntries: Int, private val maxAgeMs: Long) {
|
||||
private class Entry<T>(val value: T, val atMs: Long)
|
||||
|
||||
private val entries = object : LinkedHashMap<String, Entry<V>>(16, 0.75f, true) {
|
||||
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Entry<V>>): Boolean {
|
||||
return size > maxEntries
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun getOrPut(key: String, compute: () -> V): V {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
val cached = entries[key]
|
||||
if (cached != null && now - cached.atMs < maxAgeMs) {
|
||||
return cached.value
|
||||
}
|
||||
val value = compute()
|
||||
entries[key] = Entry(value, now)
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() {
|
||||
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
|
||||
/**
|
||||
* As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the
|
||||
* mirror's Scaled preview). The area may extend past the frame's edges; it is black
|
||||
* there.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap {
|
||||
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||
throw IllegalArgumentException("invalid gray preview buffer")
|
||||
}
|
||||
if (cropWidth <= 0 || cropHeight <= 0) {
|
||||
throw IllegalArgumentException("invalid preview crop")
|
||||
}
|
||||
val bytes = ByteArray(width * height)
|
||||
gray.get(bytes)
|
||||
val cropped = ByteArray(cropWidth * cropHeight)
|
||||
val left = maxOf(cropX, 0)
|
||||
val right = minOf(cropX + cropWidth, width)
|
||||
if (right > left) {
|
||||
for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) {
|
||||
System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left)
|
||||
}
|
||||
}
|
||||
val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green)
|
||||
return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
|
||||
/**
|
||||
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
|
||||
* alpha; black, which the glasses show as unlit, is transparent), for
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Faceclaw</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0.0</string>
|
||||
<string>0.8.4</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
@@ -30,6 +30,8 @@
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSAppleMusicUsageDescription</key>
|
||||
<string>Faceclaw shows what is playing in Apple Music on your glasses, controls playback, and lets you browse and play your music library from the glasses.</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Faceclaw uses your location for local weather, turn-by-turn navigation, and true north on the glasses compass.</string>
|
||||
<key>NSLocationTemporaryUsageDescriptionDictionary</key>
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>NSPrivacyAccessedAPITypes</key>
|
||||
<array>
|
||||
<dict>
|
||||
<!-- Faceclaw preferences in its own NSUserDefaults domain. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategoryUserDefaults</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>CA92.1</string></array>
|
||||
</dict>
|
||||
<dict>
|
||||
<!-- App-container files used by Faceclaw and its NativeScript/Kotlin runtimes. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategoryFileTimestamp</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>C617.1</string></array>
|
||||
</dict>
|
||||
<dict>
|
||||
<!-- NativeScript's monotonic clock measures elapsed time within the app. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategorySystemBootTime</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>35F9.1</string></array>
|
||||
</dict>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -2,8 +2,8 @@ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
|
||||
TARGETED_DEVICE_FAMILY = 1;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
|
||||
// Select the local signing team in Xcode or pass --teamId to the iOS scripts.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* Apple Music (the system Music app) through the MediaPlayer framework: now-playing state
|
||||
* and transport on the system player, the media library as a browse tree, and the system
|
||||
* output volume. Call from the main thread; callbacks are delivered on the main queue.
|
||||
* Nothing touches the player or library until media-library access is granted, so no
|
||||
* permission prompt appears unasked.
|
||||
*/
|
||||
@interface FaceclawAppleMusic : NSObject
|
||||
+ (instancetype)shared;
|
||||
/** Playback state, the now-playing item or authorization may have changed. */
|
||||
@property(nonatomic, copy) void (^changeHandler)(void);
|
||||
- (BOOL)hasPermission;
|
||||
- (NSString *)permissionStatus;
|
||||
- (void)requestPermission:(void (^)(BOOL granted))completion;
|
||||
/** Observe the system player (and keep observing across authorization changes). */
|
||||
- (void)start;
|
||||
- (void)stop;
|
||||
/** The system player as a JSON object; {"access": false} until access is granted. */
|
||||
- (NSString *)stateJson;
|
||||
- (void)playPause;
|
||||
- (void)skipNext;
|
||||
- (void)skipPrevious;
|
||||
/**
|
||||
* The queue Faceclaw last started, as [{"id", "title", "active"}], while the player is
|
||||
* still playing from it; "[]" otherwise (the system player does not expose queues that
|
||||
* the Music app started).
|
||||
*/
|
||||
- (NSString *)queueJson;
|
||||
- (void)skipToQueueIndex:(NSInteger)index;
|
||||
/** [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), or nil. */
|
||||
- (NSData *)albumArt:(NSInteger)maxSize;
|
||||
/** System output volume 0..100, or -1 when unknown. */
|
||||
- (double)volumePercent;
|
||||
- (void)setVolumePercent:(double)percent;
|
||||
/** One level of the library tree ("root" first) as [{"mediaId", "title", "subtitle", "browsable", "playable"}]. */
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *json, NSString *error))completion;
|
||||
/** Start playback of a browsed collection, or of a song within its collection. */
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *error))completion;
|
||||
@end
|
||||
@@ -0,0 +1,512 @@
|
||||
#import "FaceclawAppleMusic.h"
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <MediaPlayer/MediaPlayer.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <errno.h>
|
||||
#import <math.h>
|
||||
#import <stdlib.h>
|
||||
|
||||
/** How long a queue Faceclaw just set may take to become the player's now-playing context. */
|
||||
static const NSTimeInterval kQueueConfirmSeconds = 5.0;
|
||||
|
||||
static NSString *JsonString(id object, NSString *fallback) {
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
|
||||
return data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : fallback;
|
||||
}
|
||||
|
||||
/** Persistent IDs are unsigned 64-bit, beyond JavaScript's exact integers, so they cross as decimal strings. */
|
||||
static NSString *IdString(MPMediaEntityPersistentID persistentId) {
|
||||
return [NSString stringWithFormat:@"%llu", persistentId];
|
||||
}
|
||||
|
||||
static BOOL ParseId(NSString *text, MPMediaEntityPersistentID *out) {
|
||||
const char *chars = text.UTF8String;
|
||||
if (!chars || chars[0] < '0' || chars[0] > '9') return NO;
|
||||
char *end = NULL;
|
||||
errno = 0;
|
||||
unsigned long long value = strtoull(chars, &end, 10);
|
||||
if (errno || !end || *end) return NO;
|
||||
*out = value;
|
||||
return YES;
|
||||
}
|
||||
|
||||
static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *subtitle, BOOL browsable, BOOL playable) {
|
||||
return @{@"mediaId": mediaId, @"title": title ?: @"", @"subtitle": subtitle ?: @"",
|
||||
@"browsable": @(browsable), @"playable": @(playable)};
|
||||
}
|
||||
|
||||
@interface FaceclawAppleMusic ()
|
||||
@property(nonatomic, strong) dispatch_queue_t libraryQueue;
|
||||
@property(nonatomic) BOOL started;
|
||||
@property(nonatomic) BOOL observing;
|
||||
/** Bumped whenever the now-playing item changes, so cached album art is refetched. */
|
||||
@property(nonatomic) NSInteger artVersion;
|
||||
@property(nonatomic, copy) NSString *nowPlayingKey;
|
||||
/** The items of the queue Faceclaw last started, until the player leaves them. */
|
||||
@property(nonatomic, copy) NSArray<MPMediaItem *> *queueItems;
|
||||
@property(nonatomic) BOOL queueConfirmed;
|
||||
@property(nonatomic, strong) NSDate *queueSetAt;
|
||||
@property(nonatomic, strong) MPVolumeView *volumeView;
|
||||
@end
|
||||
|
||||
@implementation FaceclawAppleMusic
|
||||
+ (instancetype)shared {
|
||||
static FaceclawAppleMusic *instance;
|
||||
static dispatch_once_t once;
|
||||
dispatch_once(&once, ^{ instance = [FaceclawAppleMusic new]; });
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) {
|
||||
_libraryQueue = dispatch_queue_create("com.faceclaw.applemusic", DISPATCH_QUEUE_SERIAL);
|
||||
_nowPlayingKey = @"";
|
||||
// Access may have been granted or revoked in Settings while away.
|
||||
[NSNotificationCenter.defaultCenter addObserver:self selector:@selector(appBecameActive:)
|
||||
name:UIApplicationDidBecomeActiveNotification object:nil];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - Authorization
|
||||
|
||||
- (BOOL)hasPermission {
|
||||
return MPMediaLibrary.authorizationStatus == MPMediaLibraryAuthorizationStatusAuthorized;
|
||||
}
|
||||
|
||||
- (NSString *)permissionStatus {
|
||||
switch (MPMediaLibrary.authorizationStatus) {
|
||||
case MPMediaLibraryAuthorizationStatusAuthorized: return @"granted";
|
||||
case MPMediaLibraryAuthorizationStatusDenied: return @"denied";
|
||||
case MPMediaLibraryAuthorizationStatusRestricted: return @"restricted";
|
||||
default: return @"not-determined";
|
||||
}
|
||||
}
|
||||
|
||||
- (void)requestPermission:(void (^)(BOOL))completion {
|
||||
if ([self hasPermission]) {
|
||||
completion(YES);
|
||||
return;
|
||||
}
|
||||
[MPMediaLibrary requestAuthorization:^(MPMediaLibraryAuthorizationStatus status) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
completion(status == MPMediaLibraryAuthorizationStatusAuthorized);
|
||||
});
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)appBecameActive:(NSNotification *)notification {
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
#pragma mark - Observation
|
||||
|
||||
- (void)start {
|
||||
self.started = YES;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
self.started = NO;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)syncObserving {
|
||||
BOOL want = self.started && [self hasPermission];
|
||||
if (want == self.observing) return;
|
||||
self.observing = want;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
NSNotificationCenter *center = NSNotificationCenter.defaultCenter;
|
||||
if (want) {
|
||||
[center addObserver:self selector:@selector(playbackStateChanged:)
|
||||
name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center addObserver:self selector:@selector(nowPlayingItemChanged:)
|
||||
name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player beginGeneratingPlaybackNotifications];
|
||||
} else {
|
||||
[center removeObserver:self name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center removeObserver:self name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player endGeneratingPlaybackNotifications];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)playbackStateChanged:(NSNotification *)notification {
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)nowPlayingItemChanged:(NSNotification *)notification {
|
||||
// Streamed catalog items can share identity fields, so every change counts.
|
||||
self.artVersion++;
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)notifyChanged {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.changeHandler) self.changeHandler();
|
||||
});
|
||||
}
|
||||
|
||||
#pragma mark - Now playing
|
||||
|
||||
- (NSString *)stateJson {
|
||||
if (![self hasPermission]) return @"{\"access\":false}";
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *item = player.nowPlayingItem;
|
||||
NSString *key = item ? [NSString stringWithFormat:@"%llu|%@|%@|%@", item.persistentID,
|
||||
item.playbackStoreID ?: @"", item.title ?: @"", item.artist ?: @""] : @"";
|
||||
if (![key isEqualToString:self.nowPlayingKey]) {
|
||||
self.nowPlayingKey = key;
|
||||
self.artVersion++;
|
||||
}
|
||||
NSString *state;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
state = @"playing";
|
||||
break;
|
||||
case MPMusicPlaybackStatePaused:
|
||||
case MPMusicPlaybackStateInterrupted:
|
||||
state = @"paused";
|
||||
break;
|
||||
default:
|
||||
state = @"stopped";
|
||||
break;
|
||||
}
|
||||
NSTimeInterval position = player.currentPlaybackTime;
|
||||
NSTimeInterval duration = item.playbackDuration;
|
||||
float rate = player.currentPlaybackRate;
|
||||
return JsonString(@{
|
||||
@"access": @YES,
|
||||
@"hasItem": @(item != nil),
|
||||
@"state": state,
|
||||
@"title": item.title ?: @"",
|
||||
@"artist": item.artist ?: @"",
|
||||
@"album": item.albumTitle ?: @"",
|
||||
@"positionMs": @(item && isfinite(position) && position >= 0 ? position * 1000.0 : -1),
|
||||
@"durationMs": @(item && isfinite(duration) && duration > 0 ? duration * 1000.0 : -1),
|
||||
@"rate": @(isfinite(rate) ? rate : 0),
|
||||
@"artVersion": @(self.artVersion),
|
||||
}, @"{\"access\":true}");
|
||||
}
|
||||
|
||||
- (void)playPause {
|
||||
if (![self hasPermission]) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
[player pause];
|
||||
break;
|
||||
default:
|
||||
[player play];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)skipNext {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToNextItem];
|
||||
}
|
||||
|
||||
- (void)skipPrevious {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToPreviousItem];
|
||||
}
|
||||
|
||||
- (NSData *)albumArt:(NSInteger)maxSize {
|
||||
if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil;
|
||||
MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork;
|
||||
UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)];
|
||||
CGImageRef source = image.CGImage;
|
||||
if (!source) return nil;
|
||||
double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
|
||||
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
|
||||
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
|
||||
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
|
||||
uint8_t *pixels = bytes + 4;
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
|
||||
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) return nil;
|
||||
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), source);
|
||||
CGContextRelease(context);
|
||||
// Composited over black (the display's see-through background); now opaque.
|
||||
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
|
||||
return packet;
|
||||
}
|
||||
|
||||
#pragma mark - Queue
|
||||
|
||||
/** Index of the now-playing item in Faceclaw's queue; forgets the queue once the player has left it. */
|
||||
- (NSInteger)activeQueueIndex {
|
||||
NSArray<MPMediaItem *> *items = self.queueItems;
|
||||
if (!items.count || ![self hasPermission]) return -1;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *now = player.nowPlayingItem;
|
||||
NSInteger found = -1;
|
||||
if (now) {
|
||||
NSUInteger index = player.indexOfNowPlayingItem;
|
||||
if (index < items.count && items[index].persistentID == now.persistentID) {
|
||||
found = (NSInteger)index;
|
||||
} else {
|
||||
for (NSUInteger i = 0; i < items.count; i++) {
|
||||
if (items[i].persistentID == now.persistentID) { found = (NSInteger)i; break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found >= 0) {
|
||||
self.queueConfirmed = YES;
|
||||
return found;
|
||||
}
|
||||
// Either the player hasn't switched over yet, or it has moved on to another queue.
|
||||
if (self.queueConfirmed || -[self.queueSetAt timeIntervalSinceNow] > kQueueConfirmSeconds) {
|
||||
self.queueItems = nil;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
- (NSString *)queueJson {
|
||||
NSInteger active = [self activeQueueIndex];
|
||||
if (active < 0) return @"[]";
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:self.queueItems.count];
|
||||
[self.queueItems enumerateObjectsUsingBlock:^(MPMediaItem *item, NSUInteger index, BOOL *stop) {
|
||||
[rows addObject:@{@"id": @(index), @"title": item.title ?: @"", @"active": @((NSInteger)index == active)}];
|
||||
}];
|
||||
return JsonString(rows, @"[]");
|
||||
}
|
||||
|
||||
- (void)skipToQueueIndex:(NSInteger)index {
|
||||
if ([self activeQueueIndex] < 0 || index < 0 || index >= (NSInteger)self.queueItems.count) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
player.nowPlayingItem = self.queueItems[index];
|
||||
[player play];
|
||||
}
|
||||
|
||||
#pragma mark - Volume
|
||||
|
||||
- (UIWindow *)hostWindow {
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
if (![scene isKindOfClass:UIWindowScene.class]) continue;
|
||||
NSArray<UIWindow *> *windows = ((UIWindowScene *)scene).windows;
|
||||
for (UIWindow *window in windows) if (window.isKeyWindow) return window;
|
||||
if (windows.firstObject) return windows.firstObject;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
/**
|
||||
* The slider of an off-screen MPVolumeView: iOS has no public setter for the system volume,
|
||||
* but the volume view's slider drives it while the view is in a window.
|
||||
*/
|
||||
- (UISlider *)volumeSlider:(BOOL *)attached {
|
||||
if (!self.volumeView) {
|
||||
self.volumeView = [[MPVolumeView alloc] initWithFrame:CGRectMake(-2000, -2000, 160, 40)];
|
||||
}
|
||||
if (!self.volumeView.window) {
|
||||
UIWindow *window = [self hostWindow];
|
||||
if (!window) return nil;
|
||||
[window addSubview:self.volumeView];
|
||||
if (attached) *attached = YES;
|
||||
}
|
||||
for (UIView *view in self.volumeView.subviews) {
|
||||
if ([view isKindOfClass:UISlider.class]) return (UISlider *)view;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (double)volumePercent {
|
||||
// The slider tracks the system volume once attached; until then (nothing has been set
|
||||
// yet) the session's reading, which can lag while Faceclaw's audio session is inactive.
|
||||
UISlider *slider = self.volumeView.window ? [self volumeSlider:NULL] : nil;
|
||||
float volume = slider ? slider.value : AVAudioSession.sharedInstance.outputVolume;
|
||||
return isfinite(volume) ? MAX(0.0, MIN(100.0, round(volume * 100.0))) : -1;
|
||||
}
|
||||
|
||||
- (void)setVolumePercent:(double)percent {
|
||||
if (!isfinite(percent)) return;
|
||||
BOOL attached = NO;
|
||||
UISlider *slider = [self volumeSlider:&attached];
|
||||
if (!slider) return;
|
||||
float value = (float)MAX(0.0, MIN(1.0, percent / 100.0));
|
||||
void (^apply)(void) = ^{
|
||||
[slider setValue:value animated:NO];
|
||||
[slider sendActionsForControlEvents:UIControlEventValueChanged];
|
||||
};
|
||||
// A freshly attached volume view ignores changes until it has wired up its slider.
|
||||
if (attached) dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.15 * NSEC_PER_SEC)), dispatch_get_main_queue(), apply);
|
||||
else apply();
|
||||
}
|
||||
|
||||
#pragma mark - Library
|
||||
|
||||
/**
|
||||
* Songs of a playable collection id: "playlist:<id>", "album:<id>", "artist:<id>" (album by
|
||||
* album, in track order) or "songs" (the whole library). Runs on the library queue.
|
||||
*/
|
||||
- (NSArray<MPMediaItem *> *)itemsForCollection:(NSString *)collectionId {
|
||||
if ([collectionId isEqualToString:@"songs"]) return [MPMediaQuery songsQuery].items ?: @[];
|
||||
NSRange colon = [collectionId rangeOfString:@":"];
|
||||
MPMediaEntityPersistentID persistentId;
|
||||
if (colon.location == NSNotFound || !ParseId([collectionId substringFromIndex:colon.location + 1], &persistentId)) return @[];
|
||||
NSString *kind = [collectionId substringToIndex:colon.location];
|
||||
MPMediaQuery *query;
|
||||
NSString *property;
|
||||
if ([kind isEqualToString:@"playlist"]) {
|
||||
query = [MPMediaQuery playlistsQuery];
|
||||
property = MPMediaPlaylistPropertyPersistentID;
|
||||
} else if ([kind isEqualToString:@"album"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyAlbumPersistentID;
|
||||
} else if ([kind isEqualToString:@"artist"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyArtistPersistentID;
|
||||
} else {
|
||||
return @[];
|
||||
}
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(persistentId) forProperty:property]];
|
||||
NSMutableArray<MPMediaItem *> *items = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *collection in query.collections) [items addObjectsFromArray:collection.items];
|
||||
return items;
|
||||
}
|
||||
|
||||
- (NSArray *)songRows:(NSArray<MPMediaItem *> *)items context:(NSString *)collectionId {
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:items.count];
|
||||
for (MPMediaItem *item in items) {
|
||||
[rows addObject:BrowseRow([NSString stringWithFormat:@"song:%@@%@", IdString(item.persistentID), collectionId],
|
||||
item.title, item.artist, NO, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)albumRows:(MPMediaQuery *)query {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *album in query.collections) {
|
||||
MPMediaItem *item = album.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"album:" stringByAppendingString:IdString(item.albumPersistentID)],
|
||||
item.albumTitle, item.albumArtist.length ? item.albumArtist : item.artist, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)childrenOf:(NSString *)parentId {
|
||||
if ([parentId isEqualToString:@"root"]) {
|
||||
return @[BrowseRow(@"playlists", @"Playlists", nil, YES, NO), BrowseRow(@"artists", @"Artists", nil, YES, NO),
|
||||
BrowseRow(@"albums", @"Albums", nil, YES, NO), BrowseRow(@"songs", @"Songs", nil, YES, NO)];
|
||||
}
|
||||
if ([parentId isEqualToString:@"playlists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaPlaylist *playlist in [MPMediaQuery playlistsQuery].collections) {
|
||||
NSUInteger count = playlist.count;
|
||||
NSString *subtitle = count == 1 ? @"1 song" : [NSString stringWithFormat:@"%lu songs", (unsigned long)count];
|
||||
[rows addObject:BrowseRow([@"playlist:" stringByAppendingString:IdString(playlist.persistentID)],
|
||||
playlist.name, subtitle, YES, count > 0)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"artists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *artist in [MPMediaQuery artistsQuery].collections) {
|
||||
MPMediaItem *item = artist.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"artist:" stringByAppendingString:IdString(item.artistPersistentID)],
|
||||
item.artist.length ? item.artist : @"Unknown Artist", nil, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"albums"]) return [self albumRows:[MPMediaQuery albumsQuery]];
|
||||
if ([parentId isEqualToString:@"songs"]) return [self songRows:[MPMediaQuery songsQuery].items ?: @[] context:@"songs"];
|
||||
if ([parentId hasPrefix:@"artist:"]) {
|
||||
MPMediaEntityPersistentID artistId;
|
||||
if (!ParseId([parentId substringFromIndex:7], &artistId)) return nil;
|
||||
MPMediaQuery *query = [MPMediaQuery albumsQuery];
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(artistId)
|
||||
forProperty:MPMediaItemPropertyArtistPersistentID]];
|
||||
return [self albumRows:query];
|
||||
}
|
||||
if ([parentId hasPrefix:@"playlist:"] || [parentId hasPrefix:@"album:"]) {
|
||||
return [self songRows:[self itemsForCollection:parentId] context:parentId];
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *, NSString *))completion {
|
||||
// Library queries can take a while on large libraries; keep them off the main thread.
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *json = nil, *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
NSArray *rows = [self childrenOf:parentId ?: @""];
|
||||
if (rows) json = JsonString(rows, nil);
|
||||
if (!json) error = rows ? @"Library listing failed" : @"Unknown library item";
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ completion(json, error); });
|
||||
});
|
||||
}
|
||||
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *))completion {
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *collectionId = mediaId ?: @"";
|
||||
MPMediaEntityPersistentID startId = 0;
|
||||
BOOL hasStart = NO;
|
||||
if ([collectionId hasPrefix:@"song:"]) {
|
||||
NSRange at = [collectionId rangeOfString:@"@"];
|
||||
if (at.location == NSNotFound) collectionId = @"";
|
||||
else {
|
||||
hasStart = ParseId([collectionId substringWithRange:NSMakeRange(5, at.location - 5)], &startId);
|
||||
collectionId = [collectionId substringFromIndex:at.location + 1];
|
||||
}
|
||||
}
|
||||
NSArray<MPMediaItem *> *items = @[];
|
||||
NSString *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
items = [self itemsForCollection:collectionId];
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
if (!error && !items.count) error = @"Nothing to play";
|
||||
}
|
||||
MPMediaItem *start = nil;
|
||||
if (hasStart) {
|
||||
for (MPMediaItem *item in items) {
|
||||
if (item.persistentID == startId) { start = item; break; }
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (error) {
|
||||
completion(error);
|
||||
return;
|
||||
}
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMusicPlayerMediaItemQueueDescriptor *descriptor = [[MPMusicPlayerMediaItemQueueDescriptor alloc]
|
||||
initWithItemCollection:[MPMediaItemCollection collectionWithItems:items]];
|
||||
if (start) descriptor.startItem = start;
|
||||
[player setQueueWithDescriptor:descriptor];
|
||||
[player play];
|
||||
self.queueItems = items;
|
||||
self.queueConfirmed = NO;
|
||||
self.queueSetAt = [NSDate date];
|
||||
[self notifyChanged];
|
||||
completion(nil);
|
||||
});
|
||||
});
|
||||
}
|
||||
@end
|
||||
@@ -2,7 +2,7 @@
|
||||
#import <WebKit/WebKit.h>
|
||||
|
||||
/** One local or URL EvenHub runtime, retained by its TS session until its window closes. */
|
||||
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKURLSchemeHandler>
|
||||
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKUIDelegate, WKURLSchemeHandler>
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
@property(nonatomic, copy) NSString *packageIdentifier;
|
||||
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#import "FaceclawEvenHubWebView.h"
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <SafariServices/SafariServices.h>
|
||||
#import <UniformTypeIdentifiers/UniformTypeIdentifiers.h>
|
||||
#import <CommonCrypto/CommonDigest.h>
|
||||
|
||||
@@ -7,19 +8,49 @@
|
||||
@property(nonatomic) WKWebView *webView;
|
||||
@property(nonatomic) UIView *host;
|
||||
@property(nonatomic) UIButton *back;
|
||||
@property(nonatomic) UIButton *returnButton;
|
||||
@property(nonatomic) NSTimer *timer;
|
||||
@property(nonatomic) NSString *directory;
|
||||
@property(nonatomic) NSURL *remoteURL;
|
||||
@property(nonatomic) NSString *script;
|
||||
/** The app's start page. Only documents from its origin get the bridge. */
|
||||
@property(nonatomic) NSURL *startURL;
|
||||
/** The redirect_uri of a sign-in the app started; the provider's reply is finished inside the app. */
|
||||
@property(nonatomic) NSURL *returnURL;
|
||||
/** The main frame shows a page outside the app's origin (a sign-in page, a followed link). */
|
||||
@property(nonatomic) BOOL away;
|
||||
@property(nonatomic) BOOL committed;
|
||||
@property(nonatomic) BOOL ticking;
|
||||
@property(nonatomic) BOOL destroyed;
|
||||
@end
|
||||
|
||||
static NSInteger FCDefaultPort(NSString *scheme) {
|
||||
scheme = scheme.lowercaseString;
|
||||
return [scheme isEqual:@"https"] ? 443 : [scheme isEqual:@"http"] ? 80 : 0;
|
||||
}
|
||||
static BOOL FCIsWebURL(NSURL *url) {
|
||||
NSString *scheme = url.scheme.lowercaseString;
|
||||
return [scheme isEqual:@"http"] || [scheme isEqual:@"https"];
|
||||
}
|
||||
static BOOL FCSameOrigin(NSURL *origin, NSString *scheme, NSString *host, NSInteger port) {
|
||||
return origin && host.length && [scheme.lowercaseString isEqual:origin.scheme.lowercaseString] &&
|
||||
[host.lowercaseString isEqual:origin.host.lowercaseString] &&
|
||||
(port ?: FCDefaultPort(scheme)) == (origin.port.integerValue ?: FCDefaultPort(origin.scheme));
|
||||
}
|
||||
/** A newer navigation replaced this one, or the policy below cancelled it; the current page is unaffected. */
|
||||
static BOOL FCIsInterruption(NSError *error) {
|
||||
return ([error.domain isEqual:NSURLErrorDomain] && error.code == NSURLErrorCancelled) ||
|
||||
([error.domain isEqual:@"WebKitErrorDomain"] && error.code == 102); // FrameLoadInterruptedByPolicyChange
|
||||
}
|
||||
|
||||
@implementation FaceclawEvenHubWebView
|
||||
- (void)emit:(NSDictionary *)event {
|
||||
if (!_eventHandler || _destroyed) return;
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil];
|
||||
if (data) _eventHandler([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (void)notice:(NSString *)message {
|
||||
[self emit:@{@"kind": @"notice", @"message": message.length > 4000 ? [message substringToIndex:4000] : message}];
|
||||
}
|
||||
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script {
|
||||
_directory = directory.stringByStandardizingPath.stringByResolvingSymlinksInPath;
|
||||
NSURLComponents *url = [NSURLComponents new];
|
||||
@@ -28,14 +59,14 @@
|
||||
}
|
||||
- (void)startURL:(NSString *)url script:(NSString *)script {
|
||||
NSURL *target = [NSURL URLWithString:url];
|
||||
if (!target.host.length || ![@[@"http", @"https"] containsObject:target.scheme.lowercaseString] || target.user || target.password) {
|
||||
if (!target.host.length || !FCIsWebURL(target) || target.user || target.password) {
|
||||
[self emit:@{@"kind": @"error", @"message": @"Invalid app URL."}]; return;
|
||||
}
|
||||
_remoteURL = target;
|
||||
[self startRequest:target script:script];
|
||||
}
|
||||
- (void)startRequest:(NSURL *)url script:(NSString *)script {
|
||||
if (_destroyed) return;
|
||||
_startURL = url; _script = script;
|
||||
WKWebViewConfiguration *config = [WKWebViewConfiguration new];
|
||||
// Isolate browser storage by package, retaining it across closing/reopening.
|
||||
// SDK storage also persists independently through the shared TS session.
|
||||
@@ -49,10 +80,9 @@
|
||||
}
|
||||
[config setURLSchemeHandler:self forURLScheme:@"faceclaw-ehpk"];
|
||||
[config.userContentController addScriptMessageHandler:self name:@"faceclaw"];
|
||||
[config.userContentController addUserScript:[[WKUserScript alloc] initWithSource:script
|
||||
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
|
||||
[self installScript:config.userContentController];
|
||||
_webView = [[WKWebView alloc] initWithFrame:CGRectZero configuration:config];
|
||||
_webView.navigationDelegate = self;
|
||||
_webView.navigationDelegate = self; _webView.UIDelegate = self;
|
||||
_webView.accessibilityIdentifier = @"evenhub-webview";
|
||||
UIWindow *window = nil;
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
@@ -73,13 +103,21 @@
|
||||
[_back setTitle:@"Back to Faceclaw" forState:UIControlStateNormal];
|
||||
_back.accessibilityIdentifier = @"evenhub-back";
|
||||
[_back addTarget:self action:@selector(hideOnPhone) forControlEvents:UIControlEventTouchUpInside];
|
||||
[_host addSubview:_back]; _back.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
// Shown only while the page is outside the app, e.g. an abandoned sign-in.
|
||||
_returnButton = [UIButton buttonWithType:UIButtonTypeSystem];
|
||||
[_returnButton setTitle:@"Return to app" forState:UIControlStateNormal];
|
||||
_returnButton.accessibilityIdentifier = @"evenhub-return";
|
||||
_returnButton.hidden = YES;
|
||||
[_returnButton addTarget:self action:@selector(returnToApp) forControlEvents:UIControlEventTouchUpInside];
|
||||
UIStackView *bar = [[UIStackView alloc] initWithArrangedSubviews:@[_back, _returnButton]];
|
||||
bar.distribution = UIStackViewDistributionFillEqually;
|
||||
[_host addSubview:bar]; bar.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
[NSLayoutConstraint activateConstraints:@[
|
||||
[_back.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
|
||||
[_back.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[_back.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[_back.heightAnchor constraintEqualToConstant:44],
|
||||
[_webView.topAnchor constraintEqualToAnchor:_back.bottomAnchor],
|
||||
[bar.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
|
||||
[bar.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[bar.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[bar.heightAnchor constraintEqualToConstant:44],
|
||||
[_webView.topAnchor constraintEqualToAnchor:bar.bottomAnchor],
|
||||
[_webView.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[_webView.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[_webView.bottomAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.bottomAnchor]
|
||||
@@ -92,7 +130,7 @@
|
||||
__weak typeof(self) weakSelf = self;
|
||||
_timer = [NSTimer timerWithTimeInterval:1.0 / 60 repeats:YES block:^(NSTimer *timer) {
|
||||
typeof(self) strongSelf = weakSelf;
|
||||
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking) return;
|
||||
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking || strongSelf.away) return;
|
||||
strongSelf.ticking = YES;
|
||||
[strongSelf.webView evaluateJavaScript:@"window.__fcTimerTick && window.__fcTimerTick(); window.__fcRafTick && window.__fcRafTick();" completionHandler:^(id result, NSError *error) {
|
||||
weakSelf.ticking = NO;
|
||||
@@ -101,8 +139,18 @@
|
||||
[NSRunLoop.mainRunLoop addTimer:_timer forMode:NSRunLoopCommonModes];
|
||||
[_webView loadRequest:[NSURLRequest requestWithURL:url]];
|
||||
}
|
||||
/** The bridge scripts run only in the app's own documents, never in pages it navigates to. */
|
||||
- (void)installScript:(WKUserContentController *)controller {
|
||||
NSData *start = [NSJSONSerialization dataWithJSONObject:_startURL.absoluteString options:NSJSONWritingFragmentsAllowed error:nil];
|
||||
NSString *source = [NSString stringWithFormat:@"if ((function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(%@))) {\n%@\n}",
|
||||
[[NSString alloc] initWithData:start encoding:NSUTF8StringEncoding], _script];
|
||||
[controller removeAllUserScripts];
|
||||
[controller addUserScript:[[WKUserScript alloc] initWithSource:source
|
||||
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
|
||||
}
|
||||
- (void)evaluate:(NSString *)script {
|
||||
if (!_destroyed) [_webView evaluateJavaScript:script completionHandler:nil];
|
||||
// Host messages are for the app's documents; a page it navigated to must not receive them.
|
||||
if (!_destroyed && !_away) [_webView evaluateJavaScript:script completionHandler:nil];
|
||||
}
|
||||
- (void)showOnPhone {
|
||||
if (_destroyed) return;
|
||||
@@ -112,51 +160,152 @@
|
||||
[_host endEditing:YES]; [_host.superview sendSubviewToBack:_host]; _host.userInteractionEnabled = NO; _host.accessibilityElementsHidden = YES;
|
||||
[self emit:@{@"kind": @"hidden"}];
|
||||
}
|
||||
- (void)returnToApp {
|
||||
if (_destroyed) return;
|
||||
_returnURL = nil; [_webView loadRequest:[NSURLRequest requestWithURL:_startURL]];
|
||||
}
|
||||
- (void)destroy {
|
||||
if (_destroyed) return;
|
||||
_destroyed = YES; [_timer invalidate]; _timer = nil;
|
||||
[_webView stopLoading]; _webView.navigationDelegate = nil;
|
||||
[_webView stopLoading]; _webView.navigationDelegate = nil; _webView.UIDelegate = nil;
|
||||
[_webView.configuration.userContentController removeScriptMessageHandlerForName:@"faceclaw"];
|
||||
[_webView.configuration.userContentController removeAllUserScripts];
|
||||
[_host removeFromSuperview]; _host = nil; _back = nil; _webView = nil; _eventHandler = nil;
|
||||
[_host removeFromSuperview]; _host = nil; _back = nil; _returnButton = nil; _webView = nil; _eventHandler = nil;
|
||||
}
|
||||
- (BOOL)allowsURL:(NSURL *)url {
|
||||
if (!_remoteURL) return [url.scheme isEqual:@"faceclaw-ehpk"] && [url.host isEqual:@"app"];
|
||||
NSNumber *port = url.port ?: @([url.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
|
||||
NSNumber *remotePort = _remoteURL.port ?: @([_remoteURL.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
|
||||
return [url.scheme.lowercaseString isEqual:_remoteURL.scheme.lowercaseString] &&
|
||||
[url.host.lowercaseString isEqual:_remoteURL.host.lowercaseString] && [port isEqual:remotePort];
|
||||
- (BOOL)isAppURL:(NSURL *)url {
|
||||
return FCSameOrigin(_startURL, url.scheme, url.host, url.port.integerValue);
|
||||
}
|
||||
- (BOOL)isAppFrame:(WKFrameInfo *)frame {
|
||||
WKSecurityOrigin *origin = frame.securityOrigin;
|
||||
return frame.isMainFrame && FCSameOrigin(_startURL, origin.protocol, origin.host, origin.port);
|
||||
}
|
||||
/** An OAuth authorization request names where the provider replies. */
|
||||
- (void)rememberReturnURLFrom:(NSURL *)url {
|
||||
for (NSURLQueryItem *item in [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO].queryItems) {
|
||||
NSURL *target = [item.name isEqual:@"redirect_uri"] && item.value ? [NSURL URLWithString:item.value] : nil;
|
||||
// A redirect_uri on the app's own origin needs no help: the app handles it itself.
|
||||
if (FCIsWebURL(target) && target.host.length && ![self isAppURL:target]) _returnURL = target;
|
||||
}
|
||||
}
|
||||
- (BOOL)isReturnURL:(NSURL *)url {
|
||||
// NSURL.path drops a trailing slash, so ".../callback/" and ".../callback" match.
|
||||
NSString *path = url.path.length ? url.path : @"/", *returnPath = _returnURL.path.length ? _returnURL.path : @"/";
|
||||
return FCSameOrigin(_returnURL, url.scheme, url.host, url.port.integerValue) && [path isEqual:returnPath];
|
||||
}
|
||||
/** The app's start page carrying the provider's reply (its query and fragment). */
|
||||
- (void)loadAppWithResultOf:(NSURL *)url {
|
||||
NSURLComponents *result = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO];
|
||||
NSURLComponents *app = [NSURLComponents componentsWithURL:_startURL resolvingAgainstBaseURL:NO];
|
||||
if (result.percentEncodedQuery.length) app.percentEncodedQuery = app.percentEncodedQuery.length ?
|
||||
[NSString stringWithFormat:@"%@&%@", app.percentEncodedQuery, result.percentEncodedQuery] : result.percentEncodedQuery;
|
||||
if (result.percentEncodedFragment) app.percentEncodedFragment = result.percentEncodedFragment;
|
||||
NSURL *target = app.URL;
|
||||
_returnURL = nil;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (!self.destroyed) [self.webView loadRequest:[NSURLRequest requestWithURL:target]];
|
||||
});
|
||||
}
|
||||
- (void)userContentController:(WKUserContentController *)controller didReceiveScriptMessage:(WKScriptMessage *)message {
|
||||
// Subframes and navigated third-party documents never receive host access.
|
||||
NSURL *url = message.frameInfo.request.URL;
|
||||
if (!message.frameInfo.isMainFrame || ![self allowsURL:url]) return;
|
||||
// Only the app's own main frame gets host access: not subframes, nor pages the app navigated to.
|
||||
if (![self isAppFrame:message.frameInfo]) return;
|
||||
if ([message.body isKindOfClass:NSDictionary.class]) [self emit:message.body];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView decidePolicyForNavigationAction:(WKNavigationAction *)action decisionHandler:(void (^)(WKNavigationActionPolicy))decisionHandler {
|
||||
NSURL *url = action.request.URL;
|
||||
BOOL allowed = [self allowsURL:url] && action.targetFrame.isMainFrame;
|
||||
if (!allowed && action.targetFrame.isMainFrame)
|
||||
[self emit:@{@"kind": @"error", @"message": @"Navigation outside the app origin is blocked. Open the destination as a separate app URL."}];
|
||||
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel);
|
||||
NSString *scheme = url.scheme.lowercaseString;
|
||||
// A new window (target=_blank, window.open): createWebView below opens it outside the app.
|
||||
if (!action.targetFrame) { decisionHandler(WKNavigationActionPolicyAllow); return; }
|
||||
// Subframes load as in a browser; they never get the bridge.
|
||||
if (!action.targetFrame.isMainFrame) {
|
||||
BOOL allowed = FCIsWebURL(url) || [self isAppURL:url] || [@[@"about", @"data", @"blob"] containsObject:scheme];
|
||||
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel); return;
|
||||
}
|
||||
if ([self isAppURL:url]) { _returnURL = nil; decisionHandler(WKNavigationActionPolicyAllow); return; }
|
||||
if (FCIsWebURL(url) && !_committed && !_directory.length) {
|
||||
// The app URL redirected before anything loaded: the app lives where it lands. The
|
||||
// script update reaches the web process ahead of the policy reply.
|
||||
_startURL = url; [self installScript:webView.configuration.userContentController];
|
||||
decisionHandler(WKNavigationActionPolicyAllow); return;
|
||||
}
|
||||
if (_returnURL && [self isReturnURL:url]) {
|
||||
// A sign-in provider replying: finish in the app rather than at the page the redirect_uri names.
|
||||
decisionHandler(WKNavigationActionPolicyCancel); [self loadAppWithResultOf:url]; return;
|
||||
}
|
||||
if (!FCIsWebURL(url)) {
|
||||
[self notice:[NSString stringWithFormat:@"Blocked navigation to a %@: URL.", scheme ?: @"schemeless"]];
|
||||
decisionHandler(WKNavigationActionPolicyCancel); return;
|
||||
}
|
||||
// Leaving the app's origin. Such pages never get the bridge; "Return to app" leads back.
|
||||
if ([self isAppFrame:action.sourceFrame]) [self rememberReturnURLFrom:url];
|
||||
decisionHandler(WKNavigationActionPolicyAllow);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didCommitNavigation:(WKNavigation *)navigation {
|
||||
_committed = YES; _away = ![self isAppURL:webView.URL]; _returnButton.hidden = !_away;
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFinishNavigation:(WKNavigation *)navigation {
|
||||
[self emit:@{@"kind": @"loaded"}];
|
||||
if (!_away) [self emit:@{@"kind": @"loaded"}];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFailProvisionalNavigation:(WKNavigation *)navigation withError:(NSError *)error {
|
||||
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFailNavigation:(WKNavigation *)navigation withError:(NSError *)error {
|
||||
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
}
|
||||
- (void)webViewWebContentProcessDidTerminate:(WKWebView *)webView {
|
||||
[self emit:@{@"kind": @"error", @"message": @"The app's web process ended. Close and reopen the app."}];
|
||||
[self emit:@{@"kind": @"error", @"fatal": @YES, @"message": @"The app's web process ended. Close and reopen the app."}];
|
||||
}
|
||||
/** What to present over, only while the app's phone UI is shown and not mid-transition. */
|
||||
- (UIViewController *)presenter {
|
||||
if (_destroyed || !_host.userInteractionEnabled) return nil;
|
||||
UIViewController *controller = _host.window.rootViewController;
|
||||
while (controller.presentedViewController) controller = controller.presentedViewController;
|
||||
return controller.transitionCoordinator ? nil : controller;
|
||||
}
|
||||
- (WKWebView *)webView:(WKWebView *)webView createWebViewWithConfiguration:(WKWebViewConfiguration *)configuration forNavigationAction:(WKNavigationAction *)action windowFeatures:(WKWindowFeatures *)windowFeatures {
|
||||
NSURL *url = action.request.URL;
|
||||
UIViewController *presenter = FCIsWebURL(url) ? [self presenter] : nil;
|
||||
if (presenter) [presenter presentViewController:[[SFSafariViewController alloc] initWithURL:url] animated:YES completion:nil];
|
||||
else [self notice:[NSString stringWithFormat:@"Did not open a window for a %@: URL.", url.scheme ?: @"schemeless"]];
|
||||
return nil;
|
||||
}
|
||||
// JavaScript dialogs. WebKit requires each completion handler to run exactly once, so a
|
||||
// dialog that cannot be shown (the app is hidden) completes at once with its default.
|
||||
- (UIAlertController *)dialog:(NSString *)message {
|
||||
if ([self presenter]) return [UIAlertController alertControllerWithTitle:nil message:message preferredStyle:UIAlertControllerStyleAlert];
|
||||
[self notice:[@"Dialog while hidden: " stringByAppendingString:message ?: @""]];
|
||||
return nil;
|
||||
}
|
||||
- (BOOL)presentDialog:(UIAlertController *)alert {
|
||||
if (!alert) return NO;
|
||||
[[self presenter] presentViewController:alert animated:YES completion:nil];
|
||||
return alert.presentingViewController != nil;
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptAlertPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(void))completionHandler {
|
||||
UIAlertController *alert = [self dialog:message];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(); }]];
|
||||
if (![self presentDialog:alert]) completionHandler();
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptConfirmPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(BOOL))completionHandler {
|
||||
UIAlertController *alert = [self dialog:message];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(NO); }]];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(YES); }]];
|
||||
if (![self presentDialog:alert]) completionHandler(NO);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptTextInputPanelWithPrompt:(NSString *)prompt defaultText:(NSString *)defaultText initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(NSString *))completionHandler {
|
||||
UIAlertController *alert = [self dialog:prompt];
|
||||
[alert addTextFieldWithConfigurationHandler:^(UITextField *field) { field.text = defaultText; }];
|
||||
__weak UIAlertController *weakAlert = alert;
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(nil); }]];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) {
|
||||
completionHandler(weakAlert.textFields.firstObject.text ?: @"");
|
||||
}]];
|
||||
if (![self presentDialog:alert]) completionHandler(nil);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView startURLSchemeTask:(id<WKURLSchemeTask>)task {
|
||||
NSURL *url = task.request.URL;
|
||||
NSString *relative = [url.path stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@"/"]];
|
||||
NSString *file = [[_directory stringByAppendingPathComponent:relative] stringByStandardizingPath].stringByResolvingSymlinksInPath;
|
||||
BOOL allowed = !_destroyed && _directory.length && !_remoteURL && [url.host isEqual:@"app"] &&
|
||||
BOOL allowed = !_destroyed && _directory.length && [url.host isEqual:@"app"] &&
|
||||
[file hasPrefix:[_directory stringByAppendingString:@"/"]];
|
||||
NSData *data = allowed ? [NSData dataWithContentsOfFile:file options:NSDataReadingMappedIfSafe error:nil] : nil;
|
||||
NSString *ext = file.pathExtension.lowercaseString;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
@interface FaceclawGraphics : NSObject
|
||||
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
|
||||
/** A glasses icon as white-on-transparent (gray becomes alpha), for the dark header. */
|
||||
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height;
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
|
||||
+ (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height;
|
||||
+ (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight;
|
||||
|
||||
@@ -76,6 +76,23 @@
|
||||
CGColorSpaceRelease(space);
|
||||
return result;
|
||||
}
|
||||
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height {
|
||||
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||
const uint8_t *src = gray.bytes;
|
||||
uint8_t *dst = rgba.mutableBytes;
|
||||
// Premultiplied white: every channel equals the alpha.
|
||||
for (NSInteger i = 0; i < width * height; i++) memset(dst + i * 4, src[i], 4);
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
|
||||
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
|
||||
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
|
||||
if (image) CGImageRelease(image);
|
||||
CGDataProviderRelease(provider);
|
||||
CGColorSpaceRelease(space);
|
||||
return result;
|
||||
}
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
|
||||
if (size <= 0 || size > 1024) return nil;
|
||||
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
/** Thread-safe event queue; each NativeScript worker drains it on its own thread. */
|
||||
@interface FaceclawSocket : NSObject <NSURLSessionWebSocketDelegate>
|
||||
- (instancetype)initWithURL:(NSString *)url;
|
||||
/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue;
|
||||
- (NSString *)takeEvents;
|
||||
- (void)sendText:(NSString *)text;
|
||||
- (void)sendData:(NSData *)data;
|
||||
- (void)close;
|
||||
@end
|
||||
@@ -8,12 +8,17 @@
|
||||
@end
|
||||
@implementation FaceclawSocket
|
||||
- (instancetype)initWithURL:(NSString *)url {
|
||||
return [self initWithURL:url headerName:nil headerValue:nil];
|
||||
}
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue {
|
||||
if ((self = [super init])) {
|
||||
_events = [NSMutableArray array];
|
||||
NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration];
|
||||
config.timeoutIntervalForRequest = 20;
|
||||
_session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
|
||||
_task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]];
|
||||
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]];
|
||||
if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName];
|
||||
_task = [_session webSocketTaskWithRequest:request];
|
||||
_task.maximumMessageSize = 8 * 1024 * 1024;
|
||||
[_task resume];
|
||||
}
|
||||
@@ -25,7 +30,7 @@
|
||||
_queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
|
||||
if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) {
|
||||
[_events removeAllObjects]; _queuedBytes = 0;
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}];
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}];
|
||||
[self close];
|
||||
} else [_events addObject:event];
|
||||
}
|
||||
@@ -56,7 +61,13 @@
|
||||
[self close];
|
||||
}
|
||||
- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error {
|
||||
if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}];
|
||||
if (error) {
|
||||
// A refused handshake only says "bad response"; name the HTTP status
|
||||
// so callers can tell a rejected key (401/403) from a network error.
|
||||
NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0;
|
||||
NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription;
|
||||
[self enqueue:@{@"kind": @"error", @"text": text}];
|
||||
}
|
||||
[self close];
|
||||
}
|
||||
- (void)sendText:(NSString *)text {
|
||||
@@ -65,6 +76,12 @@
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)sendData:(NSData *)data {
|
||||
__weak FaceclawSocket *weakSelf = self;
|
||||
[_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) {
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)close {
|
||||
@synchronized(self) {
|
||||
if (_ended) return; _ended = YES;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */
|
||||
@interface FaceclawSpeech : NSObject
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */
|
||||
@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm);
|
||||
+ (NSInteger)authorizationStatus;
|
||||
+ (NSInteger)microphoneAuthorizationStatus;
|
||||
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion;
|
||||
@@ -9,6 +11,9 @@
|
||||
/** Returns an actionable error, or an empty string when capture starts. */
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing;
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing;
|
||||
/** Capture without the recognizer (for a cloud provider): audio goes to
|
||||
* pcmHandler, and "ended" follows the last chunk. No transcript events. */
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone;
|
||||
- (void)acceptPacket:(NSData *)packet;
|
||||
- (void)finish;
|
||||
- (void)cancel;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
@property(atomic) NSUInteger generation;
|
||||
@property(nonatomic) NSUInteger pendingPackets;
|
||||
@property(atomic) BOOL accepting;
|
||||
@property(nonatomic) BOOL pcmOutput;
|
||||
@property(nonatomic, strong) AVAudioEngine *phoneEngine;
|
||||
@property(nonatomic) BOOL phoneTapInstalled;
|
||||
@property(nonatomic) BOOL phoneSessionActive;
|
||||
@@ -23,6 +24,23 @@
|
||||
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
|
||||
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
|
||||
@end
|
||||
/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial
|
||||
* audio queue; the converter keeps its filter state between buffers. */
|
||||
static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) {
|
||||
AVAudioFormat *format = converter.outputFormat;
|
||||
AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32;
|
||||
AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity];
|
||||
__block BOOL supplied = NO;
|
||||
NSError *error = nil;
|
||||
AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error
|
||||
withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) {
|
||||
// NoDataNow (not EndOfStream) leaves the stream open for the next buffer.
|
||||
if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; }
|
||||
supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input;
|
||||
}];
|
||||
if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil;
|
||||
return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)];
|
||||
}
|
||||
@implementation FaceclawSpeech
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL);
|
||||
@@ -46,37 +64,51 @@
|
||||
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:NO];
|
||||
return [self startWithEndpointing:endpointing phone:NO pcm:NO];
|
||||
}
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:YES];
|
||||
return [self startWithEndpointing:endpointing phone:YES pcm:NO];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
return [self startWithEndpointing:endpointing phone:phone pcm:YES];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm {
|
||||
[self cancel];
|
||||
if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized)
|
||||
return @"Allow Microphone access for Faceclaw in iPhone Settings.";
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
if (!pcm) {
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
}
|
||||
if (!phone) {
|
||||
self.decoder = [FaceclawLc3Decoder new];
|
||||
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
|
||||
}
|
||||
self.pcmOutput = pcm;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
if (pcm) {
|
||||
if (phone) {
|
||||
NSString *error = [self startPhoneAudio];
|
||||
if (error.length) { [self cancel]; return error; }
|
||||
}
|
||||
// The PCM consumer reports its own status and has no session limit.
|
||||
return @"";
|
||||
}
|
||||
self.recognizer.queue = NSOperationQueue.mainQueue;
|
||||
self.request = [SFSpeechAudioBufferRecognitionRequest new];
|
||||
self.request.requiresOnDeviceRecognition = YES;
|
||||
self.request.shouldReportPartialResults = YES;
|
||||
self.request.taskHint = SFSpeechRecognitionTaskHintDictation;
|
||||
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.transcript = [FaceclawSpeechTranscript new];
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -124,6 +156,13 @@
|
||||
AVAudioInputNode *input = self.phoneEngine.inputNode;
|
||||
AVAudioFormat *format = [input outputFormatForBus:0];
|
||||
if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again.";
|
||||
AVAudioConverter *converter = nil;
|
||||
if (self.pcmOutput) {
|
||||
AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES];
|
||||
converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat];
|
||||
if (!converter) return @"Could not convert phone microphone audio.";
|
||||
converter.downmix = YES;
|
||||
}
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector;
|
||||
@@ -141,7 +180,10 @@
|
||||
memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize);
|
||||
dispatch_async(owner.audioQueue, ^{ @autoreleasepool {
|
||||
if (owner.generation != generation) return;
|
||||
[request appendAudioPCMBuffer:copy];
|
||||
if (converter) {
|
||||
NSData *pcm = resampleToPcm(converter, copy);
|
||||
if (pcm.length) [owner deliverPcm:pcm generation:generation];
|
||||
} else [request appendAudioPCMBuffer:copy];
|
||||
double sum = 0;
|
||||
if (copy.floatChannelData) {
|
||||
const float *samples = copy.floatChannelData[0];
|
||||
@@ -188,6 +230,7 @@
|
||||
FaceclawLc3Decoder *decoder = self.decoder;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
NSData *copy = [packet copy];
|
||||
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
|
||||
if (self.generation != generation) return;
|
||||
@@ -195,17 +238,26 @@
|
||||
double rms = 0;
|
||||
BOOL speechEnded = NO;
|
||||
if (pcm.length) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
|
||||
buffer.frameLength = buffer.frameCapacity;
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < buffer.frameLength; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
buffer.floatChannelData[0][i] = value; rms += value * value;
|
||||
AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2);
|
||||
AVAudioPCMBuffer *buffer = nil;
|
||||
if (!pcmOutput) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames];
|
||||
buffer.frameLength = frames;
|
||||
}
|
||||
rms = sqrt(rms / buffer.frameLength);
|
||||
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < frames; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
if (buffer) buffer.floatChannelData[0][i] = value;
|
||||
rms += value * value;
|
||||
}
|
||||
rms = sqrt(rms / frames);
|
||||
if (self.generation == generation) {
|
||||
// The decoder's output is already the PCM consumer's format.
|
||||
if (pcmOutput) [self deliverPcm:pcm generation:generation];
|
||||
else [request appendAudioPCMBuffer:buffer];
|
||||
}
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames];
|
||||
}
|
||||
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
@@ -223,9 +275,15 @@
|
||||
[self stopPhoneAudio];
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
[self emit:@{@"kind":@"finishing"}];
|
||||
// Drain already received audio before marking the utterance complete.
|
||||
dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; });
|
||||
dispatch_async(self.audioQueue, ^{
|
||||
if (self.generation != generation) return;
|
||||
if (!pcmOutput) { [request endAudio]; return; }
|
||||
// Drained chunks are already queued for pcmHandler; "ended" follows them.
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; });
|
||||
});
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -239,9 +297,14 @@
|
||||
if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}];
|
||||
if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}];
|
||||
}
|
||||
- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm);
|
||||
});
|
||||
}
|
||||
- (void)cancel {
|
||||
self.generation++;
|
||||
self.accepting = NO; self.pendingPackets = 0;
|
||||
self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO;
|
||||
[self stopPhoneAudio];
|
||||
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
|
||||
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
|
||||
|
||||
@@ -6,6 +6,7 @@ module FaceclawGraphics {
|
||||
header "FaceclawSpeech.h"
|
||||
header "FaceclawLocation.h"
|
||||
header "FaceclawCalendar.h"
|
||||
header "FaceclawAppleMusic.h"
|
||||
header "FaceclawConfigPort.h"
|
||||
header "FaceclawEvenHubWebView.h"
|
||||
header "FaceclawCrypto.h"
|
||||
|
||||
@@ -1,10 +1,57 @@
|
||||
# Changelog
|
||||
|
||||
## 0.8.3 - Future
|
||||
## 0.8.5 - Future
|
||||
|
||||
### Features
|
||||
|
||||
* Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover
|
||||
* Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view
|
||||
* Add an Apple Music integration on iOS
|
||||
|
||||
### Bug fixes
|
||||
|
||||
* Inverting Android notification-icons to indicate selectionis now gamma-correct
|
||||
* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks
|
||||
|
||||
## 0.8.4 - 2026-10-17
|
||||
|
||||
### Features
|
||||
|
||||
* Bottom-bar app switcher switches two a multi-row layout when it runs out of space
|
||||
* T3 Code client: Add options (worktree, host, etc) when creating threads
|
||||
* Add configuration options to the glanceboard system card and music player
|
||||
* Glanceboard compass shows calibration-accuracy indicator
|
||||
* Status-bar widgets (nightscout, weather, timer) are selectable
|
||||
* Roam app: Add a date-picker for navigating to a daily page
|
||||
* Clear notification popups when they're dismissed on the phone, and avoid showing some notifications that are spurious
|
||||
* Add depth effects to some modals that didn't have them
|
||||
* Add EvenHub icon to the phone preview screen
|
||||
* Add Hermes integration
|
||||
* Add wake-into-last-app setting (changes the default focus after screen wake)
|
||||
* Nightscout: Add a missed-bolus detection feature
|
||||
|
||||
### Bug fixes
|
||||
|
||||
* iOS: Fix several bugs that would crash the app
|
||||
* EvenHub: Fix issues that would prevent logging in with the webview on another domain
|
||||
* EvenHub app localstorage is now partitioned by app
|
||||
* Phone preview is centered on the phone screen when apps are not maximized
|
||||
* Fix a bug where album art and track display could get out of sync if you advance twice quickly
|
||||
* Fix a bug where EvenHub searches with spaces in the search term would fail
|
||||
* Crashes in a webview renderer can no longer take down the rest of the app
|
||||
* Fix issues that would prevent sign-in to EvenHub apps that rely on cross-domain navigation
|
||||
* Fix a bunch of bugs in the Microphones prototype app
|
||||
|
||||
## 0.8.3 - 2026-10-10
|
||||
|
||||
* Fix several issues that would cause disconnection and disconnect-looping
|
||||
* Add a client for T3 Code
|
||||
* Weather app shows an icon for current conditions, and can add the weather to the status bar and the Glanceboard system card (Weather menu > Settings)
|
||||
* Optional usage statistics
|
||||
* Add an on-glasses Onboarding app, which prompts you to configure some things
|
||||
* iOS Notifications have an icon based on ANCS category
|
||||
* iOS voice input can use cloud transcription providers
|
||||
* Weather app is more graphical, and can appear on the status bar and glanceboard Glanceboard system card
|
||||
* Set up an iOS TestFlight (but it may take awhile before Apple approves it)
|
||||
|
||||
## 0.8.2 - 2026-10-03
|
||||
|
||||
|
||||
@@ -1,10 +1,91 @@
|
||||
# Faceclaw Privacy Policy
|
||||
|
||||
Faceclaw is a privacy-conscious app. The developers of Faceclaw don't collect
|
||||
or store user data. Faceclaw does integrate with some third-party services,
|
||||
which have their own privacy policies; this document describes what those
|
||||
services are, what they're used for and what data they collect, and links to
|
||||
their privacy policies.
|
||||
Faceclaw's own usage statistics are off unless you opt in. Separately, Apple
|
||||
collects and shares information when you use an iOS TestFlight build, even
|
||||
when Faceclaw statistics are set to None. Information you voluntarily send
|
||||
us, such as feedback, may also identify you. This policy describes these
|
||||
separate channels and the third-party services you can choose to use.
|
||||
|
||||
|
||||
## Usage Statistics
|
||||
|
||||
The first time you connect your glasses, Faceclaw asks whether to share
|
||||
usage statistics with its developers, to help them fix problems and
|
||||
improve the app. The choices are None, Minimal and Full, and you can change
|
||||
your choice at any time in Settings > Privacy > Faceclaw statistics. Until
|
||||
you choose, it is None. This setting controls only Faceclaw's own statistics.
|
||||
|
||||
With None, Faceclaw sends and stores none of its own usage statistics. This
|
||||
does not disable Apple's TestFlight reports or services you choose to use.
|
||||
|
||||
With Minimal, Faceclaw keeps counts on your phone and sends them about once a
|
||||
day to stats.faceclaw.org, a server run by Faceclaw's developers:
|
||||
|
||||
- the Faceclaw version, whether it runs on Android or iOS, and whether it is
|
||||
an official release or a build someone compiled themselves;
|
||||
- whether glasses and a ring are paired, and which devices (glasses, ring,
|
||||
Wear OS watch) were in use each day;
|
||||
- counts of connection events: connection attempts, disconnections and
|
||||
their causes, message timeouts and retries, Bluetooth error codes, and
|
||||
how long connections lasted.
|
||||
|
||||
With Full, it also sends:
|
||||
|
||||
- which kinds of API key are set (for example, that an Anthropic key is
|
||||
set), but never the keys themselves;
|
||||
- how many terminal and T3 Code connections are set up, and which of
|
||||
Faceclaw's built-in fonts is in use;
|
||||
- the values of settings chosen from fixed options, such as on/off
|
||||
switches and pick-one lists. Settings you type in, such as addresses,
|
||||
URLs and names, are never sent.
|
||||
|
||||
Faceclaw's own statistics never include your name or contact details, your location,
|
||||
Bluetooth device addresses, anything you say or type, or what is shown on
|
||||
your glasses. Each report carries a random identifier, created when you turn
|
||||
statistics on, which links your reports to each other but not to you.
|
||||
Turning statistics off deletes that identifier and everything kept on the
|
||||
phone, and stops all reports; turning them back on starts a new identifier.
|
||||
The statistics server doesn't record the IP addresses reports come from.
|
||||
|
||||
|
||||
## iOS and TestFlight
|
||||
|
||||
For TestFlight builds, Apple automatically collects crash reports and beta
|
||||
usage information and shares them with us. This includes installations,
|
||||
sessions, crashes, app/build and OS versions, device model and identifier,
|
||||
and language. It is separate from Faceclaw statistics, with no in-app opt-out.
|
||||
|
||||
Joining only through a public TestFlight link hides your name and email from
|
||||
us, not from Apple. Email invitations can identify you to us. Apple also
|
||||
collects your device's name (not shared with us) and may use your IP address
|
||||
to estimate your region for beta availability.
|
||||
|
||||
Feedback is optional. Comments, screenshots and diagnostics submitted
|
||||
through TestFlight go to Apple and us. Public-link testers can omit their
|
||||
email in the feedback form. Email sent directly to us reveals your email
|
||||
even after public-link enrollment, but is not shared with Apple. Check
|
||||
screenshots and comments for private content; feedback can include device
|
||||
state such as battery level, free storage and network type.
|
||||
|
||||
Faceclaw has no public iOS API to switch off TestFlight's automatic reporting.
|
||||
Its statistics setting and iOS tracking permissions do not control it. To
|
||||
leave, select Stop Testing on Faceclaw's TestFlight information page, then
|
||||
stop using and uninstall the beta. This does not erase previous reports.
|
||||
Self-built installations do not use TestFlight reporting.
|
||||
|
||||
For ordinary iOS analytics, use iPhone Settings > Privacy & Security >
|
||||
Analytics & Improvements, including Share With App Developers. Faceclaw
|
||||
cannot change these system settings. TestFlight crash reporting applies
|
||||
regardless of those settings. See https://www.apple.com/legal/privacy/data/en/app-analytics/.
|
||||
|
||||
We use TestFlight reports only to improve Faceclaw, not for advertising or
|
||||
sharing with third parties. Apple retains feedback for one year and may keep
|
||||
crash/usage reports until bugs are resolved. For access/deletion requests,
|
||||
contact us using the developer email in TestFlight; manage Apple's data at
|
||||
https://privacy.apple.com/.
|
||||
|
||||
Apple's details: https://www.apple.com/legal/privacy/data/en/test-flight/
|
||||
and https://testflight.apple.com/.
|
||||
|
||||
|
||||
## Audio
|
||||
@@ -64,7 +145,12 @@ transmitted to Mapbox, and are subject to Mapbox's privacy policy.
|
||||
Mapbox: https://www.mapbox.com/legal/privacy
|
||||
|
||||
The Weather app will retrieve the user's coarse location, and will use it to
|
||||
query a third-party weather information provider.
|
||||
query a third-party weather information provider: the US National Weather
|
||||
Service, then Open-Meteo if the location is outside the US. Outside the US, it
|
||||
also asks the phone's built-in geocoder (Google's on most Android phones,
|
||||
Apple's on iOS) for the name of the place.
|
||||
|
||||
Open-Meteo: https://open-meteo.com/en/terms
|
||||
|
||||
|
||||
## EvenHub
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
/**
|
||||
* The app-wide things analytics reports (the reliability counts come from
|
||||
* Kotlin, drained in analytics.ts):
|
||||
* - Minimal: which devices are paired, and which were in use each day;
|
||||
* - Full: which kinds of API key are set, how many terminal and T3 Code
|
||||
* connections there are, the UI font, and every fixed-choice setting.
|
||||
* Never anything a person typed in: no keys, addresses, names or URLs.
|
||||
*/
|
||||
import { loadEnvironments } from "../apps/t3code/t3-environments";
|
||||
import { loadConnections } from "../apps/terminal/connections";
|
||||
import { loadDeviceAddresses } from "../g2/device-addresses";
|
||||
import { onGlassesPresenceChanged } from "../g2/glasses-presence";
|
||||
import { isPreinstalledFont } from "../graphics/installed-fonts";
|
||||
import { getUiFontSelection } from "../graphics/ui-fonts";
|
||||
import { getStringSetting } from "../native/settings-store";
|
||||
import { isPreviewOnlyMode } from "../phone-ui/onboarding-state";
|
||||
import { fixedChoiceSettingValues, ringConnectionModeSetting } from "../ui/dashboard-settings";
|
||||
import { addInputListener } from "../ui/input-monitor";
|
||||
import { flagAnalyticsEvent, registerAnalyticsSnapshot, startAnalytics } from "./analytics";
|
||||
import type { Snapshot } from "./analytics-store";
|
||||
|
||||
/** Settings holding API keys and tokens, reported only as set or not, by kind. */
|
||||
const API_KEY_SETTINGS: Record<string, string> = {
|
||||
anthropic: "llm.anthropicApiKey",
|
||||
openai: "voice.openAiApiKey",
|
||||
elevenlabs: "voice.elevenLabsApiKey",
|
||||
soniox: "voice.sonioxApiKey",
|
||||
"whisper-server": "voice.whisperServerApiKey",
|
||||
mapbox: "maps.mapboxApiKey",
|
||||
roam: "integrations.roam.apiToken",
|
||||
nightscout: "integrations.nightscout.apiToken",
|
||||
"agent-bridge": "assistant.bridgeToken",
|
||||
evenhub: "integrations.evenhub.token",
|
||||
};
|
||||
|
||||
let started = false;
|
||||
|
||||
/** Registers the sources above and starts analytics. Main thread, at boot. */
|
||||
export function startAppAnalytics(): void {
|
||||
if (started) return;
|
||||
started = true;
|
||||
registerAnalyticsSnapshot("minimal", deviceSnapshot);
|
||||
registerAnalyticsSnapshot("full", configurationSnapshot);
|
||||
|
||||
onGlassesPresenceChanged((presence) => {
|
||||
if (presence.connected) flagAnalyticsEvent("minimal", "device.g2.used");
|
||||
});
|
||||
addInputListener((event) => {
|
||||
if (!("source" in event)) return;
|
||||
if (event.source === "ring") flagAnalyticsEvent("minimal", "device.r1.used");
|
||||
else if (event.source === "left-arm" || event.source === "right-arm") flagAnalyticsEvent("minimal", "device.g2.touchpad-used");
|
||||
});
|
||||
startAnalytics();
|
||||
}
|
||||
|
||||
function deviceSnapshot(): Snapshot {
|
||||
const addresses = loadDeviceAddresses();
|
||||
return {
|
||||
"device.g2.paired": Boolean(addresses.left && addresses.right),
|
||||
"device.g2.preview-only": isPreviewOnlyMode(),
|
||||
"device.r1.paired": Boolean(addresses.ring),
|
||||
"device.r1.connection": ringConnectionModeSetting.get(),
|
||||
};
|
||||
}
|
||||
|
||||
function configurationSnapshot(): Snapshot {
|
||||
const snapshot: Snapshot = {};
|
||||
for (const [kind, storageKey] of Object.entries(API_KEY_SETTINGS)) {
|
||||
snapshot[`apikey.${kind}`] = getStringSetting(storageKey, "").trim() !== "";
|
||||
}
|
||||
snapshot["config.nightscout-site"] = getStringSetting("integrations.nightscout.siteUrl", "").trim() !== "";
|
||||
snapshot["config.terminal-connections"] = loadConnections().length;
|
||||
snapshot["config.t3code-environments"] = loadEnvironments().length;
|
||||
|
||||
const font = getUiFontSelection();
|
||||
snapshot["config.ui-font"] =
|
||||
font.kind === "bitmap"
|
||||
? font.face
|
||||
: `${isPreinstalledFont(font.file) ? font.file.replace(/\.ttf$/i, "") : "installed"}:${font.size}`;
|
||||
|
||||
for (const [storageKey, value] of Object.entries(fixedChoiceSettingValues())) {
|
||||
snapshot[`setting.${storageKey}`] = value;
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
@@ -0,0 +1,410 @@
|
||||
/**
|
||||
* Opt-in analytics, held on the phone between daily uploads to
|
||||
* stats.faceclaw.org. This is the platform-free core: storage, clock, the
|
||||
* data-collection setting and the network are passed in (analytics.ts wires
|
||||
* them up), so the tests can drive it directly.
|
||||
*
|
||||
* Data is recorded at a level, and only while the data-collection setting is
|
||||
* at that level or above:
|
||||
* - "minimal": the app version and build, which devices are paired and in
|
||||
* use, and connection-reliability counts;
|
||||
* - "full": also API-key presence (never the keys) and settings chosen from
|
||||
* fixed options.
|
||||
* Turning the setting down discards what was kept above the new level;
|
||||
* turning it off deletes everything, including the install id, and stops all
|
||||
* writes and uploads.
|
||||
*
|
||||
* Counters accumulate in pending.json (per UTC day). Once a day the pending
|
||||
* counters and a snapshot of the current state are sealed into an outbox
|
||||
* report, and every outbox report is posted; each stays until the server
|
||||
* accepts it, under a report id the server deduplicates on. The report
|
||||
* format is stats-server/PROTOCOL.md.
|
||||
*/
|
||||
|
||||
export type AnalyticsLevel = "minimal" | "full";
|
||||
export type DataCollectionLevel = "none" | AnalyticsLevel;
|
||||
export type SnapshotValue = boolean | number | string;
|
||||
export type Snapshot = Record<string, SnapshotValue>;
|
||||
|
||||
export interface AppInfo {
|
||||
version: string;
|
||||
platform: string;
|
||||
build: string;
|
||||
}
|
||||
|
||||
/** Named text files in the analytics directory. */
|
||||
export interface AnalyticsStorage {
|
||||
read(name: string): string | null;
|
||||
write(name: string, text: string): void;
|
||||
remove(name: string): void;
|
||||
list(): string[];
|
||||
/** Deletes the directory and everything in it. */
|
||||
removeAll(): void;
|
||||
}
|
||||
|
||||
export interface AnalyticsDeps {
|
||||
storage: AnalyticsStorage;
|
||||
/** The data-collection setting, read fresh on every use. */
|
||||
level(): DataCollectionLevel;
|
||||
appInfo(): AppInfo;
|
||||
now(): number;
|
||||
randomId(): string;
|
||||
/** The current state to report at `level` (each level's keys, not cumulative). */
|
||||
snapshot(level: AnalyticsLevel): Snapshot;
|
||||
/** POSTs a report body; resolves to the HTTP status, rejects on network failure. */
|
||||
send(body: string): Promise<number>;
|
||||
log?(message: string): void;
|
||||
}
|
||||
|
||||
/** UTC day (YYYY-MM-DD) → counter name → count. */
|
||||
type Counters = Record<string, Record<string, number>>;
|
||||
type ByLevel<T> = { minimal: T; full: T };
|
||||
|
||||
interface PendingFile {
|
||||
format: 1;
|
||||
/** The app version the counts were made under; a different one seals them. */
|
||||
version: string;
|
||||
counters: ByLevel<Counters>;
|
||||
}
|
||||
|
||||
interface OutboxReport {
|
||||
format: 1;
|
||||
reportId: string;
|
||||
createdAt: string;
|
||||
app: AppInfo;
|
||||
counters: ByLevel<Counters>;
|
||||
snapshot: ByLevel<Snapshot>;
|
||||
}
|
||||
|
||||
interface StateFile {
|
||||
format: 1;
|
||||
installId: string;
|
||||
lastSealAt: number;
|
||||
/** When to retry after a failed upload; 0 when none failed. */
|
||||
retryAt: number;
|
||||
}
|
||||
|
||||
export const REPORT_SCHEMA_VERSION = 1;
|
||||
export const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
export const RETRY_MS = 60 * 60 * 1000;
|
||||
/** Unsent reports kept while the server is unreachable; the oldest go first. */
|
||||
export const MAX_OUTBOX_REPORTS = 30;
|
||||
/** Distinct counter names per day and level, so a runaway name can't grow the file. */
|
||||
export const MAX_NAMES_PER_DAY = 300;
|
||||
const FLUSH_DELAY_MS = 15_000;
|
||||
|
||||
const PENDING_FILE = "pending.json";
|
||||
const STATE_FILE = "state.json";
|
||||
const OUTBOX_PREFIX = "outbox-";
|
||||
|
||||
/** Counter names and snapshot keys the server accepts (stats-server/src/report.ts). */
|
||||
const COUNTER_NAME = /^[a-z0-9][a-z0-9_.:-]{0,119}$/;
|
||||
const SNAPSHOT_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,119}$/;
|
||||
const MAX_SNAPSHOT_STRING = 200;
|
||||
|
||||
export class AnalyticsStore {
|
||||
private pending: PendingFile | null = null;
|
||||
private dirty = false;
|
||||
private flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private uploading = false;
|
||||
/** Bumped by discardAll, so an upload in flight knows not to write its state back. */
|
||||
private generation = 0;
|
||||
|
||||
constructor(private readonly deps: AnalyticsDeps) {}
|
||||
|
||||
/** Whether data at `level` is being kept. */
|
||||
allows(level: AnalyticsLevel): boolean {
|
||||
const setting = this.deps.level();
|
||||
return setting === "full" || (setting === "minimal" && level === "minimal");
|
||||
}
|
||||
|
||||
/** Adds `amount` to today's count of `name`. */
|
||||
count(level: AnalyticsLevel, name: string, amount = 1): void {
|
||||
if (!this.allows(level) || !(amount > 0)) return;
|
||||
if (!COUNTER_NAME.test(name)) {
|
||||
this.log(`analytics: dropping bad counter name ${name}`);
|
||||
return;
|
||||
}
|
||||
const day = this.dayCounters(level);
|
||||
if (day[name] === undefined && Object.keys(day).length >= MAX_NAMES_PER_DAY) return;
|
||||
day[name] = (day[name] ?? 0) + Math.round(amount);
|
||||
this.markDirty();
|
||||
}
|
||||
|
||||
/** Records that `name` happened today (a count of 1, however often it happens). */
|
||||
flag(level: AnalyticsLevel, name: string): void {
|
||||
if (!this.allows(level)) return;
|
||||
if (!COUNTER_NAME.test(name)) {
|
||||
this.log(`analytics: dropping bad counter name ${name}`);
|
||||
return;
|
||||
}
|
||||
const day = this.dayCounters(level);
|
||||
if (day[name] !== undefined || Object.keys(day).length >= MAX_NAMES_PER_DAY) return;
|
||||
day[name] = 1;
|
||||
this.markDirty();
|
||||
}
|
||||
|
||||
/**
|
||||
* Brings the files in line with the setting: off deletes everything,
|
||||
* minimal drops what was kept for full. Called at start and whenever the
|
||||
* setting changes.
|
||||
*/
|
||||
applyLevel(): void {
|
||||
const setting = this.deps.level();
|
||||
if (setting === "none") {
|
||||
this.discardAll();
|
||||
return;
|
||||
}
|
||||
if (setting === "full") return;
|
||||
const pending = this.loadPending();
|
||||
if (Object.keys(pending.counters.full).length) {
|
||||
pending.counters.full = {};
|
||||
this.writePending(pending);
|
||||
}
|
||||
for (const name of this.outboxNames()) {
|
||||
const report = this.readJson(name) as OutboxReport | null;
|
||||
if (!report) continue;
|
||||
if (Object.keys(report.counters.full).length || Object.keys(report.snapshot.full).length) {
|
||||
report.counters.full = {};
|
||||
report.snapshot.full = {};
|
||||
this.deps.storage.write(name, JSON.stringify(report));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes pending counts now, if any changed. */
|
||||
flush(): void {
|
||||
if (this.flushTimer !== null) {
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
if (!this.dirty || !this.pending) return;
|
||||
this.dirty = false;
|
||||
if (this.deps.level() === "none") {
|
||||
this.discardAll();
|
||||
return;
|
||||
}
|
||||
this.writePending(this.pending);
|
||||
}
|
||||
|
||||
/**
|
||||
* The daily upload: once a day, seal the pending counts into a report and
|
||||
* post every unsent report; after a failure, retry hourly. Does nothing,
|
||||
* and makes no connection, while the setting is off.
|
||||
*/
|
||||
async uploadIfDue(): Promise<void> {
|
||||
if (this.uploading) return;
|
||||
if (this.deps.level() === "none") {
|
||||
this.discardAll();
|
||||
return;
|
||||
}
|
||||
this.uploading = true;
|
||||
const generation = this.generation;
|
||||
try {
|
||||
const now = this.deps.now();
|
||||
const state = this.loadState();
|
||||
let sealed = false;
|
||||
// A clock set backwards shouldn't stall the schedule until it catches up.
|
||||
if (now - state.lastSealAt >= DAY_MS || now < state.lastSealAt) {
|
||||
this.seal();
|
||||
state.lastSealAt = now;
|
||||
this.writeState(state);
|
||||
sealed = true;
|
||||
}
|
||||
if (!sealed && !(state.retryAt > 0 && now >= state.retryAt)) return;
|
||||
|
||||
for (const name of this.outboxNames()) {
|
||||
const report = this.readJson(name) as OutboxReport | null;
|
||||
if (!report) {
|
||||
this.deps.storage.remove(name);
|
||||
continue;
|
||||
}
|
||||
let status: number | null;
|
||||
try {
|
||||
status = await this.deps.send(this.reportBody(report, state.installId));
|
||||
} catch (error) {
|
||||
this.log(`analytics: upload failed: ${error}`);
|
||||
status = null;
|
||||
}
|
||||
// The setting may have been turned off (and maybe on again) while the
|
||||
// request was out; then everything this upload knew about is gone.
|
||||
if (this.deps.level() === "none") {
|
||||
this.discardAll();
|
||||
return;
|
||||
}
|
||||
if (this.generation !== generation) return;
|
||||
if (status !== null && status >= 200 && status < 300) {
|
||||
this.deps.storage.remove(name);
|
||||
} else if (status === 400 || status === 413) {
|
||||
// Rejected as malformed or too large: sending it again won't help.
|
||||
// (Anything else, like a 404 from a misconfigured server, is retried.)
|
||||
this.log(`analytics: server rejected report ${report.reportId} (HTTP ${status})`);
|
||||
this.deps.storage.remove(name);
|
||||
} else {
|
||||
state.retryAt = now + RETRY_MS;
|
||||
this.writeState(state);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (state.retryAt !== 0) {
|
||||
state.retryAt = 0;
|
||||
this.writeState(state);
|
||||
}
|
||||
} finally {
|
||||
this.uploading = false;
|
||||
}
|
||||
}
|
||||
|
||||
/** Deletes everything kept, including the install id. */
|
||||
discardAll(): void {
|
||||
if (this.flushTimer !== null) {
|
||||
clearTimeout(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
this.pending = null;
|
||||
this.dirty = false;
|
||||
this.generation++;
|
||||
this.deps.storage.removeAll();
|
||||
}
|
||||
|
||||
/** The JSON body posted for `report`, including only what the current setting allows. */
|
||||
reportBody(report: OutboxReport, installId: string): string {
|
||||
const level: AnalyticsLevel = this.deps.level() === "full" ? "full" : "minimal";
|
||||
const counters: Counters = {};
|
||||
for (const source of level === "full" ? [report.counters.minimal, report.counters.full] : [report.counters.minimal]) {
|
||||
for (const [day, counts] of Object.entries(source)) {
|
||||
const target = (counters[day] ??= {});
|
||||
for (const [name, count] of Object.entries(counts)) target[name] = (target[name] ?? 0) + count;
|
||||
}
|
||||
}
|
||||
return JSON.stringify({
|
||||
schema: REPORT_SCHEMA_VERSION,
|
||||
reportId: report.reportId,
|
||||
installId,
|
||||
level,
|
||||
createdAt: report.createdAt,
|
||||
app: report.app,
|
||||
counters,
|
||||
snapshot: level === "full" ? { ...report.snapshot.minimal, ...report.snapshot.full } : report.snapshot.minimal,
|
||||
});
|
||||
}
|
||||
|
||||
private dayCounters(level: AnalyticsLevel): Record<string, number> {
|
||||
const day = new Date(this.deps.now()).toISOString().slice(0, 10);
|
||||
return (this.loadPending().counters[level][day] ??= {});
|
||||
}
|
||||
|
||||
private markDirty(): void {
|
||||
this.dirty = true;
|
||||
this.flushTimer ??= setTimeout(() => {
|
||||
this.flushTimer = null;
|
||||
this.flush();
|
||||
}, FLUSH_DELAY_MS);
|
||||
}
|
||||
|
||||
private loadPending(): PendingFile {
|
||||
if (this.pending) return this.pending;
|
||||
const version = this.deps.appInfo().version;
|
||||
const stored = this.readJson(PENDING_FILE) as PendingFile | null;
|
||||
if (stored && stored.format === 1 && stored.counters?.minimal && stored.counters?.full) {
|
||||
this.pending = stored;
|
||||
if (stored.version !== version && hasCounts(stored)) {
|
||||
// Counts from before an upgrade go out under the version they were made with.
|
||||
this.seal();
|
||||
}
|
||||
}
|
||||
return (this.pending ??= emptyPending(version));
|
||||
}
|
||||
|
||||
/** Moves the pending counts, with a snapshot of the current state, into a new outbox report. */
|
||||
private seal(): void {
|
||||
const pending = this.loadPending();
|
||||
const app = { ...this.deps.appInfo(), version: pending.version };
|
||||
const report: OutboxReport = {
|
||||
format: 1,
|
||||
reportId: this.deps.randomId(),
|
||||
createdAt: new Date(this.deps.now()).toISOString(),
|
||||
app,
|
||||
counters: pending.counters,
|
||||
snapshot: {
|
||||
minimal: this.takeSnapshot("minimal"),
|
||||
full: this.allows("full") ? this.takeSnapshot("full") : {},
|
||||
},
|
||||
};
|
||||
this.deps.storage.write(`${OUTBOX_PREFIX}${report.createdAt.replace(/[^0-9]/g, "")}-${report.reportId}.json`, JSON.stringify(report));
|
||||
this.pending = emptyPending(this.deps.appInfo().version);
|
||||
this.dirty = false;
|
||||
this.writePending(this.pending);
|
||||
const outbox = this.outboxNames();
|
||||
for (const name of outbox.slice(0, Math.max(0, outbox.length - MAX_OUTBOX_REPORTS))) this.deps.storage.remove(name);
|
||||
}
|
||||
|
||||
private takeSnapshot(level: AnalyticsLevel): Snapshot {
|
||||
const snapshot: Snapshot = {};
|
||||
let entries: [string, SnapshotValue][];
|
||||
try {
|
||||
entries = Object.entries(this.deps.snapshot(level));
|
||||
} catch (error) {
|
||||
this.log(`analytics: ${level} snapshot failed: ${error}`);
|
||||
return snapshot;
|
||||
}
|
||||
for (const [key, value] of entries) {
|
||||
const ok =
|
||||
SNAPSHOT_KEY.test(key) &&
|
||||
(typeof value === "boolean" ||
|
||||
(typeof value === "number" && Number.isFinite(value)) ||
|
||||
(typeof value === "string" && value.length <= MAX_SNAPSHOT_STRING));
|
||||
if (ok) snapshot[key] = value;
|
||||
else this.log(`analytics: dropping bad snapshot entry ${key}`);
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
/** Outbox file names, oldest first (the names start with the seal time). */
|
||||
private outboxNames(): string[] {
|
||||
return this.deps.storage
|
||||
.list()
|
||||
.filter((name) => name.startsWith(OUTBOX_PREFIX))
|
||||
.sort();
|
||||
}
|
||||
|
||||
private loadState(): StateFile {
|
||||
const stored = this.readJson(STATE_FILE) as StateFile | null;
|
||||
if (stored && stored.format === 1 && typeof stored.installId === "string") return stored;
|
||||
const state: StateFile = { format: 1, installId: this.deps.randomId(), lastSealAt: 0, retryAt: 0 };
|
||||
this.writeState(state);
|
||||
return state;
|
||||
}
|
||||
|
||||
private writeState(state: StateFile): void {
|
||||
this.deps.storage.write(STATE_FILE, JSON.stringify(state));
|
||||
}
|
||||
|
||||
private writePending(pending: PendingFile): void {
|
||||
this.deps.storage.write(PENDING_FILE, JSON.stringify(pending));
|
||||
}
|
||||
|
||||
private readJson(name: string): unknown {
|
||||
const text = this.deps.storage.read(name);
|
||||
if (text === null) return null;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
this.log(`analytics: discarding unreadable ${name}`);
|
||||
this.deps.storage.remove(name);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private log(message: string): void {
|
||||
this.deps.log?.(message);
|
||||
}
|
||||
}
|
||||
|
||||
function emptyPending(version: string): PendingFile {
|
||||
return { format: 1, version, counters: { minimal: {}, full: {} } };
|
||||
}
|
||||
|
||||
function hasCounts(pending: PendingFile): boolean {
|
||||
return Object.keys(pending.counters.minimal).length > 0 || Object.keys(pending.counters.full).length > 0;
|
||||
}
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* The app's analytics API. What is collected, when it's kept and when it's
|
||||
* sent is described in analytics-store.ts; the snapshot providers (devices,
|
||||
* API-key presence, settings) are in analytics-snapshots.ts.
|
||||
*
|
||||
* Record from the main thread only. The store is created by startAnalytics()
|
||||
* at boot, which app workers never run, so calls made in a worker are
|
||||
* dropped. Kotlin-side connection events go through ConnectionCounters
|
||||
* (drained here) instead.
|
||||
*/
|
||||
import { File, Folder, knownFolders, path as fsPath } from "@nativescript/core";
|
||||
import { buildKind } from "../native/build-info";
|
||||
import { drainConnectionCounters } from "../native/connection-counters";
|
||||
import { onSettingsStoreChanged } from "../native/settings-store";
|
||||
import { DATA_COLLECTION_KEY, dataCollectionSetting } from "../ui/dashboard-settings";
|
||||
import { fetchTextWithUserAgent } from "../util/http";
|
||||
import { FACECLAW_VERSION } from "../version";
|
||||
import {
|
||||
AnalyticsStore,
|
||||
type AnalyticsLevel,
|
||||
type AnalyticsStorage,
|
||||
type Snapshot,
|
||||
} from "./analytics-store";
|
||||
|
||||
export type { AnalyticsLevel, Snapshot } from "./analytics-store";
|
||||
|
||||
declare const global: any;
|
||||
|
||||
const REPORT_URL = "https://stats.faceclaw.org/v1/report";
|
||||
const REQUEST_TIMEOUT_MS = 30_000;
|
||||
/** How often to see whether the daily upload is due (it usually isn't). */
|
||||
const UPLOAD_CHECK_INTERVAL_MS = 60 * 60 * 1000;
|
||||
/** The first check, a little after boot or after analytics is turned on. */
|
||||
const FIRST_UPLOAD_CHECK_DELAY_MS = 2 * 60 * 1000;
|
||||
/** How often the Kotlin connection counts are moved into the store. */
|
||||
const DRAIN_INTERVAL_MS = 5 * 60 * 1000;
|
||||
|
||||
let store: AnalyticsStore | null = null;
|
||||
let firstCheckTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
const snapshotProviders: { level: AnalyticsLevel; provide: () => Snapshot }[] = [];
|
||||
|
||||
/**
|
||||
* Adds `amount` to today's count of `name`, if the data-collection setting
|
||||
* includes `level`. Names are lowercase, dot-separated (e.g. "g2.connect").
|
||||
*/
|
||||
export function countAnalyticsEvent(level: AnalyticsLevel, name: string, amount = 1): void {
|
||||
store?.count(level, name, amount);
|
||||
}
|
||||
|
||||
/** Records that `name` happened today: a count of 1 however often it happens. */
|
||||
export function flagAnalyticsEvent(level: AnalyticsLevel, name: string): void {
|
||||
store?.flag(level, name);
|
||||
}
|
||||
|
||||
/**
|
||||
* Adds state to report with each daily upload, when the setting includes
|
||||
* `level`. Called when the report is sealed; keys must be namespaced like
|
||||
* "device.g2.paired", and values booleans, numbers or short strings.
|
||||
*/
|
||||
export function registerAnalyticsSnapshot(level: AnalyticsLevel, provide: () => Snapshot): void {
|
||||
snapshotProviders.push({ level, provide });
|
||||
}
|
||||
|
||||
/** Starts the store, the daily upload schedule and the counter drain. Main thread, once, at boot. */
|
||||
export function startAnalytics(): void {
|
||||
if (store) return;
|
||||
const analytics = new AnalyticsStore({
|
||||
storage: createStorage(),
|
||||
level: () => dataCollectionSetting.get(),
|
||||
appInfo: () => ({ version: FACECLAW_VERSION, platform: global.isIOS ? "ios" : "android", build: buildKind() }),
|
||||
now: () => Date.now(),
|
||||
randomId: randomUuid,
|
||||
snapshot: collectSnapshot,
|
||||
send: sendReport,
|
||||
log: (message) => console.log(message),
|
||||
});
|
||||
store = analytics;
|
||||
analytics.applyLevel();
|
||||
|
||||
onSettingsStoreChanged((key) => {
|
||||
if (key !== DATA_COLLECTION_KEY) return;
|
||||
analytics.applyLevel();
|
||||
if (dataCollectionSetting.get() !== "none") scheduleFirstCheck(analytics);
|
||||
});
|
||||
setInterval(() => drainKotlinCounters(analytics), DRAIN_INTERVAL_MS);
|
||||
setInterval(() => void checkUpload(analytics), UPLOAD_CHECK_INTERVAL_MS);
|
||||
scheduleFirstCheck(analytics);
|
||||
}
|
||||
|
||||
function scheduleFirstCheck(analytics: AnalyticsStore): void {
|
||||
if (firstCheckTimer !== null) clearTimeout(firstCheckTimer);
|
||||
firstCheckTimer = setTimeout(() => {
|
||||
firstCheckTimer = null;
|
||||
void checkUpload(analytics);
|
||||
}, FIRST_UPLOAD_CHECK_DELAY_MS);
|
||||
}
|
||||
|
||||
async function checkUpload(analytics: AnalyticsStore): Promise<void> {
|
||||
drainKotlinCounters(analytics);
|
||||
try {
|
||||
await analytics.uploadIfDue();
|
||||
} catch (error) {
|
||||
console.warn(`analytics: upload check failed: ${error}`);
|
||||
}
|
||||
}
|
||||
|
||||
function drainKotlinCounters(analytics: AnalyticsStore): void {
|
||||
let counts: Record<string, number>;
|
||||
try {
|
||||
counts = drainConnectionCounters();
|
||||
} catch (error) {
|
||||
console.warn(`analytics: could not read connection counters: ${error}`);
|
||||
return;
|
||||
}
|
||||
// Drained either way, so counts from while analytics was off don't pile up.
|
||||
for (const [name, count] of Object.entries(counts)) analytics.count("minimal", name, count);
|
||||
analytics.flush();
|
||||
}
|
||||
|
||||
function collectSnapshot(level: AnalyticsLevel): Snapshot {
|
||||
const snapshot: Snapshot = {};
|
||||
for (const provider of snapshotProviders) {
|
||||
if (provider.level !== level) continue;
|
||||
try {
|
||||
Object.assign(snapshot, provider.provide());
|
||||
} catch (error) {
|
||||
console.warn(`analytics: snapshot provider failed: ${error}`);
|
||||
}
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
async function sendReport(body: string): Promise<number> {
|
||||
let timeout: ReturnType<typeof setTimeout> | undefined;
|
||||
const timedOut = new Promise<never>((_resolve, reject) => {
|
||||
timeout = setTimeout(() => reject(new Error("timed out")), REQUEST_TIMEOUT_MS);
|
||||
});
|
||||
try {
|
||||
const response = await Promise.race([
|
||||
fetchTextWithUserAgent(REPORT_URL, { method: "POST", headers: { "Content-Type": "application/json" }, body }),
|
||||
timedOut,
|
||||
]);
|
||||
return response.status;
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Files in an `analytics` directory: the app's private files directory on
|
||||
* Android; on iOS, Library rather than Documents, which the Files app shows.
|
||||
*/
|
||||
function createStorage(): AnalyticsStorage {
|
||||
const base = global.isIOS ? knownFolders.ios.library().path : knownFolders.documents().path;
|
||||
const dir = fsPath.join(base, "analytics");
|
||||
const filePath = (name: string) => fsPath.join(dir, name);
|
||||
return {
|
||||
read: (name) => (File.exists(filePath(name)) ? File.fromPath(filePath(name)).readTextSync() : null),
|
||||
write: (name, text) => {
|
||||
Folder.fromPath(dir);
|
||||
File.fromPath(filePath(name)).writeTextSync(text);
|
||||
},
|
||||
remove: (name) => {
|
||||
if (File.exists(filePath(name))) File.fromPath(filePath(name)).removeSync();
|
||||
},
|
||||
list: () => (Folder.exists(dir) ? Folder.fromPath(dir).getEntitiesSync().map((entity) => entity.name) : []),
|
||||
removeAll: () => {
|
||||
if (Folder.exists(dir)) Folder.fromPath(dir).removeSync();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function randomUuid(): string {
|
||||
const bytes = new Uint8Array(16);
|
||||
const crypto = (globalThis as { crypto?: { getRandomValues?: (array: Uint8Array) => Uint8Array } }).crypto;
|
||||
if (crypto?.getRandomValues) crypto.getRandomValues(bytes);
|
||||
else for (let index = 0; index < bytes.length; index++) bytes[index] = Math.floor(Math.random() * 256);
|
||||
bytes[6] = (bytes[6]! & 0x0f) | 0x40;
|
||||
bytes[8] = (bytes[8]! & 0x3f) | 0x80;
|
||||
const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join("");
|
||||
return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`;
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
/** Conditional wording: also shown on self-built iOS apps, where TestFlight does not apply. */
|
||||
export const IOS_PRIVACY_NOTICE = [
|
||||
"If installed with TestFlight, Apple automatically collects crash logs, installs, sessions, and device/app details and shares them with Faceclaw's developers. This continues even when Faceclaw statistics are None; the app cannot turn it off.",
|
||||
"Joining only through a public link hides your name and email from us, not from Apple. Email invitations can identify you. Optional feedback can disclose your email, comments, screenshots and diagnostics. Review what you send.",
|
||||
"iPhone Settings > Privacy & Security > Analytics & Improvements controls ordinary iOS analytics. It does not stop TestFlight crash reports. Faceclaw cannot change Apple's analytics settings.",
|
||||
"To leave the beta, open Faceclaw in TestFlight and choose Stop Testing, then stop using and uninstall the beta. This does not erase earlier reports. Details: Settings > Privacy > Privacy policy.",
|
||||
].join("\n\n");
|
||||
|
||||
export const IOS_PRIVACY_SUMMARY =
|
||||
"With TestFlight, crash/usage reports continue even with None. Open iOS / TestFlight privacy for details.";
|
||||
+1
-236
@@ -1,6 +1,7 @@
|
||||
@import '@nativescript/theme/css/core.css';
|
||||
@import '@nativescript/theme/css/default.css';
|
||||
@import './phone-ui/remote-controls.css';
|
||||
@import './phone-ui/pairing.css';
|
||||
|
||||
Page {
|
||||
background-color: #ffffff;
|
||||
@@ -183,242 +184,6 @@ Button.-primary {
|
||||
margin-bottom: 12;
|
||||
}
|
||||
|
||||
/*
|
||||
* Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product
|
||||
* art (or a colourway swatch) on the left, decoded SKU + proximity + serial in
|
||||
* the middle, a chevron/check on the right.
|
||||
*/
|
||||
.pairing-instructions {
|
||||
color: #555555;
|
||||
font-size: 14;
|
||||
}
|
||||
|
||||
.pairing-status {
|
||||
font-size: 14;
|
||||
margin-left: 8;
|
||||
}
|
||||
|
||||
.pairing-section {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
margin-bottom: 4;
|
||||
}
|
||||
|
||||
.pairing-empty {
|
||||
color: #777777;
|
||||
font-size: 14;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.pairing-row {
|
||||
background-color: #f4f4f4;
|
||||
border-color: #dddddd;
|
||||
border-width: 1;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
margin-bottom: 8;
|
||||
}
|
||||
|
||||
.pairing-row-selected {
|
||||
background-color: #e6f0ff;
|
||||
border-color: #3b6fd6;
|
||||
}
|
||||
|
||||
.pairing-row-incomplete {
|
||||
background-color: #fafafa;
|
||||
border-color: #e6e6e6;
|
||||
}
|
||||
|
||||
.pairing-art {
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
.pairing-title {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
.pairing-badge {
|
||||
font-size: 11;
|
||||
font-weight: bold;
|
||||
color: #2a5bc4;
|
||||
background-color: #dbe6fb;
|
||||
border-radius: 9;
|
||||
padding: 1 7;
|
||||
}
|
||||
|
||||
.pairing-variant {
|
||||
font-size: 13;
|
||||
font-weight: bold;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.pairing-proximity {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.pairing-proximity-immediate {
|
||||
color: #2a5bc4;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.pairing-serial {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.pairing-arms {
|
||||
font-size: 12;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.pairing-warning {
|
||||
font-size: 12;
|
||||
color: #a33a1f;
|
||||
margin-top: 4;
|
||||
}
|
||||
|
||||
.pairing-swatch-label {
|
||||
font-size: 11;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.pairing-chevron {
|
||||
font-size: 22;
|
||||
color: #444444;
|
||||
margin-left: 6;
|
||||
}
|
||||
|
||||
.pairing-selection {
|
||||
background-color: #eef4ff;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
}
|
||||
|
||||
Button.pairing-link {
|
||||
background-color: transparent;
|
||||
color: #2a5bc4;
|
||||
font-size: 14;
|
||||
border-width: 0;
|
||||
/* Buttons keep their Material elevation shadow even with a transparent
|
||||
* background, which shows up as a stray, downward-offset outline. */
|
||||
android-elevation: 0;
|
||||
android-dynamic-elevation-offset: 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row
|
||||
* / pairing-title / pairing-badge look above; these add the explanation body
|
||||
* and the already-granted checkmark.
|
||||
*/
|
||||
.permissions-description {
|
||||
font-size: 13;
|
||||
color: #555555;
|
||||
margin-top: 2;
|
||||
}
|
||||
|
||||
.permissions-check {
|
||||
font-size: 24;
|
||||
color: #2e7d32;
|
||||
margin-left: 10;
|
||||
}
|
||||
|
||||
/* Continue with required permissions still missing: looks disabled but stays
|
||||
* tappable (the tap raises an are-you-sure prompt instead of being ignored). */
|
||||
Button.permissions-primary-dimmed {
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
/*
|
||||
* Dark-mode pairing page. The cards keep their light-mode look above; here
|
||||
* they go dark-gray, and every text colour set above gets a light counterpart
|
||||
* so nothing renders dark-on-dark (the instructions) or dark-on-a-dark-card.
|
||||
* Order matters: the -selected/-incomplete rules must follow .pairing-row so
|
||||
* they win at equal specificity.
|
||||
*/
|
||||
.ns-dark .pairing-instructions {
|
||||
color: #b0b0b0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-empty {
|
||||
color: #999999;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row {
|
||||
background-color: #2e2e2e;
|
||||
border-color: #4a4a4a;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row-selected {
|
||||
background-color: #333c4c;
|
||||
border-color: #5b8def;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-row-incomplete {
|
||||
background-color: #282828;
|
||||
border-color: #3c3c3c;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-title {
|
||||
color: #f0f0f0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-badge {
|
||||
color: #a8c4f5;
|
||||
background-color: #2a3b5e;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-variant {
|
||||
color: #cccccc;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-proximity {
|
||||
color: #a0a0a0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-proximity-immediate {
|
||||
color: #7da7f5;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-serial {
|
||||
color: #a0a0a0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-arms {
|
||||
color: #c0c0c0;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-warning {
|
||||
color: #e08a66;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-swatch-label {
|
||||
color: #aaaaaa;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-chevron {
|
||||
color: #bbbbbb;
|
||||
}
|
||||
|
||||
.ns-dark .pairing-selection {
|
||||
background-color: #2e2e2e;
|
||||
}
|
||||
|
||||
.ns-dark Button.pairing-link {
|
||||
color: #7da7f5;
|
||||
}
|
||||
|
||||
.ns-dark .permissions-description {
|
||||
color: #b0b0b0;
|
||||
}
|
||||
|
||||
.ns-dark .permissions-check {
|
||||
color: #7bc47f;
|
||||
}
|
||||
|
||||
/*
|
||||
* Conversation review (phone-ui/conversations-page, conversation-page,
|
||||
* speakers-page). Cards follow the pairing-row look; speaker/emotion chips
|
||||
|
||||
+3
-141
@@ -1,146 +1,6 @@
|
||||
@import '@nativescript/theme/css/core.css';
|
||||
@import './phone-ui/remote-controls.css';
|
||||
|
||||
.pairing-instructions {
|
||||
color: #555555;
|
||||
font-size: 14;
|
||||
}
|
||||
|
||||
.pairing-status {
|
||||
font-size: 14;
|
||||
margin-left: 8;
|
||||
}
|
||||
|
||||
.pairing-section {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
margin-bottom: 4;
|
||||
}
|
||||
|
||||
.pairing-empty {
|
||||
color: #777777;
|
||||
font-size: 14;
|
||||
padding: 8 0;
|
||||
}
|
||||
|
||||
.pairing-row {
|
||||
background-color: #f4f4f4;
|
||||
border-color: #dddddd;
|
||||
border-width: 1;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
margin-bottom: 8;
|
||||
}
|
||||
|
||||
.pairing-row-selected {
|
||||
background-color: #e6f0ff;
|
||||
border-color: #3b6fd6;
|
||||
}
|
||||
|
||||
.pairing-row-incomplete {
|
||||
background-color: #fafafa;
|
||||
border-color: #e6e6e6;
|
||||
}
|
||||
|
||||
.pairing-art {
|
||||
vertical-align: center;
|
||||
}
|
||||
|
||||
.pairing-title {
|
||||
font-size: 16;
|
||||
font-weight: bold;
|
||||
color: #222222;
|
||||
}
|
||||
|
||||
.pairing-badge {
|
||||
font-size: 11;
|
||||
font-weight: bold;
|
||||
color: #2a5bc4;
|
||||
background-color: #dbe6fb;
|
||||
border-radius: 9;
|
||||
padding: 1 7;
|
||||
}
|
||||
|
||||
.pairing-variant {
|
||||
font-size: 13;
|
||||
font-weight: bold;
|
||||
color: #333333;
|
||||
}
|
||||
|
||||
.pairing-proximity {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
}
|
||||
|
||||
.pairing-proximity-immediate {
|
||||
color: #2a5bc4;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.pairing-serial {
|
||||
font-size: 12;
|
||||
color: #666666;
|
||||
font-family: monospace;
|
||||
}
|
||||
|
||||
.pairing-arms {
|
||||
font-size: 12;
|
||||
color: #444444;
|
||||
}
|
||||
|
||||
.pairing-warning {
|
||||
font-size: 12;
|
||||
color: #a33a1f;
|
||||
margin-top: 4;
|
||||
}
|
||||
|
||||
.pairing-swatch-label {
|
||||
font-size: 11;
|
||||
color: #555555;
|
||||
}
|
||||
|
||||
.pairing-chevron {
|
||||
font-size: 22;
|
||||
color: #444444;
|
||||
margin-left: 6;
|
||||
}
|
||||
|
||||
.pairing-selection {
|
||||
background-color: #eef4ff;
|
||||
border-radius: 10;
|
||||
padding: 10;
|
||||
}
|
||||
|
||||
Button.pairing-link {
|
||||
background-color: transparent;
|
||||
color: #2a5bc4;
|
||||
font-size: 14;
|
||||
border-width: 0;
|
||||
}
|
||||
|
||||
/*
|
||||
* Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row
|
||||
* / pairing-title / pairing-badge look above; these add the explanation body
|
||||
* and the already-granted checkmark.
|
||||
*/
|
||||
.permissions-description {
|
||||
font-size: 13;
|
||||
color: #555555;
|
||||
margin-top: 2;
|
||||
}
|
||||
|
||||
.permissions-check {
|
||||
font-size: 24;
|
||||
color: #2e7d32;
|
||||
margin-left: 10;
|
||||
}
|
||||
|
||||
/* Continue with required permissions still missing: looks disabled but stays
|
||||
* tappable (the tap raises an are-you-sure prompt instead of being ignored). */
|
||||
Button.permissions-primary-dimmed {
|
||||
opacity: 0.45;
|
||||
}
|
||||
|
||||
@import './phone-ui/pairing.css';
|
||||
|
||||
.body-text { font-size: 18; }
|
||||
.h1 { font-size: 28; font-weight: bold; }
|
||||
@@ -175,6 +35,8 @@ Page.main-page {
|
||||
.main-page .action-bar-title { font-size: 18; font-weight: bold; color: #ffffff; }
|
||||
.main-page .action-bar-status { font-size: 14; color: #ffffff; margin-right: 12; }
|
||||
.main-page .action-bar-warning { font-size: 18; padding: 4 6; }
|
||||
/* Matches Android's 32x32 padded icon; the Image inside is 21x21. */
|
||||
.main-page .action-bar-app-icon { width: 32; height: 32; margin-right: 8; }
|
||||
.main-page .overflow-button {
|
||||
background-color: transparent;
|
||||
color: #ffffff;
|
||||
|
||||
@@ -39,7 +39,7 @@ export class VoiceDraft {
|
||||
this.phase = "listening";
|
||||
this.status = "Listening... release to send";
|
||||
this.ownsVoiceActivity = true;
|
||||
voiceActivity.setActive(true);
|
||||
voiceActivity.setActive(true, "voice-draft");
|
||||
this.unsubscribe.push(voiceControlBridge.onTranscript((event) => {
|
||||
if (!this.active || generation !== this.generation) return;
|
||||
if (event.isFinal) {
|
||||
@@ -129,7 +129,7 @@ export class VoiceDraft {
|
||||
}
|
||||
if (this.ownsVoiceActivity) {
|
||||
this.ownsVoiceActivity = false;
|
||||
voiceActivity.setActive(false);
|
||||
voiceActivity.setActive(false, "voice-draft");
|
||||
}
|
||||
this.changed();
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import musicApp from "./music";
|
||||
import nightscoutApp from "./nightscout";
|
||||
import healthApp from "./health";
|
||||
import transcribeApp from "./transcribe";
|
||||
import translateApp from "./translate";
|
||||
import teleprompterApp from "./teleprompter";
|
||||
import microphonesApp from "./microphones";
|
||||
import notificationsApp from "./notifications";
|
||||
@@ -27,6 +28,7 @@ import developerApp from "./developer";
|
||||
import evenhubApp from "./evenhub";
|
||||
import glanceboardApp from "./glanceboard";
|
||||
import settingsApp from "./settings";
|
||||
import onboardingApp from "./onboarding";
|
||||
|
||||
/**
|
||||
* Every app, in launcher-grid order (the launcher itself is first but hidden
|
||||
@@ -45,6 +47,7 @@ export const ALL_APPS: readonly AppDefinition[] = [
|
||||
nightscoutApp,
|
||||
healthApp,
|
||||
transcribeApp,
|
||||
translateApp,
|
||||
teleprompterApp,
|
||||
microphonesApp,
|
||||
notificationsApp,
|
||||
@@ -62,4 +65,5 @@ export const ALL_APPS: readonly AppDefinition[] = [
|
||||
evenhubApp,
|
||||
glanceboardApp,
|
||||
settingsApp,
|
||||
onboardingApp,
|
||||
];
|
||||
@@ -4,7 +4,7 @@ import { type IconName } from "../graphics/icons";
|
||||
import { type LayerActions } from "../ui/layers";
|
||||
import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window";
|
||||
import { WorkerAppHost } from "../ui/shell/worker-window";
|
||||
import { shell } from "../ui/shell/shell";
|
||||
import { shell, type TrayWidget } from "../ui/shell/shell";
|
||||
|
||||
/**
|
||||
* An app, as seen from outside while it isn't running. Each app directory's
|
||||
@@ -96,6 +96,17 @@ export type AppContext = {
|
||||
surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow,
|
||||
) => Promise<void>;
|
||||
/**
|
||||
* Open a main-thread singleton window in the background (neither
|
||||
* foregrounded nor focused) unless it is open already; `opened` says this
|
||||
* call opened it. For windows the shell brings forward itself, such as an
|
||||
* app widget's (see inProcessTrayWidget). Hosts that leave it out can't.
|
||||
*/
|
||||
openInProcessAppInBackground?: (
|
||||
windowId: string,
|
||||
surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow,
|
||||
) => { window: InProcessWindow; opened: boolean };
|
||||
/** Get this app's shared worker host, spawning the worker on first use. */
|
||||
ensureWorkerHost: (createWorker: () => Worker) => WorkerAppHost;
|
||||
/** Submit a window surface's painted frame, as planes (windows created outside launchInProcessApp). */
|
||||
@@ -120,6 +131,32 @@ export type TextEditorHost = {
|
||||
requestRender: () => void;
|
||||
};
|
||||
|
||||
/**
|
||||
* A tray widget (see Shell.setTrayIcon) whose window is a main-thread
|
||||
* singleton app window, opened in the background while the switcher's
|
||||
* selection is on it; undefined on hosts that can't do that. `show`, if
|
||||
* given, runs each time the selection moves onto the widget, once the
|
||||
* window is open, to bring what the widget stands for on screen in it (the
|
||||
* shell repaints the window after).
|
||||
*/
|
||||
export function inProcessTrayWidget(
|
||||
ctx: AppContext,
|
||||
windowId: string,
|
||||
surfaceId: string,
|
||||
create: (options: InProcessAppOptions) => InProcessWindow,
|
||||
show?: () => void,
|
||||
): TrayWidget | undefined {
|
||||
const open = ctx.openInProcessAppInBackground;
|
||||
if (!open) return undefined;
|
||||
return {
|
||||
openWindow: () => {
|
||||
const { window, opened } = open(windowId, surfaceId, create);
|
||||
show?.();
|
||||
return { window: window.window, opened };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Launch-or-focus flow shared by the worker-hosted apps: focus the app's
|
||||
* existing window if one is open, otherwise open a window in the app's shared
|
||||
|
||||
@@ -49,7 +49,7 @@ const PANEL_X = 230;
|
||||
/**
|
||||
* Height the layout above is drawn for: a min-height window's viewport under
|
||||
* the standard top bar. A taller viewport (the status bar folded into the
|
||||
* switcher row, a full-panel display) centers it vertically.
|
||||
* switcher row, a Tall or Full Screen display) centers it vertically.
|
||||
*/
|
||||
const LAYOUT_HEIGHT = 260;
|
||||
|
||||
@@ -290,8 +290,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
updateTickTimer(window);
|
||||
}
|
||||
break;
|
||||
case "resize-window": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
window.viewportWidth = message.viewport.width;
|
||||
window.viewportHeight = message.viewport.height;
|
||||
window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight });
|
||||
window.lastSubmittedFingerprint = "";
|
||||
if (window.foreground) renderAndSubmit(window, 0);
|
||||
break;
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "resize-window":
|
||||
case "text-input":
|
||||
case "tool-call":
|
||||
break;
|
||||
|
||||
@@ -116,6 +116,9 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce
|
||||
},
|
||||
removeSurface: options.removeSurface,
|
||||
onClosed: () => {
|
||||
// Closing keeps the base layer (clearToBase), so stop its continuous
|
||||
// listening explicitly; otherwise the mic capture outlives the window.
|
||||
layer.onRemoved();
|
||||
toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID);
|
||||
options.onClosed();
|
||||
},
|
||||
|
||||
@@ -16,6 +16,7 @@ import { type Layer, type LayerActions, type LayerContext } from "../../ui/layer
|
||||
import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control";
|
||||
import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||
import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator";
|
||||
import { type LongRunOffer, type LongRunProgress, offerGlassesPrompt, progressText } from "./math/workload";
|
||||
import { plain } from "./math/text-renderer";
|
||||
@@ -56,6 +57,11 @@ export const calculatorDegreesSetting = new ConfigSettingBoolean({
|
||||
description: "Evaluate sin/cos/tan in degrees instead of radians.",
|
||||
});
|
||||
|
||||
// The calculator's id for shared continuous capture, and its claim on the mic
|
||||
// while a dictation is armed.
|
||||
const CONTINUOUS_CLIENT = "calculator";
|
||||
const DICTATION_HOLDER = "calculator-dictation";
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Layer
|
||||
|
||||
@@ -131,22 +137,22 @@ export class CalculatorLayer implements Layer {
|
||||
if (this.listening === "continuous") {
|
||||
if (!this.continuousActive) {
|
||||
this.continuousActive = true;
|
||||
void this.actions.startContinuousVoiceCapture();
|
||||
void this.actions.startContinuousVoiceCapture(CONTINUOUS_CLIENT);
|
||||
}
|
||||
} else if (this.continuousActive) {
|
||||
this.continuousActive = false;
|
||||
void this.actions.stopContinuousVoiceCapture();
|
||||
void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT);
|
||||
}
|
||||
}
|
||||
|
||||
private stopListening(): void {
|
||||
if (this.dictating) {
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.setDictating(false);
|
||||
}
|
||||
if (this.continuousActive) {
|
||||
this.continuousActive = false;
|
||||
void this.actions.stopContinuousVoiceCapture();
|
||||
void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -248,15 +254,25 @@ export class CalculatorLayer implements Layer {
|
||||
beginDictation(): void {
|
||||
if (this.textInput) { void this.textInput(); return; }
|
||||
if (this.dictating) return;
|
||||
this.dictating = true;
|
||||
this.setDictating(true);
|
||||
void this.actions.startVoiceCapture(true);
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
/**
|
||||
* Arm or disarm dictation, holding the mic claim for it as the voice dialog
|
||||
* does: the Microphones app and EvenHub mic apps stand down meanwhile (in
|
||||
* 4-mic mode their stream would otherwise reach the recognizer).
|
||||
*/
|
||||
private setDictating(dictating: boolean): void {
|
||||
this.dictating = dictating;
|
||||
voiceActivity.setActive(dictating, DICTATION_HOLDER);
|
||||
}
|
||||
|
||||
cancelDictation(): void {
|
||||
if (!this.dictating) return;
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.setDictating(false);
|
||||
this.requestRender();
|
||||
}
|
||||
|
||||
@@ -277,8 +293,8 @@ export class CalculatorLayer implements Layer {
|
||||
const continuous = this.continuousActive && this.foreground;
|
||||
if (!armed && !continuous) return;
|
||||
if (armed) {
|
||||
this.dictating = false;
|
||||
void this.actions.stopVoiceCapture();
|
||||
this.setDictating(false);
|
||||
}
|
||||
// A standing offer or a run in flight owns the next yes/no said in the
|
||||
// room ("keep going", "never mind", "stop").
|
||||
|
||||
@@ -23,7 +23,14 @@ import {
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../native/location-permissions";
|
||||
import { isCompassCalibrated, normalizeHeading } from "./calibration";
|
||||
import { cardinalDirection, createCompassBackground, drawCompassRose, layoutCompassRose, TICK_HEIGHT } from "./compass-rose";
|
||||
import {
|
||||
cardinalDirection,
|
||||
createCompassBackground,
|
||||
drawCompassRose,
|
||||
layoutCompassRose,
|
||||
magneticAccuracyDots,
|
||||
TICK_HEIGHT,
|
||||
} from "./compass-rose";
|
||||
import { CompassCalibrationLayer } from "./calibration-layer";
|
||||
import { getDeclinationAvailability, onDeclinationChanged, refreshDeclination } from "./declination";
|
||||
import { getNorthReference, resolveHeading, setNorthReference, type NorthReference } from "./heading";
|
||||
@@ -277,14 +284,7 @@ function frameStatusText(): string {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The firmware's magnetometer calibration readiness (0-3) as filled/empty
|
||||
* dots, e.g. "Calibration: ●●○" for 2/3. This is the glasses' own sensor
|
||||
* calibration, not the wearer-fit offset set on the calibration screen.
|
||||
*/
|
||||
/** The firmware's magnetometer calibration readiness, e.g. "Calibration: ●●○" for 2/3. */
|
||||
function magneticAccuracyText(diagnostics: CompassDiagnostics | null): string {
|
||||
const accuracy = diagnostics && (diagnostics.flags & 0x80) ? diagnostics.magneticAccuracy : -1;
|
||||
if (accuracy < 0) return "Calibration: --";
|
||||
const filled = Math.min(3, accuracy);
|
||||
return `Calibration: ${"●".repeat(filled)}${"○".repeat(3 - filled)}`;
|
||||
return `Calibration: ${magneticAccuracyDots(diagnostics) ?? "--"}`;
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
* own the magnetometer lifecycle and the readout text around it.
|
||||
*/
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { type CompassDiagnostics } from "../../native/compass-types";
|
||||
import { normalizeHeading } from "./calibration";
|
||||
|
||||
/**
|
||||
@@ -102,6 +103,19 @@ export function cardinalDirection(heading: number): string {
|
||||
return names[Math.round(normalizeHeading(heading) / 45) % names.length]!;
|
||||
}
|
||||
|
||||
/**
|
||||
* The firmware's magnetometer calibration readiness (0-3) as filled/empty
|
||||
* dots, e.g. "●●○" for 2/3, or null when the firmware doesn't report it. This
|
||||
* is the glasses' own sensor calibration, not the wearer-fit offset set on the
|
||||
* calibration screen.
|
||||
*/
|
||||
export function magneticAccuracyDots(diagnostics: CompassDiagnostics | null): string | null {
|
||||
const accuracy = diagnostics && (diagnostics.flags & 0x80) ? diagnostics.magneticAccuracy : -1;
|
||||
if (accuracy < 0) return null;
|
||||
const filled = Math.min(3, accuracy);
|
||||
return "●".repeat(filled) + "○".repeat(3 - filled);
|
||||
}
|
||||
|
||||
interface Point {
|
||||
x: number;
|
||||
y: number;
|
||||
|
||||
@@ -65,6 +65,14 @@ type ArmStats = {
|
||||
statusAtMs: number | null;
|
||||
};
|
||||
|
||||
/** Drop arrivals older than the rate window. */
|
||||
function trimRecent(stats: ArmStats, nowMs: number): void {
|
||||
const cutoff = nowMs - RATE_WINDOW_MS;
|
||||
let drop = 0;
|
||||
while (drop < stats.recent.length && stats.recent[drop]!.wallMs < cutoff) drop++;
|
||||
if (drop > 0) stats.recent.splice(0, drop);
|
||||
}
|
||||
|
||||
function emptyStats(): ArmStats {
|
||||
return {
|
||||
packets: 0,
|
||||
@@ -169,6 +177,8 @@ export class MicrophonesMonitorLayer implements Layer {
|
||||
this.advanceBuckets(nowMs);
|
||||
stats.packets++;
|
||||
stats.buckets[BUCKETS - 1]!++;
|
||||
// Trim here too: painting (the other trim) stops while the page is hidden.
|
||||
trimRecent(stats, nowMs);
|
||||
stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length });
|
||||
stats.lastArrivalWallMs = nowMs;
|
||||
|
||||
@@ -268,8 +278,7 @@ export class MicrophonesMonitorLayer implements Layer {
|
||||
const font = getDefaultSmallFont();
|
||||
const line = font.lineHeight;
|
||||
const stats = this.stats[arm];
|
||||
const cutoff = nowMs - RATE_WINDOW_MS;
|
||||
while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift();
|
||||
trimRecent(stats, nowMs);
|
||||
const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS;
|
||||
const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS;
|
||||
let maxGap = 0;
|
||||
|
||||
@@ -14,7 +14,7 @@ import { isCompassCalibrated } from "../compass/calibration";
|
||||
import { refreshDeclination } from "../compass/declination";
|
||||
import { resolveHeading } from "../compass/heading";
|
||||
|
||||
/** Wire shape of the `compass` extension event (faceclaw-extensions `CompassReading`). */
|
||||
/** Wire shape of the `compass` extension event (@faceclaw/evenhub-extensions `CompassReading`). */
|
||||
export type EvenHubCompassReading = {
|
||||
headingDegrees: number;
|
||||
magneticHeadingDegrees: number;
|
||||
|
||||
@@ -4,7 +4,8 @@
|
||||
*
|
||||
* Text uses Even's extracted 20px firmware font (via EvenHubFont), measured
|
||||
* and wrapped with @evenrealities/pretext, so glyph widths and line breaks
|
||||
* match what apps expect.
|
||||
* match what apps expect. Extended layouts may pick other fonts per container
|
||||
* (fonts.ts), which wrap with Faceclaw's own text wrapping.
|
||||
*
|
||||
* Known deviations from stock, acceptable for now:
|
||||
* - List rendering is a plain vertical list. A non-zero itemWidth sets the
|
||||
@@ -13,7 +14,7 @@
|
||||
* - Lists scroll with our menus' animation and bounce (see list-menu.ts).
|
||||
*/
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { EvenHubFont } from "../../graphics/evenhub-font";
|
||||
import { containerFace } from "./fonts";
|
||||
import {
|
||||
type EvenHubContainer,
|
||||
type EvenHubImageContainer,
|
||||
@@ -47,14 +48,11 @@ function paintTextContainer(image: GrayImage, container: EvenHubTextContainer):
|
||||
// the container's border at full brightness regardless.
|
||||
paintBorder(image, container);
|
||||
const inset = container.borderWidth + container.paddingLength;
|
||||
EvenHubFont.get().drawTextWrapped(
|
||||
image,
|
||||
container.x + inset,
|
||||
container.y + inset,
|
||||
Math.max(1, container.width - 2 * inset),
|
||||
container.content,
|
||||
textGrey(container.textColor),
|
||||
);
|
||||
const face = containerFace(container.font);
|
||||
const lines = face.wrap(container.content, Math.max(1, container.width - 2 * inset));
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
face.drawText(image, container.x + inset, container.y + inset + i * face.lineHeight, lines[i]!, textGrey(container.textColor));
|
||||
}
|
||||
}
|
||||
|
||||
function paintBorder(
|
||||
@@ -94,7 +92,7 @@ function paintImageContainer(image: GrayImage, container: EvenHubImageContainer)
|
||||
|
||||
function paintListContainer(image: GrayImage, container: EvenHubListContainer, focused: boolean): void {
|
||||
paintBorder(image, container);
|
||||
paintListItems(image, container, EvenHubFont.get(), focused);
|
||||
paintListItems(image, container, containerFace(container.font), focused);
|
||||
}
|
||||
|
||||
/** Bottom-to-top paint order for a page's containers. */
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
* (Container_ID), and numbers sometimes arrive as strings, so all field
|
||||
* reads go through loose key matching (the SDK's pickLoose behavior).
|
||||
*/
|
||||
import { mergeFontSpecs, parseFontSpec, type EvenHubFontSpec } from "./font-spec";
|
||||
|
||||
export type EvenHubTextContainer = {
|
||||
kind: "text";
|
||||
@@ -24,6 +25,8 @@ export type EvenHubTextContainer = {
|
||||
content: string;
|
||||
/** SDK 0.0.14 textColor: brightness level 0..4; undefined means the device default (4). */
|
||||
textColor: number | undefined;
|
||||
/** Extended-layout font (merged with the layout's default); undefined is the stock font. */
|
||||
font: EvenHubFontSpec | undefined;
|
||||
};
|
||||
|
||||
export type EvenHubImageContainer = {
|
||||
@@ -63,6 +66,8 @@ export type EvenHubListContainer = {
|
||||
selectBorder: boolean;
|
||||
/** Selection is host-local: scroll moves it with no app round-trip. */
|
||||
selectedIndex: number;
|
||||
/** Extended-layout font (merged with the layout's default); undefined is the stock font. */
|
||||
font: EvenHubFontSpec | undefined;
|
||||
};
|
||||
|
||||
export type EvenHubContainer = EvenHubTextContainer | EvenHubImageContainer | EvenHubListContainer;
|
||||
@@ -130,6 +135,17 @@ export function readString(obj: Record<string, unknown>, key: string, fallback:
|
||||
return fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* A 0/1 protobuf flag. Stock apps send numbers; extension layouts may send
|
||||
* booleans (the extension types allow both), and numeric strings show up in
|
||||
* the wild.
|
||||
*/
|
||||
function readFlag(obj: Record<string, unknown>, key: string): boolean {
|
||||
const value = pickLoose(obj, key);
|
||||
if (typeof value === "boolean") return value;
|
||||
return readNumber(obj, key, 0) !== 0;
|
||||
}
|
||||
|
||||
export function asRecord(value: unknown): Record<string, unknown> {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? (value as Record<string, unknown>) : {};
|
||||
}
|
||||
@@ -138,7 +154,12 @@ function readBorderWidth(json: Record<string, unknown>): number {
|
||||
return Math.max(0, Math.min(MAX_BORDER_WIDTH, readNumber(json, "borderWidth", 0)));
|
||||
}
|
||||
|
||||
function parseTextContainer(json: Record<string, unknown>): EvenHubTextContainer {
|
||||
/** A container's font: its own `font` over the layout-wide default. */
|
||||
function readFont(json: Record<string, unknown>, layoutFont: EvenHubFontSpec | undefined): EvenHubFontSpec | undefined {
|
||||
return mergeFontSpecs(layoutFont, parseFontSpec(pickLoose(json, "font")));
|
||||
}
|
||||
|
||||
function parseTextContainer(json: Record<string, unknown>, layoutFont: EvenHubFontSpec | undefined): EvenHubTextContainer {
|
||||
return {
|
||||
kind: "text",
|
||||
id: readNumber(json, "containerID", 0),
|
||||
@@ -150,11 +171,12 @@ function parseTextContainer(json: Record<string, unknown>): EvenHubTextContainer
|
||||
borderWidth: readBorderWidth(json),
|
||||
borderRadius: readNumber(json, "borderRadius", 0),
|
||||
paddingLength: readNumber(json, "paddingLength", 0),
|
||||
isEventCapture: readNumber(json, "isEventCapture", 0) !== 0,
|
||||
isEventCapture: readFlag(json, "isEventCapture"),
|
||||
zOrderIndex: readOptionalNumber(json, "zOrderIndex"),
|
||||
preserve: readNumber(json, "preserve", 0) !== 0,
|
||||
preserve: readFlag(json, "preserve"),
|
||||
content: readString(json, "content", ""),
|
||||
textColor: readTextBrightness(json),
|
||||
font: readFont(json, layoutFont),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -220,14 +242,14 @@ function parseImageContainer(json: Record<string, unknown>): EvenHubImageContain
|
||||
width: readNumber(json, "width", 0),
|
||||
height: readNumber(json, "height", 0),
|
||||
zOrderIndex: readOptionalNumber(json, "zOrderIndex"),
|
||||
preserve: readNumber(json, "preserve", 0) !== 0,
|
||||
preserve: readFlag(json, "preserve"),
|
||||
pixels: null,
|
||||
pixelsWidth: 0,
|
||||
pixelsHeight: 0,
|
||||
};
|
||||
}
|
||||
|
||||
function parseListContainer(json: Record<string, unknown>): EvenHubListContainer {
|
||||
function parseListContainer(json: Record<string, unknown>, layoutFont: EvenHubFontSpec | undefined): EvenHubListContainer {
|
||||
const itemContainer = asRecord(pickLoose(json, "itemContainer"));
|
||||
const itemNamesRaw = pickLoose(itemContainer, "itemName");
|
||||
const itemNames = Array.isArray(itemNamesRaw) ? itemNamesRaw.map(String) : [];
|
||||
@@ -242,13 +264,14 @@ function parseListContainer(json: Record<string, unknown>): EvenHubListContainer
|
||||
borderWidth: readBorderWidth(json),
|
||||
borderRadius: readNumber(json, "borderRadius", 0),
|
||||
paddingLength: readNumber(json, "paddingLength", 0),
|
||||
isEventCapture: readNumber(json, "isEventCapture", 0) !== 0,
|
||||
isEventCapture: readFlag(json, "isEventCapture"),
|
||||
zOrderIndex: readOptionalNumber(json, "zOrderIndex"),
|
||||
preserve: readNumber(json, "preserve", 0) !== 0,
|
||||
preserve: readFlag(json, "preserve"),
|
||||
itemNames,
|
||||
itemWidth: readNumber(itemContainer, "itemWidth", 0),
|
||||
selectBorder: readNumber(itemContainer, "isItemSelectBorderEn", 0) !== 0,
|
||||
selectBorder: readFlag(itemContainer, "isItemSelectBorderEn"),
|
||||
selectedIndex: 0,
|
||||
font: readFont(json, layoutFont),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -263,9 +286,11 @@ function parseListContainer(json: Record<string, unknown>): EvenHubListContainer
|
||||
*/
|
||||
export function parsePage(data: Record<string, unknown>): EvenHubPage {
|
||||
const containers: EvenHubContainer[] = [];
|
||||
// Extended layouts may set a default font for every text and list container.
|
||||
const layoutFont = parseFontSpec(pickLoose(data, "font"));
|
||||
const lists = pickLoose(data, "listObject");
|
||||
if (Array.isArray(lists)) {
|
||||
for (const item of lists) containers.push(parseListContainer(asRecord(item)));
|
||||
for (const item of lists) containers.push(parseListContainer(asRecord(item), layoutFont));
|
||||
}
|
||||
const images = pickLoose(data, "imageObject");
|
||||
if (Array.isArray(images)) {
|
||||
@@ -273,7 +298,7 @@ export function parsePage(data: Record<string, unknown>): EvenHubPage {
|
||||
}
|
||||
const texts = pickLoose(data, "textObject");
|
||||
if (Array.isArray(texts)) {
|
||||
for (const item of texts) containers.push(parseTextContainer(asRecord(item)));
|
||||
for (const item of texts) containers.push(parseTextContainer(asRecord(item), layoutFont));
|
||||
}
|
||||
return { containers, menuItems: parseMenuItems(data) };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
/**
|
||||
* The live data behind an EvenHub app's device status: glasses presence (fed
|
||||
* by the connection controller), the headset battery (from the shell's top
|
||||
* bar state), and the paired glasses' serial.
|
||||
*/
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { getGlassesPresence, onGlassesPresenceChanged } from "../../g2/glasses-presence";
|
||||
import { loadPairedGlassesIdentity } from "../../g2/device-addresses";
|
||||
import { type EvenHubDeviceStatusSource } from "./device-status";
|
||||
|
||||
export const liveDeviceStatusSource: EvenHubDeviceStatusSource = {
|
||||
read: () => {
|
||||
const presence = getGlassesPresence();
|
||||
return {
|
||||
serial: loadPairedGlassesIdentity()?.serial ?? null,
|
||||
connected: presence.connected,
|
||||
worn: presence.worn,
|
||||
battery: shell.getBatteryLevels().headset,
|
||||
charging: presence.charging,
|
||||
};
|
||||
},
|
||||
subscribe: (listener) => {
|
||||
const offs = [onGlassesPresenceChanged(listener), shell.onBatteryLevelsChanged(listener)];
|
||||
return () => {
|
||||
for (const off of offs) off();
|
||||
};
|
||||
},
|
||||
};
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* The stock SDK's DeviceStatus (getGlassesInfo().status and the
|
||||
* deviceStatusChanged push), built from what Faceclaw actually knows about
|
||||
* the glasses. Pure, so it can be tested without NativeScript.
|
||||
*/
|
||||
|
||||
/** Used when no glasses have been paired (preview-only mode). */
|
||||
export const PLACEHOLDER_SERIAL = "FACECLAW-G2";
|
||||
|
||||
/** Wire shape of the SDK's DeviceStatus. Unknown fields are omitted rather than guessed. */
|
||||
export type EvenHubDeviceStatus = {
|
||||
sn: string;
|
||||
connectType: "connected" | "disconnected";
|
||||
isWearing?: boolean;
|
||||
batteryLevel?: number;
|
||||
isCharging?: boolean;
|
||||
isInCase?: boolean;
|
||||
};
|
||||
|
||||
export type EvenHubDeviceStatusInput = {
|
||||
/** The paired glasses' serial number, or null when none are paired. */
|
||||
serial: string | null;
|
||||
connected: boolean;
|
||||
/** null until the firmware reports a wear state this session. */
|
||||
worn: boolean | null;
|
||||
/** Last known headset battery percentage, or null if never reported. */
|
||||
battery: number | null;
|
||||
charging: boolean;
|
||||
};
|
||||
|
||||
/** Where a session reads the device status from (live in the app; fixed in tests). */
|
||||
export type EvenHubDeviceStatusSource = {
|
||||
read(): EvenHubDeviceStatusInput;
|
||||
/** Call `listener` after anything read() reports may have changed. Returns an unsubscribe function. */
|
||||
subscribe(listener: () => void): () => void;
|
||||
};
|
||||
|
||||
/** No glasses: what a session reports when it isn't given a live source. */
|
||||
export const OFFLINE_DEVICE_STATUS_SOURCE: EvenHubDeviceStatusSource = {
|
||||
read: () => ({ serial: null, connected: false, worn: null, battery: null, charging: false }),
|
||||
subscribe: () => () => {},
|
||||
};
|
||||
|
||||
export function buildEvenHubDeviceStatus(input: EvenHubDeviceStatusInput): EvenHubDeviceStatus {
|
||||
const status: EvenHubDeviceStatus = {
|
||||
// The SDK only applies a status push whose sn matches getGlassesInfo's,
|
||||
// so both must come from here.
|
||||
sn: input.serial?.trim() || PLACEHOLDER_SERIAL,
|
||||
connectType: input.connected ? "connected" : "disconnected",
|
||||
isCharging: input.charging,
|
||||
// The G2 only charges in its case, and Faceclaw has no separate
|
||||
// in-case report.
|
||||
isInCase: input.charging,
|
||||
};
|
||||
if (input.worn !== null) status.isWearing = input.worn;
|
||||
if (input.battery !== null) status.batteryLevel = input.battery;
|
||||
return status;
|
||||
}
|
||||
|
||||
export function sameDeviceStatus(a: EvenHubDeviceStatus, b: EvenHubDeviceStatus): boolean {
|
||||
return a.sn === b.sn && a.connectType === b.connectType && a.isWearing === b.isWearing &&
|
||||
a.batteryLevel === b.batteryLevel && a.isCharging === b.isCharging && a.isInCase === b.isInCase;
|
||||
}
|
||||
@@ -235,7 +235,7 @@ export class EvenHubApiClient {
|
||||
const token = options.token ?? "";
|
||||
const headers: Record<string, string> = {
|
||||
common,
|
||||
sign: hmacSign(signingParts(method, path, common, query, token, body)),
|
||||
sign: hmacSign(signingParts(method, path, common, signedQuery(options.query ?? {}), token, body)),
|
||||
"Content-Type": "application/json",
|
||||
};
|
||||
if (token) headers.token = token;
|
||||
@@ -266,7 +266,7 @@ export function signingParts(
|
||||
body = "",
|
||||
): string {
|
||||
const parts = [method.toUpperCase(), path, common];
|
||||
if (query) parts.push(normalizeQuery(query));
|
||||
if (query) parts.push(query);
|
||||
if (token) parts.push(token);
|
||||
if (body) parts.push(body);
|
||||
return parts.sort().join("\n");
|
||||
@@ -320,19 +320,26 @@ function buildCommon(version: 1 | 3): string {
|
||||
return Object.entries(fields).map(([key, value]) => `${formEncode(key)}=${formEncode(String(value))}`).join("&");
|
||||
}
|
||||
|
||||
function encodePairs(values: Record<string, string | number>): string {
|
||||
/** Query entries sorted by key in code-unit order (Dart's default String sort). */
|
||||
function sortedPairs(values: Record<string, string | number>): [string, string][] {
|
||||
return Object.entries(values)
|
||||
.sort(([a], [b]) => a.localeCompare(b))
|
||||
.map(([key, value]) => `${formEncode(key)}=${formEncode(String(value))}`)
|
||||
.join("&");
|
||||
.map(([key, value]): [string, string] => [key, String(value)])
|
||||
.sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
|
||||
}
|
||||
|
||||
function normalizeQuery(query: string): string {
|
||||
return query
|
||||
.split("&")
|
||||
.filter(Boolean)
|
||||
.sort((a, b) => a.split("=", 1)[0]!.localeCompare(b.split("=", 1)[0]!))
|
||||
.join("&");
|
||||
function encodePairs(values: Record<string, string | number>): string {
|
||||
return sortedPairs(values).map(([key, value]) => `${formEncode(key)}=${formEncode(value)}`).join("&");
|
||||
}
|
||||
|
||||
/**
|
||||
* The query as it participates in signing: sorted raw `key=value` pairs, NOT
|
||||
* URL-encoded (SignatureUtil.calculateSignature in the official 2.2.8 Dart
|
||||
* image). The server signs the decoded parameters, so signing the encoded
|
||||
* form only matched when encoding was a no-op (a search for "two words" was
|
||||
* rejected with HTTP 406).
|
||||
*/
|
||||
function signedQuery(values: Record<string, string | number>): string {
|
||||
return sortedPairs(values).map(([key, value]) => `${key}=${value}`).join("&");
|
||||
}
|
||||
|
||||
function formEncode(value: string): string {
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
*/
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { EVENHUB_SCREEN_WIDTH } from "./compositor";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { directionalFallback, isDirectionalInput, type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import {
|
||||
createInProcessWindow,
|
||||
@@ -18,13 +18,20 @@ import { EvenHubSession } from "./session";
|
||||
import { renderInstalledEvenHubIcon } from "./installed-apps";
|
||||
|
||||
class EvenHubAppLayer implements Layer {
|
||||
/**
|
||||
* Watch swipes become the usual stock gestures here rather than in the
|
||||
* stack's fallback, which labels them as ring input: an app reading the
|
||||
* extension's source field should see that they came from the watch.
|
||||
*/
|
||||
readonly acceptsDirectional = true;
|
||||
|
||||
constructor(private readonly session: EvenHubSession) {}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const size = ctx.stack.getBaseSize();
|
||||
// EvenHub apps lay out for the stock 576-wide surface. In the full-panel
|
||||
// display mode the window is 640 wide; paint at stock width and centre
|
||||
// it rather than leaving all the slack on the right.
|
||||
// EvenHub apps lay out for the stock 576-wide surface. In the Full
|
||||
// Screen display mode the window is 640 wide; paint at stock width and
|
||||
// centre it rather than leaving all the slack on the right.
|
||||
if (size.width > EVENHUB_SCREEN_WIDTH) {
|
||||
const content = this.session.paint({ width: EVENHUB_SCREEN_WIDTH, height: size.height }, ctx.stack.isFocused());
|
||||
const image = new GrayImage(size.width, size.height, 0);
|
||||
@@ -40,7 +47,7 @@ class EvenHubAppLayer implements Layer {
|
||||
// menu intercepts tap-then-hold), which is the guaranteed way out since
|
||||
// EvenHub apps own double-click. Tap-then-hold is still reported to the
|
||||
// app as LONG_PRESS_EVENT — see the handleInput wrapper below.
|
||||
this.session.handleGesture(event);
|
||||
this.session.handleGesture(isDirectionalInput(event) ? ({ ...directionalFallback(event), source: "watch" } as InputEvent) : event);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +57,7 @@ export function createEvenHubWindow(
|
||||
session: EvenHubSession,
|
||||
options: InProcessAppOptions,
|
||||
onShowPhone: () => void,
|
||||
requestShellRender: () => void,
|
||||
): InProcessWindow {
|
||||
const fallbackIcon = windowIcon("package", session.manifest.name.charAt(0).toUpperCase() || "E");
|
||||
const created = createInProcessWindow({
|
||||
@@ -58,11 +66,14 @@ export function createEvenHubWindow(
|
||||
title: session.manifest.name,
|
||||
iconLetter: session.manifest.name.charAt(0).toUpperCase() || "E",
|
||||
closeable: true,
|
||||
// An icon the app set at runtime (extension setWindowIcon) wins over the
|
||||
// package's.
|
||||
drawIcon: makeImageWindowIcon(
|
||||
(size) => renderInstalledEvenHubIcon(session.manifest.packageId, size),
|
||||
(size) => session.renderWindowIcon(size) ?? renderInstalledEvenHubIcon(session.manifest.packageId, size),
|
||||
fallbackIcon,
|
||||
),
|
||||
heightMode: "medium",
|
||||
keepsScreenOn: () => session.keepsScreenOn(),
|
||||
// The window's context menu doubles as the OS contextual menu EvenHub
|
||||
// apps register with `menuObject` (SDK 0.0.14): the app's own entries come
|
||||
// first, then the host entries. Evaluated at open time, so a rebuild that
|
||||
@@ -99,6 +110,14 @@ export function createEvenHubWindow(
|
||||
// The extended layout uses the full 576x452 app area ("max"); stock apps
|
||||
// stay in the 576x288 band ("medium").
|
||||
setTallCanvas: (tall) => created.setHeightMode(tall ? "max" : "medium"),
|
||||
setAttention: (attention) => shell.setWindowAttention(windowId, attention),
|
||||
// Only while the app listens for text: otherwise the shell's Type Into App
|
||||
// destinations (voice dialog, phone keyboard, watch) would send into a void.
|
||||
setAcceptsTextInput: (accept) => {
|
||||
created.window.receiveTextInput = accept ? (text, textOptions) => session.deliverTextInput(text, textOptions) : undefined;
|
||||
},
|
||||
startVoiceInput: () => shell.foregroundWindow()?.windowId === windowId && shell.startVoiceInput(),
|
||||
iconChanged: requestShellRender,
|
||||
windowId,
|
||||
});
|
||||
|
||||
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Font options for EvenHub text and list containers (the extension layout's
|
||||
* `font: { family, size, weight }`): parsing, inheritance from the layout's
|
||||
* default, and choosing a face from what is installed. Pure, so it can be
|
||||
* tested without NativeScript; app/apps/evenhub/fonts.ts loads the result.
|
||||
*
|
||||
* A container with no font anywhere keeps the stock firmware font, exactly as
|
||||
* on stock. Naming any font option switches to Faceclaw's fonts, with the
|
||||
* defaults below filling the options left out.
|
||||
*/
|
||||
|
||||
/** A container's font request, after parsing. Absent fields take the defaults. */
|
||||
export type EvenHubFontSpec = {
|
||||
family?: string;
|
||||
size?: number;
|
||||
weight?: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_FONT_FAMILY = "Roboto";
|
||||
/** The stock firmware font's size, so switching family alone keeps text about the same size. */
|
||||
export const DEFAULT_FONT_SIZE = 20;
|
||||
export const DEFAULT_FONT_WEIGHT = 400;
|
||||
export const MIN_FONT_SIZE = 6;
|
||||
/** Larger glyphs leave the glyph-cache path (255px limit), so text stops being cheap to send. */
|
||||
export const MAX_FONT_SIZE = 128;
|
||||
|
||||
/**
|
||||
* Parse a `font` object from a layout or container. Undefined when there is
|
||||
* none; unusable fields are dropped, so `{}` means "Faceclaw's default font".
|
||||
*/
|
||||
export function parseFontSpec(raw: unknown): EvenHubFontSpec | undefined {
|
||||
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined;
|
||||
const record = raw as Record<string, unknown>;
|
||||
const spec: EvenHubFontSpec = {};
|
||||
if (typeof record.family === "string" && record.family.trim()) spec.family = record.family.trim();
|
||||
const size = typeof record.size === "string" ? Number(record.size) : record.size;
|
||||
if (typeof size === "number" && Number.isFinite(size)) {
|
||||
spec.size = Math.min(MAX_FONT_SIZE, Math.max(MIN_FONT_SIZE, Math.round(size)));
|
||||
}
|
||||
const weight = parseWeight(record.weight);
|
||||
if (weight !== undefined) spec.weight = weight;
|
||||
return spec;
|
||||
}
|
||||
|
||||
function parseWeight(raw: unknown): number | undefined {
|
||||
if (typeof raw === "string") {
|
||||
const keyword = raw.trim().toLowerCase();
|
||||
if (keyword === "normal") return 400;
|
||||
if (keyword === "bold") return 700;
|
||||
raw = Number(keyword);
|
||||
}
|
||||
if (typeof raw !== "number" || !Number.isFinite(raw)) return undefined;
|
||||
return Math.min(1000, Math.max(1, Math.round(raw)));
|
||||
}
|
||||
|
||||
/** A container's font: its own options over the layout default's, field by field. */
|
||||
export function mergeFontSpecs(
|
||||
layoutDefault: EvenHubFontSpec | undefined,
|
||||
own: EvenHubFontSpec | undefined,
|
||||
): EvenHubFontSpec | undefined {
|
||||
if (!layoutDefault) return own;
|
||||
if (!own) return layoutDefault;
|
||||
return { ...layoutDefault, ...own };
|
||||
}
|
||||
|
||||
/** One loadable face: a TTF file, a Terminus weight, or the stock font. */
|
||||
export type FontFaceCandidate = {
|
||||
/** Family as the font names itself (e.g. "Inter 18pt"). */
|
||||
family: string;
|
||||
/** 100..900. */
|
||||
weight: number;
|
||||
/** The sizes a bitmap face comes in; null for a scalable face. */
|
||||
sizes: readonly number[] | null;
|
||||
monospace: boolean;
|
||||
/** Identifies the face to the loader. */
|
||||
key: string;
|
||||
};
|
||||
|
||||
/**
|
||||
* The family name apps use: an optical-size suffix ("Inter 18pt") is dropped,
|
||||
* since the bundled Inter is its 18pt cut and apps would ask for "Inter".
|
||||
*/
|
||||
export function displayFamily(family: string): string {
|
||||
return family.replace(/\s+\d+\s*pt$/i, "").trim() || family;
|
||||
}
|
||||
|
||||
function normalizeFamily(name: string): string {
|
||||
return name.toLowerCase().replace(/[^a-z0-9]/g, "");
|
||||
}
|
||||
|
||||
export function familyMatches(candidate: string, requested: string): boolean {
|
||||
const wanted = normalizeFamily(requested);
|
||||
return normalizeFamily(candidate) === wanted || normalizeFamily(displayFamily(candidate)) === wanted;
|
||||
}
|
||||
|
||||
/**
|
||||
* Pick the face and pixel size for a spec. An unknown family falls back to
|
||||
* DEFAULT_FONT_FAMILY; null when even that is missing (the caller then uses
|
||||
* the stock font). Weight follows CSS's font matching: 400–500 look up to
|
||||
* 500 first, lighter requests look lighter first, heavier ones heavier
|
||||
* first. A bitmap face takes its nearest size (the smaller on a tie).
|
||||
*/
|
||||
export function selectFontFace(
|
||||
spec: EvenHubFontSpec,
|
||||
candidates: readonly FontFaceCandidate[],
|
||||
): { face: FontFaceCandidate; size: number } | null {
|
||||
const family = spec.family ?? DEFAULT_FONT_FAMILY;
|
||||
let faces = candidates.filter((candidate) => familyMatches(candidate.family, family));
|
||||
if (faces.length === 0) faces = candidates.filter((candidate) => familyMatches(candidate.family, DEFAULT_FONT_FAMILY));
|
||||
if (faces.length === 0) return null;
|
||||
const desired = spec.weight ?? DEFAULT_FONT_WEIGHT;
|
||||
const face = [...faces].sort((a, b) => weightRank(a.weight, desired) - weightRank(b.weight, desired))[0]!;
|
||||
const requestedSize = spec.size ?? DEFAULT_FONT_SIZE;
|
||||
return { face, size: face.sizes ? nearestSize(face.sizes, requestedSize) : requestedSize };
|
||||
}
|
||||
|
||||
/** Lower is better: CSS font-weight matching as a sort key. */
|
||||
function weightRank(weight: number, desired: number): number {
|
||||
const distance = Math.abs(weight - desired);
|
||||
if (weight === desired) return 0;
|
||||
if (desired >= 400 && desired <= 500) {
|
||||
if (weight > desired && weight <= 500) return distance;
|
||||
if (weight < desired) return 1000 + distance;
|
||||
return 2000 + distance;
|
||||
}
|
||||
const preferLighter = desired < 400;
|
||||
return (weight < desired) === preferLighter ? distance : 1000 + distance;
|
||||
}
|
||||
|
||||
function nearestSize(sizes: readonly number[], requested: number): number {
|
||||
let best = sizes[0]!;
|
||||
for (const size of sizes) {
|
||||
const better = Math.abs(size - requested) < Math.abs(best - requested) ||
|
||||
(Math.abs(size - requested) === Math.abs(best - requested) && size < best);
|
||||
if (better) best = size;
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
/** A family as the extension's getFonts() reports it. */
|
||||
export type FontFamilyInfo = {
|
||||
family: string;
|
||||
weights: number[];
|
||||
/** Bitmap families' sizes; absent for scalable families. */
|
||||
sizes?: number[];
|
||||
monospace: boolean;
|
||||
};
|
||||
|
||||
/** Group faces into families for getFonts(), sorted by name. */
|
||||
export function describeFontFamilies(candidates: readonly FontFaceCandidate[]): FontFamilyInfo[] {
|
||||
const families = new Map<string, FontFamilyInfo>();
|
||||
for (const candidate of candidates) {
|
||||
const name = displayFamily(candidate.family);
|
||||
const key = normalizeFamily(name);
|
||||
let info = families.get(key);
|
||||
if (!info) {
|
||||
info = { family: name, weights: [], monospace: candidate.monospace };
|
||||
families.set(key, info);
|
||||
}
|
||||
if (!info.weights.includes(candidate.weight)) info.weights.push(candidate.weight);
|
||||
if (candidate.sizes) info.sizes = Array.from(new Set([...(info.sizes ?? []), ...candidate.sizes]));
|
||||
info.monospace = info.monospace && candidate.monospace;
|
||||
}
|
||||
return Array.from(families.values())
|
||||
.map((info) => ({
|
||||
...info,
|
||||
weights: [...info.weights].sort((a, b) => a - b),
|
||||
...(info.sizes ? { sizes: [...info.sizes].sort((a, b) => a - b) } : {}),
|
||||
}))
|
||||
.sort((a, b) => a.family.localeCompare(b.family));
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
/**
|
||||
* Loads the fonts EvenHub text and list containers ask for (see font-spec.ts
|
||||
* for the matching rules): the stock firmware font, Terminus at its bitmap
|
||||
* sizes, and every installed TTF/OTF — the bundled Roboto, Inter, Montserrat
|
||||
* and Roboto Mono, plus any the user installed from the Files app.
|
||||
*/
|
||||
import { EvenHubFont } from "../../graphics/evenhub-font";
|
||||
import { getTerminusFont, TERMINUS_SIZES } from "../../graphics/bdffont";
|
||||
import { TtfFont } from "../../graphics/ttf-font";
|
||||
import { listInstalledFonts } from "../../graphics/installed-fonts";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import type { GrayImage, UiFont } from "../../graphics/image";
|
||||
import {
|
||||
describeFontFamilies,
|
||||
familyMatches,
|
||||
selectFontFace,
|
||||
type EvenHubFontSpec,
|
||||
type FontFaceCandidate,
|
||||
type FontFamilyInfo,
|
||||
} from "./font-spec";
|
||||
|
||||
/** What containers draw text with: the stock font's interface, which UiFonts are adapted to. */
|
||||
export type EvenHubTextFace = {
|
||||
readonly lineHeight: number;
|
||||
measureLine(text: string): number;
|
||||
wrap(text: string, maxWidth: number): string[];
|
||||
drawText(image: GrayImage, x: number, y: number, text: string, value?: number): void;
|
||||
};
|
||||
|
||||
class UiFontFace implements EvenHubTextFace {
|
||||
constructor(private readonly font: UiFont) {}
|
||||
|
||||
get lineHeight(): number {
|
||||
return this.font.lineHeight;
|
||||
}
|
||||
|
||||
measureLine(text: string): number {
|
||||
return this.font.measureText(text);
|
||||
}
|
||||
|
||||
wrap(text: string, maxWidth: number): string[] {
|
||||
return wrapText(this.font, text, maxWidth);
|
||||
}
|
||||
|
||||
drawText(image: GrayImage, x: number, y: number, text: string, value = 255): void {
|
||||
this.font.drawText(image, x, y, text, value);
|
||||
}
|
||||
}
|
||||
|
||||
const STOCK_KEY = "stock";
|
||||
const TERMINUS_KEY_PREFIX = "terminus:";
|
||||
const TTF_KEY_PREFIX = "ttf:";
|
||||
|
||||
const BUILT_IN_FACES: readonly FontFaceCandidate[] = [
|
||||
// The firmware font: one size, one weight.
|
||||
{ family: "EvenHub", weight: 400, sizes: [20], monospace: false, key: STOCK_KEY },
|
||||
{ family: "Terminus", weight: 400, sizes: TERMINUS_SIZES, monospace: true, key: `${TERMINUS_KEY_PREFIX}400` },
|
||||
{ family: "Terminus", weight: 700, sizes: TERMINUS_SIZES, monospace: true, key: `${TERMINUS_KEY_PREFIX}700` },
|
||||
];
|
||||
|
||||
/** A family the listing lacks is re-listed at most this often (the user may have just installed it). */
|
||||
const RELIST_INTERVAL_MS = 30_000;
|
||||
|
||||
let faces: FontFaceCandidate[] | null = null;
|
||||
let listedAtMs = 0;
|
||||
|
||||
function listFaces(): FontFaceCandidate[] {
|
||||
const installed = listInstalledFonts()
|
||||
// Upright faces only: there is no italic option yet.
|
||||
.filter((font) => font.weightOrder < 1000)
|
||||
.map((font): FontFaceCandidate => ({
|
||||
family: font.family,
|
||||
weight: font.weightOrder,
|
||||
sizes: null,
|
||||
monospace: font.monospace,
|
||||
key: `${TTF_KEY_PREFIX}${font.path}`,
|
||||
}));
|
||||
faces = [...BUILT_IN_FACES, ...installed];
|
||||
listedAtMs = Date.now();
|
||||
return faces;
|
||||
}
|
||||
|
||||
/** Whether some listed face has the family (no family asks for the default, which is bundled). */
|
||||
function hasFamily(available: readonly FontFaceCandidate[], family: string | undefined): boolean {
|
||||
return !family || available.some((face) => familyMatches(face.family, family));
|
||||
}
|
||||
|
||||
/** Resolved faces by spec; null means the stock font (asked for, or nothing loadable). */
|
||||
const resolved = new Map<string, EvenHubTextFace | null>();
|
||||
|
||||
/** The face for a spec, and whether it is in the family asked for (rather than the fallback). */
|
||||
function loadFace(spec: EvenHubFontSpec): { face: EvenHubTextFace | null; exact: boolean } {
|
||||
let available = faces ?? listFaces();
|
||||
if (!hasFamily(available, spec.family) && Date.now() - listedAtMs > RELIST_INTERVAL_MS) available = listFaces();
|
||||
const exact = hasFamily(available, spec.family);
|
||||
return { face: openFace(selectFontFace(spec, available)), exact };
|
||||
}
|
||||
|
||||
function openFace(choice: { face: FontFaceCandidate; size: number } | null): EvenHubTextFace | null {
|
||||
if (!choice || choice.face.key === STOCK_KEY) return null;
|
||||
const { face, size } = choice;
|
||||
if (face.key.startsWith(TERMINUS_KEY_PREFIX)) {
|
||||
const font = getTerminusFont(size, face.weight >= 600);
|
||||
return font ? new UiFontFace(font) : null;
|
||||
}
|
||||
const font = TtfFont.load(face.key.slice(TTF_KEY_PREFIX.length), size);
|
||||
return font ? new UiFontFace(font) : null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The face a container draws with: the stock font when it names no font,
|
||||
* otherwise the closest match to its spec.
|
||||
*/
|
||||
export function containerFace(spec: EvenHubFontSpec | undefined): EvenHubTextFace {
|
||||
if (!spec) return EvenHubFont.get();
|
||||
const key = `${spec.family ?? ""}|${spec.size ?? ""}|${spec.weight ?? ""}`;
|
||||
let face = resolved.get(key);
|
||||
if (face === undefined) {
|
||||
const loaded = loadFace(spec);
|
||||
face = loaded.face;
|
||||
// A fallback isn't kept, so a family the user installs later gets picked up.
|
||||
if (loaded.exact) resolved.set(key, face);
|
||||
}
|
||||
// Not cached as an object: EvenHubFont.get() changes once firmware fonts are extracted.
|
||||
return face ?? EvenHubFont.get();
|
||||
}
|
||||
|
||||
/** The font families apps can ask for (extension getFonts). Re-lists installed fonts. */
|
||||
export function availableFontFamilies(): FontFamilyInfo[] {
|
||||
return describeFontFamilies(listFaces());
|
||||
}
|
||||
|
||||
/** Width of the widest line of `text` and the line height, in the font a container would use. */
|
||||
export function measureContainerText(text: string, spec: EvenHubFontSpec | undefined): { width: number; lineHeight: number } {
|
||||
const face = containerFace(spec);
|
||||
let width = 0;
|
||||
for (const line of text.split("\n")) width = Math.max(width, face.measureLine(line));
|
||||
return { width: Math.ceil(width), lineHeight: face.lineHeight };
|
||||
}
|
||||
@@ -22,6 +22,7 @@ import { unpackRuntime } from "./unpack-runtime";
|
||||
import { type EvenHubPermission } from "./permissions";
|
||||
import { EvenHubSession } from "./session";
|
||||
import { createEvenHubWindow } from "./evenhub-window";
|
||||
import { liveDeviceStatusSource } from "./device-status-source";
|
||||
import { createEvenHubWebView, type EvenHubWebView } from "./webview";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
@@ -126,7 +127,7 @@ async function startApp(
|
||||
): Promise<void> {
|
||||
const windowId = `evenhub:app:${nextSerial++}`;
|
||||
const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl,
|
||||
payload => ctx.actions.playBuzzerSequence(payload));
|
||||
payload => ctx.actions.playBuzzerSequence(payload), liveDeviceStatusSource);
|
||||
const webView = createEvenHubWebView(session);
|
||||
session.attachWebView({
|
||||
evaluateJs: webView.evaluateJs,
|
||||
@@ -147,7 +148,7 @@ async function startApp(
|
||||
|
||||
try {
|
||||
await ctx.launchInProcessApp(windowId, `window:${windowId}`, (options) =>
|
||||
createEvenHubWindow(windowId, appId, session, options, () => showOnPhone(windowId)),
|
||||
createEvenHubWindow(windowId, appId, session, options, () => showOnPhone(windowId), () => ctx.requestShellRender()),
|
||||
);
|
||||
} catch (error) { session.close(); throw error; }
|
||||
}
|
||||
|
||||
@@ -17,6 +17,9 @@ import { voiceActivity } from "../../ui/shell/voice-activity";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
// The router's claim on the shared raw-PCM tap (the Microphones app holds its own).
|
||||
const RAW_CAPTURE_OWNER = "evenhub-mic-router";
|
||||
|
||||
export type EvenHubMicClient = {
|
||||
readonly windowId: string;
|
||||
isForeground(): boolean;
|
||||
@@ -83,9 +86,9 @@ class EvenHubMicRouter {
|
||||
// eligibility change so it self-heals after an STT preemption. Returns
|
||||
// false if STT currently owns the mic; a later trigger (assistant modal
|
||||
// close, foreground/screen change) re-tries.
|
||||
voiceControlBridge.startRawCapture({ communicator: activeCommunicator() });
|
||||
voiceControlBridge.startRawCapture({ communicator: activeCommunicator(), owner: RAW_CAPTURE_OWNER });
|
||||
} else {
|
||||
voiceControlBridge.stopRawCapture();
|
||||
voiceControlBridge.stopRawCapture(RAW_CAPTURE_OWNER);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+283
-67
@@ -63,6 +63,16 @@ import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/to
|
||||
import { getCurrentLocation } from "../../native/location";
|
||||
import { LocationTracker, type TrackedLocation } from "../../native/location-tracker";
|
||||
import { ensureFineLocationPermission } from "../../native/location-permissions";
|
||||
import {
|
||||
buildEvenHubDeviceStatus,
|
||||
OFFLINE_DEVICE_STATUS_SOURCE,
|
||||
sameDeviceStatus,
|
||||
type EvenHubDeviceStatus,
|
||||
type EvenHubDeviceStatusSource,
|
||||
} from "./device-status";
|
||||
import { parseWindowIcon, WindowIconRenderer } from "./window-icon";
|
||||
import { availableFontFamilies, containerFace, measureContainerText } from "./fonts";
|
||||
import { parseFontSpec } from "./font-spec";
|
||||
|
||||
const UPNG = require("upng-js");
|
||||
|
||||
@@ -226,9 +236,15 @@ export const EVENHUB_BRIDGE_INJECT_SCRIPT = `
|
||||
/**
|
||||
* Injected at document-start (after the stock bridge shim): defines
|
||||
* `window.getFaceclawExtensions`, the single entry point for Faceclaw-only
|
||||
* capabilities (see the faceclaw-extensions package). It never touches the
|
||||
* standard SDK or any prototype — everything lives behind this one global, so
|
||||
* apps stay compatible with the stock Even app (where the global is absent).
|
||||
* capabilities. Its public contract (the `FaceclawExtensions` interface) is
|
||||
* the @faceclaw/evenhub-extensions package in evenhub-extensions/. Keep the
|
||||
* two in step (tests/evenhub-extensions-shim.test.cjs checks the member list).
|
||||
* When adding members, bump API_VERSION and tag them `@since apiVersion N` in
|
||||
* the package.
|
||||
*
|
||||
* It never touches the standard SDK or any prototype — everything lives behind
|
||||
* this one global, so apps stay compatible with the stock Even app (where the
|
||||
* global is absent).
|
||||
*
|
||||
* RPC uses its own id-keyed promise map and `__fcExtResolve`, kept separate
|
||||
* from the stock `__fcResolve` channel. Events arrive via `__fcExtEvent`.
|
||||
@@ -239,13 +255,15 @@ export function buildFaceclawExtensionsScript(versionString: string): string {
|
||||
(function () {
|
||||
if (window.getFaceclawExtensions) return;
|
||||
var VERSION = ${JSON.stringify(versionString)};
|
||||
var API_VERSION = 1;
|
||||
var pending = {};
|
||||
var nextId = 1;
|
||||
window.__fcExtResolve = function (id, ok, value) {
|
||||
var entry = pending[id];
|
||||
if (!entry) return;
|
||||
delete pending[id];
|
||||
(ok ? entry[0] : entry[1])(value);
|
||||
if (ok) entry[0](value);
|
||||
else entry[1](new Error(String(value)));
|
||||
};
|
||||
function call(method, params) {
|
||||
return new Promise(function (resolve, reject) {
|
||||
@@ -260,9 +278,10 @@ export function buildFaceclawExtensionsScript(versionString: string): string {
|
||||
}
|
||||
var listeners = {};
|
||||
window.__fcExtEvent = function (name, data) {
|
||||
var arr = listeners[name];
|
||||
if (!arr) return;
|
||||
for (var i = 0; i < arr.slice().length; i++) {
|
||||
// Iterate a snapshot, so a listener that unsubscribes mid-dispatch
|
||||
// doesn't make the next one get skipped.
|
||||
var arr = (listeners[name] || []).slice();
|
||||
for (var i = 0; i < arr.length; i++) {
|
||||
try { arr[i](data); } catch (e) { if (window.console) console.error(e); }
|
||||
}
|
||||
};
|
||||
@@ -275,19 +294,29 @@ export function buildFaceclawExtensionsScript(versionString: string): string {
|
||||
if (i >= 0) arr.splice(i, 1);
|
||||
};
|
||||
}
|
||||
// Compass: enable on first listener, disable when the last one leaves.
|
||||
var compassListeners = [];
|
||||
function addCompass(cb) {
|
||||
compassListeners.push(cb);
|
||||
if (compassListeners.length === 1) send("setCompass", [true]);
|
||||
var off = on("compass", cb);
|
||||
return function () {
|
||||
off();
|
||||
var i = compassListeners.indexOf(cb);
|
||||
if (i >= 0) compassListeners.splice(i, 1);
|
||||
if (compassListeners.length === 0) send("setCompass", [false]);
|
||||
// Events the host only produces on request (compass, touch-down, text
|
||||
// input): it is told to start on the first listener and to stop when the
|
||||
// last one leaves. Each unsubscribe function works once.
|
||||
function counted(name, setMethod) {
|
||||
var count = 0;
|
||||
return function (cb) {
|
||||
var off = on(name, cb);
|
||||
if (++count === 1) send(setMethod, [true]);
|
||||
var active = true;
|
||||
return function () {
|
||||
if (!active) return;
|
||||
active = false;
|
||||
off();
|
||||
if (--count === 0) send(setMethod, [false]);
|
||||
};
|
||||
};
|
||||
}
|
||||
function setWindowIcon(icon) {
|
||||
if (!icon) return call("setWindowIcon", [null]);
|
||||
// Typed arrays (ImageData.data) don't survive JSON; send a plain array.
|
||||
var data = Array.prototype.slice.call(icon.data || []);
|
||||
return call("setWindowIcon", [{ width: icon.width, height: icon.height, data: data }]);
|
||||
}
|
||||
// Voice-assistant tools: specs go to the host; handlers stay here and run when
|
||||
// the host invokes a tool via __fcExtInvokeTool.
|
||||
var toolHandlers = {};
|
||||
@@ -318,6 +347,7 @@ export function buildFaceclawExtensionsScript(versionString: string): string {
|
||||
send("setAssistantTools", [specs]);
|
||||
}
|
||||
var extensions = {
|
||||
apiVersion: API_VERSION,
|
||||
getVersion: function () { return VERSION; },
|
||||
returnToAppSwitcher: function () { send("returnToAppSwitcher"); },
|
||||
quit: function () { send("quit"); },
|
||||
@@ -325,10 +355,18 @@ export function buildFaceclawExtensionsScript(versionString: string): string {
|
||||
getConfiguredApiKeys: function () { return call("getConfiguredApiKeys"); },
|
||||
requestApiKeyAccess: function (services) { return call("requestApiKeyAccess", [services || []]); },
|
||||
playBuzzer: function (steps) { return call("playBuzzer", [steps || []]); },
|
||||
addCompassListener: addCompass,
|
||||
addCompassListener: counted("compass", "setCompass"),
|
||||
createLayout: function (layout) { return call("createLayout", [layout || {}]); },
|
||||
replaceLayout: function (layout) { return call("replaceLayout", [layout || {}]); },
|
||||
setAssistantTools: setAssistantTools
|
||||
setAssistantTools: setAssistantTools,
|
||||
addTouchDownListener: counted("touchDown", "setTouchDown"),
|
||||
addTextInputListener: counted("textInput", "setTextInput"),
|
||||
startVoiceInput: function () { return call("startVoiceInput"); },
|
||||
setWindowIcon: setWindowIcon,
|
||||
setAttention: function (attention) { send("setAttention", [!!attention]); },
|
||||
setKeepScreenOn: function (keepOn) { send("setKeepScreenOn", [!!keepOn]); },
|
||||
getFonts: function () { return call("getFonts"); },
|
||||
measureText: function (text, font) { return call("measureText", [String(text), font || null]); }
|
||||
};
|
||||
window.getFaceclawExtensions = function () { return extensions; };
|
||||
})();
|
||||
@@ -354,6 +392,14 @@ export type EvenHubWindowHooks = {
|
||||
pushOverlay: (layer: Layer) => void;
|
||||
/** Switch the window between the stock 576x288 band and the full 576x452 canvas. */
|
||||
setTallCanvas: (tall: boolean) => void;
|
||||
/** Show or clear the attention dot on the window's switcher icon. */
|
||||
setAttention: (attention: boolean) => void;
|
||||
/** Offer (or stop offering) the window as the shell's "Type Into App" destination. */
|
||||
setAcceptsTextInput: (accept: boolean) => void;
|
||||
/** Open the shell's voice dialog aimed at the window; false if it can't open now. */
|
||||
startVoiceInput: () => boolean;
|
||||
/** The app changed its window icon: repaint the switcher. */
|
||||
iconChanged: () => void;
|
||||
/** The shell window id (unique per running instance; used for tool registration). */
|
||||
windowId: string;
|
||||
};
|
||||
@@ -404,6 +450,21 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
private shellFocused = true;
|
||||
/** API-key services the user has granted this app this session. */
|
||||
private readonly grantedApiKeys = new Set<string>();
|
||||
/** The app listens for touch-down (extension addTouchDownListener). */
|
||||
private touchDownRequested = false;
|
||||
/** The app listens for text input, so the shell offers it as a Type Into App destination. */
|
||||
private textInputRequested = false;
|
||||
/** The app asked for the attention dot on its switcher icon. */
|
||||
private attention = false;
|
||||
/** The app asked the idle screen timeout to wait while it is in the foreground. */
|
||||
private keepScreenOnRequested = false;
|
||||
/** The app's own switcher icon; null draws the package icon. */
|
||||
private windowIcon: WindowIconRenderer | null = null;
|
||||
/** The paired glasses' serial, read once so getGlassesInfo and every status push agree. */
|
||||
private glassesSerial: string | null | undefined = undefined;
|
||||
/** The last deviceStatusChanged sent, and the subscriptions that refresh it. */
|
||||
private lastDeviceStatus: EvenHubDeviceStatus | null = null;
|
||||
private readonly deviceStatusOffs: (() => void)[] = [];
|
||||
private log: (message: string) => void;
|
||||
|
||||
constructor(
|
||||
@@ -412,6 +473,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
log: (message: string) => void,
|
||||
remoteUrl = "",
|
||||
private readonly sendBuzzerSequence: (payload: Uint8Array) => Promise<void> | void = () => {},
|
||||
private readonly deviceStatusSource: EvenHubDeviceStatusSource = OFFLINE_DEVICE_STATUS_SOURCE,
|
||||
) {
|
||||
this.manifest = manifest;
|
||||
this.distDir = distDir;
|
||||
@@ -425,26 +487,28 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
this.webViewHandle = handle;
|
||||
}
|
||||
|
||||
webViewFailed(message: string): void {
|
||||
/**
|
||||
* A WebView load failed. Once the app has loaded, a failed or blocked
|
||||
* navigation leaves its page running, so only a failure before then (or one
|
||||
* the host marks `fatal`) replaces the glasses window with the error.
|
||||
*/
|
||||
webViewFailed(message: string, fatal = false): void {
|
||||
if (this.closed) return;
|
||||
this.loadError = message; this.log(`evenhub: ${message}`); this.windowHooks?.requestRender();
|
||||
this.log(`evenhub: ${message}`);
|
||||
if (this.launchContextPushed && !fatal) return;
|
||||
this.loadError = message; this.windowHooks?.requestRender();
|
||||
}
|
||||
|
||||
/** The WebView finished loading the app: push the one-shot launch context. */
|
||||
webViewLoaded(): void {
|
||||
if (this.loadError) { this.loadError = ''; this.windowHooks?.requestRender(); }
|
||||
if (this.launchContextPushed) return;
|
||||
this.launchContextPushed = true;
|
||||
// Launched to drive the glasses (from the on-glasses file browser), so
|
||||
// apps that branch on this should take their glasses path.
|
||||
this.pushMessage("evenAppLaunchSource", { launchSource: "glassesMenu" });
|
||||
this.pushMessage("deviceStatusChanged", {
|
||||
sn: "FACECLAW-G2",
|
||||
connectType: "connected",
|
||||
isWearing: true,
|
||||
batteryLevel: 100,
|
||||
isCharging: false,
|
||||
isInCase: false,
|
||||
});
|
||||
this.pushDeviceStatus();
|
||||
this.deviceStatusOffs.push(this.deviceStatusSource.subscribe(() => this.pushDeviceStatus()));
|
||||
}
|
||||
|
||||
// ----- glasses window wiring -----
|
||||
@@ -452,6 +516,10 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
attachWindow(hooks: EvenHubWindowHooks): void {
|
||||
this.windowHooks = hooks;
|
||||
this.shellWindowId = hooks.windowId;
|
||||
// Requests the app made before the window existed.
|
||||
if (this.textInputRequested) hooks.setAcceptsTextInput(true);
|
||||
if (this.attention) hooks.setAttention(true);
|
||||
if (this.windowIcon) hooks.iconChanged();
|
||||
}
|
||||
|
||||
windowClosed(): void {
|
||||
@@ -481,18 +549,29 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
* locally with events only for clicks and boundary scrolls.
|
||||
*/
|
||||
handleGesture(event: InputEvent): void {
|
||||
// Touch-down has no stock event (an unknown eventType would parse as a
|
||||
// click), so it goes only to apps that subscribed through the extension.
|
||||
if (event.type === "ring-press") {
|
||||
if (this.touchDownRequested) {
|
||||
this.pushExtEvent("touchDown", { source: extensionInputSource(event.source), timestampMs: event.timestampMs });
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!this.page) return;
|
||||
const capture = this.page ? eventCaptureContainer(this.page) : undefined;
|
||||
const capture = eventCaptureContainer(this.page);
|
||||
// The stock eventSource can't tell the watch from the ring, and scrolls
|
||||
// carry none; the extension field names the device when it is known.
|
||||
const inputSource = "source" in event && event.source ? extensionInputSource(event.source) : undefined;
|
||||
switch (event.type) {
|
||||
case "click":
|
||||
if (capture?.kind === "list") {
|
||||
this.emitListEvent(capture, CLICK_EVENT);
|
||||
this.emitListEvent(capture, CLICK_EVENT, inputSource);
|
||||
} else {
|
||||
this.emitSysEvent(CLICK_EVENT, gestureSource(event.source));
|
||||
this.emitSysEvent(CLICK_EVENT, gestureSource(event.source), inputSource);
|
||||
}
|
||||
break;
|
||||
case "double-click":
|
||||
this.emitSysEvent(DOUBLE_CLICK_EVENT, gestureSource(event.source));
|
||||
this.emitSysEvent(DOUBLE_CLICK_EVENT, gestureSource(event.source), inputSource);
|
||||
break;
|
||||
case "long-press":
|
||||
case "short-then-long-press":
|
||||
@@ -506,9 +585,9 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
// given exclusive use of the gesture.
|
||||
const eventType = event.type === "long-press-release" ? LONG_PRESS_RELEASE_EVENT : LONG_PRESS_EVENT;
|
||||
if (capture?.kind === "list") {
|
||||
this.emitListEvent(capture, eventType);
|
||||
this.emitListEvent(capture, eventType, inputSource);
|
||||
} else {
|
||||
this.emitSysEvent(eventType, gestureSource(event.source));
|
||||
this.emitSysEvent(eventType, gestureSource(event.source), inputSource);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -516,9 +595,9 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
case "scroll-down": {
|
||||
const eventType = event.type === "scroll-up" ? SCROLL_TOP_EVENT : SCROLL_BOTTOM_EVENT;
|
||||
if (capture?.kind === "list") {
|
||||
this.scrollList(capture, eventType);
|
||||
this.scrollList(capture, eventType, inputSource);
|
||||
} else {
|
||||
this.emitSysEvent(eventType, 0);
|
||||
this.emitSysEvent(eventType, 0, inputSource);
|
||||
}
|
||||
break;
|
||||
}
|
||||
@@ -541,11 +620,11 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
this.pushMessage("evenHubEvent", { type: "menuItemClickEvent", jsonData: { itemID } });
|
||||
}
|
||||
|
||||
private scrollList(list: EvenHubListContainer, eventType: number): void {
|
||||
private scrollList(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void {
|
||||
// Swipe up selects the previous item, swipe down the next (stock direction).
|
||||
const delta = eventType === SCROLL_TOP_EVENT ? -1 : 1;
|
||||
// At a boundary the selection stays put (the list bounces) and the app hears about it.
|
||||
if (!stepListSelection(list, EvenHubFont.get(), delta)) this.emitListEvent(list, eventType);
|
||||
if (!stepListSelection(list, containerFace(list.font), delta)) this.emitListEvent(list, eventType, inputSource);
|
||||
this.windowHooks?.requestRender();
|
||||
}
|
||||
|
||||
@@ -810,19 +889,10 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
return ApplicationSettings.getString(this.storageKey(readString(data, "key", "")), "");
|
||||
case "getUserInfo":
|
||||
return { uid: 0, name: "Faceclaw", avatar: "", country: "" };
|
||||
case "getGlassesInfo":
|
||||
return {
|
||||
model: "g2",
|
||||
sn: "FACECLAW-G2",
|
||||
status: {
|
||||
sn: "FACECLAW-G2",
|
||||
connectType: "connected",
|
||||
isWearing: true,
|
||||
batteryLevel: 100,
|
||||
isCharging: false,
|
||||
isInCase: false,
|
||||
},
|
||||
};
|
||||
case "getGlassesInfo": {
|
||||
const status = this.deviceStatus();
|
||||
return { model: "g2", sn: status.sn, status };
|
||||
}
|
||||
case "audioControl":
|
||||
return this.audioControl(data);
|
||||
case "getAppLocation":
|
||||
@@ -956,7 +1026,12 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
}
|
||||
|
||||
private storageKey(key: string): string {
|
||||
return `evenhub:${this.manifest.packageId}:ls:${key}`;
|
||||
// The package ID must end at the first ':' so that no other package can
|
||||
// reach this one's keys (manifest IDs are unvalidated: package "a" asking
|
||||
// for key "b:ls:c" must not read package "a:ls:b"'s "c"). IDs containing
|
||||
// ':' or '%' are encoded, so they can't collide with the plain IDs either.
|
||||
const id = this.manifest.packageId;
|
||||
return `evenhub:${/[:%]/.test(id) ? encodeURIComponent(id) : id}:ls:${key}`;
|
||||
}
|
||||
|
||||
// ----- pushes (host -> web) -----
|
||||
@@ -980,6 +1055,12 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
);
|
||||
}
|
||||
|
||||
/** Log a problem with the app's own calls where its developer will see it: the app's console. */
|
||||
private warnApp(message: string): void {
|
||||
this.log(`evenhub: ${message}`);
|
||||
this.webViewHandle?.evaluateJs(`window.console && console.warn(${JSON.stringify(`Faceclaw: ${message}`)})`);
|
||||
}
|
||||
|
||||
/** Deliver an extension event to the app (e.g. windowLifecycle). */
|
||||
private pushExtEvent(name: string, data: unknown): void {
|
||||
this.webViewHandle?.evaluateJs(
|
||||
@@ -991,17 +1072,18 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
this.pushMessage("evenHubEvent", { type: payloadType, jsonData: elideZeroFields(jsonData) });
|
||||
}
|
||||
|
||||
private emitSysEvent(eventType: number, eventSource: number): void {
|
||||
this.pushEvenHubEvent("sysEvent", { eventType, eventSource });
|
||||
private emitSysEvent(eventType: number, eventSource: number, inputSource?: ExtensionInputSource): void {
|
||||
this.pushEvenHubEvent("sysEvent", { eventType, eventSource, [INPUT_SOURCE_FIELD]: inputSource });
|
||||
}
|
||||
|
||||
private emitListEvent(list: EvenHubListContainer, eventType: number): void {
|
||||
private emitListEvent(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void {
|
||||
this.pushEvenHubEvent("listEvent", {
|
||||
containerID: list.id,
|
||||
containerName: list.name,
|
||||
currentSelectItemIndex: list.selectedIndex,
|
||||
currentSelectItemName: list.itemNames[list.selectedIndex] ?? "",
|
||||
eventType,
|
||||
[INPUT_SOURCE_FIELD]: inputSource,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1017,7 +1099,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
result = await this.dispatchExtension(method, args.slice(1));
|
||||
} catch (error) {
|
||||
ok = false;
|
||||
result = String(error);
|
||||
result = error instanceof Error ? error.message : String(error);
|
||||
this.log(`evenhub: extension call failed: ${error}`);
|
||||
}
|
||||
// id 0 is a fire-and-forget send; nothing is awaiting it.
|
||||
@@ -1050,12 +1132,96 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
case "setAssistantTools":
|
||||
this.setAssistantTools(Array.isArray(params[0]) ? params[0] : []);
|
||||
return null;
|
||||
case "setTouchDown":
|
||||
this.touchDownRequested = params[0] === true;
|
||||
return null;
|
||||
case "setTextInput":
|
||||
this.setTextInputRequested(params[0] === true);
|
||||
return null;
|
||||
case "startVoiceInput":
|
||||
// Without a text listener the dialog would have nowhere in the app to send to.
|
||||
return this.textInputRequested && (this.windowHooks?.startVoiceInput() ?? false);
|
||||
case "setWindowIcon":
|
||||
this.setWindowIcon(params[0]);
|
||||
return null;
|
||||
case "setAttention":
|
||||
this.attention = params[0] === true;
|
||||
this.windowHooks?.setAttention(this.attention);
|
||||
return null;
|
||||
case "setKeepScreenOn":
|
||||
this.keepScreenOnRequested = params[0] === true;
|
||||
return null;
|
||||
case "getFonts":
|
||||
return availableFontFamilies();
|
||||
case "measureText":
|
||||
// No font measures in the stock font, as a container without one draws.
|
||||
return measureContainerText(String(params[0] ?? ""), parseFontSpec(params[1]));
|
||||
default:
|
||||
this.log(`evenhub: unknown extension method ${method}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ----- device status (stock getGlassesInfo / deviceStatusChanged) -----
|
||||
|
||||
private deviceStatus(): EvenHubDeviceStatus {
|
||||
const input = this.deviceStatusSource.read();
|
||||
if (this.glassesSerial === undefined) this.glassesSerial = input.serial;
|
||||
return buildEvenHubDeviceStatus({ ...input, serial: this.glassesSerial });
|
||||
}
|
||||
|
||||
/** Push deviceStatusChanged if the status differs from the last one sent. */
|
||||
private pushDeviceStatus(): void {
|
||||
if (this.closed) return;
|
||||
const status = this.deviceStatus();
|
||||
if (this.lastDeviceStatus && sameDeviceStatus(status, this.lastDeviceStatus)) return;
|
||||
this.lastDeviceStatus = status;
|
||||
this.pushMessage("deviceStatusChanged", status);
|
||||
}
|
||||
|
||||
// ----- window controls and text input (extensions) -----
|
||||
|
||||
/** Whether the idle screen timeout should wait while this window is in the foreground. */
|
||||
keepsScreenOn(): boolean {
|
||||
return this.keepScreenOnRequested;
|
||||
}
|
||||
|
||||
/** The app's own switcher icon at `size`, or null to draw the package icon. */
|
||||
renderWindowIcon(size: number): GrayImage | null {
|
||||
return this.windowIcon?.render(size) ?? null;
|
||||
}
|
||||
|
||||
private setWindowIcon(raw: unknown): void {
|
||||
if (raw == null) {
|
||||
this.windowIcon = null;
|
||||
} else {
|
||||
const image = parseWindowIcon(raw);
|
||||
if (!image) {
|
||||
throw new Error(
|
||||
"setWindowIcon: expected {width, height, data} of at most 256x256, with data holding " +
|
||||
"width*height grayscale or width*height*4 RGBA bytes",
|
||||
);
|
||||
}
|
||||
this.windowIcon = new WindowIconRenderer(image);
|
||||
}
|
||||
this.windowHooks?.iconChanged();
|
||||
}
|
||||
|
||||
private setTextInputRequested(accept: boolean): void {
|
||||
if (this.textInputRequested === accept) return;
|
||||
this.textInputRequested = accept;
|
||||
this.windowHooks?.setAcceptsTextInput(accept);
|
||||
}
|
||||
|
||||
/**
|
||||
* Text the shell routed to this window: the voice dialog's Type Into App,
|
||||
* the phone keyboard, the watch, or remote input. Only arrives while the
|
||||
* app listens (setAcceptsTextInput).
|
||||
*/
|
||||
deliverTextInput(text: string, options?: { submit?: boolean }): void {
|
||||
this.pushExtEvent("textInput", options?.submit === undefined ? { text } : { text, submit: options.submit });
|
||||
}
|
||||
|
||||
// ----- compass (EvenHubCompassClient) -----
|
||||
|
||||
deliverCompass(reading: EvenHubCompassReading): void {
|
||||
@@ -1106,12 +1272,16 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
for (const raw of rawSpecs) {
|
||||
const record = asRecord(raw);
|
||||
const name = readString(record, "name", "");
|
||||
if (!name) continue;
|
||||
const problem = appToolProblem(name, record.parameters);
|
||||
if (problem) {
|
||||
this.warnApp(`setAssistantTools: skipped tool ${JSON.stringify(name)}: ${problem}`);
|
||||
continue;
|
||||
}
|
||||
const availability = record.availability === "open" ? "open" : "foreground";
|
||||
specs.push({
|
||||
name,
|
||||
description: readString(record, "description", ""),
|
||||
inputSchema: asRecord(record.parameters),
|
||||
inputSchema: record.parameters == null ? { type: "object", properties: {} } : asRecord(record.parameters),
|
||||
availability,
|
||||
});
|
||||
}
|
||||
@@ -1202,13 +1372,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
if (steps.length === 0) return;
|
||||
for (let i = 0; i < steps.length; i += CFW_SEQ_MAX) {
|
||||
const chunk = steps.slice(i, i + CFW_SEQ_MAX);
|
||||
const payload = buildSoundSequencePayload(chunk);
|
||||
try {
|
||||
await this.sendBuzzerSequence(payload);
|
||||
} catch (error) {
|
||||
this.log(`evenhub: playBuzzer failed: ${error}`);
|
||||
return;
|
||||
}
|
||||
// A send failure rejects the app's promise (handleExtensionCall logs it).
|
||||
await this.sendBuzzerSequence(buildSoundSequencePayload(chunk));
|
||||
if (i + CFW_SEQ_MAX < steps.length) await delay(phraseDurationMs(chunk));
|
||||
}
|
||||
}
|
||||
@@ -1239,6 +1404,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
this.pendingToolCalls.clear();
|
||||
this.locationTracker?.stop();
|
||||
this.locationTracker = null;
|
||||
for (const off of this.deviceStatusOffs.splice(0)) off();
|
||||
if (!this.systemExitSent) {
|
||||
// Best effort: lets the app's teardown handlers run before the webview dies.
|
||||
this.emitSysEvent(SYSTEM_EXIT_EVENT, 0);
|
||||
@@ -1350,6 +1516,35 @@ function configuredApiKeyServices(): string[] {
|
||||
return API_KEY_SERVICES.filter((service) => service.setting.get().trim()).map((service) => service.id);
|
||||
}
|
||||
|
||||
/** App assistant-tool names: what every model provider accepts in a tool name. */
|
||||
const APP_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
|
||||
|
||||
/**
|
||||
* Why an app's assistant tool can't be offered, or null if it can. Provider
|
||||
* APIs reject a whole request over one bad tool definition, so offering it
|
||||
* would break every assistant turn while the app runs.
|
||||
*/
|
||||
function appToolProblem(name: string, parameters: unknown): string | null {
|
||||
if (!APP_TOOL_NAME.test(name)) return "name must be 1-64 letters, digits, underscores or hyphens";
|
||||
if (parameters == null) return null;
|
||||
const schema = parameters as Record<string, unknown>;
|
||||
if (typeof parameters !== "object" || Array.isArray(parameters) || schema.type !== "object") {
|
||||
return 'parameters must be a JSON Schema with type "object"';
|
||||
}
|
||||
if ("oneOf" in schema || "anyOf" in schema || "allOf" in schema) {
|
||||
return "parameters can't use oneOf, anyOf or allOf at the top level";
|
||||
}
|
||||
const properties = schema.properties;
|
||||
if (properties !== undefined && (!properties || typeof properties !== "object" || Array.isArray(properties))) {
|
||||
return "parameters.properties must be an object";
|
||||
}
|
||||
const required = schema.required;
|
||||
if (required !== undefined && !(Array.isArray(required) && required.every((key) => typeof key === "string"))) {
|
||||
return "parameters.required must be an array of property names";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Coerce an app-supplied buzzer step list into validated {freq, ms, duty?}. */
|
||||
function normalizeBuzzerSteps(raw: unknown[]): Step[] {
|
||||
const out: Step[] = [];
|
||||
@@ -1441,6 +1636,27 @@ function luminance(r: number, g: number, b: number): number {
|
||||
return Math.round(0.2126 * r + 0.7152 * g + 0.0722 * b);
|
||||
}
|
||||
|
||||
/**
|
||||
* Input-device names in the extension API: the touchDown event's source, and
|
||||
* the `faceclawInputSource` field added to stock gesture events.
|
||||
*/
|
||||
type ExtensionInputSource = "ring" | "left-touchpad" | "right-touchpad" | "watch";
|
||||
|
||||
const INPUT_SOURCE_FIELD = "faceclawInputSource";
|
||||
|
||||
function extensionInputSource(source: InputSource): ExtensionInputSource {
|
||||
switch (source) {
|
||||
case "ring":
|
||||
return "ring";
|
||||
case "left-arm":
|
||||
return "left-touchpad";
|
||||
case "right-arm":
|
||||
return "right-touchpad";
|
||||
case "watch":
|
||||
return "watch";
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* ring/left-arm/right-arm -> PB EventSourceType. The watch is reported to
|
||||
* EvenHub apps as the ring: they only know stock hardware, and the watch's
|
||||
|
||||
@@ -23,14 +23,19 @@ window.__faceclawEvenHub = { postMessage: function(name, args, id) {
|
||||
export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView {
|
||||
const host = FaceclawEvenHubWebView.new()
|
||||
host.packageIdentifier = session.manifest.packageId
|
||||
// The manager's close callback for the shown phone UI; the native back
|
||||
// button hides the host itself and reports it as a 'hidden' event.
|
||||
let onClose: (() => void) | null = null
|
||||
host.eventHandler = (json: string) => {
|
||||
try {
|
||||
const event = JSON.parse(json)
|
||||
if (event.kind === 'call' && typeof event.name === 'string' && typeof event.args === 'string' && Number.isSafeInteger(event.id))
|
||||
session.handleBridgeCall(event.name, event.args, event.id)
|
||||
else if (event.kind === 'loaded') session.webViewLoaded()
|
||||
else if (event.kind === 'error') session.webViewFailed(String(event.message))
|
||||
else if (event.kind === 'console') console.log(`[evenhub:${session.manifest.name}] ${event.message}`)
|
||||
else if (event.kind === 'error') session.webViewFailed(String(event.message), event.fatal === true)
|
||||
else if (event.kind === 'hidden') { const close = onClose; onClose = null; close?.() }
|
||||
// 'notice': host-side events such as a blocked navigation, logged with the app's console.
|
||||
else if (event.kind === 'console' || event.kind === 'notice') console.log(`[evenhub:${session.manifest.name}] ${event.message}`)
|
||||
} catch (error) { console.warn(`EvenHub bridge: ${error}`) }
|
||||
}
|
||||
// Defer loading until manager.ts has attached the session/window handles.
|
||||
@@ -41,5 +46,6 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView {
|
||||
}, 0)
|
||||
return { native: host, evaluateJs: js => host.evaluate(js),
|
||||
destroy: () => { clearTimeout(loading); host.destroy() },
|
||||
showOnPhone: () => host.showOnPhone(), hideOnPhone: () => host.hideOnPhone() }
|
||||
showOnPhone: close => { onClose = close; host.showOnPhone() },
|
||||
hideOnPhone: () => { onClose = null; host.hideOnPhone() } }
|
||||
}
|
||||
+68
-18
@@ -15,6 +15,11 @@
|
||||
* the offline serving entirely and loads that URL from the network; the shim
|
||||
* is registered as a document-start script instead of being spliced into the
|
||||
* served HTML.
|
||||
*
|
||||
* The app can navigate its WebView to other sites, so host access is tied to
|
||||
* the app's origin: the bridge port is injected only into the app's own frames
|
||||
* (FaceclawEvenHubJsBridge), and every script the host runs in the WebView is
|
||||
* guarded by an origin check.
|
||||
*/
|
||||
import { Application, Utils } from "@nativescript/core";
|
||||
import { EVENHUB_BRIDGE_INJECT_SCRIPT, buildFaceclawExtensionsScript, type EvenHubSession } from "./session";
|
||||
@@ -34,19 +39,40 @@ export type EvenHubWebView = {
|
||||
native: unknown;
|
||||
};
|
||||
|
||||
/** A distinct (fake, intercepted) https origin per app isolates web storage. */
|
||||
function originHost(session: EvenHubSession): string {
|
||||
const safe = session.manifest.packageId.toLowerCase().replace(/[^a-z0-9.-]/g, "-");
|
||||
return `${safe}.evenhub.invalid`;
|
||||
/**
|
||||
* Wraps the bridge port (window.__faceclawEvenHubPort, which WebView injects
|
||||
* only into the app's own frames) as the window.__faceclawEvenHub interface
|
||||
* the bridge shim and the extensions call.
|
||||
*/
|
||||
export const ANDROID_EVENHUB_TRANSPORT_SCRIPT = `
|
||||
(function () {
|
||||
var port = window.__faceclawEvenHubPort;
|
||||
if (!port || window.__faceclawEvenHub) return;
|
||||
window.__faceclawEvenHub = {
|
||||
postMessage: function (name, args, id) { port.postMessage(JSON.stringify(['call', String(name), String(args), id])); },
|
||||
wakeTimers: function () { port.postMessage('["wake"]'); }
|
||||
};
|
||||
})();
|
||||
`;
|
||||
|
||||
/** A JS expression that is true only in a document from `startUrl`'s origin. */
|
||||
export function appOriginCheck(startUrl: string): string {
|
||||
return `(function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(${JSON.stringify(startUrl)}))`;
|
||||
}
|
||||
|
||||
/** `js`, run only when the WebView's document is from `startUrl`'s origin. */
|
||||
export function guardToAppOrigin(startUrl: string, js: string): string {
|
||||
return `if (${appOriginCheck(startUrl)}) {\n${js}\n}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* The allowed-origin rule for a remote app's document-start script: the
|
||||
* loaded URL's own origin, so the shim isn't handed to third-party frames.
|
||||
* A distinct (fake, intercepted) https origin per app. The per-app profile is
|
||||
* what keeps storage apart; without one, this still separates origin-scoped
|
||||
* storage, though not cookies scoped to the shared evenhub.invalid domain.
|
||||
*/
|
||||
function originRule(url: string): string {
|
||||
const match = /^(https?:\/\/[^/?#]+)/i.exec(url);
|
||||
return match ? match[1]! : "*";
|
||||
function originHost(session: EvenHubSession): string {
|
||||
const safe = session.manifest.packageId.toLowerCase().replace(/[^a-z0-9.-]/g, "-");
|
||||
return `${safe}.evenhub.invalid`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -59,12 +85,20 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView {
|
||||
// A mutable wrapper, so the host can re-point the WebView at a recreated
|
||||
// activity (see FaceclawEvenHubWebViewHost).
|
||||
const webView = new com.faceclaw.app.FaceclawEvenHubWebView(new android.content.MutableContextWrapper(context));
|
||||
// The app's own WebView profile, so its browser storage (and that of sites it
|
||||
// navigates to) is separate from every other app's. This must come before
|
||||
// any other use of the WebView.
|
||||
if (!com.faceclaw.app.FaceclawEvenHubProfile.apply(webView, session.manifest.packageId)) {
|
||||
console.warn(`evenhub: this WebView can't separate profiles; ${session.manifest.packageId} shares browser storage with other apps`);
|
||||
}
|
||||
const settings = webView.getSettings();
|
||||
settings.setJavaScriptEnabled(true);
|
||||
settings.setDomStorageEnabled(true);
|
||||
settings.setAllowFileAccess(false);
|
||||
settings.setMediaPlaybackRequiresUserGesture(false);
|
||||
|
||||
// Set once the renderer is gone: the WebView is destroyed and must not be used.
|
||||
let gone = false;
|
||||
const listener = new com.faceclaw.app.FaceclawEvenHubListener({
|
||||
onEvenAppMessage: (handlerName: string, argsJson: string, callId: number) => {
|
||||
session.handleBridgeCall(String(handlerName), String(argsJson), Number(callId));
|
||||
@@ -72,17 +106,31 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView {
|
||||
onPageFinished: () => {
|
||||
session.webViewLoaded();
|
||||
},
|
||||
onRenderProcessGone: (crashed: boolean) => {
|
||||
gone = true;
|
||||
session.webViewFailed(crashed
|
||||
? "The app's web page crashed. Close and reopen the app."
|
||||
: "Android stopped the app's web page to free memory. Close and reopen the app.", true);
|
||||
},
|
||||
});
|
||||
|
||||
const remoteUrl = session.remoteUrl;
|
||||
// Remote apps load from their own server, so nothing is intercepted and the
|
||||
// fake per-app origin is unused.
|
||||
const host = remoteUrl ? "" : originHost(session);
|
||||
// Stock bridge shim first, then the Faceclaw extensions global (both at
|
||||
// document-start, before any app JS).
|
||||
const injectScript = EVENHUB_BRIDGE_INJECT_SCRIPT + buildFaceclawExtensionsScript(`Faceclaw/${FACECLAW_VERSION}`);
|
||||
const documentStartInstalled = remoteUrl
|
||||
? com.faceclaw.app.FaceclawEvenHubDocumentStart.install(webView, injectScript, originRule(remoteUrl)) === true
|
||||
const startUrl = remoteUrl || `https://${host}/${session.manifest.entrypoint}`;
|
||||
const originRule: string | null = com.faceclaw.app.FaceclawEvenHubJsBridge.originRule(startUrl);
|
||||
const appCheck = appOriginCheck(startUrl);
|
||||
const guarded = (js: string) => guardToAppOrigin(startUrl, js);
|
||||
// The transport, the stock bridge shim, then the Faceclaw extensions global
|
||||
// (all at document-start, before any app JS).
|
||||
const injectScript = guarded(ANDROID_EVENHUB_TRANSPORT_SCRIPT + EVENHUB_BRIDGE_INJECT_SCRIPT +
|
||||
buildFaceclawExtensionsScript(`Faceclaw/${FACECLAW_VERSION}`));
|
||||
const bridgeError: string | null = originRule
|
||||
? com.faceclaw.app.FaceclawEvenHubJsBridge.install(webView, originRule, listener)
|
||||
: `Not a web address: ${startUrl}`;
|
||||
const documentStartInstalled = remoteUrl && originRule
|
||||
? com.faceclaw.app.FaceclawEvenHubDocumentStart.install(webView, injectScript, originRule) === true
|
||||
: false;
|
||||
webView.setWebViewClient(
|
||||
new com.faceclaw.app.FaceclawEvenHubWebViewClient(
|
||||
@@ -95,20 +143,22 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView {
|
||||
listener,
|
||||
),
|
||||
);
|
||||
webView.addJavascriptInterface(new com.faceclaw.app.FaceclawEvenHubJsBridge(listener), "__faceclawEvenHub");
|
||||
|
||||
// Surface the app's console in logcat (tag FaceclawEvenHubConsole).
|
||||
webView.setWebChromeClient(new com.faceclaw.app.FaceclawEvenHubChromeClient());
|
||||
|
||||
const nativeHost = com.faceclaw.app.FaceclawEvenHubWebViewHost.getInstance();
|
||||
nativeHost.attach(activity, webView);
|
||||
webView.loadUrl(remoteUrl || `https://${host}/${session.manifest.entrypoint}`);
|
||||
nativeHost.attach(activity, webView, appCheck);
|
||||
// Without the bridge the app could only hang waiting for it.
|
||||
if (bridgeError) session.webViewFailed(String(bridgeError), true);
|
||||
else webView.loadUrl(startUrl);
|
||||
|
||||
return {
|
||||
native: webView,
|
||||
evaluateJs: (js: string) => {
|
||||
if (gone) return;
|
||||
try {
|
||||
webView.evaluateJavascript(js, null);
|
||||
webView.evaluateJavascript(guarded(js), null);
|
||||
} catch (error) {
|
||||
console.warn(`evenhub evaluateJs failed: ${error}`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,119 @@
|
||||
/**
|
||||
* App-supplied window icons (the extension setWindowIcon): parse the pixel
|
||||
* buffer an app sends and scale it to the sizes the app switcher draws at.
|
||||
* Pure, so it can be tested without NativeScript.
|
||||
*/
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
|
||||
/** Larger icons are refused: they only cost bridge traffic, since switcher icons are ~32px. */
|
||||
export const MAX_WINDOW_ICON_DIMENSION = 256;
|
||||
|
||||
/**
|
||||
* Parse `{width, height, data}` from the bridge into a grayscale image.
|
||||
* `data` is width*height brightness bytes, or width*height*4 RGBA bytes (an
|
||||
* ImageData), which are reduced to luminance times alpha. Arrays arrive as
|
||||
* plain arrays or as JSON-ified typed arrays ({"0": n, ...}). Returns null for
|
||||
* anything malformed.
|
||||
*/
|
||||
export function parseWindowIcon(raw: unknown): GrayImage | null {
|
||||
if (!raw || typeof raw !== "object") return null;
|
||||
const record = raw as Record<string, unknown>;
|
||||
const width = Number(record.width);
|
||||
const height = Number(record.height);
|
||||
if (!Number.isInteger(width) || !Number.isInteger(height)) return null;
|
||||
if (width < 1 || height < 1 || width > MAX_WINDOW_ICON_DIMENSION || height > MAX_WINDOW_ICON_DIMENSION) return null;
|
||||
const data = toNumberArray(record.data);
|
||||
if (!data) return null;
|
||||
const area = width * height;
|
||||
const image = new GrayImage(width, height, 0);
|
||||
if (data.length === area) {
|
||||
for (let i = 0; i < area; i++) image.pixels[i] = clampByte(data[i]!);
|
||||
} else if (data.length === area * 4) {
|
||||
for (let i = 0; i < area; i++) {
|
||||
const o = i * 4;
|
||||
const luminance = 0.2126 * data[o]! + 0.7152 * data[o + 1]! + 0.0722 * data[o + 2]!;
|
||||
image.pixels[i] = clampByte((luminance * data[o + 3]!) / 255);
|
||||
}
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
return image;
|
||||
}
|
||||
|
||||
/**
|
||||
* Scale an icon to fit a size x size box, keeping its aspect ratio, with an
|
||||
* area-averaging filter. Results are cached per size, since the switcher
|
||||
* repaints often.
|
||||
*/
|
||||
export class WindowIconRenderer {
|
||||
private readonly scaled = new Map<number, GrayImage>();
|
||||
|
||||
constructor(private readonly source: GrayImage) {}
|
||||
|
||||
render(size: number): GrayImage {
|
||||
const box = Math.max(1, Math.round(size));
|
||||
let image = this.scaled.get(box);
|
||||
if (!image) {
|
||||
image = scaleToFit(this.source, box);
|
||||
this.scaled.set(box, image);
|
||||
}
|
||||
return image;
|
||||
}
|
||||
}
|
||||
|
||||
function scaleToFit(source: GrayImage, box: number): GrayImage {
|
||||
const scale = Math.min(box / source.width, box / source.height);
|
||||
const width = Math.max(1, Math.round(source.width * scale));
|
||||
const height = Math.max(1, Math.round(source.height * scale));
|
||||
if (width === source.width && height === source.height) return source;
|
||||
const out = new GrayImage(width, height, 0);
|
||||
for (let y = 0; y < height; y++) {
|
||||
const y0 = (y * source.height) / height;
|
||||
const y1 = ((y + 1) * source.height) / height;
|
||||
for (let x = 0; x < width; x++) {
|
||||
const x0 = (x * source.width) / width;
|
||||
const x1 = ((x + 1) * source.width) / width;
|
||||
out.pixels[y * width + x] = clampByte(areaAverage(source, x0, y0, x1, y1));
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Mean of the source over the rectangle [x0,x1) x [y0,y1), weighting partly covered pixels. */
|
||||
function areaAverage(source: GrayImage, x0: number, y0: number, x1: number, y1: number): number {
|
||||
let sum = 0;
|
||||
let weight = 0;
|
||||
for (let sy = Math.floor(y0); sy < Math.min(source.height, Math.ceil(y1)); sy++) {
|
||||
const wy = Math.min(y1, sy + 1) - Math.max(y0, sy);
|
||||
for (let sx = Math.floor(x0); sx < Math.min(source.width, Math.ceil(x1)); sx++) {
|
||||
const w = wy * (Math.min(x1, sx + 1) - Math.max(x0, sx));
|
||||
sum += source.pixels[sy * source.width + sx]! * w;
|
||||
weight += w;
|
||||
}
|
||||
}
|
||||
return weight > 0 ? sum / weight : 0;
|
||||
}
|
||||
|
||||
function toNumberArray(raw: unknown): number[] | null {
|
||||
let values: unknown[];
|
||||
if (Array.isArray(raw)) {
|
||||
values = raw;
|
||||
} else if (raw && typeof raw === "object") {
|
||||
const record = raw as Record<string, unknown>;
|
||||
values = [];
|
||||
for (let i = 0; String(i) in record; i++) values.push(record[String(i)]);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
const out: number[] = [];
|
||||
for (const entry of values) {
|
||||
const value = Number(entry);
|
||||
if (!Number.isFinite(value)) return null;
|
||||
out.push(value);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function clampByte(value: number): number {
|
||||
return Math.max(0, Math.min(255, Math.round(value)));
|
||||
}
|
||||
@@ -205,9 +205,18 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
if (window.foreground) renderAndSubmit(window, 0);
|
||||
break;
|
||||
}
|
||||
case "resize-window": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
window.viewportWidth = message.viewport.width;
|
||||
window.viewportHeight = message.viewport.height;
|
||||
window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight });
|
||||
window.lastSubmittedFingerprint = "";
|
||||
if (window.foreground) renderAndSubmit(window, 0);
|
||||
break;
|
||||
}
|
||||
case "input-focus":
|
||||
case "navigation-sensors":
|
||||
case "resize-window":
|
||||
case "screen":
|
||||
case "text-input":
|
||||
case "tool-call":
|
||||
|
||||
@@ -15,6 +15,11 @@ export function isTerminalWidgetConfigured(): boolean {
|
||||
return isGlanceWidgetConfigured("terminal");
|
||||
}
|
||||
|
||||
/** Whether the system card can show: it is in a slot and the Glanceboard is enabled. */
|
||||
export function isSystemCardWidgetInUse(): boolean {
|
||||
return getBooleanSetting("glanceboard.enabled", false) && isGlanceWidgetConfigured("system-card");
|
||||
}
|
||||
|
||||
/**
|
||||
* Whether the T3 Code widget can show: it is in a slot and the Glanceboard is
|
||||
* enabled (the T3 worker holds its server connections open only then).
|
||||
|
||||
@@ -5,7 +5,7 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from
|
||||
import { uiFontOverridePickerMenuItem } from "../../ui/font-picker";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu";
|
||||
import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu";
|
||||
import { Menu } from "../../ui/menu-core";
|
||||
import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics";
|
||||
import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window";
|
||||
@@ -27,7 +27,8 @@ import {
|
||||
type GlanceSlotChoice,
|
||||
} from "./glanceboard-settings";
|
||||
import { resolveGlanceRegions, slotDividers } from "./layout";
|
||||
import { glanceWidgetSpans } from "./widgets";
|
||||
import { type GlanceWidgetDefinition } from "./widget";
|
||||
import { findGlanceWidget, glanceWidgetSpans } from "./widgets";
|
||||
|
||||
export const GLANCEBOARD_WINDOW_ID = "glanceboard";
|
||||
export const GLANCEBOARD_SURFACE_ID = "window:glanceboard";
|
||||
@@ -300,10 +301,14 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void {
|
||||
openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current)));
|
||||
}
|
||||
|
||||
/** Settings: layout, sleep gestures, duration, slot lines, widget font, and stereo depth. */
|
||||
/**
|
||||
* Settings: layout, sleep gestures, duration, slot lines, widget font, and
|
||||
* stereo depth, then a row per widget on the board for its own settings.
|
||||
*/
|
||||
function openGlanceSettings(ctx: LayerContext): void {
|
||||
const items: MenuItem[] = [
|
||||
enumSettingMenuItem(glanceLayoutSetting),
|
||||
const items = (): MenuItem[] => [
|
||||
// The layout decides which slots (and so which widgets) are on the board.
|
||||
enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }),
|
||||
enumSettingMenuItem(glanceTapDurationSetting),
|
||||
toggleSettingMenuItem(glanceShowOnLongPressSetting),
|
||||
toggleSettingMenuItem(glanceShowOnHeadTiltSetting),
|
||||
@@ -314,8 +319,31 @@ function openGlanceSettings(ctx: LayerContext): void {
|
||||
"Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.",
|
||||
}),
|
||||
enumSettingMenuItem(glanceDepthSetting),
|
||||
...widgetSettingsMenuItems(),
|
||||
];
|
||||
ctx.stack.push(new MenuLayer("Glanceboard settings", items, pageMenuLayout(ctx.stack.getBaseSize().width)));
|
||||
const layer = new MenuLayer("Glanceboard settings", items(), pageMenuLayout(ctx.stack.getBaseSize().width));
|
||||
ctx.stack.push(layer);
|
||||
}
|
||||
|
||||
/**
|
||||
* One row per widget on the board, in slot order: a submenu of the widget's
|
||||
* settings, or a disabled row for a widget that has none.
|
||||
*/
|
||||
function widgetSettingsMenuItems(): MenuItem[] {
|
||||
const widgets = new Set<GlanceWidgetDefinition>();
|
||||
for (const setting of glanceSlotSettings()) {
|
||||
const widget = findGlanceWidget(setting.get());
|
||||
if (widget) widgets.add(widget);
|
||||
}
|
||||
return Array.from(widgets, (widget): MenuItem => {
|
||||
const settingsMenuItems = widget.settingsMenuItems;
|
||||
if (!settingsMenuItems) {
|
||||
return { label: widget.label, description: `${widget.label} has no settings.`, disabled: true, onSelect: () => {} };
|
||||
}
|
||||
return submenuItem(widget.label, (ctx) => {
|
||||
ctx.stack.push(new MenuLayer(`${widget.label} settings`, settingsMenuItems(), pageMenuLayout(ctx.stack.getBaseSize().width)));
|
||||
}, { description: `Settings for the ${widget.label} widget.` });
|
||||
});
|
||||
}
|
||||
|
||||
/** The live board in the window; any click or double-click returns to the list. */
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { type GrayImage } from "../../graphics/image";
|
||||
import { type MenuItem } from "../../ui/menu";
|
||||
|
||||
/**
|
||||
* A Glanceboard widget: a small self-contained view that paints into a slot
|
||||
@@ -39,5 +40,11 @@ export type GlanceWidgetDefinition = {
|
||||
* gets the taller canvas; the divider between the slots is not drawn.
|
||||
*/
|
||||
tall?: boolean;
|
||||
/**
|
||||
* The rows of this widget's submenu in the Glanceboard app's Settings. A
|
||||
* widget without settings still gets a row there (disabled) while it is
|
||||
* on the board.
|
||||
*/
|
||||
settingsMenuItems?: () => MenuItem[];
|
||||
create: () => GlanceWidget;
|
||||
};
|
||||
@@ -1,8 +1,15 @@
|
||||
import { type GrayImage } from "../../../graphics/image";
|
||||
import { hasLocationPermission } from "../../../native/location-permissions";
|
||||
import { addCompassListener, COMPASS_CHANGED, setCompassEnabled } from "../../../native/compass";
|
||||
import { addCompassListener, COMPASS_CHANGED, setCompassEnabled, type CompassDiagnostics } from "../../../native/compass";
|
||||
import { isCompassCalibrated, normalizeHeading } from "../../compass/calibration";
|
||||
import { cardinalDirection, createCompassBackground, drawCompassRose, layoutCompassRose, TICK_HEIGHT } from "../../compass/compass-rose";
|
||||
import {
|
||||
cardinalDirection,
|
||||
createCompassBackground,
|
||||
drawCompassRose,
|
||||
layoutCompassRose,
|
||||
magneticAccuracyDots,
|
||||
TICK_HEIGHT,
|
||||
} from "../../compass/compass-rose";
|
||||
import { getDeclinationAvailability, onDeclinationChanged, refreshDeclination } from "../../compass/declination";
|
||||
import { getNorthReference, resolveHeading } from "../../compass/heading";
|
||||
import { glanceFont } from "../glance-font";
|
||||
@@ -14,16 +21,21 @@ const TOP_PAD = 4;
|
||||
const BOTTOM_PAD = 4;
|
||||
const EDGE_PAD = 8;
|
||||
const MAX_ROSE_RADIUS = 60;
|
||||
/** Inset of the top-right magnetometer calibration dots from the card edges. */
|
||||
const ACCURACY_TOP_PAD = TOP_PAD;
|
||||
const ACCURACY_RIGHT_PAD = EDGE_PAD;
|
||||
|
||||
/**
|
||||
* The Compass app's rose at card size: heading readout above, tilted rose
|
||||
* below. Turns the magnetometer on for as long as the board shows it (the
|
||||
* Compass app keeps its own hold, so the two never switch each other off).
|
||||
* Never prompts for location: true north is used when a declination is
|
||||
* already known, otherwise the readout says it is magnetic.
|
||||
* already known, otherwise the readout says it is magnetic. The firmware's
|
||||
* magnetometer calibration shows as the Compass app's dots, without a label.
|
||||
*/
|
||||
export class CompassWidget implements GlanceWidget {
|
||||
private rawHeading: number | null = null;
|
||||
private diagnostics: CompassDiagnostics | null = null;
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private unsubscribeDeclination: (() => void) | null = null;
|
||||
private enabled = false;
|
||||
@@ -33,6 +45,7 @@ export class CompassWidget implements GlanceWidget {
|
||||
this.unsubscribe = addCompassListener((event) => {
|
||||
if (event.command === COMPASS_CHANGED && event.headingDegrees >= 0) {
|
||||
this.rawHeading = normalizeHeading(event.headingDegrees);
|
||||
this.diagnostics = event.diagnostics ?? null;
|
||||
requestRender();
|
||||
}
|
||||
});
|
||||
@@ -96,5 +109,9 @@ export class CompassWidget implements GlanceWidget {
|
||||
if (status) {
|
||||
image.drawText(small, Math.round(cx - small.measureText(status) / 2), y + medium.lineHeight + 2, status, 125);
|
||||
}
|
||||
const accuracyDots = magneticAccuracyDots(this.diagnostics);
|
||||
if (accuracyDots) {
|
||||
image.drawText(small, width - ACCURACY_RIGHT_PAD - small.measureText(accuracyDots), ACCURACY_TOP_PAD, accuracyDots, 175);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,22 +1,22 @@
|
||||
import { type GlanceWidgetDefinition, type GlanceWidgetId } from "../widget";
|
||||
import { CalendarWidget } from "./calendar-widget";
|
||||
import { CompassWidget } from "./compass-widget";
|
||||
import { MusicWidget } from "./music-widget";
|
||||
import { MusicWidget, musicSettingsMenuItems } from "./music-widget";
|
||||
import { NightscoutWidget } from "./nightscout-widget";
|
||||
import { SystemCardWidget } from "./system-card";
|
||||
import { SystemCardWidget, systemCardSettingsMenuItems } from "./system-card";
|
||||
import { T3CodeWidget } from "./t3code-widget";
|
||||
import { TerminalWidget } from "./terminal-widget";
|
||||
import { HealthWidget } from "./health-widget";
|
||||
|
||||
/** Every widget a board slot can hold. */
|
||||
export const GLANCE_WIDGETS: readonly GlanceWidgetDefinition[] = [
|
||||
{ id: "system-card", label: "System card", create: () => new SystemCardWidget() },
|
||||
{ id: "system-card", label: "System card", settingsMenuItems: systemCardSettingsMenuItems, create: () => new SystemCardWidget() },
|
||||
{ id: "calendar", label: "Calendar", tall: true, create: () => new CalendarWidget() },
|
||||
{ id: "terminal", label: "Terminal", tall: true, create: () => new TerminalWidget() },
|
||||
{ id: "t3code", label: "T3 Code", tall: true, create: () => new T3CodeWidget() },
|
||||
{ id: "nightscout", label: "Nightscout", create: () => new NightscoutWidget() },
|
||||
{ id: "compass", label: "Compass", create: () => new CompassWidget() },
|
||||
{ id: "music", label: "Music", create: () => new MusicWidget() },
|
||||
{ id: "music", label: "Music", settingsMenuItems: musicSettingsMenuItems, create: () => new MusicWidget() },
|
||||
{ id: "health", label: "Health", create: () => new HealthWidget() },
|
||||
];
|
||||
|
||||
|
||||
@@ -2,6 +2,8 @@ import { GrayImage } from "../../../graphics/image";
|
||||
import { renderIcon } from "../../../graphics/icons";
|
||||
import { truncateText } from "../../../graphics/textwrap";
|
||||
import { mediaControllerBridge, type MediaControllerState } from "../../../native/media-controller";
|
||||
import { ConfigSettingBoolean, onAnySettingChanged, toggleSettingMenuItem } from "../../../ui/dashboard-settings";
|
||||
import { type MenuItem } from "../../../ui/menu";
|
||||
import { clamp } from "../../../util/numeric-util";
|
||||
import { glanceFont } from "../glance-font";
|
||||
import { type GlanceWidget } from "../widget";
|
||||
@@ -36,20 +38,37 @@ function idleNoteImage(): GrayImage | null {
|
||||
return note;
|
||||
}
|
||||
const ART_X = PAD;
|
||||
const ART_TEXT_GAP = 12;
|
||||
const PROGRESS_BAR_HEIGHT = 5;
|
||||
|
||||
export const musicShowAlbumArtSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-music-show-album-art",
|
||||
label: "Show album art",
|
||||
storageKey: "glanceboard.music.showAlbumArt",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.",
|
||||
});
|
||||
|
||||
/** The music card's submenu in Glanceboard > Settings. */
|
||||
export function musicSettingsMenuItems(): MenuItem[] {
|
||||
return [toggleSettingMenuItem(musicShowAlbumArtSetting)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Now playing: album art, title/artist/album and a progress bar for the
|
||||
* active Android media session, the Music app's header at card size. The
|
||||
* media bridge runs for the whole session; the widget subscribes and ticks
|
||||
* once a second only while something is playing.
|
||||
* Now playing: album art (unless turned off), title/artist/album and a
|
||||
* progress bar for the active media session (Apple Music on iOS), the Music
|
||||
* app's header at card size. The media bridge runs for the whole session; the
|
||||
* widget subscribes and ticks once a second only while something is playing.
|
||||
*/
|
||||
export class MusicWidget implements GlanceWidget {
|
||||
private media: MediaControllerState = mediaControllerBridge.snapshot();
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private unsubscribeSettings: (() => void) | null = null;
|
||||
private progressTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private art: GrayImage | null = null;
|
||||
private artKey = "";
|
||||
/** The artVersion `art` was fetched for; -1 = not fetched yet. */
|
||||
private artVersion = -1;
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribe = mediaControllerBridge.onStateChange((state) => {
|
||||
@@ -61,11 +80,14 @@ export class MusicWidget implements GlanceWidget {
|
||||
}
|
||||
requestRender();
|
||||
});
|
||||
this.unsubscribeSettings = onAnySettingChanged(requestRender);
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.unsubscribe?.();
|
||||
this.unsubscribe = null;
|
||||
this.unsubscribeSettings?.();
|
||||
this.unsubscribeSettings = null;
|
||||
this.stopProgressTimer();
|
||||
}
|
||||
|
||||
@@ -83,7 +105,7 @@ export class MusicWidget implements GlanceWidget {
|
||||
if (!media.accessEnabled) {
|
||||
// Actionable, so it stays as text.
|
||||
image.drawText(small, PAD, PAD, "Music", 150);
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, "Notification access needed", 140);
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
|
||||
return;
|
||||
}
|
||||
// Nothing playing: a dim note, centred, and nothing else.
|
||||
@@ -92,10 +114,12 @@ export class MusicWidget implements GlanceWidget {
|
||||
return;
|
||||
}
|
||||
|
||||
// Without the art, the text and progress bar keep its rows and take its columns.
|
||||
const artY = Math.max(PAD, ((image.height - ART_SIZE) / 2) | 0);
|
||||
this.drawArt(image, media, artY);
|
||||
const showArt = musicShowAlbumArtSetting.get();
|
||||
if (showArt) this.drawArt(image, media, artY);
|
||||
|
||||
const textX = ART_X + ART_SIZE + 12;
|
||||
const textX = showArt ? ART_X + ART_SIZE + ART_TEXT_GAP : PAD;
|
||||
const textWidth = image.width - textX - PAD;
|
||||
let y = artY;
|
||||
image.drawText(medium, textX, y, truncateText(medium, media.title || "Unknown title", textWidth), 230);
|
||||
@@ -129,10 +153,8 @@ export class MusicWidget implements GlanceWidget {
|
||||
}
|
||||
|
||||
private drawArt(image: GrayImage, media: MediaControllerState, artY: number): void {
|
||||
const key = `${media.packageName}|${media.title}|${media.album}`;
|
||||
// Retry while null: players often publish metadata before the bitmap.
|
||||
if (key !== this.artKey || this.art === null) {
|
||||
this.artKey = key;
|
||||
if (media.artVersion !== this.artVersion) {
|
||||
this.artVersion = media.artVersion;
|
||||
this.art = mediaControllerBridge.getAlbumArt(ART_SIZE);
|
||||
}
|
||||
if (this.art) {
|
||||
|
||||
@@ -1,25 +1,30 @@
|
||||
import {
|
||||
BATTERY_ICON_WIDTH,
|
||||
DENSE_BATTERY_BLOCK_HEIGHT,
|
||||
denseBatteriesWidth,
|
||||
drawBattery,
|
||||
drawDenseBatteries,
|
||||
type BatteryDevice,
|
||||
} from "../../../graphics/battery";
|
||||
import { type GrayImage, type UiFont } from "../../../graphics/image";
|
||||
import { onAndroidNotificationPosted, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../../native/location-permissions";
|
||||
import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons";
|
||||
import { readPhoneBatteryState } from "../../../native/phone-battery";
|
||||
import { weatherBridge } from "../../../native/weather";
|
||||
import { formatClockDate, formatClockTime } from "../../../ui/clock-format";
|
||||
import {
|
||||
batteryDisplayModeSetting,
|
||||
batteryIndicatorVisible,
|
||||
ConfigSettingBoolean,
|
||||
glassesBatteryVisibilitySetting,
|
||||
onAnySettingChanged,
|
||||
phoneBatteryVisibilitySetting,
|
||||
ringBatteryVisibilitySetting,
|
||||
toggleSettingMenuItem,
|
||||
watchBatteryVisibilitySetting,
|
||||
type BatteryIndicatorVisibility,
|
||||
} from "../../../ui/dashboard-settings";
|
||||
import { type MenuItem } from "../../../ui/menu";
|
||||
import { shell } from "../../../ui/shell/shell";
|
||||
import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness";
|
||||
import { truncateText } from "../../../graphics/textwrap";
|
||||
@@ -36,20 +41,113 @@ const NOTIFICATION_ICON_SIZE = 24;
|
||||
const NOTIFICATION_ICON_GAP = 4;
|
||||
const BATTERY_ITEM_GAP = 12;
|
||||
const BATTERY_LABEL_VALUE = 150;
|
||||
/** Gap between the battery block and the weather reading below it. */
|
||||
const WEATHER_GAP = 6;
|
||||
/** Gap between the battery block and the weather reading stacked after it. */
|
||||
const SIDE_BLOCK_GAP = 6;
|
||||
/** Rows between the time and the date. */
|
||||
const CLOCK_LINE_GAP = 2;
|
||||
/** Rows between the clock and the timers, and between the timers and the notification icons. */
|
||||
const TIMERS_GAP = 4;
|
||||
const TIMER_NAME_GAP = 8;
|
||||
|
||||
type BatteryItem = { label: string; device: BatteryDevice; percent: number; charging: boolean };
|
||||
/** A block drawn against the card's right edge: its left edge and its rows. */
|
||||
type RightBlock = { left: number; top: number; bottom: number };
|
||||
type Rect = { x: number; y: number; width: number; height: number };
|
||||
/** A side-column block, measured so it can be placed against either edge before it is drawn. */
|
||||
type SideBlock = { width: number; height: number; draw: (image: GrayImage, left: number, top: number) => void };
|
||||
/** The main column's horizontal extent on some rows. */
|
||||
type Span = { left: number; right: number };
|
||||
|
||||
export const systemCardShowTimeSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-show-time",
|
||||
label: "Time",
|
||||
storageKey: "glanceboard.systemCard.showTime",
|
||||
defaultValue: true,
|
||||
description: "Show the clock on the system card.",
|
||||
});
|
||||
|
||||
export const systemCardShowDateSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-show-date",
|
||||
label: "Date",
|
||||
storageKey: "glanceboard.systemCard.showDate",
|
||||
defaultValue: true,
|
||||
description: "Show the date on the system card, under the time.",
|
||||
});
|
||||
|
||||
export const systemCardShowBatterySetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-show-battery",
|
||||
label: "Battery",
|
||||
storageKey: "glanceboard.systemCard.showBattery",
|
||||
defaultValue: true,
|
||||
description:
|
||||
"Show battery indicators on the system card. Their style and which devices appear follow Settings > Customization > Battery indicators.",
|
||||
});
|
||||
|
||||
export const systemCardShowNotificationsSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-show-notifications",
|
||||
label: "Notifications",
|
||||
storageKey: "glanceboard.systemCard.showNotifications",
|
||||
defaultValue: true,
|
||||
description: "Show the icons of the phone's notifications along the edge of the system card.",
|
||||
});
|
||||
|
||||
export const systemCardFlipHorizontalSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-flip-horizontal",
|
||||
label: "Flip layout horizontally",
|
||||
storageKey: "glanceboard.systemCard.flipHorizontal",
|
||||
defaultValue: false,
|
||||
description:
|
||||
"Mirror the system card left to right: time, date, timers and notifications on the right, batteries and weather on the left.",
|
||||
});
|
||||
|
||||
export const systemCardFlipVerticalSetting = new ConfigSettingBoolean({
|
||||
id: "glanceboard-system-card-flip-vertical",
|
||||
label: "Flip layout vertically",
|
||||
storageKey: "glanceboard.systemCard.flipVertical",
|
||||
defaultValue: false,
|
||||
description:
|
||||
"Mirror the system card top to bottom: notifications along the top; time, date, batteries and weather along the bottom.",
|
||||
});
|
||||
|
||||
/**
|
||||
* The system card's submenu in Glanceboard > Settings. Weather is the
|
||||
* Weather app's "Show on system card" setting under a shorter label.
|
||||
*/
|
||||
export function systemCardSettingsMenuItems(): MenuItem[] {
|
||||
return [
|
||||
toggleSettingMenuItem(systemCardShowTimeSetting),
|
||||
toggleSettingMenuItem(systemCardShowDateSetting),
|
||||
toggleSettingMenuItem(systemCardShowBatterySetting),
|
||||
toggleSettingMenuItem(weatherShowOnSystemCardSetting, {
|
||||
label: "Weather",
|
||||
onChange: (_ctx, enabled) => {
|
||||
if (enabled) requestWeatherLocationAccess();
|
||||
},
|
||||
}),
|
||||
toggleSettingMenuItem(systemCardShowNotificationsSetting),
|
||||
toggleSettingMenuItem(systemCardFlipHorizontalSetting),
|
||||
toggleSettingMenuItem(systemCardFlipVerticalSetting),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Weather switched on without location access: ask for it now, as the
|
||||
* Weather app's own toggle does, since the background refresh has no screen
|
||||
* to prompt from.
|
||||
*/
|
||||
function requestWeatherLocationAccess(): void {
|
||||
if (hasLocationPermission()) return;
|
||||
void ensureLocationPermission().then((granted) => {
|
||||
if (granted) void weatherBridge.refreshNow();
|
||||
}).catch((error) => console.warn(`Weather permission: ${error}`));
|
||||
}
|
||||
|
||||
/**
|
||||
* Date and time, the phone's notification icons, the phone/watch/G2/R1
|
||||
* battery indicators (the Wear OS watch's only while one is reachable), the
|
||||
* current weather (when the Weather app's "Show on system card" is on), and
|
||||
* the Timers app's countdowns: the top bar's contents plus timers, laid out
|
||||
* for a card. Battery style and per-device visibility
|
||||
* follow Settings > Customization > Battery indicators, so the card agrees with
|
||||
* current weather, and the Timers app's countdowns: the top bar's contents
|
||||
* plus timers, laid out for a card. Every part but the timers can be turned
|
||||
* off, and the layout mirrored either way, in the card's submenu of
|
||||
* Glanceboard > Settings. Battery style and per-device visibility follow
|
||||
* Settings > Customization > Battery indicators, so the card agrees with
|
||||
* the bar. Countdowns tick once a second only while one is running.
|
||||
*/
|
||||
export class SystemCardWidget implements GlanceWidget {
|
||||
@@ -64,7 +162,7 @@ export class SystemCardWidget implements GlanceWidget {
|
||||
start(requestRender: () => void): void {
|
||||
this.requestRender = requestRender;
|
||||
this.scheduleMinuteTick();
|
||||
this.unsubscribeNotifications = onAndroidNotificationPosted(() => this.requestRender?.());
|
||||
this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.());
|
||||
this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.());
|
||||
this.unsubscribeTimers = timerEngine.onChange(() => {
|
||||
this.syncSecondTick();
|
||||
@@ -120,35 +218,93 @@ export class SystemCardWidget implements GlanceWidget {
|
||||
const medium = glanceFont.medium();
|
||||
const small = glanceFont.small();
|
||||
const now = new Date();
|
||||
const { width, height } = image;
|
||||
|
||||
// Clock and date down the left; batteries take the right edge, with the
|
||||
// weather right-aligned beneath them. Text on the left stops short of
|
||||
// whichever of those shares its rows.
|
||||
const battery = drawBatteryBlock(image, small, image.width - PAD, PAD);
|
||||
const weatherTop = battery.bottom > battery.top ? battery.bottom + WEATHER_GAP : PAD;
|
||||
const weather = drawWeatherBlock(image, medium, image.width - PAD, weatherTop, now.getTime());
|
||||
const textRight = (top: number, bottom: number): number =>
|
||||
weather && weather.top < bottom && weather.bottom > top ? Math.min(battery.left, weather.left) : battery.left;
|
||||
const textWidth = Math.max(0, textRight(PAD, PAD + large.lineHeight) - BATTERY_ITEM_GAP - PAD);
|
||||
// Unflipped, the time and date run down the left from the top, the
|
||||
// batteries and then the weather stack down the right edge, and the
|
||||
// notification icons take the bottom row, with the timers in the band
|
||||
// between the date and the icons. The flips mirror where each block sits
|
||||
// (positions below are given for the unflipped card), not what is inside
|
||||
// it: the time stays above the date, and timers read soonest first.
|
||||
const flipX = systemCardFlipHorizontalSetting.get();
|
||||
const flipY = systemCardFlipVerticalSetting.get();
|
||||
const placeX = (x: number, w: number): number => (flipX ? width - x - w : x);
|
||||
const placeY = (y: number, h: number): number => (flipY ? height - y - h : y);
|
||||
|
||||
const sideBlocks: SideBlock[] = [];
|
||||
const batteryItems = systemCardShowBatterySetting.get() ? collectBatteryItems() : [];
|
||||
if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems));
|
||||
const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null;
|
||||
if (weather) sideBlocks.push(weather);
|
||||
const side: Rect[] = [];
|
||||
let sideOffset = PAD;
|
||||
for (const block of sideBlocks) {
|
||||
const rect = {
|
||||
x: placeX(width - PAD - block.width, block.width),
|
||||
y: placeY(sideOffset, block.height),
|
||||
width: block.width,
|
||||
height: block.height,
|
||||
};
|
||||
block.draw(image, rect.x, rect.y);
|
||||
side.push(rect);
|
||||
sideOffset += block.height + SIDE_BLOCK_GAP;
|
||||
}
|
||||
// Text in the main column stops short of whichever side blocks share its
|
||||
// rows, and hugs the card's outer edge.
|
||||
const span = (top: number, bottom: number): Span => {
|
||||
let left = PAD;
|
||||
let right = width - PAD;
|
||||
for (const rect of side) {
|
||||
if (rect.y >= bottom || rect.y + rect.height <= top) continue;
|
||||
if (flipX) left = Math.max(left, rect.x + rect.width + BATTERY_ITEM_GAP);
|
||||
else right = Math.min(right, rect.x - BATTERY_ITEM_GAP);
|
||||
}
|
||||
return { left, right };
|
||||
};
|
||||
const lineX = ({ left, right }: Span, lineWidth: number): number => (flipX ? right - lineWidth : left);
|
||||
|
||||
const showTime = systemCardShowTimeSetting.get();
|
||||
const showDate = systemCardShowDateSetting.get();
|
||||
const timeText = formatClockTime(now);
|
||||
const timeFont = large.measureText(timeText) <= textWidth ? large : medium;
|
||||
let y = PAD;
|
||||
image.drawText(timeFont, PAD, y, timeText, 230);
|
||||
y += timeFont.lineHeight + 2;
|
||||
image.drawText(medium, PAD, y, formatClockDate(now), 170);
|
||||
y += medium.lineHeight + 4;
|
||||
const dateText = formatClockDate(now);
|
||||
const clockHeight = (timeFont: UiFont): number =>
|
||||
(showTime ? timeFont.lineHeight : 0) + (showTime && showDate ? CLOCK_LINE_GAP : 0) + (showDate ? medium.lineHeight : 0);
|
||||
// The time drops to the medium font when the large one would run into a side block.
|
||||
let timeFont = large;
|
||||
if (showTime) {
|
||||
const top = placeY(PAD, clockHeight(large));
|
||||
const { left, right } = span(top, top + large.lineHeight);
|
||||
if (large.measureText(timeText) > right - left) timeFont = medium;
|
||||
}
|
||||
const clockSize = clockHeight(timeFont);
|
||||
let y = placeY(PAD, clockSize);
|
||||
if (showTime) {
|
||||
image.drawText(timeFont, lineX(span(y, y + timeFont.lineHeight), timeFont.measureText(timeText)), y, timeText, 230);
|
||||
y += timeFont.lineHeight + CLOCK_LINE_GAP;
|
||||
}
|
||||
if (showDate) {
|
||||
image.drawText(medium, lineX(span(y, y + medium.lineHeight), medium.measureText(dateText)), y, dateText, 170);
|
||||
}
|
||||
|
||||
// Notification icons along the bottom, as many as fit; the Timers app's
|
||||
// countdowns take the band between the date and the icons.
|
||||
const iconY = image.height - PAD - NOTIFICATION_ICON_SIZE;
|
||||
const timersWidth = Math.max(0, textRight(y, iconY - 4) - BATTERY_ITEM_GAP - PAD);
|
||||
drawTimers(image, small, PAD, y, timersWidth, iconY - 4, now.getTime());
|
||||
const maxIcons = Math.max(0, ((image.width - 2 * PAD + NOTIFICATION_ICON_GAP) / (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP)) | 0);
|
||||
if (maxIcons > 0) {
|
||||
const showNotifications = systemCardShowNotificationsSetting.get();
|
||||
const bandTop = clockSize > 0 ? PAD + clockSize + TIMERS_GAP : PAD;
|
||||
const bandBottom = height - PAD - (showNotifications ? NOTIFICATION_ICON_SIZE + TIMERS_GAP : 0);
|
||||
drawTimers(image, small, {
|
||||
top: placeY(bandTop, bandBottom - bandTop),
|
||||
height: bandBottom - bandTop,
|
||||
fromBottom: flipY,
|
||||
alignRight: flipX,
|
||||
span,
|
||||
}, now.getTime());
|
||||
|
||||
// Notification icons, as many as fit, the first nearest the main column's edge.
|
||||
const maxIcons = Math.max(0, ((width - 2 * PAD + NOTIFICATION_ICON_GAP) / (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP)) | 0);
|
||||
if (showNotifications && maxIcons > 0) {
|
||||
const { icons, stale } = readActiveNotificationIcons(maxIcons, renderPassAllowsStaleData());
|
||||
if (stale) noteStaleDataUsed();
|
||||
const iconY = placeY(height - PAD - NOTIFICATION_ICON_SIZE, NOTIFICATION_ICON_SIZE);
|
||||
for (let index = 0; index < icons.length; index++) {
|
||||
image.drawImage(icons[index]!, PAD + index * (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP), iconY);
|
||||
image.drawImage(icons[index]!, placeX(PAD + index * (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP), NOTIFICATION_ICON_SIZE), iconY);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -180,59 +336,62 @@ function collectBatteryItems(): BatteryItem[] {
|
||||
}
|
||||
|
||||
/**
|
||||
* The battery indicators right-aligned at `right`, in the configured style.
|
||||
* Side-by-side styles put the label beside the gauge (or percentage) on one
|
||||
* line; stacked styles centre the label above it; the dense style swaps
|
||||
* labels for device icons and stacks two indicators per column, as in the
|
||||
* top bar. Returns the block's extent (zero-height at `right` when nothing
|
||||
* is shown).
|
||||
* The battery indicators in the configured style. Side-by-side styles put
|
||||
* the label beside the gauge (or percentage) on one line; stacked styles
|
||||
* centre the label above it; the dense style swaps labels for device icons
|
||||
* and stacks two indicators per column, as in the top bar.
|
||||
*/
|
||||
function drawBatteryBlock(image: GrayImage, font: UiFont, right: number, top: number): RightBlock {
|
||||
const items = collectBatteryItems();
|
||||
if (!items.length) return { left: right, top, bottom: top };
|
||||
function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock {
|
||||
const mode = batteryDisplayModeSetting.get();
|
||||
if (mode === "dense") {
|
||||
return { left: drawDenseBatteries(image, items, right, top), top, bottom: top + DENSE_BATTERY_BLOCK_HEIGHT };
|
||||
const width = denseBatteriesWidth(items);
|
||||
return {
|
||||
width,
|
||||
height: DENSE_BATTERY_BLOCK_HEIGHT,
|
||||
draw: (image, left, top) => {
|
||||
drawDenseBatteries(image, items, left + width, top);
|
||||
},
|
||||
};
|
||||
}
|
||||
const percentage = mode === "percentage" || mode === "stacked-percentage";
|
||||
const stacked = mode === "stacked" || mode === "stacked-percentage";
|
||||
const gaugeHeight = drawBattery(0, false).height;
|
||||
const valueHeight = percentage ? font.lineHeight : Math.max(font.lineHeight, gaugeHeight);
|
||||
let x = right;
|
||||
for (let index = items.length - 1; index >= 0; index--) {
|
||||
const item = items[index]!;
|
||||
const percentText = `${item.percent}%`;
|
||||
const cells = items.map((item) => {
|
||||
const labelWidth = font.measureText(item.label);
|
||||
const valueWidth = percentage ? font.measureText(percentText) : BATTERY_ICON_WIDTH;
|
||||
if (stacked) {
|
||||
const itemWidth = Math.max(labelWidth, valueWidth);
|
||||
x -= itemWidth;
|
||||
image.drawText(font, x + (((itemWidth - labelWidth) / 2) | 0), top, item.label, BATTERY_LABEL_VALUE);
|
||||
drawBatteryValue(image, font, x + (((itemWidth - valueWidth) / 2) | 0), top + font.lineHeight + 2, item, percentage, gaugeHeight);
|
||||
} else {
|
||||
x -= labelWidth + 5 + valueWidth;
|
||||
image.drawText(font, x, top, item.label, BATTERY_LABEL_VALUE);
|
||||
drawBatteryValue(image, font, x + labelWidth + 5, top, item, percentage, gaugeHeight);
|
||||
}
|
||||
x -= BATTERY_ITEM_GAP;
|
||||
}
|
||||
return { left: x + BATTERY_ITEM_GAP, top, bottom: top + (stacked ? font.lineHeight + 2 : 0) + valueHeight };
|
||||
const valueWidth = percentage ? font.measureText(`${item.percent}%`) : BATTERY_ICON_WIDTH;
|
||||
return { item, labelWidth, valueWidth, width: stacked ? Math.max(labelWidth, valueWidth) : labelWidth + 5 + valueWidth };
|
||||
});
|
||||
return {
|
||||
width: cells.reduce((total, cell) => total + cell.width, 0) + BATTERY_ITEM_GAP * (cells.length - 1),
|
||||
height: (stacked ? font.lineHeight + 2 : 0) + valueHeight,
|
||||
draw: (image, left, top) => {
|
||||
let x = left;
|
||||
for (const { item, labelWidth, valueWidth, width } of cells) {
|
||||
if (stacked) {
|
||||
image.drawText(font, x + (((width - labelWidth) / 2) | 0), top, item.label, BATTERY_LABEL_VALUE);
|
||||
drawBatteryValue(image, font, x + (((width - valueWidth) / 2) | 0), top + font.lineHeight + 2, item, percentage, gaugeHeight);
|
||||
} else {
|
||||
image.drawText(font, x, top, item.label, BATTERY_LABEL_VALUE);
|
||||
drawBatteryValue(image, font, x + labelWidth + 5, top, item, percentage, gaugeHeight);
|
||||
}
|
||||
x += width + BATTERY_ITEM_GAP;
|
||||
}
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The current weather's condition icon and temperature, right-aligned at
|
||||
* `right` from `top`, while the system-card weather setting is on and a
|
||||
* fresh reading exists. Returns the block's extent, or null when not drawn.
|
||||
*/
|
||||
function drawWeatherBlock(image: GrayImage, font: UiFont, right: number, top: number, nowMs: number): RightBlock | null {
|
||||
if (!weatherShowOnSystemCardSetting.get()) return null;
|
||||
/** The current weather's condition icon and temperature, or null without a fresh reading. */
|
||||
function weatherBlock(font: UiFont, nowMs: number): SideBlock | null {
|
||||
const reading = weatherReading(weatherBridge.snapshot(), nowMs);
|
||||
if (!reading) return null;
|
||||
const iconSize = font.lineHeight + 4;
|
||||
const height = Math.max(iconSize, font.lineHeight);
|
||||
const left = right - Math.ceil(measureWeatherReading(reading, font, iconSize));
|
||||
drawWeatherReading(image, reading, font, iconSize, left, top, height, 200);
|
||||
return { left, top, bottom: top + height };
|
||||
return {
|
||||
width: Math.ceil(measureWeatherReading(reading, font, iconSize)),
|
||||
height,
|
||||
draw: (image, left, top) => drawWeatherReading(image, reading, font, iconSize, left, top, height, 200),
|
||||
};
|
||||
}
|
||||
|
||||
function drawBatteryValue(
|
||||
@@ -262,26 +421,36 @@ function drawBatteryValue(
|
||||
/**
|
||||
* The Timers app's countdowns, soonest first: remaining time (or "Done" once
|
||||
* rung, inverted so it is noticed; "paused" for a paused one) and the
|
||||
* timer's name, one per line while they fit between `top` and `bottom`.
|
||||
* timer's name, one per line, as many as fit in the band. The lines sit
|
||||
* against the band's top (or its bottom), each as wide as the main column
|
||||
* allows on its rows and hugging its left (or right) edge.
|
||||
*/
|
||||
function drawTimers(image: GrayImage, font: UiFont, x: number, top: number, width: number, bottom: number, nowMs: number): void {
|
||||
const timers = sortTimers(timerEngine.timers(), nowMs);
|
||||
function drawTimers(
|
||||
image: GrayImage,
|
||||
font: UiFont,
|
||||
band: { top: number; height: number; fromBottom: boolean; alignRight: boolean; span: (top: number, bottom: number) => Span },
|
||||
nowMs: number,
|
||||
): void {
|
||||
const step = lineStep(font);
|
||||
let y = top;
|
||||
const fit = band.height < font.lineHeight ? 0 : 1 + Math.floor((band.height - font.lineHeight) / step);
|
||||
const timers = sortTimers(timerEngine.timers(), nowMs).slice(0, fit);
|
||||
if (!timers.length) return;
|
||||
let y = band.fromBottom ? band.top + band.height - (timers.length - 1) * step - font.lineHeight : band.top;
|
||||
for (const timer of timers) {
|
||||
if (y + font.lineHeight > bottom) break;
|
||||
const { left, right } = band.span(y, y + font.lineHeight);
|
||||
const phase = timerPhase(timer);
|
||||
const clock = phase === "rung" ? "Done" : formatCountdown(timerRemainingMs(timer, nowMs));
|
||||
const clockWidth = font.measureText(clock);
|
||||
const nameOffset = font.measureText(clock) + TIMER_NAME_GAP;
|
||||
const fullName = phase === "paused" ? `${timerDisplayName(timer)} (paused)` : timerDisplayName(timer);
|
||||
const name = truncateText(font, fullName, Math.max(0, right - left - nameOffset));
|
||||
const x = band.alignRight ? right - nameOffset - font.measureText(name) : left;
|
||||
if (phase === "rung") {
|
||||
image.fillRect(x - 2, y - 1, clockWidth + 4, font.lineHeight + 2, 230);
|
||||
image.fillRect(x - 2, y - 1, nameOffset - TIMER_NAME_GAP + 4, font.lineHeight + 2, 230);
|
||||
image.drawText(font, x, y, clock, 1);
|
||||
} else {
|
||||
image.drawText(font, x, y, clock, phase === "paused" ? 120 : 200);
|
||||
}
|
||||
const nameX = x + clockWidth + 8;
|
||||
const name = phase === "paused" ? `${timerDisplayName(timer)} (paused)` : timerDisplayName(timer);
|
||||
image.drawText(font, nameX, y, truncateText(font, name, Math.max(0, x + width - nameX)), 160);
|
||||
image.drawText(font, x + nameOffset, y, name, 160);
|
||||
y += step;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,10 @@ const BOTTOM_MARGIN = 2;
|
||||
* The indicator animates on the hub's cadence while any session is active;
|
||||
* each row carries its own activity deadline, so nothing needs the worker
|
||||
* once the snapshot is in.
|
||||
*
|
||||
* The worker sorts sessions most-recently-updated first; the widget keeps the
|
||||
* order it first sees after the board opens so rows don't jump around while
|
||||
* it is showing. Sessions that appear later go to the end of their host.
|
||||
*/
|
||||
export class TerminalWidget implements GlanceWidget {
|
||||
private snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined;
|
||||
@@ -31,12 +35,17 @@ export class TerminalWidget implements GlanceWidget {
|
||||
private requestRender: (() => void) | null = null;
|
||||
private phase = 0;
|
||||
private animationTimer: ReturnType<typeof setInterval> | null = null;
|
||||
/** Row key -> display rank, fixed from the first snapshot seen since start. */
|
||||
private readonly rowOrder = new Map<string, number>();
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.requestRender = requestRender;
|
||||
this.rowOrder.clear();
|
||||
this.snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined;
|
||||
this.rankNewRows();
|
||||
this.unsubscribe = onWorkerStateChanged(TERMINAL_SESSIONS_STATE_KEY, (snapshot) => {
|
||||
this.snapshot = snapshot as TerminalSessionsSnapshot;
|
||||
this.rankNewRows();
|
||||
this.syncAnimation();
|
||||
requestRender();
|
||||
});
|
||||
@@ -51,6 +60,15 @@ export class TerminalWidget implements GlanceWidget {
|
||||
this.animationTimer = null;
|
||||
}
|
||||
|
||||
/** Give rows not seen yet the next ranks, in the snapshot's (recency) order. */
|
||||
private rankNewRows(): void {
|
||||
for (const host of this.snapshot?.hosts ?? []) {
|
||||
for (const row of host.sessions) {
|
||||
if (!this.rowOrder.has(row.key)) this.rowOrder.set(row.key, this.rowOrder.size);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private anyActive(nowMs = Date.now()): boolean {
|
||||
return Boolean(this.snapshot?.hosts.some((host) => host.sessions.some((row) => isSessionRowActive(row, nowMs))));
|
||||
}
|
||||
@@ -107,7 +125,10 @@ export class TerminalWidget implements GlanceWidget {
|
||||
image.drawText(font, PAD, y, truncateText(font, host.name, textWidth), 110);
|
||||
y += step;
|
||||
}
|
||||
for (const row of host.sessions) {
|
||||
const rows = host.sessions
|
||||
.slice()
|
||||
.sort((a, b) => (this.rowOrder.get(a.key) ?? 0) - (this.rowOrder.get(b.key) ?? 0));
|
||||
for (const row of rows) {
|
||||
if (!fits(y)) return;
|
||||
drawSessionRow(image, font, {
|
||||
x: GUTTER_X,
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
/** Native services still outside the current iOS app-screen milestone. */
|
||||
const unavailable: Record<string, string> = {
|
||||
music: 'Media-player integration is not available on iOS yet.',
|
||||
transcribe: 'Voice capture and transcription are not available on iOS yet.',
|
||||
microphones: 'Glasses microphone capture is not available on iOS yet.',
|
||||
}
|
||||
export function iosAppUnavailableReason(appId: string): string | null {
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// Utterance times come from the caption engine's sample clock, which only
|
||||
// advances with audio fed to it. Audio that never reaches the engine (beam
|
||||
// filtered, paused for the voice dialog, lost to a stall) would otherwise
|
||||
// shift every later line earlier, so the clock is re-anchored to wall time
|
||||
// (arrival time, so ordinary BLE jitter is tolerated) whenever the two drift
|
||||
// this far apart, in either direction: missing audio puts wall time ahead,
|
||||
// and a backlog delivered late in a burst puts it behind once the stream
|
||||
// catches up.
|
||||
export const CAPTION_CLOCK_DRIFT_MS = 300;
|
||||
const MAX_ANCHORS = 1000;
|
||||
|
||||
/** Maps a caption engine's clock (ms of audio fed to it) to wall time. */
|
||||
export class CaptionClock {
|
||||
private fedSamples = 0;
|
||||
// Engine-clock times with the wall time each corresponds to, in order.
|
||||
private readonly anchors: Array<{ engineMs: number; wallMs: number }> = [];
|
||||
|
||||
constructor(
|
||||
private readonly sampleRate: number,
|
||||
private readonly startWallMs: number,
|
||||
) {}
|
||||
|
||||
/** `bytes` of 16-bit mono PCM were fed to the engine, arriving at `nowMs`. */
|
||||
noteFed(bytes: number, nowMs: number): void {
|
||||
const samples = Math.floor(bytes / 2);
|
||||
const engineMs = (this.fedSamples * 1000) / this.sampleRate;
|
||||
this.fedSamples += samples;
|
||||
// The chunk arrived as it ended; its first sample is that much older.
|
||||
const wallMs = nowMs - (samples * 1000) / this.sampleRate;
|
||||
const last = this.anchors[this.anchors.length - 1];
|
||||
if (last && Math.abs(wallMs - (last.wallMs + engineMs - last.engineMs)) <= CAPTION_CLOCK_DRIFT_MS) return;
|
||||
this.anchors.push({ engineMs, wallMs });
|
||||
if (this.anchors.length > MAX_ANCHORS) this.anchors.shift();
|
||||
}
|
||||
|
||||
/** Wall time of a moment on the engine's clock. */
|
||||
wallMs(engineMs: number): number {
|
||||
for (let i = this.anchors.length - 1; i >= 0; i--) {
|
||||
const anchor = this.anchors[i]!;
|
||||
if (anchor.engineMs <= engineMs) return anchor.wallMs + engineMs - anchor.engineMs;
|
||||
}
|
||||
return this.startWallMs + engineMs;
|
||||
}
|
||||
}
|
||||
@@ -15,12 +15,15 @@ export class CaptionsLayer implements Layer {
|
||||
private state: MicSessionState = micSession.getState();
|
||||
private unsubscribe: (() => void) | null = null;
|
||||
private scrollback = 0;
|
||||
private lastLineCount = 0;
|
||||
private lastLine: CaptionLine | null = null;
|
||||
|
||||
start(requestRender: () => void): void {
|
||||
this.unsubscribe = micSession.onState((state) => {
|
||||
if (state.captionLines.length !== this.lastLineCount) {
|
||||
this.lastLineCount = state.captionLines.length;
|
||||
// Watch the newest line itself: once the session's line cap is reached
|
||||
// the count stops changing even though new speech keeps arriving.
|
||||
const last = state.captionLines[state.captionLines.length - 1] ?? null;
|
||||
if (last !== this.lastLine) {
|
||||
this.lastLine = last;
|
||||
this.scrollback = 0;
|
||||
}
|
||||
this.state = state;
|
||||
@@ -36,7 +39,7 @@ export class CaptionsLayer implements Layer {
|
||||
|
||||
image.drawText(font, 12, 4, "Captions", 220);
|
||||
if (!state.captionsActive) {
|
||||
image.drawText(font, 12, 26, "Captions are off (enable in the Microphones menu).", 130);
|
||||
image.drawText(font, 12, 26, state.captionsNote, 130);
|
||||
}
|
||||
|
||||
const rows = this.buildRows(font, width - 36, state.captionLines);
|
||||
|
||||
@@ -16,6 +16,10 @@ declare const com: any;
|
||||
|
||||
// Qwen runs with a 4096-token context; leave room for the answer.
|
||||
const MAX_CONTEXT_CHARS = 7000;
|
||||
// The speaker roster grows with every voice ever heard; cap it and the
|
||||
// conversation list so they can't crowd the transcript excerpts out.
|
||||
const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2;
|
||||
const EXCERPTS_HEADING = "# Transcript excerpts\n";
|
||||
const MAX_KEYWORDS = 6;
|
||||
const SEGMENTS_PER_KEYWORD = 30;
|
||||
const RECENT_SEGMENT_LIMIT = 120;
|
||||
@@ -156,31 +160,57 @@ function buildContext(question: string): { context: string; note: string } {
|
||||
}
|
||||
}
|
||||
|
||||
const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
|
||||
let header =
|
||||
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
|
||||
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
|
||||
`${sessionLines.join("\n") || "(none in range)"}\n\n`;
|
||||
let truncated = false;
|
||||
if (header.length > MAX_HEADER_CHARS) {
|
||||
header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`;
|
||||
truncated = true;
|
||||
}
|
||||
|
||||
const sessionHeading = (sessionId: number) =>
|
||||
`\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`;
|
||||
const segmentText = (segment: SegmentRow) => {
|
||||
const speaker =
|
||||
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
|
||||
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
|
||||
const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`;
|
||||
return segment.translation ? `${line}\n (translated: ${segment.translation})` : line;
|
||||
};
|
||||
|
||||
// Fill the remaining budget newest-first, so an overflowing history drops
|
||||
// its oldest excerpts rather than its most recent ones.
|
||||
const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
|
||||
let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length;
|
||||
const segments: SegmentRow[] = [];
|
||||
let runSession = -1;
|
||||
for (let index = sorted.length - 1; index >= 0; index--) {
|
||||
const segment = sorted[index]!;
|
||||
let cost = segmentText(segment).length + 1;
|
||||
if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1;
|
||||
if (cost > budget) {
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
budget -= cost;
|
||||
runSession = segment.sessionId;
|
||||
segments.push(segment);
|
||||
}
|
||||
segments.reverse();
|
||||
|
||||
const excerptLines: string[] = [];
|
||||
let currentSession = -1;
|
||||
for (const segment of segments) {
|
||||
if (segment.sessionId !== currentSession) {
|
||||
currentSession = segment.sessionId;
|
||||
excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`);
|
||||
excerptLines.push(sessionHeading(currentSession));
|
||||
}
|
||||
const speaker =
|
||||
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
|
||||
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
|
||||
excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`);
|
||||
if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`);
|
||||
excerptLines.push(segmentText(segment));
|
||||
}
|
||||
|
||||
let context =
|
||||
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
|
||||
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
|
||||
`${sessionLines.join("\n") || "(none in range)"}\n\n` +
|
||||
`# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`;
|
||||
let truncated = false;
|
||||
if (context.length > MAX_CONTEXT_CHARS) {
|
||||
context = context.slice(0, MAX_CONTEXT_CHARS);
|
||||
truncated = true;
|
||||
}
|
||||
const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`;
|
||||
const note =
|
||||
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
|
||||
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
|
||||
|
||||
@@ -286,6 +286,12 @@ export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: num
|
||||
export class TempleAudioPipeline {
|
||||
private readonly beamformer: DelaySumBeamformer;
|
||||
private readonly anc = new InterMicNoiseCanceller();
|
||||
// Steering delays reach back before the start of each frame. Carrying the
|
||||
// previous frame's last samples keeps those real; zero-filling them (as a
|
||||
// standalone beam() does) dips the output at every frame boundary.
|
||||
private readonly carry: number;
|
||||
private frontTail: Float32Array;
|
||||
private rearTail: Float32Array;
|
||||
enableANC = true;
|
||||
|
||||
constructor(
|
||||
@@ -294,19 +300,38 @@ export class TempleAudioPipeline {
|
||||
geometry: MicGeometry = G2_DEFAULT_GEOMETRY,
|
||||
) {
|
||||
this.beamformer = new DelaySumBeamformer(sampleRate, geometry);
|
||||
const { front, rear } = geometry.positions[side];
|
||||
const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z);
|
||||
this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1;
|
||||
this.frontTail = new Float32Array(this.carry);
|
||||
this.rearTail = new Float32Array(this.carry);
|
||||
}
|
||||
|
||||
/**
|
||||
* Process one frame's deinterleaved front/rear channels toward a
|
||||
* device-frame target, returning the enhanced mono beam for this temple.
|
||||
* Frames must be consecutive: each one continues the previous.
|
||||
*/
|
||||
process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array {
|
||||
const beam = this.beamformer.beam(this.side, front, rear, target);
|
||||
const n = Math.min(front.length, rear.length);
|
||||
const extendedFront = this.withTail(this.frontTail, front, n);
|
||||
const extendedRear = this.withTail(this.rearTail, rear, n);
|
||||
this.frontTail = extendedFront.slice(n);
|
||||
this.rearTail = extendedRear.slice(n);
|
||||
const beam = this.beamformer
|
||||
.beam(this.side, extendedFront, extendedRear, target)
|
||||
.slice(this.carry);
|
||||
if (!this.enableANC) return beam;
|
||||
// Null-steer reference: the difference channel carries off-axis noise.
|
||||
const n = Math.min(front.length, rear.length);
|
||||
const reference = new Float32Array(n);
|
||||
for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!;
|
||||
return this.anc.process(beam, reference);
|
||||
}
|
||||
|
||||
private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array {
|
||||
const out = new Float32Array(tail.length + n);
|
||||
out.set(tail);
|
||||
out.set(frame.subarray(0, n), tail.length);
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,9 @@ const microphonesApp: AppDefinition = {
|
||||
title: "Microphones",
|
||||
icon: "mic",
|
||||
boot: () => {
|
||||
// Prune expired captions/recordings per the retention settings.
|
||||
// Close out conversations and recordings a dead process left open, then
|
||||
// prune expired captions/recordings per the retention settings.
|
||||
micSession.recoverInterruptedConversations();
|
||||
void micSession.applyRetentionSweep();
|
||||
},
|
||||
launch: async (ctx) => {
|
||||
|
||||
@@ -110,6 +110,7 @@ class MicArrayController {
|
||||
* doesn't spend BLE bandwidth.
|
||||
*/
|
||||
apply(config: MicConfig): boolean {
|
||||
if (config.armHardware && !micControlSupported()) return false;
|
||||
const leftActive = templeActive(config, "left");
|
||||
const rightActive = templeActive(config, "right");
|
||||
if (leftActive || rightActive) {
|
||||
@@ -124,6 +125,17 @@ class MicArrayController {
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-send CONFIGURE to one temple whose stream went quiet (a lost write, or
|
||||
* a temple that restarted on its own), leaving the other temple's stream
|
||||
* untouched.
|
||||
*/
|
||||
reconfigureTemple(config: MicConfig, side: MicSide): boolean {
|
||||
if (!templeActive(config, side)) return false;
|
||||
if (config.armHardware && !micControlSupported()) return false;
|
||||
return this.send("configure", config, `configure-${side}`, side === "right", side === "left");
|
||||
}
|
||||
|
||||
query(): boolean {
|
||||
return this.send("query", undefined, "query");
|
||||
}
|
||||
@@ -148,11 +160,11 @@ class MicArrayController {
|
||||
): boolean {
|
||||
const communicator = activeCommunicator();
|
||||
if (!communicator) return false;
|
||||
this.ensureStatusProxy(communicator);
|
||||
try {
|
||||
this.ensureStatusProxy(communicator);
|
||||
const record = encodeMicControl(op, config);
|
||||
communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple);
|
||||
return true;
|
||||
// False when the session wasn't ready and nothing was queued.
|
||||
return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false;
|
||||
} catch (error) {
|
||||
console.warn(`mic control ${label} failed: ${error}`);
|
||||
return false;
|
||||
|
||||
@@ -75,6 +75,9 @@ export type MicModelState = {
|
||||
};
|
||||
|
||||
const downloaders = new Map<MicModelId, any>();
|
||||
// Each download's listener is referenced only from Java while it runs; hold
|
||||
// it here so it isn't GC'd before its callbacks.
|
||||
const downloadListeners = new Map<MicModelId, any>();
|
||||
const progress = new Map<MicModelId, number>();
|
||||
const stateListeners = new Set<(id: MicModelId, state: MicModelState) => void>();
|
||||
|
||||
@@ -142,6 +145,7 @@ function notifyStateChanged(id: MicModelId): void {
|
||||
|
||||
export function startMicModelDownload(id: MicModelId): void {
|
||||
if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return;
|
||||
progress.delete(id);
|
||||
downloadNextFile(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
@@ -151,6 +155,7 @@ function downloadNextFile(id: MicModelId): void {
|
||||
const nextFile = definition.files.find((file) => !isFilePresent(id, file));
|
||||
if (!nextFile) {
|
||||
downloaders.delete(id);
|
||||
downloadListeners.delete(id);
|
||||
notifyStateChanged(id);
|
||||
return;
|
||||
}
|
||||
@@ -168,18 +173,28 @@ function downloadNextFile(id: MicModelId): void {
|
||||
onError: (message: string) => {
|
||||
console.error(`Mic model download failed (${id}/${nextFile.name}): ${message}`);
|
||||
downloaders.delete(id);
|
||||
downloadListeners.delete(id);
|
||||
notifyStateChanged(id);
|
||||
},
|
||||
});
|
||||
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${definition.baseUrl}${nextFile.remoteName}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloaders.set(id, downloader);
|
||||
downloader.start();
|
||||
downloadListeners.set(id, listener);
|
||||
try {
|
||||
const downloader = new com.faceclaw.app.FaceclawModelDownloader(
|
||||
`${definition.baseUrl}${nextFile.remoteName}`,
|
||||
filePath(id, nextFile),
|
||||
nextFile.sha256,
|
||||
nextFile.sizeBytes,
|
||||
listener,
|
||||
);
|
||||
downloaders.set(id, downloader);
|
||||
downloader.start();
|
||||
} catch (error) {
|
||||
// Without this, a dead entry would pin the model at "downloading".
|
||||
console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`);
|
||||
downloaders.delete(id);
|
||||
downloadListeners.delete(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
}
|
||||
|
||||
export function cancelMicModelDownload(id: MicModelId): void {
|
||||
@@ -187,6 +202,7 @@ export function cancelMicModelDownload(id: MicModelId): void {
|
||||
if (!downloader) return;
|
||||
downloader.cancel();
|
||||
downloaders.delete(id);
|
||||
downloadListeners.delete(id);
|
||||
notifyStateChanged(id);
|
||||
}
|
||||
|
||||
|
||||
+844
-236
File diff suppressed because it is too large.
Load diff
@@ -22,7 +22,7 @@ import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../nat
|
||||
import { CaptionsLayer } from "./captions-layer";
|
||||
import { LevelsLayer } from "./levels-layer";
|
||||
import { RadarLayer } from "./radar-layer";
|
||||
import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control";
|
||||
import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control";
|
||||
import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models";
|
||||
import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol";
|
||||
import { micSession } from "./mic-session";
|
||||
@@ -72,13 +72,9 @@ const MENU_LAYOUT = {
|
||||
opaque: true,
|
||||
};
|
||||
|
||||
/** Persist a config change and re-apply it live when the array is streaming. */
|
||||
/** Persist a config change; the running session applies it live. */
|
||||
function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void {
|
||||
const updated = mutate(loadMicConfig());
|
||||
saveMicConfig(updated);
|
||||
if (micSession.getState().mode === "extended") {
|
||||
micArrayController.apply(updated);
|
||||
}
|
||||
micSession.updateConfig(mutate);
|
||||
}
|
||||
|
||||
function micChannelToggleItem(
|
||||
@@ -457,8 +453,9 @@ export function createMicrophonesAppWindow(options: InProcessAppOptions): InProc
|
||||
}),
|
||||
configToggleItem(
|
||||
"Captions",
|
||||
() => captionsEnabledSetting.get(),
|
||||
() => micSession.setCaptionsEnabled(!captionsEnabledSetting.get()),
|
||||
// Shows off after repeated engine failures, so turning it on retries.
|
||||
() => micSession.captionsEnabled(),
|
||||
() => micSession.setCaptionsEnabled(!micSession.captionsEnabled()),
|
||||
undefined,
|
||||
captionsEnabledSetting.description,
|
||||
),
|
||||
|
||||
@@ -55,7 +55,9 @@ const SELF_PATTERNS = [
|
||||
new RegExp(`\\b[Mm]y name is ${NAME}`),
|
||||
new RegExp(`\\b[Ii](?:'| a)m ${NAME}`),
|
||||
new RegExp(`\\b[Cc]all me ${NAME}`),
|
||||
new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`),
|
||||
// Phone-style: "this is Dana calling about..." is the speaker, not a
|
||||
// third party being introduced.
|
||||
new RegExp(`\\b[Tt]his is ${NAME} (?:[Ss]peaking|[Cc]alling)\\b`),
|
||||
];
|
||||
|
||||
const THIRD_PARTY_PATTERNS = [
|
||||
|
||||
@@ -35,6 +35,10 @@ function store(): any {
|
||||
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
|
||||
}
|
||||
|
||||
// Each run's listener is referenced only from Java while the diarizer works;
|
||||
// hold it here so it isn't GC'd before its callbacks.
|
||||
const activeListeners = new Set<any>();
|
||||
|
||||
export function rediarizeSession(
|
||||
sessionId: number,
|
||||
options: { numSpeakers?: number; threshold?: number },
|
||||
@@ -54,6 +58,7 @@ export function rediarizeSession(
|
||||
const listener = new com.faceclaw.app.FaceclawDiarizer.Listener({
|
||||
onProgress: (progress: number) => callbacks.onProgress(Number(progress)),
|
||||
onDone: (turnsJson: string) => {
|
||||
activeListeners.delete(listener);
|
||||
try {
|
||||
const turns = JSON.parse(String(turnsJson)) as Turn[];
|
||||
callbacks.onDone(applyTurns(sessionId, turns));
|
||||
@@ -61,17 +66,26 @@ export function rediarizeSession(
|
||||
callbacks.onError(`Applying diarization failed: ${String(error)}`);
|
||||
}
|
||||
},
|
||||
onError: (message: string) => callbacks.onError(String(message)),
|
||||
onError: (message: string) => {
|
||||
activeListeners.delete(listener);
|
||||
callbacks.onError(String(message));
|
||||
},
|
||||
});
|
||||
const diarizer = new com.faceclaw.app.FaceclawDiarizer(
|
||||
segmentationModel,
|
||||
embeddingModel,
|
||||
audioPath,
|
||||
options.numSpeakers ?? -1,
|
||||
options.threshold ?? 0.5,
|
||||
listener,
|
||||
);
|
||||
diarizer.start();
|
||||
activeListeners.add(listener);
|
||||
try {
|
||||
const diarizer = new com.faceclaw.app.FaceclawDiarizer(
|
||||
segmentationModel,
|
||||
embeddingModel,
|
||||
audioPath,
|
||||
options.numSpeakers ?? -1,
|
||||
options.threshold ?? 0.5,
|
||||
listener,
|
||||
);
|
||||
diarizer.start();
|
||||
} catch (error) {
|
||||
activeListeners.delete(listener);
|
||||
callbacks.onError(`Re-diarization failed to start: ${String(error)}`);
|
||||
}
|
||||
}
|
||||
|
||||
type StoredSegment = {
|
||||
@@ -140,6 +154,8 @@ function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult {
|
||||
} else {
|
||||
const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid)));
|
||||
speakerRegistry.reload();
|
||||
// A failed insert returns -1; leave that cluster's segments as they were.
|
||||
if (id < 0) continue;
|
||||
clusterSpeaker.set(cluster, id);
|
||||
newSpeakers += 1;
|
||||
}
|
||||
|
||||
@@ -139,8 +139,9 @@ export class SpeakerRegistry {
|
||||
}));
|
||||
this.loaded = true;
|
||||
} catch (error) {
|
||||
// Keep the last good cache: emptying it would make the next utterance
|
||||
// enroll an already-known voice as a duplicate "Speaker N".
|
||||
console.error(`speaker reload failed: ${String(error)}`);
|
||||
this.profiles = [];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -194,7 +195,8 @@ export class SpeakerRegistry {
|
||||
const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding)));
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
const profile = this.byId(id)!;
|
||||
const profile = this.byId(id);
|
||||
if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`);
|
||||
return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 };
|
||||
}
|
||||
|
||||
@@ -213,12 +215,14 @@ export class SpeakerRegistry {
|
||||
store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP);
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
return this.byId(existing.id)!;
|
||||
return this.byId(existing.id) ?? existing;
|
||||
}
|
||||
const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding)));
|
||||
this.reload();
|
||||
this.notifyChanged();
|
||||
return this.byId(id)!;
|
||||
const profile = this.byId(id);
|
||||
if (!profile) throw new Error(`wearer profile was not created (id ${id})`);
|
||||
return profile;
|
||||
}
|
||||
|
||||
/** Is this utterance the wearer's own voice? Used to gate command recognition. */
|
||||
|
||||
@@ -18,7 +18,10 @@ const downloadedPairs = new Set<string>();
|
||||
export function deviceLanguage(): string {
|
||||
if (!global.isAndroid) return "en";
|
||||
try {
|
||||
return String(java.util.Locale.getDefault().getLanguage() || "en");
|
||||
// toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit
|
||||
// uses; getLanguage() returns the legacy "iw", "in", "ji".
|
||||
const tag = String(java.util.Locale.getDefault().toLanguageTag() || "");
|
||||
return tag.split("-")[0] || "en";
|
||||
} catch {
|
||||
return "en";
|
||||
}
|
||||
@@ -95,11 +98,18 @@ export function translateText(text: string, sourceLang: string, targetLang: stri
|
||||
if (!translator) return Promise.resolve("");
|
||||
const key = `${sourceLang}>${targetLang}`;
|
||||
return new Promise((resolve) => {
|
||||
// Also runs from the download listener, where a throw would escape into
|
||||
// the Play Services callback and leave this promise unsettled.
|
||||
const runTranslate = () => {
|
||||
translator
|
||||
.translate(text)
|
||||
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
|
||||
.addOnFailureListener(failureListener(() => resolve("")));
|
||||
try {
|
||||
translator
|
||||
.translate(text)
|
||||
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
|
||||
.addOnFailureListener(failureListener(() => resolve("")));
|
||||
} catch (error) {
|
||||
console.warn(`translation failed (${key}): ${error}`);
|
||||
resolve("");
|
||||
}
|
||||
};
|
||||
if (downloadedPairs.has(key)) {
|
||||
runTranslate();
|
||||
|
||||
@@ -250,8 +250,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => {
|
||||
syncClock(window);
|
||||
}
|
||||
break;
|
||||
case "resize-window": {
|
||||
const window = windows.get(message.windowId);
|
||||
if (!window) break;
|
||||
window.viewportWidth = message.viewport.width;
|
||||
window.viewportHeight = message.viewport.height;
|
||||
window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight });
|
||||
window.lastSubmittedFingerprint = "";
|
||||
if (window.foreground) renderAndSubmit(window, 0);
|
||||
break;
|
||||
}
|
||||
case "navigation-sensors":
|
||||
case "resize-window":
|
||||
case "text-input":
|
||||
case "tool-call":
|
||||
break;
|
||||
|
||||
+26
-25
@@ -46,10 +46,11 @@ type FocusColumn = "actions" | "playlist";
|
||||
|
||||
/**
|
||||
* Music controller app: metadata + album art + transport controls for the
|
||||
* active Android media session (any player that publishes one), plus the
|
||||
* player's queue when it exposes one (scroll to a track, click to jump).
|
||||
* With no session, the front page lists the apps whose library can be
|
||||
* browsed (click one to open its MediaBrowseLayer).
|
||||
* active Android media session (any player that publishes one) or, on iOS,
|
||||
* the Apple Music system player, plus the player's queue when it exposes one
|
||||
* (scroll to a track, click to jump). With no session, the front page lists
|
||||
* the apps whose library can be browsed (click one to open its
|
||||
* MediaBrowseLayer).
|
||||
*/
|
||||
class MusicAppLayer implements Layer {
|
||||
// Watch swipes skip tracks: right = next, left = previous.
|
||||
@@ -83,7 +84,8 @@ class MusicAppLayer implements Layer {
|
||||
*/
|
||||
private libraryAppsFresh = false;
|
||||
private art: GrayImage | null = null;
|
||||
private artKey = "";
|
||||
/** The artVersion `art` was fetched for; -1 = not fetched yet. */
|
||||
private artVersion = -1;
|
||||
|
||||
isPlaylistFocused(): boolean {
|
||||
return this.focusColumn === "playlist";
|
||||
@@ -96,15 +98,11 @@ class MusicAppLayer implements Layer {
|
||||
const media = mediaControllerBridge.snapshot();
|
||||
|
||||
if (!media.accessEnabled) {
|
||||
const lines = wrapText(
|
||||
font,
|
||||
"Notification access is required to control Android media sessions. Tap to open settings.",
|
||||
width - 48,
|
||||
);
|
||||
const lines = wrapText(font, mediaControllerBridge.accessPrompt, width - 48);
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
image.drawText(font, 24, 16 + index * lineStep(font), lines[index]!, 180);
|
||||
}
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings`, 110);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} ${mediaControllerBridge.accessAction}`, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -112,10 +110,12 @@ class MusicAppLayer implements Layer {
|
||||
const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh);
|
||||
this.libraryAppsFresh = true;
|
||||
this.libraryMenu.setItems(apps);
|
||||
image.drawText(font, 24, 16, "No active media session.", 180);
|
||||
const prompt = apps.length
|
||||
? "Browse a music app's library, or start playback on the phone."
|
||||
: "Start playback in another app on the phone.";
|
||||
image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180);
|
||||
const prompt = global.isIOS
|
||||
? "Browse your library, or start playback on the phone."
|
||||
: apps.length
|
||||
? "Browse a music app's library, or start playback on the phone."
|
||||
: "Start playback in another app on the phone.";
|
||||
let y = 22 + lineStep(font);
|
||||
for (const line of wrapText(font, prompt, width - 48)) {
|
||||
image.drawText(font, 24, y, line, 150);
|
||||
@@ -228,7 +228,7 @@ class MusicAppLayer implements Layer {
|
||||
}
|
||||
if (!media.accessEnabled) {
|
||||
if (event.type === "click") {
|
||||
mediaControllerBridge.openNotificationAccessSettings();
|
||||
void mediaControllerBridge.requestAccess();
|
||||
} else if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -267,8 +267,8 @@ class MusicAppLayer implements Layer {
|
||||
const item = this.queueMenu.selectedItem;
|
||||
if (item) {
|
||||
await mediaControllerBridge.skipToQueueItem(item.id);
|
||||
// The player rebuilds the queue with the chosen track at index 0.
|
||||
this.queueMenu.select(0);
|
||||
// Android players rebuild the queue with the chosen track at index 0.
|
||||
if (mediaControllerBridge.skipRebuildsQueue) this.queueMenu.select(0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -371,12 +371,12 @@ class MusicAppLayer implements Layer {
|
||||
}
|
||||
|
||||
private drawArt(image: GrayImage, media: MediaControllerState): void {
|
||||
const key = `${media.packageName}|${media.title}|${media.album}`;
|
||||
// Retry while null: players (e.g. Spotify) often publish metadata before
|
||||
// the art bitmap is downloaded, then re-emit the same strings with the
|
||||
// bitmap attached. The no-art fetch path is a cheap near-no-op.
|
||||
if (key !== this.artKey || this.art === null) {
|
||||
this.artKey = key;
|
||||
// Keyed on artVersion, not the strings: players (e.g. Spotify) publish
|
||||
// metadata before the art bitmap is downloaded, or with the previous
|
||||
// track's bitmap after a quick double skip, then re-emit the same
|
||||
// strings with the right bitmap attached.
|
||||
if (media.artVersion !== this.artVersion) {
|
||||
this.artVersion = media.artVersion;
|
||||
this.art = mediaControllerBridge.getAlbumArt(ART_SIZE);
|
||||
}
|
||||
if (this.art) {
|
||||
@@ -496,7 +496,8 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin
|
||||
iconLetter: "M",
|
||||
icon: "music",
|
||||
closeable: true,
|
||||
menuItems: () => [{
|
||||
// Music settings choose among Android media apps; iOS has only Apple Music.
|
||||
menuItems: () => global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
|
||||
@@ -33,8 +33,7 @@ import {
|
||||
navigateDestinationAddressDraftSetting,
|
||||
navigateDestinationNameDraftSetting,
|
||||
navigateRememberRecentSetting,
|
||||
navigateDisplayModeSetting,
|
||||
navigateVerticalPositionSetting,
|
||||
navigateDisplayLayoutSetting,
|
||||
mapboxApiKeySetting,
|
||||
enumSettingMenuItem,
|
||||
type ConfigSettingString,
|
||||
@@ -935,10 +934,7 @@ function windowMenuItems(): MenuItem[] {
|
||||
},
|
||||
});
|
||||
}
|
||||
items.push(
|
||||
enumSettingMenuItem(navigateDisplayModeSetting),
|
||||
enumSettingMenuItem(navigateVerticalPositionSetting),
|
||||
);
|
||||
items.push(enumSettingMenuItem(navigateDisplayLayoutSetting));
|
||||
items.push(submenuItem("Saved destinations", (ctx) => {
|
||||
ctx.stack.push(new WindowMenuLayer("Saved destinations", savedDestinationMenuItems()));
|
||||
}));
|
||||
|
||||
@@ -1,11 +1,18 @@
|
||||
import { type AppDefinition } from "../app-definition";
|
||||
import { inProcessTrayWidget, type AppDefinition } from "../app-definition";
|
||||
import { createNightscoutAppWindow, startNightscoutTrayIcon, NIGHTSCOUT_SURFACE_ID, NIGHTSCOUT_WINDOW_ID } from "./nightscout-app";
|
||||
import { startMissedBolusMonitor } from "./missed-bolus-monitor";
|
||||
|
||||
const nightscoutApp: AppDefinition = {
|
||||
appId: "nightscout",
|
||||
title: "Nightscout",
|
||||
icon: "nightscout",
|
||||
boot: () => startNightscoutTrayIcon(),
|
||||
boot: (ctx) => {
|
||||
// Selected in the switcher, the tray readout shows the app's window.
|
||||
startNightscoutTrayIcon(
|
||||
inProcessTrayWidget(ctx, NIGHTSCOUT_WINDOW_ID, NIGHTSCOUT_SURFACE_ID, createNightscoutAppWindow),
|
||||
);
|
||||
startMissedBolusMonitor(ctx);
|
||||
},
|
||||
launch: (ctx) => ctx.launchInProcessApp(NIGHTSCOUT_WINDOW_ID, NIGHTSCOUT_SURFACE_ID, createNightscoutAppWindow),
|
||||
};
|
||||
|
||||
|
||||
@@ -0,0 +1,336 @@
|
||||
/**
|
||||
* Missed-bolus detector: given a recent window of CGM readings and
|
||||
* treatments, decide whether glucose is rising the way an unannounced meal
|
||||
* does, with no meal bolus, manual bolus or carb entry to account for it.
|
||||
*
|
||||
* Deliberately self-contained (no imports; plain data in, plain data out) so
|
||||
* it can move to the server side of Nightscout unchanged, and so it can be
|
||||
* iterated on against recorded history with
|
||||
* scripts/nightscout-missed-bolus-replay.cjs. The Faceclaw side (when to run
|
||||
* it, cooldown, the glasses notification) is in missed-bolus-monitor.ts.
|
||||
*
|
||||
* Glucose is mg/dL throughout: Nightscout stores sgv in mg/dL whatever the
|
||||
* site's display units.
|
||||
*/
|
||||
|
||||
export type GlucoseReading = { timeMs: number; mgdl: number };
|
||||
|
||||
/**
|
||||
* - meal: a meal/snack/wizard bolus, or any bolus entered together with carbs.
|
||||
* - manual: a bolus explicitly marked user-initiated (e.g. Loop's `automatic: false`).
|
||||
* - automatic: a bolus the loop issued (SMB, automatic bolus).
|
||||
* - ambiguous: not marked either way. OpenAPS on a Medtronic pump uploads its
|
||||
* SMBs as plain "Correction Bolus" records, indistinguishable from a manual
|
||||
* correction except by size; `ambiguousBolusMinUnits` decides.
|
||||
*/
|
||||
export type BolusKind = "meal" | "manual" | "automatic" | "ambiguous";
|
||||
export type BolusEvent = { timeMs: number; units: number; kind: BolusKind };
|
||||
export type CarbEntry = { timeMs: number; grams: number };
|
||||
|
||||
export type MissedBolusInput = {
|
||||
/** The time to classify as of. Readings and treatments after it are ignored. */
|
||||
nowMs: number;
|
||||
glucose: readonly GlucoseReading[];
|
||||
boluses: readonly BolusEvent[];
|
||||
carbs: readonly CarbEntry[];
|
||||
};
|
||||
|
||||
export type MissedBolusOptions = {
|
||||
/** The latest reading must be at most this old, or there is nothing current to judge. */
|
||||
maxReadingAgeMs: number;
|
||||
/** How far back from the latest reading to look for where the rise started. */
|
||||
lookbackMs: number;
|
||||
/** A gap between readings longer than this ends the window (the CGM dropped out). */
|
||||
maxGapMs: number;
|
||||
/** Recent span the rate of rise is fitted over (least squares). */
|
||||
slopeWindowMs: number;
|
||||
/** Fewest readings the rate fit needs. */
|
||||
minSlopeReadings: number;
|
||||
/** Rise from the window's lowest reading to the latest. */
|
||||
minRiseMgdl: number;
|
||||
/** The latest reading must be at least this high. */
|
||||
minGlucoseMgdl: number;
|
||||
/** Rises from below this are taken as recovering from a low (rescue carbs go unbolused). */
|
||||
hypoNadirMgdl: number;
|
||||
/** The rise must have lasted at least this long, to rule out sensor noise. */
|
||||
minRiseDurationMs: number;
|
||||
/** Rate of rise over the slope window, mg/dL per minute. */
|
||||
minRateMgdlPerMin: number;
|
||||
/** A bolus or carb entry this long before the rise began still accounts for it (pre-bolusing). */
|
||||
preBolusWindowMs: number;
|
||||
/** Smallest meal or manual bolus that accounts for a rise. */
|
||||
minManualBolusUnits: number;
|
||||
/** Smallest unmarked bolus taken as manual rather than an SMB. */
|
||||
ambiguousBolusMinUnits: number;
|
||||
/** Whether a carb entry with no bolus accounts for a rise (the user evidently knows they ate). */
|
||||
carbsCountAsAnnouncement: boolean;
|
||||
};
|
||||
|
||||
export const DEFAULT_MISSED_BOLUS_OPTIONS: Readonly<MissedBolusOptions> = {
|
||||
maxReadingAgeMs: 12 * 60_000,
|
||||
lookbackMs: 90 * 60_000,
|
||||
maxGapMs: 16 * 60_000,
|
||||
slopeWindowMs: 20 * 60_000,
|
||||
minSlopeReadings: 3,
|
||||
minRiseMgdl: 40,
|
||||
minGlucoseMgdl: 140,
|
||||
hypoNadirMgdl: 70,
|
||||
minRiseDurationMs: 20 * 60_000,
|
||||
minRateMgdlPerMin: 1.5,
|
||||
preBolusWindowMs: 60 * 60_000,
|
||||
minManualBolusUnits: 0.5,
|
||||
ambiguousBolusMinUnits: 1.5,
|
||||
carbsCountAsAnnouncement: true,
|
||||
};
|
||||
|
||||
export type GlucoseRise = {
|
||||
/** When the rise started: the lowest reading in the window (the latest, if tied). */
|
||||
onsetMs: number;
|
||||
nadirMgdl: number;
|
||||
latestMs: number;
|
||||
latestMgdl: number;
|
||||
riseMgdl: number;
|
||||
/** Least-squares rate over the slope window, mg/dL per minute. */
|
||||
rateMgdlPerMin: number;
|
||||
};
|
||||
|
||||
export type MissedBolusClassification = "missed-bolus" | "no-alert" | "insufficient-data";
|
||||
|
||||
export type MissedBolusResult = {
|
||||
classification: MissedBolusClassification;
|
||||
/** One-line explanation, for logs and the replay tool. */
|
||||
reason: string;
|
||||
/** The rise that was judged, when there was enough data to measure one. */
|
||||
rise: GlucoseRise | null;
|
||||
};
|
||||
|
||||
export function detectMissedBolus(
|
||||
input: MissedBolusInput,
|
||||
overrides: Partial<MissedBolusOptions> = {},
|
||||
): MissedBolusResult {
|
||||
const options: MissedBolusOptions = { ...DEFAULT_MISSED_BOLUS_OPTIONS, ...overrides };
|
||||
const { nowMs } = input;
|
||||
const readings = usableReadings(input.glucose, nowMs);
|
||||
const latest = readings[readings.length - 1];
|
||||
if (!latest) return insufficient("no glucose readings");
|
||||
if (nowMs - latest.timeMs > options.maxReadingAgeMs) {
|
||||
return insufficient(`latest reading is ${formatMinutes(nowMs - latest.timeMs)} old`);
|
||||
}
|
||||
|
||||
// The contiguous run of readings ending at the latest, within the lookback.
|
||||
const windowStartMs = latest.timeMs - options.lookbackMs;
|
||||
let first = readings.length - 1;
|
||||
while (first > 0) {
|
||||
const previous = readings[first - 1]!;
|
||||
if (previous.timeMs < windowStartMs || readings[first]!.timeMs - previous.timeMs > options.maxGapMs) break;
|
||||
first--;
|
||||
}
|
||||
const window = readings.slice(first);
|
||||
let nadir = window[0]!;
|
||||
for (const reading of window) {
|
||||
if (reading.mgdl <= nadir.mgdl) nadir = reading;
|
||||
}
|
||||
|
||||
const slopeStartMs = Math.max(nadir.timeMs, latest.timeMs - options.slopeWindowMs);
|
||||
const slopeReadings = window.filter((reading) => reading.timeMs >= slopeStartMs);
|
||||
const rise: GlucoseRise = {
|
||||
onsetMs: nadir.timeMs,
|
||||
nadirMgdl: nadir.mgdl,
|
||||
latestMs: latest.timeMs,
|
||||
latestMgdl: latest.mgdl,
|
||||
riseMgdl: latest.mgdl - nadir.mgdl,
|
||||
rateMgdlPerMin: slopeReadings.length >= 2 ? slopePerMinute(slopeReadings) : 0,
|
||||
};
|
||||
const noAlert = (reason: string): MissedBolusResult => ({ classification: "no-alert", reason, rise });
|
||||
|
||||
if (rise.riseMgdl < options.minRiseMgdl) {
|
||||
return noAlert(`rise of ${rise.riseMgdl} mg/dL is under ${options.minRiseMgdl}`);
|
||||
}
|
||||
if (latest.mgdl < options.minGlucoseMgdl) {
|
||||
return noAlert(`glucose ${latest.mgdl} mg/dL is under ${options.minGlucoseMgdl}`);
|
||||
}
|
||||
if (nadir.mgdl < options.hypoNadirMgdl) {
|
||||
return noAlert(`rising from a low of ${nadir.mgdl} mg/dL`);
|
||||
}
|
||||
if (latest.timeMs - nadir.timeMs < options.minRiseDurationMs) {
|
||||
return noAlert(`rise has lasted only ${formatMinutes(latest.timeMs - nadir.timeMs)}`);
|
||||
}
|
||||
if (slopeReadings.length < options.minSlopeReadings) {
|
||||
return { classification: "insufficient-data", reason: `only ${slopeReadings.length} readings to fit a rate`, rise };
|
||||
}
|
||||
if (rise.rateMgdlPerMin < options.minRateMgdlPerMin) {
|
||||
return noAlert(`rising ${formatRate(rise.rateMgdlPerMin)}, under ${formatRate(options.minRateMgdlPerMin)}`);
|
||||
}
|
||||
|
||||
const coverStartMs = nadir.timeMs - options.preBolusWindowMs;
|
||||
const inCoverWindow = (timeMs: number) => timeMs >= coverStartMs && timeMs <= nowMs;
|
||||
const bolus = latestMatching(input.boluses, (event) => inCoverWindow(event.timeMs) && accountsForRise(event, options));
|
||||
if (bolus) {
|
||||
return noAlert(`${formatUnits(bolus.units)} U ${bolus.kind} bolus ${formatMinutes(nowMs - bolus.timeMs)} ago`);
|
||||
}
|
||||
if (options.carbsCountAsAnnouncement) {
|
||||
const carbs = latestMatching(input.carbs, (entry) => inCoverWindow(entry.timeMs) && entry.grams > 0);
|
||||
if (carbs) return noAlert(`${carbs.grams} g carbs entered ${formatMinutes(nowMs - carbs.timeMs)} ago`);
|
||||
}
|
||||
return {
|
||||
classification: "missed-bolus",
|
||||
reason: `up ${rise.riseMgdl} mg/dL in ${formatMinutes(latest.timeMs - nadir.timeMs)}, rising ${formatRate(rise.rateMgdlPerMin)}, no bolus`,
|
||||
rise,
|
||||
};
|
||||
}
|
||||
|
||||
function insufficient(reason: string): MissedBolusResult {
|
||||
return { classification: "insufficient-data", reason, rise: null };
|
||||
}
|
||||
|
||||
function accountsForRise(bolus: BolusEvent, options: MissedBolusOptions): boolean {
|
||||
switch (bolus.kind) {
|
||||
case "meal":
|
||||
case "manual":
|
||||
return bolus.units >= options.minManualBolusUnits;
|
||||
case "ambiguous":
|
||||
return bolus.units >= Math.max(options.minManualBolusUnits, options.ambiguousBolusMinUnits);
|
||||
case "automatic":
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Finite, in-range readings at or before now, oldest first, one per
|
||||
* timestamp. Dexcom reports 39 for "LOW" and smaller values as error codes.
|
||||
*/
|
||||
function usableReadings(glucose: readonly GlucoseReading[], nowMs: number): GlucoseReading[] {
|
||||
const sorted = glucose
|
||||
.filter((reading) => Number.isFinite(reading.timeMs) && Number.isFinite(reading.mgdl)
|
||||
&& reading.mgdl >= 39 && reading.mgdl <= 600 && reading.timeMs <= nowMs)
|
||||
.slice()
|
||||
.sort((a, b) => a.timeMs - b.timeMs);
|
||||
return sorted.filter((reading, index) => index === 0 || reading.timeMs !== sorted[index - 1]!.timeMs);
|
||||
}
|
||||
|
||||
/** Least-squares slope through the readings, in mg/dL per minute. */
|
||||
function slopePerMinute(readings: readonly GlucoseReading[]): number {
|
||||
const t0 = readings[0]!.timeMs;
|
||||
const n = readings.length;
|
||||
let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0;
|
||||
for (const reading of readings) {
|
||||
const x = (reading.timeMs - t0) / 60_000;
|
||||
sumX += x;
|
||||
sumY += reading.mgdl;
|
||||
sumXY += x * reading.mgdl;
|
||||
sumXX += x * x;
|
||||
}
|
||||
const denominator = n * sumXX - sumX * sumX;
|
||||
return denominator > 0 ? (n * sumXY - sumX * sumY) / denominator : 0;
|
||||
}
|
||||
|
||||
function latestMatching<T extends { timeMs: number }>(events: readonly T[], predicate: (event: T) => boolean): T | null {
|
||||
let found: T | null = null;
|
||||
for (const event of events) {
|
||||
if (predicate(event) && (!found || event.timeMs > found.timeMs)) found = event;
|
||||
}
|
||||
return found;
|
||||
}
|
||||
|
||||
function formatMinutes(ms: number): string {
|
||||
return `${Math.round(ms / 60_000)} min`;
|
||||
}
|
||||
|
||||
function formatRate(mgdlPerMin: number): string {
|
||||
return `${mgdlPerMin.toFixed(1)} mg/dL/min`;
|
||||
}
|
||||
|
||||
function formatUnits(units: number): string {
|
||||
return `${Math.round(units * 100) / 100}`;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Nightscout record normalization, shared by the Faceclaw bridge and a
|
||||
// server-side port.
|
||||
|
||||
/** The fields of a Nightscout treatment record the detector reads. */
|
||||
export type NightscoutTreatmentRecord = {
|
||||
eventType?: unknown;
|
||||
created_at?: unknown;
|
||||
timestamp?: unknown;
|
||||
insulin?: unknown;
|
||||
carbs?: unknown;
|
||||
automatic?: unknown;
|
||||
isSMB?: unknown;
|
||||
type?: unknown;
|
||||
wizard?: unknown;
|
||||
};
|
||||
|
||||
/** The fields of a Nightscout entry record the detector reads. */
|
||||
export type NightscoutEntryRecord = { date?: unknown; sgv?: unknown };
|
||||
|
||||
const MEAL_BOLUS_EVENT_TYPES = new Set(["Meal Bolus", "Snack Bolus", "Bolus Wizard", "Combo Bolus"]);
|
||||
|
||||
/**
|
||||
* How a treatment's insulin was given. Uploaders mark automatic boluses
|
||||
* differently: Loop sets `automatic`, AndroidAPS `isSMB`, Trio/OpenAPS
|
||||
* `type: "SMB"` or eventType "SMB". Meal boluses carry carbs or a wizard
|
||||
* record, or a meal-type eventType.
|
||||
*/
|
||||
export function bolusKindFromTreatment(treatment: NightscoutTreatmentRecord): BolusKind {
|
||||
const eventType = typeof treatment.eventType === "string" ? treatment.eventType : "";
|
||||
if (treatment.automatic === true || treatment.isSMB === true || treatment.type === "SMB" || eventType === "SMB") {
|
||||
return "automatic";
|
||||
}
|
||||
if (MEAL_BOLUS_EVENT_TYPES.has(eventType) || positiveNumber(treatment.carbs) !== null
|
||||
|| (typeof treatment.wizard === "object" && treatment.wizard !== null)) {
|
||||
return "meal";
|
||||
}
|
||||
if (treatment.automatic === false || treatment.isSMB === false) return "manual";
|
||||
return "ambiguous";
|
||||
}
|
||||
|
||||
/** Boluses and carb entries from raw Nightscout treatment records, oldest first. */
|
||||
export function normalizeNightscoutTreatments(treatments: readonly NightscoutTreatmentRecord[]): {
|
||||
boluses: BolusEvent[];
|
||||
carbs: CarbEntry[];
|
||||
} {
|
||||
const boluses: BolusEvent[] = [];
|
||||
const carbs: CarbEntry[] = [];
|
||||
for (const treatment of treatments) {
|
||||
const timeMs = treatmentTimeMs(treatment);
|
||||
if (timeMs === null) continue;
|
||||
const units = positiveNumber(treatment.insulin);
|
||||
if (units !== null) boluses.push({ timeMs, units, kind: bolusKindFromTreatment(treatment) });
|
||||
const grams = positiveNumber(treatment.carbs);
|
||||
if (grams !== null) carbs.push({ timeMs, grams });
|
||||
}
|
||||
boluses.sort((a, b) => a.timeMs - b.timeMs);
|
||||
carbs.sort((a, b) => a.timeMs - b.timeMs);
|
||||
return { boluses, carbs };
|
||||
}
|
||||
|
||||
/** Glucose readings from raw Nightscout sgv entries, oldest first. */
|
||||
export function normalizeNightscoutEntries(entries: readonly NightscoutEntryRecord[]): GlucoseReading[] {
|
||||
const readings: GlucoseReading[] = [];
|
||||
for (const entry of entries) {
|
||||
const timeMs = Number(entry.date);
|
||||
const mgdl = Number(entry.sgv);
|
||||
if (Number.isFinite(timeMs) && timeMs > 0 && Number.isFinite(mgdl)) readings.push({ timeMs, mgdl });
|
||||
}
|
||||
return readings.sort((a, b) => a.timeMs - b.timeMs);
|
||||
}
|
||||
|
||||
function treatmentTimeMs(treatment: NightscoutTreatmentRecord): number | null {
|
||||
for (const value of [treatment.created_at, treatment.timestamp]) {
|
||||
if (typeof value === "number" && Number.isFinite(value) && value > 0) return value;
|
||||
if (typeof value === "string" && value.trim()) {
|
||||
const numeric = Number(value);
|
||||
const parsed = Number.isFinite(numeric) ? numeric : Date.parse(value);
|
||||
if (Number.isFinite(parsed) && parsed > 0) return parsed;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function positiveNumber(value: unknown): number | null {
|
||||
if (typeof value !== "number" && typeof value !== "string") return null;
|
||||
if (typeof value === "string" && !value.trim()) return null;
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) && numeric > 0 ? numeric : null;
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
import { type AppContext } from "../app-definition";
|
||||
import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge";
|
||||
import { getStringSetting, setStringSetting } from "../../native/settings-store";
|
||||
import { nightscoutMissedBolusAlertsSetting, nightscoutMissedBolusCooldownSetting } from "../../ui/dashboard-settings";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { detectMissedBolus, type MissedBolusInput } from "./missed-bolus-detector";
|
||||
import { MissedBolusNotificationLayer } from "./missed-bolus-notification";
|
||||
|
||||
/** When the last alert was shown, persisted so a restart doesn't reset the cooldown. */
|
||||
const LAST_ALERT_STORAGE_KEY = "integrations.nightscout.missedBolusLastAlertMs";
|
||||
|
||||
let started = false;
|
||||
let lastInputKey = "";
|
||||
let lastSummary = "";
|
||||
|
||||
/**
|
||||
* Runs the missed-bolus detector each time a Nightscout sync brings new
|
||||
* readings or treatments, and shows a glasses notification when it fires
|
||||
* outside the cooldown. Shell-lifetime, started from the app's boot hook.
|
||||
*/
|
||||
export function startMissedBolusMonitor(ctx: AppContext): void {
|
||||
if (started) return;
|
||||
started = true;
|
||||
nightscoutBridge.onStateChange((state) => checkForMissedBolus(state, ctx));
|
||||
}
|
||||
|
||||
function checkForMissedBolus(state: NightscoutState, ctx: AppContext): void {
|
||||
if (!nightscoutMissedBolusAlertsSetting.get() || !state.available) return;
|
||||
const key = inputKey(state);
|
||||
if (key === lastInputKey) return;
|
||||
lastInputKey = key;
|
||||
|
||||
const nowMs = Date.now();
|
||||
const result = detectMissedBolus(detectorInput(state, nowMs));
|
||||
const summary = `${result.classification}: ${result.reason}`;
|
||||
if (summary !== lastSummary) {
|
||||
lastSummary = summary;
|
||||
console.log(`missed-bolus detector: ${summary}`);
|
||||
}
|
||||
if (result.classification !== "missed-bolus" || !result.rise) return;
|
||||
|
||||
const lastAlertMs = Number(getStringSetting(LAST_ALERT_STORAGE_KEY, "0")) || 0;
|
||||
const cooldownMs = Number(nightscoutMissedBolusCooldownSetting.get()) * 60_000;
|
||||
// A last-alert time in the future means the clock moved back; don't let it block alerts.
|
||||
if (lastAlertMs <= nowMs && nowMs - lastAlertMs < cooldownMs) return;
|
||||
setStringSetting(LAST_ALERT_STORAGE_KEY, String(nowMs));
|
||||
ctx.appendLog(`missed-bolus alert: ${result.reason}`);
|
||||
const rise = result.rise;
|
||||
shell.postAppNotification({
|
||||
key: `nightscout-missed-bolus:${nowMs}`,
|
||||
createLayer: (close) => new MissedBolusNotificationLayer(rise, nowMs, close, () => {
|
||||
void ctx.launchApp("nightscout");
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
/** Changes whenever a sync brings a new reading or a treatment the detector reads. */
|
||||
function inputKey(state: NightscoutState): string {
|
||||
return [
|
||||
state.latest?.timestampMs ?? 0,
|
||||
state.boluses.map((bolus) => bolus.timestampMs).join(","),
|
||||
state.carbs.map((entry) => entry.timestampMs).join(","),
|
||||
].join("|");
|
||||
}
|
||||
|
||||
function detectorInput(state: NightscoutState, nowMs: number): MissedBolusInput {
|
||||
return {
|
||||
nowMs,
|
||||
glucose: state.history.map((point) => ({ timeMs: point.timestampMs, mgdl: point.sgv })),
|
||||
boluses: state.boluses.map((bolus) => ({ timeMs: bolus.timestampMs, units: bolus.insulin, kind: bolus.kind })),
|
||||
carbs: state.carbs.map((entry) => ({ timeMs: entry.timestampMs, grams: entry.carbs })),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,111 @@
|
||||
import { getDefaultSmallFont } from "../../graphics/ui-fonts";
|
||||
import { GrayImage } from "../../graphics/image";
|
||||
import { renderIcon } from "../../graphics/icons";
|
||||
import { wrapText } from "../../graphics/textwrap";
|
||||
import { formatRelativeTime } from "../../util/date-util";
|
||||
import { nightscoutBridge } from "../../native/nightscout-bridge";
|
||||
import { type InputEvent } from "../../ui/gestures";
|
||||
import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { lineStep } from "../../ui/metrics";
|
||||
import {
|
||||
NOTIFICATION_CONTENT_X,
|
||||
createNotificationActionMenu,
|
||||
drawNotificationHeader,
|
||||
notificationActionMenuBox,
|
||||
notificationContentWidth,
|
||||
} from "../../ui/notifications";
|
||||
import { type GlucoseRise } from "./missed-bolus-detector";
|
||||
import {
|
||||
drawDirectionIndicator,
|
||||
drawNightscoutGraph,
|
||||
drawNightscoutValueStrikeThrough,
|
||||
formatDelta,
|
||||
isNightscoutPointStale,
|
||||
} from "./nightscout";
|
||||
|
||||
const ICON_SIZE = 24;
|
||||
const GRAPH_GAP = 6;
|
||||
const GRAPH_BOTTOM_MARGIN = 12;
|
||||
const MIN_GRAPH_HEIGHT = 32;
|
||||
|
||||
type ActionItem = { label: string; run: () => void };
|
||||
|
||||
/**
|
||||
* The missed-bolus alert, laid out like an Android notification's detail view
|
||||
* (header, text, right-hand action column), with the live glucose readout and
|
||||
* 2-hour graph below the text.
|
||||
*/
|
||||
export class MissedBolusNotificationLayer implements Layer {
|
||||
private readonly menu = createNotificationActionMenu<ActionItem>();
|
||||
|
||||
constructor(
|
||||
private readonly rise: GlucoseRise,
|
||||
private readonly postedAtMs: number,
|
||||
private readonly close: () => void,
|
||||
openNightscout: () => void,
|
||||
) {
|
||||
this.menu.setItems([
|
||||
{ label: "Dismiss", run: close },
|
||||
{ label: "Open Nightscout", run: () => {
|
||||
close();
|
||||
openNightscout();
|
||||
} },
|
||||
]);
|
||||
}
|
||||
|
||||
paint(ctx: LayerContext): GrayImage {
|
||||
const font = getDefaultSmallFont();
|
||||
const step = lineStep(font);
|
||||
const { width, height } = ctx.stack.getBaseSize();
|
||||
const image = new GrayImage(width, height, 0);
|
||||
const contentX = NOTIFICATION_CONTENT_X;
|
||||
const contentWidth = notificationContentWidth(width);
|
||||
const nowMs = Date.now();
|
||||
|
||||
let y = drawNotificationHeader(image, font, renderIcon("nightscout", ICON_SIZE), `Nightscout ${formatRelativeTime(this.postedAtMs)}`);
|
||||
image.drawText(font, contentX, y, "Missed bolus?", 230);
|
||||
y += step;
|
||||
const minutes = Math.round((this.rise.latestMs - this.rise.onsetMs) / 60_000);
|
||||
const body = `Glucose rose ${this.rise.riseMgdl} mg/dL in ${minutes} min with no bolus or carbs entered.`;
|
||||
for (const line of wrapText(font, body, contentWidth)) {
|
||||
image.drawText(font, contentX, y, line, 190);
|
||||
y += step;
|
||||
}
|
||||
|
||||
// Live readout and graph: the modal repaints with each shell render, so
|
||||
// these follow new readings (and a bolus, once entered) while it is up.
|
||||
const nightscout = nightscoutBridge.snapshot();
|
||||
const latest = nightscout.latest;
|
||||
if (latest) {
|
||||
const value = `${latest.sgv}`;
|
||||
image.drawText(font, contentX, y, value, 230);
|
||||
if (isNightscoutPointStale(latest, nowMs)) {
|
||||
drawNightscoutValueStrikeThrough(image, contentX, y + (font.lineHeight >> 1), font.measureText(value));
|
||||
}
|
||||
let x = contentX + font.measureText(`${value} `);
|
||||
x += drawDirectionIndicator(image, font, x, y, nightscout.direction, 200);
|
||||
x += font.measureText(" ");
|
||||
const iob = nightscout.iob === null ? "" : ` IOB ${nightscout.iob.toFixed(2)}`;
|
||||
image.drawText(font, x, y, `${formatDelta(nightscout.delta)}${iob}`, 180);
|
||||
y += step;
|
||||
}
|
||||
const graphTop = y + GRAPH_GAP;
|
||||
const graphHeight = height - GRAPH_BOTTOM_MARGIN - graphTop;
|
||||
if (graphHeight >= MIN_GRAPH_HEIGHT) {
|
||||
drawNightscoutGraph(image, { x: contentX, y: graphTop, width: contentWidth, height: graphHeight }, nightscout, nowMs, font);
|
||||
}
|
||||
|
||||
this.menu.paint(image, notificationActionMenuBox(width, height), ctx.stack.isFocused());
|
||||
return image;
|
||||
}
|
||||
|
||||
async handleInput(event: InputEvent): Promise<void> {
|
||||
if (event.type === "double-click") {
|
||||
this.close();
|
||||
} else if (event.type === "click") {
|
||||
this.menu.selectedItem?.run();
|
||||
} else if (event.type === "scroll-up" || event.type === "scroll-down") {
|
||||
await this.menu.handleInput(event);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ import {
|
||||
} from "../../ui/shell/in-process-window";
|
||||
import { evaluateNightscoutAlerts } from "./nightscout-alerts";
|
||||
import { loadNightscoutThresholds, nightscoutAlwaysShowInTopBarSetting, onAnySettingChanged } from "../../ui/dashboard-settings";
|
||||
import { shell } from "../../ui/shell/shell";
|
||||
import { shell, type TrayWidget } from "../../ui/shell/shell";
|
||||
|
||||
export const NIGHTSCOUT_WINDOW_ID = "nightscout";
|
||||
export const NIGHTSCOUT_SURFACE_ID = "window:nightscout";
|
||||
@@ -23,11 +23,12 @@ const TRAY_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000;
|
||||
|
||||
const windowRenders = new Set<() => void>();
|
||||
let trayStarted = false;
|
||||
let trayWidget: TrayWidget | undefined;
|
||||
let trayTick: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
function syncNightscout(): void {
|
||||
const visible = windowRenders.size > 0 || nightscoutAlwaysShowInTopBarSetting.get();
|
||||
shell.setTrayIcon(TRAY_ICON_ID, visible ? buildNightscoutTrayIcon(nightscoutBridge.snapshot()) : null);
|
||||
shell.setTrayIcon(TRAY_ICON_ID, visible ? buildNightscoutTrayIcon(nightscoutBridge.snapshot()) : null, trayWidget);
|
||||
for (const render of windowRenders) render();
|
||||
// Age warnings and stale glucose must advance even during a stalled fetch.
|
||||
if (visible && trayTick === null) trayTick = setInterval(syncNightscout, 60_000);
|
||||
@@ -37,8 +38,12 @@ function syncNightscout(): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** Shell-lifetime subscription so the tray can be enabled before any window opens. */
|
||||
export function startNightscoutTrayIcon(): void {
|
||||
/**
|
||||
* Shell-lifetime subscription so the tray can be enabled before any window
|
||||
* opens; `widget` makes it selectable in the switcher.
|
||||
*/
|
||||
export function startNightscoutTrayIcon(widget?: TrayWidget): void {
|
||||
trayWidget ??= widget;
|
||||
if (trayStarted) return;
|
||||
trayStarted = true;
|
||||
nightscoutBridge.onStateChange(syncNightscout);
|
||||
|
||||
Loaded 100 of 368 files, more files were not shown because too many files have changed in this diff.
Show more
Reference in new issue
Block a user