diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..a2e63bb4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +# Check text out with LF everywhere, so Windows checkouts keep runnable bash +# scripts and the LF-anchored source regexes in tests. Batch files are the +# exception: cmd.exe can miss goto/call labels in LF-only batch files. +* text=auto eol=lf +*.bat text eol=crlf +*.cmd text eol=crlf diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dce2bfd9..65ba1739 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,3 +26,48 @@ jobs: - uses: actions/checkout@v4 - run: npm ci - run: npm run test:kotlin + + # The stats.faceclaw.org server: typecheck, and the tests including the + # database one against a throwaway Postgres. + stats-server: + runs-on: ubuntu-latest + defaults: + run: + working-directory: stats-server + services: + postgres: + image: postgres:16 + env: + POSTGRES_PASSWORD: postgres + ports: ["5432:5432"] + options: >- + --health-cmd pg_isready --health-interval 5s --health-timeout 5s --health-retries 10 + env: + PGHOST: localhost + PGUSER: postgres + PGPASSWORD: postgres + STATS_TEST_DATABASE: postgres + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: 22 + - run: npm ci + - run: npm run check + - run: npm test + + # The @faceclaw/evenhub-extensions npm package: typecheck (including + # compatibility with the stock SDK's container classes), build, and check + # that the committed API reference matches the source. + evenhub-extensions: + runs-on: ubuntu-latest + defaults: + run: + working-directory: evenhub-extensions + steps: + - uses: actions/checkout@v4 + - run: npm ci + - run: npm run check + - run: npm run build + - run: npm run docs + - run: git diff --exit-code -- docs diff --git a/.gitignore b/.gitignore index e928afc2..95246319 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ typings/ App_Resources/Android/src/main/jniLibs/**/*.so App_Resources/Android/src/main/assets/faceclaw-voice-asr/ old +__pycache__ # Visual Studio Code .vscode/* diff --git a/.oxlintrc.json b/.oxlintrc.json index 5f63cfb3..1be2a6ac 100644 --- a/.oxlintrc.json +++ b/.oxlintrc.json @@ -78,7 +78,7 @@ "node_modules/**", "platforms/**", "dist/**", - "faceclaw-extensions/dist/**", + "evenhub-extensions/dist/**", "old/**", "tmp/**" ] diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index a5584773..fe2fac36 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -13,6 +13,9 @@ and others who contributed to documenting the G2's bluetooth protocol and firmwa G2 microphone LC3 decoding uses Google's liblc3: https://github.com/google/liblc3 +Weather data outside the US is by Open-Meteo.com: https://open-meteo.com/. It is +licensed under CC BY 4.0 (https://creativecommons.org/licenses/by/4.0/). + The Terminus font is by Dimitar Zhekov's Terminus: https://framagit.org/ohnonot/terv-terc and is distributed under the SIL Open Font License; see app/fonts/terminus/LICENSE. diff --git a/App_Resources/Android/app.gradle b/App_Resources/Android/app.gradle index 65d42a3c..997e1f4d 100644 --- a/App_Resources/Android/app.gradle +++ b/App_Resources/Android/app.gradle @@ -46,6 +46,17 @@ def faceclawNdkHostTag = { return "linux-x86_64" } +// Host tools carry .exe on Windows. Call the binaries themselves: the NDK's +// target-prefixed clang wrappers are bash scripts (or .cmd files) that only add +// --target. +def faceclawHostExecutable = { File dir, String name -> + return file("$dir/$name${Os.isFamily(Os.FAMILY_WINDOWS) ? ".exe" : ""}") +} + +def faceclawNdkTool = { File ndkRoot, String name -> + return faceclawHostExecutable(file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin"), name) +} + def faceclawFindNdkRoot = { def explicitNdk = System.getenv("ANDROID_NDK_HOME") if (explicitNdk != null && file(explicitNdk).exists()) { @@ -111,7 +122,7 @@ tasks.register("prepareFaceclawNativeLibs") { if (ndkRoot == null || !ndkRoot.exists()) { throw new GradleException("Android NDK is required to build libfaceclaw_lc3.so") } - def clang = file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/aarch64-linux-android24-clang") + def clang = faceclawNdkTool(ndkRoot, "clang") if (!clang.exists()) { throw new GradleException("Could not find Android clang at $clang") } @@ -125,6 +136,7 @@ tasks.register("prepareFaceclawNativeLibs") { exec { commandLine([ clang.absolutePath, + "--target=aarch64-linux-android24", "-shared", "-fPIC", "-O3", @@ -192,8 +204,8 @@ tasks.register("prepareFaceclawLlama") { if (cmakeDir == null) { throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so") } - def cmake = file("$cmakeDir/bin/cmake") - def ninja = file("$cmakeDir/bin/ninja") + def cmake = faceclawHostExecutable(file("$cmakeDir/bin"), "cmake") + def ninja = faceclawHostExecutable(file("$cmakeDir/bin"), "ninja") def march = "-march=armv8.2-a+dotprod+fp16" // Configure once; afterwards the ninja no-op build keeps this cheap while @@ -227,14 +239,13 @@ tasks.register("prepareFaceclawLlama") { } // The static-linked lib carries llama.cpp's full debug info (~50MB); // strip it so the APK only pays for the code (~8MB). - def ndkRootForStrip = faceclawFindNdkRoot() - if (ndkRootForStrip != null) { - def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip") - if (llvmStrip.exists()) { - exec { - commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath]) - } + def llvmStrip = faceclawNdkTool(ndkRoot, "llvm-strip") + if (llvmStrip.exists()) { + exec { + commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath]) } + } else { + logger.warn("Could not find llvm-strip at $llvmStrip; packaging an unstripped libfaceclaw_llama.so (~50MB)") } } } @@ -251,7 +262,8 @@ if (faceclawVersionFile.exists()) { } } def faceclawVersionCode = { - def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger() + // Prerelease labels belong in versionName, not the numeric versionCode. + def parts = (faceclawVersionName.split('-')[0].tokenize('.') + ["0", "0"]).take(3)*.toInteger() return parts[0] * 10000 + parts[1] * 100 + parts[2] }() diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt index 58e016cd..f53ba1ac 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawAudioTranscoder.kt @@ -13,7 +13,8 @@ import java.io.IOException /** * Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A * container, cutting speech recordings to roughly a tenth of their WAV size. - * Runs on its own thread; the source WAV is deleted on success when requested. + * Runs on its own thread; when requested, the source WAV is deleted once the + * listener's onDone has returned normally. */ class FaceclawAudioTranscoder( private val wavPath: String?, @@ -66,19 +67,38 @@ class FaceclawAudioTranscoder( private fun run() { try { transcode() - val size = File(outputPath).length() - if (deleteSourceOnSuccess) { - File(wavPath).delete() - } - if (listener != null) { - listener.onDone(outputPath, size) - } } catch (t: Throwable) { Log.w(TAG, "transcode failed for $wavPath", t) File(outputPath).delete() - if (listener != null) { - listener.onError(if (t.message == null) t.toString() else t.message) + try { + listener?.onError(if (t.message == null) t.toString() else t.message) + } catch (listenerError: Throwable) { + Log.w(TAG, "transcode error listener failed", listenerError) } + return + } + val size = File(outputPath).length() + if (size <= 0) { + Log.w(TAG, "transcode produced no output for $wavPath") + File(outputPath).delete() + try { + listener?.onError("transcode produced no output") + } catch (listenerError: Throwable) { + Log.w(TAG, "transcode error listener failed", listenerError) + } + return + } + try { + listener?.onDone(outputPath, size) + } catch (t: Throwable) { + // The caller never took up the new file, so whatever it recorded still names the + // source: keep that and drop the output, rather than lose both. + Log.w(TAG, "transcode listener failed for $wavPath", t) + File(outputPath).delete() + return + } + if (deleteSourceOnSuccess) { + File(wavPath).delete() } } @@ -93,6 +113,7 @@ class FaceclawAudioTranscoder( val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC) var muxer: MediaMuxer? = null + var muxerStopped = false var input: FileInputStream? = null try { codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) @@ -159,6 +180,11 @@ class FaceclawAudioTranscoder( } } } + // stop() writes the index (moov); without it the file won't play, so its failure + // (storage full, I/O error, a muxer that never started) fails the transcode rather + // than reporting a broken output as done. + muxerStopped = true + muxer.stop() } finally { if (input != null) { try { @@ -172,9 +198,12 @@ class FaceclawAudioTranscoder( } codec.release() if (muxer != null) { - try { - muxer.stop() - } catch (ignored: Throwable) { + if (!muxerStopped) { + // Already failing; the output is deleted either way. + try { + muxer.stop() + } catch (ignored: Throwable) { + } } muxer.release() } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt new file mode 100644 index 00000000..a2b358a9 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBinaryPost.kt @@ -0,0 +1,103 @@ +package com.faceclaw.app + +import android.os.Handler +import android.os.Looper + +import java.io.IOException +import java.nio.ByteBuffer +import java.util.concurrent.TimeUnit + +import okhttp3.Call +import okhttp3.Callback +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response + +/** + * One POST of a binary body (a Whisper server WAV upload), sent by start(). + * NativeScript's http module hands an ArrayBuffer body to Java one bridge call + * per byte, which for a ~1 MB upload stalls the JS thread; here the body + * arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk. + * headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and + * may carry the Content-Type. The response text and status reach the listener + * on the constructing thread's Looper. + */ +class FaceclawBinaryPost( + url: String?, + body: ByteBuffer?, + headers: Array?, + timeoutMs: Long, + listener: FaceclawBinaryPostListener? +) { + private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper()) + private val listener: FaceclawBinaryPostListener = + listener ?: throw IllegalArgumentException("listener is required") + private val call: Call + + @Volatile + private var cancelled = false + + init { + if (url == null || url.trim().isEmpty()) { + throw IllegalArgumentException("url is required") + } + val bytes = ByteArray(body?.remaining() ?: 0) + body?.duplicate()?.get(bytes) + // No media type on the body, so okhttp keeps the caller's Content-Type header. + val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody()) + for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value) + call = getClient().newCall(builder.build()) + call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS) + } + + fun start() { + call.enqueue(object : Callback { + override fun onFailure(call: Call, e: IOException) { + post { listener.onFailure(e.message ?: e.toString()) } + } + + override fun onResponse(call: Call, response: Response) { + val text = try { + response.use { it.body?.string() ?: "" } + } catch (e: IOException) { + post { listener.onFailure(e.message ?: e.toString()) } + return + } + post { listener.onResponse(response.code, text) } + } + }) + } + + /** Aborts the request; no callback follows. */ + fun cancel() { + cancelled = true + call.cancel() + } + + private fun post(runnable: () -> Unit) { + callbackHandler.post { + if (!cancelled) runnable() + } + } + + companion object { + @Volatile + private var sharedClient: OkHttpClient? = null + + private fun getClient(): OkHttpClient { + if (sharedClient == null) { + synchronized(FaceclawBinaryPost::class.java) { + if (sharedClient == null) { + sharedClient = OkHttpClient.Builder() + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(60, TimeUnit.SECONDS) + .addInterceptor(FaceclawHttp.userAgentInterceptor()) + .build() + } + } + } + return sharedClient!! + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt index 89448026..e384d0da 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleCommunicator.kt @@ -294,7 +294,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener) - fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) = + fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean = core.sendFaceclawMicControl(record, label, rightTemple, leftTemple) fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener) @@ -329,6 +329,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? { + val composite = core.previewComposite() ?: return null + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green, + cropX, cropY, cropWidth, cropHeight) + } + /** * Posted to the main looper after each step of an animation the preview is replaying (menu * highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt index 4ebe81ca..89baca33 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawBleManager.kt @@ -550,6 +550,7 @@ class FaceclawBleManager(context: Context) { override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) { val address = gatt.device.address Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState") + if (status != BluetoothGatt.GATT_SUCCESS) ConnectionCounters.increment("ble.gatt-status.$status") val link = links[address] val current = link?.gatt if (link == null || (current != null && current !== gatt)) { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt index d985766e..c76bd02d 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawCaptionEngine.kt @@ -68,6 +68,11 @@ class FaceclawCaptionEngine { session.stop() } + /** Stop without blocking; the listener's onStopped follows the trailing utterance. */ + fun requestStop() { + session.requestStop() + } + /** Push decoded 16 kHz mono S16LE PCM (any chunking). */ fun acceptPcm(pcm16le: ByteArray?) { session.acceptPcm(pcm16le) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt index 2d9f316f..2f309ad8 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawConversationStore.kt @@ -718,6 +718,94 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp } } + // ---- crash recovery ---- + + /** + * Close out sessions a dead process left open (ended_at NULL): the end becomes the last + * stored segment's end (or the start), and the tone average comes from the stored segments. + * excludeIdsJson is a JSON array of session ids that are legitimately still open. Returns + * the sessions it closed as a JSON array of {id, audioPath, audioCodec}. + */ + fun closeInterruptedSessions(excludeIdsJson: String?): String { + val exclude = HashSet() + if (excludeIdsJson != null && !excludeIdsJson.isEmpty()) { + val ids = JSONArray(excludeIdsJson) + for (i in 0 until ids.length()) { + exclude.add(ids.getLong(i)) + } + } + val db = writableDatabase + val closed = JSONArray() + val closedIds = ArrayList() + db.rawQuery("SELECT id, audio_path, audio_codec FROM sessions WHERE ended_at IS NULL", null).use { c -> + while (c.moveToNext()) { + val id = c.getLong(0) + if (id in exclude) continue + closedIds.add(id) + val row = JSONObject() + row.put("id", id) + row.put("audioPath", if (c.isNull(1)) JSONObject.NULL else c.getString(1)) + row.put("audioCodec", if (c.isNull(2)) JSONObject.NULL else c.getString(2)) + closed.put(row) + } + } + for (id in closedIds) { + db.execSQL( + "UPDATE sessions SET" + + " ended_at = COALESCE((SELECT MAX(ended_at) FROM segments WHERE session_id = ?), started_at)," + + " avg_sentiment = COALESCE((SELECT AVG(sentiment) FROM segments WHERE session_id = ?), 0)" + + " WHERE id = ?", + arrayOf(id, id, id)) + } + return closed.toString() + } + + /** + * Closed sessions whose audio is still the WAV: a transcode that never finished (the process + * died mid-transcode, or recovery failed partway on an earlier boot). Returns a JSON array of + * {id, audioPath}. + */ + fun listUntranscodedSessions(): String { + val sessions = JSONArray() + readableDatabase.rawQuery( + "SELECT id, audio_path FROM sessions" + + " WHERE ended_at IS NOT NULL AND audio_codec = 'wav' AND audio_path IS NOT NULL", + null, + ).use { c -> + while (c.moveToNext()) { + val row = JSONObject() + row.put("id", c.getLong(0)) + row.put("audioPath", c.getString(1)) + sessions.put(row) + } + } + return sessions.toString() + } + + /** + * Delete recordings in dirPath that no session's audio_path names: files a session row never + * learned about, or partial transcoder output. Matched by file name, so a storage move that + * changed the directory's absolute path can't make live recordings look orphaned. Only call + * while no recording or transcode is in flight. Returns how many files were deleted. + */ + fun deleteUnreferencedRecordings(dirPath: String?): Int { + if (dirPath == null) return 0 + val files = File(dirPath).listFiles() ?: return 0 + val referenced = HashSet() + readableDatabase.rawQuery("SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL", null).use { c -> + while (c.moveToNext()) { + referenced.add(File(c.getString(0)).name) + } + } + var deleted = 0 + for (file in files) { + if (file.isFile && file.name !in referenced && file.delete()) { + deleted++ + } + } + return deleted + } + fun deleteSession(sessionId: Long) { val db = writableDatabase val path = getSessionAudioPath(sessionId) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt index d099d419..770a1115 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubJsBridge.kt @@ -1,32 +1,100 @@ package com.faceclaw.app +import android.net.Uri import android.os.Handler import android.os.Looper -import android.webkit.JavascriptInterface +import android.util.Log +import android.webkit.WebView +import androidx.webkit.JavaScriptReplyProxy +import androidx.webkit.WebMessageCompat +import androidx.webkit.WebViewCompat +import androidx.webkit.WebViewFeature + +import org.json.JSONArray +import org.json.JSONException /** - * The object injected into a hosted EvenHub app's WebView as - * window.__faceclawEvenHub. The document-start shim (served inline by - * FaceclawEvenHubWebViewClient) wraps it in a promise-returning - * window.flutter_inappwebview.callHandler, which is what the EvenHub SDK - * actually talks to. + * The host end of a hosted EvenHub app's bridge: window.__faceclawEvenHubPort, + * a WebMessageListener object that WebView injects only into frames from the + * app's own origin. Sites the app navigates to, and other-origin iframes it + * embeds, never see it, and messages from the app's own subframes are dropped, + * so only the app's main document reaches the host (its SDK storage, sensors, + * glasses display and the Faceclaw extensions, API keys included). This + * replaced an addJavascriptInterface object, which WebView exposes to every + * frame of every page. * - * JavascriptInterface methods run on a WebView-internal thread; everything - * is bounced to the main thread before touching the NativeScript listener. + * The transport script (webview.ts) wraps the port as window.__faceclawEvenHub, + * whose postMessage(name, argsJson, id) and wakeTimers() the stock bridge shim + * and the extensions call. Messages are JSON arrays: ["call", name, argsJson, + * id] or ["wake"]. */ -class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) { +class FaceclawEvenHubJsBridge private constructor( + private val listener: FaceclawEvenHubListener +) : WebViewCompat.WebMessageListener { private val mainHandler = Handler(Looper.getMainLooper()) - @JavascriptInterface - fun postMessage(handlerName: String?, argsJson: String?, callId: Int) { - mainHandler.post { - listener.onEvenAppMessage(handlerName, argsJson, callId) + override fun onPostMessage( + view: WebView, + message: WebMessageCompat, + sourceOrigin: Uri, + isMainFrame: Boolean, + replyProxy: JavaScriptReplyProxy + ) { + if (!isMainFrame) return + val parts = try { + JSONArray(message.data ?: return) + } catch (e: JSONException) { + return + } + when (parts.optString(0)) { + "call" -> { + val name = parts.optString(1) + val argsJson = parts.optString(2) + val callId = parts.optInt(3) + mainHandler.post { listener.onEvenAppMessage(name, argsJson, callId) } + } + // The page's timer shim has work due before the host's next tick. + "wake" -> FaceclawEvenHubWebViewHost.getInstance().wakeTimers() } } - /** The page's timer shim has work due before the host's next tick. */ - @JavascriptInterface - fun wakeTimers() { - FaceclawEvenHubWebViewHost.getInstance().wakeTimers() + companion object { + private const val TAG = "FaceclawEvenHub" + private const val OBJECT_NAME = "__faceclawEvenHubPort" + + /** + * The origin rule matching exactly [url]'s origin (default port elided), + * or null when it isn't an http(s) URL with a host. + */ + @JvmStatic + fun originRule(url: String): String? { + val uri = Uri.parse(url) + val scheme = uri.scheme?.lowercase() + val host = uri.host?.lowercase() + if ((scheme != "http" && scheme != "https") || host.isNullOrEmpty()) return null + val defaultPort = if (scheme == "https") 443 else 80 + return if (uri.port == -1 || uri.port == defaultPort) "$scheme://$host" else "$scheme://$host:${uri.port}" + } + + /** + * Expose the bridge to [webView]'s frames from [originRule]. Must run + * before the WebView loads anything. + * + * @return null on success, else why the bridge could not be installed. + */ + @JvmStatic + fun install(webView: WebView, originRule: String, listener: FaceclawEvenHubListener): String? { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) { + Log.w(TAG, "WebMessageListener unsupported; not exposing the EvenHub bridge") + return "This Android System WebView is too old to run EvenHub apps safely. Update it, then reopen the app." + } + return try { + WebViewCompat.addWebMessageListener(webView, OBJECT_NAME, setOf(originRule), FaceclawEvenHubJsBridge(listener)) + null + } catch (e: IllegalArgumentException) { + Log.w(TAG, "bridge origin rule rejected: $originRule", e) + "The app's address ($originRule) can't be used as a web origin." + } + } } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubProfile.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubProfile.kt new file mode 100644 index 00000000..f985685c --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubProfile.kt @@ -0,0 +1,54 @@ +package com.faceclaw.app + +import android.util.Log +import android.webkit.WebView +import androidx.webkit.ProfileStore +import androidx.webkit.WebViewCompat +import androidx.webkit.WebViewFeature + +import java.security.MessageDigest + +/** + * Gives each EvenHub package its own WebView profile, so one app's browser + * storage (cookies, localStorage, IndexedDB, caches, service workers) is out of + * reach of every other app. That includes the storage of third-party sites an + * app navigates to, such as a sign-in provider's session cookies, which in the + * shared default profile any other app's WebView could reuse. The per-app fake + * origins alone don't cover that, and `*.evenhub.invalid` hosts share a site, + * so domain cookies would cross between packaged apps too. + * + * The profile persists across launches, and is shared by every instance of the + * same package, like the iOS per-package WKWebsiteDataStore. + */ +class FaceclawEvenHubProfile private constructor() { + companion object { + private const val TAG = "FaceclawEvenHub" + + /** Stable per package; the hash input matches the iOS data-store identifier's. */ + @JvmStatic + fun nameFor(packageId: String): String { + val digest = MessageDigest.getInstance("SHA-256") + .digest("com.faceclaw.ehpk:$packageId".toByteArray(Charsets.UTF_8)) + return "evenhub-" + digest.take(16).joinToString("") { "%02x".format(it) } + } + + /** + * Put a new WebView in [packageId]'s profile. Must run on the main thread + * before anything else is done with the WebView. + * + * @return false when the WebView provider lacks multi-profile support, + * leaving the WebView in the shared default profile. + */ + @JvmStatic + fun apply(webView: WebView, packageId: String): Boolean { + if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) { + Log.w(TAG, "multi-profile WebView unsupported; $packageId shares the default profile") + return false + } + val name = nameFor(packageId) + ProfileStore.getInstance().getOrCreateProfile(name) + WebViewCompat.setProfile(webView, name) + return true + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt index af821b0b..9160baf4 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewClient.kt @@ -2,6 +2,7 @@ package com.faceclaw.app import android.os.Handler import android.os.Looper +import android.webkit.RenderProcessGoneDetail import android.webkit.WebResourceRequest import android.webkit.WebResourceResponse import android.webkit.WebView @@ -33,6 +34,11 @@ import java.nio.charset.StandardCharsets * The shim is registered as a document-start script instead * (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for * WebView versions that lack that API. + * + * An app can crash its renderer, or the system can kill it for memory. Unless + * every WebView that renderer served handles that, WebView aborts the whole + * process, taking Faceclaw down with it; so onRenderProcessGone retires the + * affected WebView and reports it instead. */ class FaceclawEvenHubWebViewClient( private val rootDir: String?, @@ -93,6 +99,15 @@ class FaceclawEvenHubWebViewClient( } } + override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean { + // Called on the main thread for each WebView the renderer served. This + // one can't be used again: drop it, and the app shows as failed. + Log.w(TAG, "renderer gone (crashed=${detail.didCrash()}) for $host") + FaceclawEvenHubWebViewHost.getInstance().detach(view) + listener.onRenderProcessGone(detail.didCrash()) + return true + } + companion object { private const val TAG = "FaceclawEvenHub" diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt index 7e5cad69..18591b3e 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawEvenHubWebViewHost.kt @@ -59,8 +59,8 @@ import java.util.HashMap * requestAnimationFrame) with JS queues fired by window.__fcTick(), and this * host ticks those on the main thread regardless of screen state. The tick * returns when the page next has work, so an idle page isn't evaluated at - * 60Hz; work scheduled between ticks wakes the host through - * FaceclawEvenHubJsBridge.wakeTimers. The main Looper keeps running under the + * 60Hz; work scheduled between ticks wakes the host through the bridge's + * "wake" message (FaceclawEvenHubJsBridge). The main Looper keeps running under the * foreground service, so the tick survives the screen turning off. (Matches the * official Even app's approach.) * @@ -105,8 +105,12 @@ class FaceclawEvenHubWebViewHost { * the page wakes it. It also re-ticks after IDLE_POLL_MS regardless, so a * lost evaluateJavascript callback (e.g. mid-navigation) or a page whose * wake didn't reach us stalls its timers for at most that long. + * + * The tick only runs in the app's own documents ([appCheck], a JS + * expression): after a navigation the WebView may show another site. */ - private inner class Ticker(private val web: WebView) : Runnable { + private inner class Ticker(private val web: WebView, appCheck: String) : Runnable { + private val tickJs = "if ($appCheck) { $TICK_JS }" private var inFlight = false private var wakePending = false private var stopped = false @@ -114,7 +118,7 @@ class FaceclawEvenHubWebViewHost { override fun run() { if (stopped) return inFlight = true - web.evaluateJavascript(TICK_JS) { result -> onTicked(result) } + web.evaluateJavascript(tickJs) { result -> onTicked(result) } schedule(IDLE_POLL_MS) } @@ -278,8 +282,11 @@ class FaceclawEvenHubWebViewHost { (web.context as? MutableContextWrapper)?.baseContext = activity } - /** Add a WebView to the host, full-size and rendering, hidden behind the UI. */ - fun attach(activity: Activity, web: WebView) { + /** + * Add a WebView to the host, full-size and rendering, hidden behind the UI. + * [appCheck] is a JS expression that is true only in the app's own documents. + */ + fun attach(activity: Activity, web: WebView, appCheck: String) { ensureOverlay(activity) rebaseContext(web, activity) web.visibility = View.VISIBLE @@ -301,7 +308,7 @@ class FaceclawEvenHubWebViewHost { } if (!webViews.contains(web)) webViews.add(web) if (!tickers.containsKey(web)) { - val ticker = Ticker(web) + val ticker = Ticker(web, appCheck) tickers[web] = ticker mainHandler.postDelayed(ticker, TICK_MS) } @@ -348,9 +355,9 @@ class FaceclawEvenHubWebViewHost { return shown } - /** Remove and destroy a WebView (its app is closing). */ + /** Remove and destroy a WebView (its app is closing, or its renderer is gone). Idempotent. */ fun detach(web: WebView) { - webViews.remove(web) + if (!webViews.remove(web)) return tickers.remove(web)?.stop() stack?.removeView(web) web.destroy() diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt index b7f8ca5a..d7311dba 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaController.kt @@ -46,6 +46,7 @@ class FaceclawMediaController(context: Context) { override fun onMetadataChanged(metadata: MediaMetadata?) { synchronized(lock) { + artVersion++ emitStateLocked() } } @@ -92,6 +93,15 @@ class FaceclawMediaController(context: Context) { private var sessionsListenerRegistered = false private var ignoredPackages: Set = HashSet() + /** + * Bumped whenever the art getAlbumArtGray would return may have changed: + * every metadata update and every switch of active session. Players + * re-publish the same title/album with a different bitmap (art attached + * after download, or a stale bitmap from the previous track replaced + * after a quick double skip), so the strings can't key an art cache. + */ + private var artVersion = 0 + init { this.listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java) this.sessionManager = appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as MediaSessionManager? @@ -404,6 +414,7 @@ class FaceclawMediaController(context: Context) { previous.unregisterCallback(controllerCallback) } activeController = controller + artVersion++ if (controller != null) { controller.registerCallback(controllerCallback, mainHandler) } @@ -503,6 +514,7 @@ class FaceclawMediaController(context: Context) { val finalCanSkipPrevious = canSkipPrevious val finalAccessEnabled = accessEnabled val finalStatus = status + val finalArtVersion = artVersion mainHandler.post { currentListener.onStateChange( @@ -519,7 +531,8 @@ class FaceclawMediaController(context: Context) { finalCanSkipNext, finalCanSkipPrevious, finalAccessEnabled, - finalStatus + finalStatus, + finalArtVersion ) } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt index c0f6f229..83c294be 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawMediaNotificationListenerService.kt @@ -3,15 +3,20 @@ package com.faceclaw.app import android.app.Notification import android.app.NotificationManager import android.app.PendingIntent +import android.content.ComponentName +import android.content.Context +import android.database.ContentObserver import android.graphics.Bitmap import android.graphics.Canvas -import android.graphics.Color import android.graphics.drawable.BitmapDrawable import android.graphics.drawable.Drawable import android.graphics.drawable.Icon +import android.os.Build import android.os.Bundle import android.os.Handler import android.os.Looper +import android.os.SystemClock +import android.provider.Settings import android.service.notification.NotificationListenerService import android.service.notification.StatusBarNotification import android.util.Log @@ -29,15 +34,177 @@ import org.json.JSONObject class FaceclawMediaNotificationListenerService : NotificationListenerService() { companion object { private const val TAG = "FaceclawNotify" - private const val NOTIFICATION_ICON_GAMMA = 1.6 private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName" + /** + * How long notification access can be granted with no listener + * connected before the watchdog forces a rebind: longer than a + * normal bind at process start, and than the system's own single + * retry 10 s after a binding dies. Later attempts double from here. + */ + private const val REBIND_GRACE_MS = 15_000L + private const val REBIND_MAX_DELAY_MS = 15 * 60_000L + /** Missing this long, the listener needs the user to toggle notification access. */ + private const val LISTENER_STUCK_MS = 30_000L + /** Notifications posted while disconnected alert on reconnect only if this recent. */ + private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L + /** The connected listener; null until onListenerConnected and after onListenerDisconnected. */ @Volatile private var activeService: FaceclawMediaNotificationListenerService? = null private val mainHandler = Handler(Looper.getMainLooper()) private val notificationListeners: MutableSet = CopyOnWriteArraySet() + private val listenerStateObservers: MutableSet = CopyOnWriteArraySet() private val activeNotificationWakeKeys: MutableSet = HashSet() + // Binding watchdog (see watchListenerBinding); written on the main thread only. + private var watchdogContext: Context? = null + private var rebindScheduled = false + private var rebindDelayMs = REBIND_GRACE_MS + /** elapsedRealtime when the watchdog first saw access granted with no listener; 0 when not missing. */ + @Volatile + private var listenerMissingSinceMs = 0L + /** Wall-clock time of the last disconnect in this process (compared with postTime); 0 if none. */ + private var lastListenerDisconnectAtMs = 0L + private val forceRebindTask = Runnable { forceRebindIfStillMissing() } + private val accessObserver = object : ContentObserver(mainHandler) { + override fun onChange(selfChange: Boolean) { + checkListenerBinding() + notifyListenerStateChanged() + } + } + + /** + * Start watching for the listener going missing while notification + * access is granted. Android can drop a listener's binding, or give + * up rebinding it after the process dies, and not bring it back; + * every read then comes back empty for the life of the process. + * Called at app boot; disconnects and access changes re-check. + */ + @JvmStatic + fun watchListenerBinding(context: Context?) { + if (context == null) { + return + } + val appContext = context.applicationContext + mainHandler.post { + if (watchdogContext == null) { + watchdogContext = appContext + try { + appContext.contentResolver.registerContentObserver( + Settings.Secure.getUriFor("enabled_notification_listeners"), false, accessObserver) + } catch (t: Throwable) { + Log.w(TAG, "notification access observer registration failed", t) + } + } + checkListenerBinding() + } + } + + /** + * "connected"; "no-access" (notification access not granted); + * "connecting" (granted, listener not connected yet); or "stuck" + * (granted, but still missing past LISTENER_STUCK_MS, which forced + * rebinds can't always fix: the user has to toggle access). + */ + @JvmStatic + fun getListenerState(context: Context?): String { + if (activeService != null) { + return "connected" + } + if (!FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) { + return "no-access" + } + val missingSince = listenerMissingSinceMs + if (missingSince > 0 && SystemClock.elapsedRealtime() - missingSince >= LISTENER_STUCK_MS) { + return "stuck" + } + return "connecting" + } + + /** Run on the main thread whenever getListenerState may have changed. */ + @JvmStatic + fun addListenerStateObserver(observer: Runnable?) { + if (observer != null) { + listenerStateObservers.add(observer) + } + } + + @JvmStatic + fun removeListenerStateObserver(observer: Runnable?) { + if (observer != null) { + listenerStateObservers.remove(observer) + } + } + + private fun notifyListenerStateChanged() { + for (observer in listenerStateObservers) { + mainHandler.post { + try { + observer.run() + } catch (t: Throwable) { + Log.w(TAG, "listener state observer failed", t) + } + } + } + } + + /** Main thread. Starts the forced-rebind countdown when access is granted and no listener is connected. */ + private fun checkListenerBinding() { + val context = watchdogContext ?: return + if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) { + resetRebindWatchdog() + return + } + if (listenerMissingSinceMs == 0L) { + listenerMissingSinceMs = SystemClock.elapsedRealtime() + } + scheduleForcedRebind() + } + + private fun scheduleForcedRebind() { + if (!rebindScheduled) { + rebindScheduled = true + mainHandler.postDelayed(forceRebindTask, rebindDelayMs) + } + } + + private fun resetRebindWatchdog() { + mainHandler.removeCallbacks(forceRebindTask) + rebindScheduled = false + rebindDelayMs = REBIND_GRACE_MS + listenerMissingSinceMs = 0L + } + + private fun forceRebindIfStillMissing() { + rebindScheduled = false + val context = watchdogContext ?: return + if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) { + resetRebindWatchdog() + return + } + val missingSeconds = (SystemClock.elapsedRealtime() - listenerMissingSinceMs) / 1000 + Log.w(TAG, "notification access granted but no listener connected for ${missingSeconds}s; forcing a rebind") + val component = ComponentName(context, FaceclawMediaNotificationListenerService::class.java) + try { + // requestRebind only undoes a requestUnbind, so alone it can't + // bring back a binding the system dropped. Unbinding by + // component first marks the listener snoozed and drops any + // record of it; the rebind then un-snoozes it and binds afresh. + // Below API 34 there is no public equivalent (and flipping the + // component's enabled state can make the system revoke + // access), so there the state goes "stuck" for the user. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) { + NotificationListenerService.requestUnbind(component) + } + NotificationListenerService.requestRebind(component) + } catch (t: Throwable) { + Log.w(TAG, "forced notification listener rebind failed", t) + } + rebindDelayMs = Math.min(rebindDelayMs * 2, REBIND_MAX_DELAY_MS) + scheduleForcedRebind() + notifyListenerStateChanged() + } + @JvmStatic fun addNotificationListener(listener: FaceclawNotificationListener?) { if (listener != null) { @@ -127,15 +294,16 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) { continue } - val drawable = loadNotificationIcon(service, statusBarNotification.notification) - if (drawable == null) { - Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.packageName) + val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) { + getNotificationAppName(service, statusBarNotification) + } + if (icon == null) { + Log.i(TAG, "icon skipped (nothing to draw): " + statusBarNotification.packageName) continue } Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.packageName - + " drawable=" + drawable.javaClass.simpleName - + " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight) - appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName) + + " source=" + icon.source + " kind=" + icon.art.kind) + out.write(icon.art.gray) keysOut?.add(statusBarNotification.key ?: "") if (dedupeGroupKey != null) { emittedGroupKeys.add(dedupeGroupKey) @@ -165,13 +333,37 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { if (statusBarNotification == null || statusBarNotification.notification == null) { return ByteArray(0) } - val drawable = loadNotificationIcon(service, statusBarNotification.notification) - if (drawable == null) { + val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) { + getNotificationAppName(service, statusBarNotification) + } + return icon?.art?.gray ?: ByteArray(0) + } + + /** + * The picture attached to one active notification (BigPictureStyle: a + * photo, a screenshot, Dexcom's glucose graph), scaled to fit + * maxWidth x maxHeight as a gray packet with the tone handling (and + * light-background inversion) of ImageFileLoader.bitmapToGrayPacket; + * empty when the notification or its picture is unavailable. + */ + @JvmStatic + fun getNotificationPictureGray( + key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean, + ): ByteArray { + val service = activeService + if (service == null || maxWidth <= 0 || maxHeight <= 0) { return ByteArray(0) } - val out = ByteArrayOutputStream(size * size) - appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.packageName) - return out.toByteArray() + val statusBarNotification = findActiveNotificationByKey(service, key) + if (statusBarNotification == null || statusBarNotification.notification == null) { + return ByteArray(0) + } + val picture = loadNotificationPicture(service, statusBarNotification.notification) + if (picture == null) { + return ByteArray(0) + } + return ImageFileLoader.bitmapToGrayPacket( + halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground) } @JvmStatic @@ -267,13 +459,21 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { } private fun emitNotificationPosted(key: String?) { + emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) } + } + + private fun emitNotificationRemoved(key: String?) { + emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationRemoved(eventKey) } + } + + private fun emitNotificationEvent(key: String?, deliver: (FaceclawNotificationListener, String) -> Unit) { if (key == null || key.isEmpty() || notificationListeners.isEmpty()) { return } for (listener in notificationListeners) { mainHandler.post { try { - listener.onNotificationPosted(key) + deliver(listener, key) } catch (t: Throwable) { Log.w(TAG, "notification listener failed", t) } @@ -281,7 +481,14 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { } } - private fun refreshActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) { + /** + * Re-seed the wake keys from what is active now. After a reconnect + * this also catches the glasses up on the gap: keys that went away + * are reported removed (closing a popup for a notification + * dismissed meanwhile), and recent notifications posted since the + * disconnect are reported posted. A first connect only seeds. + */ + private fun reconcileActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) { val notifications: Array? try { notifications = service.activeNotifications @@ -289,19 +496,40 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { Log.w(TAG, "failed to refresh active notification wake keys", t) return } + if (notifications == null) { + return + } + val disconnectedAtMs = lastListenerDisconnectAtMs + val alertAfterMs = Math.max(disconnectedAtMs, System.currentTimeMillis() - RECONNECT_ALERT_WINDOW_MS) + val posted = ArrayList() + val removed = ArrayList() synchronized(activeNotificationWakeKeys) { + val previous = HashSet(activeNotificationWakeKeys) activeNotificationWakeKeys.clear() - if (notifications == null) { - return - } for (statusBarNotification in notifications) { - if (shouldShowNotificationInList(service, statusBarNotification)) { - val key: String? = statusBarNotification!!.key - if (key != null && !key.isEmpty()) { - activeNotificationWakeKeys.add(key) - } + if (!shouldShowNotificationInList(service, statusBarNotification)) { + continue + } + val key: String? = statusBarNotification!!.key + if (key == null || key.isEmpty()) { + continue + } + activeNotificationWakeKeys.add(key) + if (disconnectedAtMs > 0 && !previous.contains(key) && statusBarNotification.postTime > alertAfterMs) { + posted.add(key) } } + for (key in previous) { + if (!activeNotificationWakeKeys.contains(key)) { + removed.add(key) + } + } + } + for (key in removed) { + emitNotificationRemoved(key) + } + for (key in posted) { + emitNotificationPosted(key) } } @@ -319,16 +547,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { return !alreadyActive || !isPersistentNotification(statusBarNotification) } - private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?) { + /** Stop tracking a notification; returns whether it was tracked (i.e. listed on the glasses). */ + private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?): Boolean { if (statusBarNotification == null) { - return + return false } val key: String? = statusBarNotification.key if (key == null || key.isEmpty()) { - return + return false } synchronized(activeNotificationWakeKeys) { - activeNotificationWakeKeys.remove(key) + return activeNotificationWakeKeys.remove(key) } } @@ -397,106 +626,77 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { return groupKey } - private fun loadNotificationIcon(service: FaceclawMediaNotificationListenerService, notification: Notification): Drawable? { + /** Whether a notification carries a picture for loadNotificationPicture, without loading it. */ + @Suppress("DEPRECATION") + private fun hasNotificationPicture(extras: Bundle): Boolean { + return extras.get(Notification.EXTRA_PICTURE) != null + || (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && extras.get(Notification.EXTRA_PICTURE_ICON) != null) + } + + /** + * A notification's attached picture: BigPictureStyle's bitmap + * (EXTRA_PICTURE) or, from Android 12, its Icon (EXTRA_PICTURE_ICON). + * Newer releases swap large pictures for a content URI to a bitmap + * the system has offloaded (Dexcom's graph arrives that way); the + * offload provider lets enabled notification listeners open those. + */ + @Suppress("DEPRECATION") + private fun loadNotificationPicture(service: FaceclawMediaNotificationListenerService, notification: Notification): Bitmap? { + val extras: Bundle = notification.extras ?: return null try { - val smallIcon: Icon? = notification.smallIcon - if (smallIcon != null) { - val drawable: Drawable? = smallIcon.loadDrawable(service) - if (drawable != null) { - // Status-bar small icons are alpha templates: the platform - // draws them tinted and ignores their color channels, which - // apps may fill with garbage (Discord ships noise there). - // Tint white so the shape comes from alpha alone, matching - // how the status bar renders them. - drawable.mutate() - drawable.setTint(Color.WHITE) - if (drawable is BitmapDrawable) { - drawable.setFilterBitmap(true) - } - return drawable - } + val picture: Bitmap? = extras.getParcelable(Notification.EXTRA_PICTURE) + if (picture != null) { + return softwareBitmap(picture) } } catch (t: Throwable) { - Log.w(TAG, "failed to load small notification icon", t) + Log.w(TAG, "failed to read notification picture", t) + } + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) { + return null } try { - // Large icons are real color images (avatars, album art); keep color. - val largeIcon: Icon? = notification.getLargeIcon() - if (largeIcon != null) { - return largeIcon.loadDrawable(service) + val icon: Icon? = extras.getParcelable(Notification.EXTRA_PICTURE_ICON) + val drawable: Drawable? = icon?.loadDrawable(service) + if (drawable is BitmapDrawable && drawable.bitmap != null) { + return softwareBitmap(drawable.bitmap) + } + if (drawable != null && drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) { + val bitmap = Bitmap.createBitmap(drawable.intrinsicWidth, drawable.intrinsicHeight, Bitmap.Config.ARGB_8888) + drawable.setBounds(0, 0, bitmap.width, bitmap.height) + drawable.draw(Canvas(bitmap)) + return bitmap } } catch (t: Throwable) { - Log.w(TAG, "failed to load large notification icon", t) + Log.w(TAG, "failed to load notification picture icon", t) } return null } - private fun appendIconGrayBytes(drawable: Drawable, size: Int, out: ByteArrayOutputStream, - service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?) { - val bitmap = renderIconScaled(drawable, size) - dumpIconDebugPng(service, index, packageName, bitmap) - for (y in 0 until size) { - for (x in 0 until size) { - val color = bitmap.getPixel(x, y) - val alpha = Color.alpha(color) - val grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0) - val gray = Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)).toInt() - out.write(gray and 0xff) - } - } - bitmap.recycle() - } - - /** - * Render a drawable at the target size with proper downscaling. Detailed - * sources (e.g. avatar bitmaps used as notification icons) are rendered at - * native resolution and reduced by repeated halving: a single filtered pass - * from, say, 126px to 24px samples too sparsely and turns fine detail into - * speckle that reads as a garbled icon. - */ - private fun renderIconScaled(drawable: Drawable, size: Int): Bitmap { - val renderW = Math.max(size, drawable.intrinsicWidth) - val renderH = Math.max(size, drawable.intrinsicHeight) - var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888) - val canvas = Canvas(bitmap) - drawable.setBounds(0, 0, renderW, renderH) - drawable.draw(canvas) - while (bitmap.width >= size * 2 && bitmap.height >= size * 2) { - val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true) - bitmap.recycle() - bitmap = halved - } - if (bitmap.width != size || bitmap.height != size) { - val scaled = Bitmap.createScaledBitmap(bitmap, size, size, true) - bitmap.recycle() - bitmap = scaled + /** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */ + private fun softwareBitmap(bitmap: Bitmap): Bitmap { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) { + return bitmap.copy(Bitmap.Config.ARGB_8888, false) } return bitmap } /** - * Debug aid for garbled-icon reports: saves each rendered icon to - * /debug-icons/ (adb-pullable) so extraction problems can - * be told apart from downstream compositing/transmission problems. Cheap: - * runs at most once per icon-cache refresh on tiny bitmaps. + * Halve a picture while it is still at least twice the size it will + * be shown at, leaving the last (under 2x) step of fitting it into + * maxWidth x maxHeight to bitmapToGrayPacket. As with icons + * (NotificationIconLoader.renderArgb), one filtered pass from a much larger source + * samples too sparsely: a graph's thin lines break up into dots. */ - private fun dumpIconDebugPng(service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?, bitmap: Bitmap) { - if (index < 0) { - return - } - try { - val dir = java.io.File(service.getExternalFilesDir(null), "debug-icons") - if (!dir.exists() && !dir.mkdirs()) { - return + private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap { + var bitmap = source + while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) { + val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true) + if (bitmap !== source) { + bitmap.recycle() } - val safeName = if (packageName == null) "unknown" else packageName.replace(Regex("[^A-Za-z0-9._-]"), "_") - val file = java.io.File(dir, "icon-$index-$safeName.png") - java.io.FileOutputStream(file).use { stream -> - bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream) - } - } catch (t: Throwable) { - Log.w(TAG, "failed to dump debug icon", t) + bitmap = halved } + return bitmap } private fun findActiveNotificationByKey(service: FaceclawMediaNotificationListenerService?, key: String?): StatusBarNotification? { @@ -531,6 +731,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { out.put("appName", getNotificationAppName(service, statusBarNotification)) out.put("postTime", statusBarNotification.postTime) out.put("when", notification.`when`) + out.put("foregroundService", (notification.flags and Notification.FLAG_FOREGROUND_SERVICE) != 0) putString(out, "category", notification.category) if (extras != null) { putCharSequence(out, "title", firstNonEmpty( @@ -553,8 +754,10 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { } } out.put("lines", lines) + out.put("hasPicture", hasNotificationPicture(extras)) } else { out.put("lines", JSONArray()) + out.put("hasPicture", false) } val actionsJson = JSONArray() @@ -626,35 +829,34 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { } } - override fun onCreate() { - super.onCreate() - activeService = this - } - - override fun onDestroy() { - if (activeService === this) { - activeService = null - } - super.onDestroy() - } - override fun onListenerConnected() { activeService = this super.onListenerConnected() - refreshActiveNotificationWakeKeys(this) + Log.i(TAG, "notification listener connected") + resetRebindWatchdog() + reconcileActiveNotificationWakeKeys(this) + notifyListenerStateChanged() } + // Also called from onDestroy, which is how an unbind usually arrives. override fun onListenerDisconnected() { if (activeService === this) { activeService = null + lastListenerDisconnectAtMs = System.currentTimeMillis() + Log.w(TAG, "notification listener disconnected") + notifyListenerStateChanged() } super.onListenerDisconnected() + watchListenerBinding(applicationContext) } override fun onNotificationPosted(statusBarNotification: StatusBarNotification?) { super.onNotificationPosted(statusBarNotification) if (!shouldShowNotificationInList(this, statusBarNotification)) { - forgetActiveNotificationWakeKey(statusBarNotification) + // An update that takes a listed notification off the list removes it as far as the glasses go. + if (forgetActiveNotificationWakeKey(statusBarNotification)) { + emitNotificationRemoved(statusBarNotification!!.key) + } return } if (shouldEmitNotificationPosted(statusBarNotification!!)) { @@ -663,7 +865,12 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() { } override fun onNotificationRemoved(statusBarNotification: StatusBarNotification?) { - forgetActiveNotificationWakeKey(statusBarNotification) + // Includes notifications their source cancels on its own, e.g. a + // short-lived foreground-service notification that was standing in + // for one posted moments later under another key. + if (forgetActiveNotificationWakeKey(statusBarNotification)) { + emitNotificationRemoved(statusBarNotification!!.key) + } super.onNotificationRemoved(statusBarNotification) } } diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt index 6ae3b8ab..6c6d4a60 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawPreviewCompositor.kt @@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) { java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green) } + /** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */ + fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? { + val composite = compositor.previewComposite() ?: return null + return PreviewBitmapUtil.fromGray( + java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green, + cropX, cropY, cropWidth, cropHeight) + } + /** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */ @Throws(java.io.IOException::class) fun saveCompositePngScreenshot(): String { diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawReverseGeocoder.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawReverseGeocoder.kt new file mode 100644 index 00000000..f482eb24 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawReverseGeocoder.kt @@ -0,0 +1,51 @@ +package com.faceclaw.app + +import android.content.Context +import android.location.Address +import android.location.Geocoder +import android.os.Handler +import android.os.Looper +import android.util.Log +import java.util.Locale +import kotlin.concurrent.thread + +/** + * Place name for the Weather header where the weather source doesn't supply + * one (outside National Weather Service coverage), from Android's own + * Geocoder. Its lookup blocks on the network, so it runs on a thread of its + * own; the listener hears back on the main thread, with nulls when the phone + * has no geocoder or the lookup fails. + */ +class FaceclawReverseGeocoder(context: Context) { + interface Listener { + fun onResult(locality: String?, region: String?, country: String?) + } + + companion object { + private const val TAG = "FaceclawGeocoder" + } + + private val context: Context = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + + fun lookup(latitude: Double, longitude: Double, listener: Listener) { + thread(name = "FaceclawReverseGeocoder", isDaemon = true) { + val address = lookupBlocking(latitude, longitude) + mainHandler.post { listener.onResult(address?.locality, address?.adminArea, address?.countryName) } + } + } + + private fun lookupBlocking(latitude: Double, longitude: Double): Address? { + if (!Geocoder.isPresent()) { + return null + } + return try { + // The listener form needs API 33; this one is fine off the main thread. + @Suppress("DEPRECATION") + Geocoder(context, Locale.getDefault()).getFromLocation(latitude, longitude, 1)?.firstOrNull() + } catch (error: Throwable) { + Log.w(TAG, "reverse geocode failed: $error") + null + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt index b73baee1..1df754f9 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/ImageFileLoader.kt @@ -132,10 +132,15 @@ class ImageFileLoader private constructor() { * round on its own, which turns smooth gradients into visible bands. * Dithered output is already quantized: every byte is a level times 16, * which BmpUtil re-quantizes back to exactly that level. + * + * invertLightBackground shows a picture drawn on a light background + * (a chart, a screenshot) as its negative; see GrayPacket.fromArgb. */ @JvmStatic + @JvmOverloads fun bitmapToGrayPacket( - source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray { + source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, + invertLightBackground: Boolean = false): ByteArray { if (source == null || source.width <= 0 || source.height <= 0 || maxWidth <= 0 || maxHeight <= 0) { return ByteArray(0) @@ -152,7 +157,7 @@ class ImageFileLoader private constructor() { } val pixels = IntArray(width * height) scaled.getPixels(pixels, 0, width, 0, 0, width, height) - return GrayPacket.fromArgb(pixels, width, height, gamma, dither) + return GrayPacket.fromArgb(pixels, width, height, gamma, dither, invertLightBackground) } catch (e: Exception) { Log.w(TAG, "bitmap conversion failed", e) return ByteArray(0) diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationIconLoader.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationIconLoader.kt new file mode 100644 index 00000000..9d4a8eef --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/NotificationIconLoader.kt @@ -0,0 +1,290 @@ +package com.faceclaw.app + +import android.app.Notification +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint +import android.graphics.Rect +import android.graphics.Typeface +import android.graphics.drawable.AdaptiveIconDrawable +import android.graphics.drawable.BitmapDrawable +import android.graphics.drawable.Drawable +import android.graphics.drawable.Icon +import android.os.Build +import android.os.SystemClock +import android.service.notification.StatusBarNotification +import android.util.Log + +import com.faceclaw.app.NotificationIconArt.Kind + +import java.io.File +import java.io.FileOutputStream + +import kotlin.math.max +import kotlin.math.roundToInt + +/** + * Picks and renders the glasses' status-bar icon for a notification (see iconFor). + * NotificationIconArt does the pixel analysis; this object owns the drawables, package lookups, + * caching and the debug dump. + */ +internal object NotificationIconLoader { + private const val TAG = "FaceclawNotifyIcon" + + /** AdaptiveIconDrawable draws its layers this share of the icon's size past each edge. */ + private const val ADAPTIVE_LAYER_OUTSET = 0.25f + + private const val CACHE_MAX_ENTRIES = 128 + /** Cached art is redone after this long, so an updated app's new icon shows up. */ + private const val CACHE_MAX_AGE_MS = 30 * 60 * 1000L + + /** The debug dump keeps at most this many files, deleting the oldest. */ + private const val DEBUG_DUMP_MAX_FILES = 96 + + class Rendered( + /** The drawable it came from: small, monochrome, launcher, foreground, large or letter. */ + val source: String, + val art: NotificationIconArt.Art, + ) + + /** + * An app's launcher icon art: its themed monochrome layer, else a glyph or shape read from + * the icon (or its adaptive foreground layer), else the icon read by brightness. + */ + private class LauncherArt(val monochrome: Rendered?, val shape: Rendered?, val picture: Rendered?) + + private val smallIconCache = ArtCache(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS) + private val launcherCache = ArtCache(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS) + + /** + * The icon for a notification, size x size gray, from the first of these that works: + * 1. the small icon, when its alpha carries a shape (nearly every app); + * 2. the launcher icon's themed monochrome layer (Android 13+), which is made to be tinted; + * 3. the small icon's glyph, when it is a color tile with one background color; + * 4. the launcher icon's glyph or shape, or its adaptive foreground layer's shape; + * 5. the small icon, then the launcher icon, read by brightness; + * 6. the large icon (usually the sender's avatar: the notification's content, not its app); + * 7. the app name's initial in a rounded box. + * Null only when even the initial can't be drawn. + */ + fun iconFor(context: Context, statusBarNotification: StatusBarNotification, size: Int, appName: () -> String): Rendered? { + val packageName = statusBarNotification.packageName ?: "" + val notification = statusBarNotification.notification + val small = smallIconArt(context, packageName, notification.smallIcon, size) + val smallKind = small?.art?.kind + if (smallKind == Kind.TEMPLATE || smallKind == Kind.SOLID) return small + val launcher = launcherArt(context, packageName, size) + launcher.monochrome?.let { return it } + if (smallKind == Kind.GLYPH) return small + launcher.shape?.let { return it } + if (smallKind == Kind.PICTURE) return small + launcher.picture?.let { return it } + largeIconArt(context, packageName, notification, size)?.let { return it } + return letterArt(appName(), size) + } + + private fun smallIconArt(context: Context, packageName: String, icon: Icon?, size: Int): Rendered? { + if (icon == null) return null + val key = resourceKey(icon) ?: return loadSmallIcon(context, packageName, icon, size) + return smallIconCache.getOrPut("$packageName|$key|$size") { loadSmallIcon(context, packageName, icon, size) } + } + + /** Resource icons are cached by resource; bitmap and URI icons are converted every time. */ + private fun resourceKey(icon: Icon): String? { + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P || icon.type != Icon.TYPE_RESOURCE) return null + return icon.resPackage + "/" + icon.resId + } + + private fun loadSmallIcon(context: Context, packageName: String, icon: Icon, size: Int): Rendered? = + tryRender("small", packageName) { + val drawable = icon.loadDrawable(context) ?: return@tryRender null + // A tint (the Icon's own, or a vector's theme tint) recolors the very pixels that tell + // a template from a color tile, and can fade a template's alpha. Templates are read + // by alpha alone, so dropping it costs nothing. + drawable.mutate() + drawable.setTintList(null) + renderSource(context, packageName, "small", drawable, size) + } + + private fun launcherArt(context: Context, packageName: String, size: Int): LauncherArt = + launcherCache.getOrPut("$packageName|$size") { loadLauncherArt(context, packageName, size) } + + private fun loadLauncherArt(context: Context, packageName: String, size: Int): LauncherArt { + val icon: Drawable = try { + context.packageManager.getApplicationIcon(packageName) + } catch (t: Throwable) { + Log.w(TAG, "no launcher icon for $packageName", t) + return LauncherArt(null, null, null) + } + val adaptive = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) icon as? AdaptiveIconDrawable else null + if (adaptive != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) { + val layer = adaptive.monochrome + val monochrome = if (layer == null) null else tryRender("monochrome", packageName) { + renderSource(context, packageName, "monochrome", layer, size, ADAPTIVE_LAYER_OUTSET, template = true) + } + if (monochrome != null) return LauncherArt(monochrome, null, null) + } + val whole = tryRender("launcher", packageName) { renderSource(context, packageName, "launcher", icon, size) } + var shape = whole?.takeIf { it.art.kind == Kind.TEMPLATE || it.art.kind == Kind.GLYPH } + val foreground = adaptive?.foreground + if (shape == null && foreground != null) { + // A gradient or picture background hides the glyph; the foreground layer alone may carry it as alpha. + shape = tryRender("foreground", packageName) { + renderSource(context, packageName, "foreground", foreground, size, ADAPTIVE_LAYER_OUTSET) + }?.takeIf { it.art.kind == Kind.TEMPLATE } + } + return LauncherArt(null, shape, whole?.takeIf { it.art.kind == Kind.PICTURE }) + } + + private fun largeIconArt(context: Context, packageName: String, notification: Notification, size: Int): Rendered? = + tryRender("large", packageName) { + val drawable = notification.getLargeIcon()?.loadDrawable(context) ?: return@tryRender null + renderSource(context, packageName, "large", drawable, size) + } + + /** The app name's first letter or digit in a rounded box, like the shell's letter window icons. */ + private fun letterArt(appName: String, size: Int): Rendered? = tryRender("letter", appName) { + val letter = appName.firstOrNull { it.isLetterOrDigit() }?.uppercaseChar()?.toString() ?: "?" + val canvasSize = size * NotificationIconArt.ANALYSIS_SCALE + val bitmap = Bitmap.createBitmap(canvasSize, canvasSize, Bitmap.Config.ARGB_8888) + val canvas = Canvas(bitmap) + val stroke = NotificationIconArt.ANALYSIS_SCALE.toFloat() + val box = Paint(Paint.ANTI_ALIAS_FLAG).apply { + style = Paint.Style.STROKE + strokeWidth = stroke + color = Color.argb(160, 255, 255, 255) + } + val radius = canvasSize / 4f + canvas.drawRoundRect(stroke / 2, stroke / 2, canvasSize - stroke / 2, canvasSize - stroke / 2, radius, radius, box) + val text = Paint(Paint.ANTI_ALIAS_FLAG).apply { + color = Color.WHITE + typeface = Typeface.DEFAULT_BOLD + textSize = canvasSize * 0.6f + textAlign = Paint.Align.CENTER + } + val bounds = Rect() + text.getTextBounds(letter, 0, letter.length, bounds) + canvas.drawText(letter, canvasSize / 2f, canvasSize / 2f - bounds.exactCenterY(), text) + val argb = IntArray(canvasSize * canvasSize) + bitmap.getPixels(argb, 0, canvasSize, 0, 0, canvasSize, canvasSize) + bitmap.recycle() + val art = NotificationIconArt.template(argb, size) + if (art.kind == Kind.EMPTY) null else Rendered("letter", art) + } + + private inline fun tryRender(source: String, owner: String, render: () -> Rendered?): Rendered? { + return try { + render() + } catch (t: Throwable) { + Log.w(TAG, "failed to render $source icon for $owner", t) + null + } + } + + /** Render, dump and convert one source drawable; null when nothing visible came out. */ + private fun renderSource( + context: Context, packageName: String, source: String, drawable: Drawable, size: Int, + outset: Float = 0f, template: Boolean = false, + ): Rendered? { + val analysisSize = size * NotificationIconArt.ANALYSIS_SCALE + val argb = renderArgb(drawable, analysisSize, outset) + dumpDebugPng(context, packageName, source, argb, analysisSize) + val art = if (template) NotificationIconArt.template(argb, size) else NotificationIconArt.convert(argb, size) + return if (art.kind == Kind.EMPTY) null else Rendered(source, art) + } + + /** + * Render a drawable to target x target ARGB with proper downscaling. Detailed sources (e.g. + * avatar bitmaps used as notification icons) are rendered at native resolution and reduced + * by repeated halving: a single filtered pass from, say, 126px to 24px samples too sparsely + * and turns fine detail into speckle that reads as a garbled icon. An outset draws the + * drawable that share of the canvas past each edge, as AdaptiveIconDrawable draws its layers. + */ + private fun renderArgb(drawable: Drawable, target: Int, outset: Float): IntArray { + drawable.mutate() + if (drawable is BitmapDrawable) { + drawable.setFilterBitmap(true) + } + val scale = 1f + 2f * outset + val renderW = max(target, (drawable.intrinsicWidth / scale).roundToInt()) + val renderH = max(target, (drawable.intrinsicHeight / scale).roundToInt()) + var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888) + val dx = (renderW * outset).roundToInt() + val dy = (renderH * outset).roundToInt() + drawable.setBounds(-dx, -dy, renderW + dx, renderH + dy) + drawable.draw(Canvas(bitmap)) + while (bitmap.width >= target * 2 && bitmap.height >= target * 2) { + val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true) + bitmap.recycle() + bitmap = halved + } + if (bitmap.width != target || bitmap.height != target) { + val scaled = Bitmap.createScaledBitmap(bitmap, target, target, true) + bitmap.recycle() + bitmap = scaled + } + val pixels = IntArray(target * target) + bitmap.getPixels(pixels, 0, target, 0, 0, target, target) + bitmap.recycle() + return pixels + } + + /** + * Debug aid for icon reports: saves each source as rendered for analysis, before any + * conversion, to /debug-icons/-.png (adb-pullable), so a + * bad icon can be rerun through NotificationIconArt off-device. Runs only when a source is + * converted, not on cached icons; the oldest files past DEBUG_DUMP_MAX_FILES are deleted. + */ + private fun dumpDebugPng(context: Context, packageName: String, source: String, argb: IntArray, size: Int) { + try { + val base = context.getExternalFilesDir(null) ?: return + val dir = File(base, "debug-icons") + if (!dir.exists() && !dir.mkdirs()) { + return + } + val safeName = packageName.replace(Regex("[^A-Za-z0-9._-]"), "_") + val bitmap = Bitmap.createBitmap(argb, size, size, Bitmap.Config.ARGB_8888) + try { + FileOutputStream(File(dir, "$safeName-$source.png")).use { stream -> + bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream) + } + } finally { + bitmap.recycle() + } + val files = dir.listFiles() ?: return + if (files.size > DEBUG_DUMP_MAX_FILES) { + files.sortBy { it.lastModified() } + for (index in 0 until files.size - DEBUG_DUMP_MAX_FILES) { + files[index].delete() + } + } + } catch (t: Throwable) { + Log.w(TAG, "failed to dump debug icon", t) + } + } + + /** A small LRU of converted art with an age limit. Conversions run under its lock. */ + private class ArtCache(private val maxEntries: Int, private val maxAgeMs: Long) { + private class Entry(val value: T, val atMs: Long) + + private val entries = object : LinkedHashMap>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry>): Boolean { + return size > maxEntries + } + } + + @Synchronized + fun getOrPut(key: String, compute: () -> V): V { + val now = SystemClock.elapsedRealtime() + val cached = entries[key] + if (cached != null && now - cached.atMs < maxAgeMs) { + return cached.value + } + val value = compute() + entries[key] = Entry(value, now) + return value + } + } +} diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt index 6c98b9b0..75135480 100644 --- a/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/PreviewBitmapUtil.kt @@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() { return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888) } + /** + * As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the + * mirror's Scaled preview). The area may extend past the frame's edges; it is black + * there. + */ + @JvmStatic + fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean, + cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap { + if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) { + throw IllegalArgumentException("invalid gray preview buffer") + } + if (cropWidth <= 0 || cropHeight <= 0) { + throw IllegalArgumentException("invalid preview crop") + } + val bytes = ByteArray(width * height) + gray.get(bytes) + val cropped = ByteArray(cropWidth * cropHeight) + val left = maxOf(cropX, 0) + val right = minOf(cropX + cropWidth, width) + if (right > left) { + for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) { + System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left) + } + } + val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green) + return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888) + } + /** * A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes * alpha; black, which the glasses show as unlit, is transparent), for diff --git a/App_Resources/iOS/Info.plist b/App_Resources/iOS/Info.plist index 0f165231..61957b6e 100644 --- a/App_Resources/iOS/Info.plist +++ b/App_Resources/iOS/Info.plist @@ -13,7 +13,7 @@ CFBundleDisplayName Faceclaw CFBundleShortVersionString - 1.0.0 + 0.8.4 CFBundleVersion 1 LSRequiresIPhoneOS @@ -30,6 +30,8 @@ Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses. NSCalendarsFullAccessUsageDescription Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses. + NSAppleMusicUsageDescription + Faceclaw shows what is playing in Apple Music on your glasses, controls playback, and lets you browse and play your music library from the glasses. NSLocationWhenInUseUsageDescription Faceclaw uses your location for local weather, turn-by-turn navigation, and true north on the glasses compass. NSLocationTemporaryUsageDescriptionDictionary diff --git a/App_Resources/iOS/PrivacyInfo.xcprivacy b/App_Resources/iOS/PrivacyInfo.xcprivacy new file mode 100644 index 00000000..eb1d1561 --- /dev/null +++ b/App_Resources/iOS/PrivacyInfo.xcprivacy @@ -0,0 +1,30 @@ + + + + + NSPrivacyAccessedAPITypes + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryUserDefaults + NSPrivacyAccessedAPITypeReasons + CA92.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategoryFileTimestamp + NSPrivacyAccessedAPITypeReasons + C617.1 + + + + NSPrivacyAccessedAPIType + NSPrivacyAccessedAPICategorySystemBootTime + NSPrivacyAccessedAPITypeReasons + 35F9.1 + + + + diff --git a/App_Resources/iOS/build.xcconfig b/App_Resources/iOS/build.xcconfig index f657ddce..03805280 100644 --- a/App_Resources/iOS/build.xcconfig +++ b/App_Resources/iOS/build.xcconfig @@ -2,8 +2,8 @@ IPHONEOS_DEPLOYMENT_TARGET = 15.1; TARGETED_DEVICE_FAMILY = 1; CODE_SIGN_STYLE = Automatic; -// Select the local signing team in Xcode or pass --teamId to the iOS scripts. -OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit +// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team. +OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; diff --git a/App_Resources/iOS/src/FaceclawAppleMusic.h b/App_Resources/iOS/src/FaceclawAppleMusic.h new file mode 100644 index 00000000..50ca5c4d --- /dev/null +++ b/App_Resources/iOS/src/FaceclawAppleMusic.h @@ -0,0 +1,41 @@ +#import + +/** + * Apple Music (the system Music app) through the MediaPlayer framework: now-playing state + * and transport on the system player, the media library as a browse tree, and the system + * output volume. Call from the main thread; callbacks are delivered on the main queue. + * Nothing touches the player or library until media-library access is granted, so no + * permission prompt appears unasked. + */ +@interface FaceclawAppleMusic : NSObject ++ (instancetype)shared; +/** Playback state, the now-playing item or authorization may have changed. */ +@property(nonatomic, copy) void (^changeHandler)(void); +- (BOOL)hasPermission; +- (NSString *)permissionStatus; +- (void)requestPermission:(void (^)(BOOL granted))completion; +/** Observe the system player (and keep observing across authorization changes). */ +- (void)start; +- (void)stop; +/** The system player as a JSON object; {"access": false} until access is granted. */ +- (NSString *)stateJson; +- (void)playPause; +- (void)skipNext; +- (void)skipPrevious; +/** + * The queue Faceclaw last started, as [{"id", "title", "active"}], while the player is + * still playing from it; "[]" otherwise (the system player does not expose queues that + * the Music app started). + */ +- (NSString *)queueJson; +- (void)skipToQueueIndex:(NSInteger)index; +/** [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), or nil. */ +- (NSData *)albumArt:(NSInteger)maxSize; +/** System output volume 0..100, or -1 when unknown. */ +- (double)volumePercent; +- (void)setVolumePercent:(double)percent; +/** One level of the library tree ("root" first) as [{"mediaId", "title", "subtitle", "browsable", "playable"}]. */ +- (void)browse:(NSString *)parentId completion:(void (^)(NSString *json, NSString *error))completion; +/** Start playback of a browsed collection, or of a song within its collection. */ +- (void)play:(NSString *)mediaId completion:(void (^)(NSString *error))completion; +@end diff --git a/App_Resources/iOS/src/FaceclawAppleMusic.m b/App_Resources/iOS/src/FaceclawAppleMusic.m new file mode 100644 index 00000000..d6b8cef0 --- /dev/null +++ b/App_Resources/iOS/src/FaceclawAppleMusic.m @@ -0,0 +1,512 @@ +#import "FaceclawAppleMusic.h" +#import +#import +#import +#import +#import +#import + +/** How long a queue Faceclaw just set may take to become the player's now-playing context. */ +static const NSTimeInterval kQueueConfirmSeconds = 5.0; + +static NSString *JsonString(id object, NSString *fallback) { + NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil]; + return data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : fallback; +} + +/** Persistent IDs are unsigned 64-bit, beyond JavaScript's exact integers, so they cross as decimal strings. */ +static NSString *IdString(MPMediaEntityPersistentID persistentId) { + return [NSString stringWithFormat:@"%llu", persistentId]; +} + +static BOOL ParseId(NSString *text, MPMediaEntityPersistentID *out) { + const char *chars = text.UTF8String; + if (!chars || chars[0] < '0' || chars[0] > '9') return NO; + char *end = NULL; + errno = 0; + unsigned long long value = strtoull(chars, &end, 10); + if (errno || !end || *end) return NO; + *out = value; + return YES; +} + +static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *subtitle, BOOL browsable, BOOL playable) { + return @{@"mediaId": mediaId, @"title": title ?: @"", @"subtitle": subtitle ?: @"", + @"browsable": @(browsable), @"playable": @(playable)}; +} + +@interface FaceclawAppleMusic () +@property(nonatomic, strong) dispatch_queue_t libraryQueue; +@property(nonatomic) BOOL started; +@property(nonatomic) BOOL observing; +/** Bumped whenever the now-playing item changes, so cached album art is refetched. */ +@property(nonatomic) NSInteger artVersion; +@property(nonatomic, copy) NSString *nowPlayingKey; +/** The items of the queue Faceclaw last started, until the player leaves them. */ +@property(nonatomic, copy) NSArray *queueItems; +@property(nonatomic) BOOL queueConfirmed; +@property(nonatomic, strong) NSDate *queueSetAt; +@property(nonatomic, strong) MPVolumeView *volumeView; +@end + +@implementation FaceclawAppleMusic ++ (instancetype)shared { + static FaceclawAppleMusic *instance; + static dispatch_once_t once; + dispatch_once(&once, ^{ instance = [FaceclawAppleMusic new]; }); + return instance; +} + +- (instancetype)init { + if ((self = [super init])) { + _libraryQueue = dispatch_queue_create("com.faceclaw.applemusic", DISPATCH_QUEUE_SERIAL); + _nowPlayingKey = @""; + // Access may have been granted or revoked in Settings while away. + [NSNotificationCenter.defaultCenter addObserver:self selector:@selector(appBecameActive:) + name:UIApplicationDidBecomeActiveNotification object:nil]; + } + return self; +} + +#pragma mark - Authorization + +- (BOOL)hasPermission { + return MPMediaLibrary.authorizationStatus == MPMediaLibraryAuthorizationStatusAuthorized; +} + +- (NSString *)permissionStatus { + switch (MPMediaLibrary.authorizationStatus) { + case MPMediaLibraryAuthorizationStatusAuthorized: return @"granted"; + case MPMediaLibraryAuthorizationStatusDenied: return @"denied"; + case MPMediaLibraryAuthorizationStatusRestricted: return @"restricted"; + default: return @"not-determined"; + } +} + +- (void)requestPermission:(void (^)(BOOL))completion { + if ([self hasPermission]) { + completion(YES); + return; + } + [MPMediaLibrary requestAuthorization:^(MPMediaLibraryAuthorizationStatus status) { + dispatch_async(dispatch_get_main_queue(), ^{ + [self syncObserving]; + [self notifyChanged]; + completion(status == MPMediaLibraryAuthorizationStatusAuthorized); + }); + }]; +} + +- (void)appBecameActive:(NSNotification *)notification { + [self syncObserving]; + [self notifyChanged]; +} + +#pragma mark - Observation + +- (void)start { + self.started = YES; + [self syncObserving]; +} + +- (void)stop { + self.started = NO; + [self syncObserving]; +} + +- (void)syncObserving { + BOOL want = self.started && [self hasPermission]; + if (want == self.observing) return; + self.observing = want; + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + NSNotificationCenter *center = NSNotificationCenter.defaultCenter; + if (want) { + [center addObserver:self selector:@selector(playbackStateChanged:) + name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player]; + [center addObserver:self selector:@selector(nowPlayingItemChanged:) + name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player]; + [player beginGeneratingPlaybackNotifications]; + } else { + [center removeObserver:self name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player]; + [center removeObserver:self name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player]; + [player endGeneratingPlaybackNotifications]; + } +} + +- (void)playbackStateChanged:(NSNotification *)notification { + [self notifyChanged]; +} + +- (void)nowPlayingItemChanged:(NSNotification *)notification { + // Streamed catalog items can share identity fields, so every change counts. + self.artVersion++; + [self notifyChanged]; +} + +- (void)notifyChanged { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self.changeHandler) self.changeHandler(); + }); +} + +#pragma mark - Now playing + +- (NSString *)stateJson { + if (![self hasPermission]) return @"{\"access\":false}"; + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + MPMediaItem *item = player.nowPlayingItem; + NSString *key = item ? [NSString stringWithFormat:@"%llu|%@|%@|%@", item.persistentID, + item.playbackStoreID ?: @"", item.title ?: @"", item.artist ?: @""] : @""; + if (![key isEqualToString:self.nowPlayingKey]) { + self.nowPlayingKey = key; + self.artVersion++; + } + NSString *state; + switch (player.playbackState) { + case MPMusicPlaybackStatePlaying: + case MPMusicPlaybackStateSeekingForward: + case MPMusicPlaybackStateSeekingBackward: + state = @"playing"; + break; + case MPMusicPlaybackStatePaused: + case MPMusicPlaybackStateInterrupted: + state = @"paused"; + break; + default: + state = @"stopped"; + break; + } + NSTimeInterval position = player.currentPlaybackTime; + NSTimeInterval duration = item.playbackDuration; + float rate = player.currentPlaybackRate; + return JsonString(@{ + @"access": @YES, + @"hasItem": @(item != nil), + @"state": state, + @"title": item.title ?: @"", + @"artist": item.artist ?: @"", + @"album": item.albumTitle ?: @"", + @"positionMs": @(item && isfinite(position) && position >= 0 ? position * 1000.0 : -1), + @"durationMs": @(item && isfinite(duration) && duration > 0 ? duration * 1000.0 : -1), + @"rate": @(isfinite(rate) ? rate : 0), + @"artVersion": @(self.artVersion), + }, @"{\"access\":true}"); +} + +- (void)playPause { + if (![self hasPermission]) return; + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + switch (player.playbackState) { + case MPMusicPlaybackStatePlaying: + case MPMusicPlaybackStateSeekingForward: + case MPMusicPlaybackStateSeekingBackward: + [player pause]; + break; + default: + [player play]; + break; + } +} + +- (void)skipNext { + if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToNextItem]; +} + +- (void)skipPrevious { + if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToPreviousItem]; +} + +- (NSData *)albumArt:(NSInteger)maxSize { + if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil; + MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork; + UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)]; + CGImageRef source = image.CGImage; + if (!source) return nil; + double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source); + if (sourceWidth <= 0 || sourceHeight <= 0) return nil; + double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight)); + NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale)); + NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4]; + uint8_t *bytes = packet.mutableBytes; + bytes[0] = width & 255; bytes[1] = (width >> 8) & 255; + bytes[2] = height & 255; bytes[3] = (height >> 8) & 255; + uint8_t *pixels = bytes + 4; + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + // Memory order B, G, R, A: one little-endian ARGB word per pixel. + CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space, + kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst); + CGColorSpaceRelease(space); + if (!context) return nil; + CGContextSetInterpolationQuality(context, kCGInterpolationHigh); + CGContextDrawImage(context, CGRectMake(0, 0, width, height), source); + CGContextRelease(context); + // Composited over black (the display's see-through background); now opaque. + for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255; + return packet; +} + +#pragma mark - Queue + +/** Index of the now-playing item in Faceclaw's queue; forgets the queue once the player has left it. */ +- (NSInteger)activeQueueIndex { + NSArray *items = self.queueItems; + if (!items.count || ![self hasPermission]) return -1; + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + MPMediaItem *now = player.nowPlayingItem; + NSInteger found = -1; + if (now) { + NSUInteger index = player.indexOfNowPlayingItem; + if (index < items.count && items[index].persistentID == now.persistentID) { + found = (NSInteger)index; + } else { + for (NSUInteger i = 0; i < items.count; i++) { + if (items[i].persistentID == now.persistentID) { found = (NSInteger)i; break; } + } + } + } + if (found >= 0) { + self.queueConfirmed = YES; + return found; + } + // Either the player hasn't switched over yet, or it has moved on to another queue. + if (self.queueConfirmed || -[self.queueSetAt timeIntervalSinceNow] > kQueueConfirmSeconds) { + self.queueItems = nil; + } + return -1; +} + +- (NSString *)queueJson { + NSInteger active = [self activeQueueIndex]; + if (active < 0) return @"[]"; + NSMutableArray *rows = [NSMutableArray arrayWithCapacity:self.queueItems.count]; + [self.queueItems enumerateObjectsUsingBlock:^(MPMediaItem *item, NSUInteger index, BOOL *stop) { + [rows addObject:@{@"id": @(index), @"title": item.title ?: @"", @"active": @((NSInteger)index == active)}]; + }]; + return JsonString(rows, @"[]"); +} + +- (void)skipToQueueIndex:(NSInteger)index { + if ([self activeQueueIndex] < 0 || index < 0 || index >= (NSInteger)self.queueItems.count) return; + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + player.nowPlayingItem = self.queueItems[index]; + [player play]; +} + +#pragma mark - Volume + +- (UIWindow *)hostWindow { + for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { + if (![scene isKindOfClass:UIWindowScene.class]) continue; + NSArray *windows = ((UIWindowScene *)scene).windows; + for (UIWindow *window in windows) if (window.isKeyWindow) return window; + if (windows.firstObject) return windows.firstObject; + } + return nil; +} + +/** + * The slider of an off-screen MPVolumeView: iOS has no public setter for the system volume, + * but the volume view's slider drives it while the view is in a window. + */ +- (UISlider *)volumeSlider:(BOOL *)attached { + if (!self.volumeView) { + self.volumeView = [[MPVolumeView alloc] initWithFrame:CGRectMake(-2000, -2000, 160, 40)]; + } + if (!self.volumeView.window) { + UIWindow *window = [self hostWindow]; + if (!window) return nil; + [window addSubview:self.volumeView]; + if (attached) *attached = YES; + } + for (UIView *view in self.volumeView.subviews) { + if ([view isKindOfClass:UISlider.class]) return (UISlider *)view; + } + return nil; +} + +- (double)volumePercent { + // The slider tracks the system volume once attached; until then (nothing has been set + // yet) the session's reading, which can lag while Faceclaw's audio session is inactive. + UISlider *slider = self.volumeView.window ? [self volumeSlider:NULL] : nil; + float volume = slider ? slider.value : AVAudioSession.sharedInstance.outputVolume; + return isfinite(volume) ? MAX(0.0, MIN(100.0, round(volume * 100.0))) : -1; +} + +- (void)setVolumePercent:(double)percent { + if (!isfinite(percent)) return; + BOOL attached = NO; + UISlider *slider = [self volumeSlider:&attached]; + if (!slider) return; + float value = (float)MAX(0.0, MIN(1.0, percent / 100.0)); + void (^apply)(void) = ^{ + [slider setValue:value animated:NO]; + [slider sendActionsForControlEvents:UIControlEventValueChanged]; + }; + // A freshly attached volume view ignores changes until it has wired up its slider. + if (attached) dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.15 * NSEC_PER_SEC)), dispatch_get_main_queue(), apply); + else apply(); +} + +#pragma mark - Library + +/** + * Songs of a playable collection id: "playlist:", "album:", "artist:" (album by + * album, in track order) or "songs" (the whole library). Runs on the library queue. + */ +- (NSArray *)itemsForCollection:(NSString *)collectionId { + if ([collectionId isEqualToString:@"songs"]) return [MPMediaQuery songsQuery].items ?: @[]; + NSRange colon = [collectionId rangeOfString:@":"]; + MPMediaEntityPersistentID persistentId; + if (colon.location == NSNotFound || !ParseId([collectionId substringFromIndex:colon.location + 1], &persistentId)) return @[]; + NSString *kind = [collectionId substringToIndex:colon.location]; + MPMediaQuery *query; + NSString *property; + if ([kind isEqualToString:@"playlist"]) { + query = [MPMediaQuery playlistsQuery]; + property = MPMediaPlaylistPropertyPersistentID; + } else if ([kind isEqualToString:@"album"]) { + query = [MPMediaQuery albumsQuery]; + property = MPMediaItemPropertyAlbumPersistentID; + } else if ([kind isEqualToString:@"artist"]) { + query = [MPMediaQuery albumsQuery]; + property = MPMediaItemPropertyArtistPersistentID; + } else { + return @[]; + } + [query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(persistentId) forProperty:property]]; + NSMutableArray *items = [NSMutableArray new]; + for (MPMediaItemCollection *collection in query.collections) [items addObjectsFromArray:collection.items]; + return items; +} + +- (NSArray *)songRows:(NSArray *)items context:(NSString *)collectionId { + NSMutableArray *rows = [NSMutableArray arrayWithCapacity:items.count]; + for (MPMediaItem *item in items) { + [rows addObject:BrowseRow([NSString stringWithFormat:@"song:%@@%@", IdString(item.persistentID), collectionId], + item.title, item.artist, NO, YES)]; + } + return rows; +} + +- (NSArray *)albumRows:(MPMediaQuery *)query { + NSMutableArray *rows = [NSMutableArray new]; + for (MPMediaItemCollection *album in query.collections) { + MPMediaItem *item = album.representativeItem; + if (!item) continue; + [rows addObject:BrowseRow([@"album:" stringByAppendingString:IdString(item.albumPersistentID)], + item.albumTitle, item.albumArtist.length ? item.albumArtist : item.artist, YES, YES)]; + } + return rows; +} + +- (NSArray *)childrenOf:(NSString *)parentId { + if ([parentId isEqualToString:@"root"]) { + return @[BrowseRow(@"playlists", @"Playlists", nil, YES, NO), BrowseRow(@"artists", @"Artists", nil, YES, NO), + BrowseRow(@"albums", @"Albums", nil, YES, NO), BrowseRow(@"songs", @"Songs", nil, YES, NO)]; + } + if ([parentId isEqualToString:@"playlists"]) { + NSMutableArray *rows = [NSMutableArray new]; + for (MPMediaPlaylist *playlist in [MPMediaQuery playlistsQuery].collections) { + NSUInteger count = playlist.count; + NSString *subtitle = count == 1 ? @"1 song" : [NSString stringWithFormat:@"%lu songs", (unsigned long)count]; + [rows addObject:BrowseRow([@"playlist:" stringByAppendingString:IdString(playlist.persistentID)], + playlist.name, subtitle, YES, count > 0)]; + } + return rows; + } + if ([parentId isEqualToString:@"artists"]) { + NSMutableArray *rows = [NSMutableArray new]; + for (MPMediaItemCollection *artist in [MPMediaQuery artistsQuery].collections) { + MPMediaItem *item = artist.representativeItem; + if (!item) continue; + [rows addObject:BrowseRow([@"artist:" stringByAppendingString:IdString(item.artistPersistentID)], + item.artist.length ? item.artist : @"Unknown Artist", nil, YES, YES)]; + } + return rows; + } + if ([parentId isEqualToString:@"albums"]) return [self albumRows:[MPMediaQuery albumsQuery]]; + if ([parentId isEqualToString:@"songs"]) return [self songRows:[MPMediaQuery songsQuery].items ?: @[] context:@"songs"]; + if ([parentId hasPrefix:@"artist:"]) { + MPMediaEntityPersistentID artistId; + if (!ParseId([parentId substringFromIndex:7], &artistId)) return nil; + MPMediaQuery *query = [MPMediaQuery albumsQuery]; + [query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(artistId) + forProperty:MPMediaItemPropertyArtistPersistentID]]; + return [self albumRows:query]; + } + if ([parentId hasPrefix:@"playlist:"] || [parentId hasPrefix:@"album:"]) { + return [self songRows:[self itemsForCollection:parentId] context:parentId]; + } + return nil; +} + +- (void)browse:(NSString *)parentId completion:(void (^)(NSString *, NSString *))completion { + // Library queries can take a while on large libraries; keep them off the main thread. + dispatch_async(self.libraryQueue, ^{ + NSString *json = nil, *error = nil; + if (![self hasPermission]) { + error = @"Apple Music access is not allowed"; + } else { + @try { + NSArray *rows = [self childrenOf:parentId ?: @""]; + if (rows) json = JsonString(rows, nil); + if (!json) error = rows ? @"Library listing failed" : @"Unknown library item"; + } @catch (NSException *exception) { + error = exception.reason ?: @"Library unavailable"; + } + } + dispatch_async(dispatch_get_main_queue(), ^{ completion(json, error); }); + }); +} + +- (void)play:(NSString *)mediaId completion:(void (^)(NSString *))completion { + dispatch_async(self.libraryQueue, ^{ + NSString *collectionId = mediaId ?: @""; + MPMediaEntityPersistentID startId = 0; + BOOL hasStart = NO; + if ([collectionId hasPrefix:@"song:"]) { + NSRange at = [collectionId rangeOfString:@"@"]; + if (at.location == NSNotFound) collectionId = @""; + else { + hasStart = ParseId([collectionId substringWithRange:NSMakeRange(5, at.location - 5)], &startId); + collectionId = [collectionId substringFromIndex:at.location + 1]; + } + } + NSArray *items = @[]; + NSString *error = nil; + if (![self hasPermission]) { + error = @"Apple Music access is not allowed"; + } else { + @try { + items = [self itemsForCollection:collectionId]; + } @catch (NSException *exception) { + error = exception.reason ?: @"Library unavailable"; + } + if (!error && !items.count) error = @"Nothing to play"; + } + MPMediaItem *start = nil; + if (hasStart) { + for (MPMediaItem *item in items) { + if (item.persistentID == startId) { start = item; break; } + } + } + dispatch_async(dispatch_get_main_queue(), ^{ + if (error) { + completion(error); + return; + } + MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer; + MPMusicPlayerMediaItemQueueDescriptor *descriptor = [[MPMusicPlayerMediaItemQueueDescriptor alloc] + initWithItemCollection:[MPMediaItemCollection collectionWithItems:items]]; + if (start) descriptor.startItem = start; + [player setQueueWithDescriptor:descriptor]; + [player play]; + self.queueItems = items; + self.queueConfirmed = NO; + self.queueSetAt = [NSDate date]; + [self notifyChanged]; + completion(nil); + }); + }); +} +@end diff --git a/App_Resources/iOS/src/FaceclawEvenHubWebView.h b/App_Resources/iOS/src/FaceclawEvenHubWebView.h index 0634e501..ef711e60 100644 --- a/App_Resources/iOS/src/FaceclawEvenHubWebView.h +++ b/App_Resources/iOS/src/FaceclawEvenHubWebView.h @@ -2,7 +2,7 @@ #import /** One local or URL EvenHub runtime, retained by its TS session until its window closes. */ -@interface FaceclawEvenHubWebView : NSObject +@interface FaceclawEvenHubWebView : NSObject @property(nonatomic, copy) void (^eventHandler)(NSString *json); @property(nonatomic, copy) NSString *packageIdentifier; - (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script; diff --git a/App_Resources/iOS/src/FaceclawEvenHubWebView.m b/App_Resources/iOS/src/FaceclawEvenHubWebView.m index 448a8da2..317588c0 100644 --- a/App_Resources/iOS/src/FaceclawEvenHubWebView.m +++ b/App_Resources/iOS/src/FaceclawEvenHubWebView.m @@ -1,5 +1,6 @@ #import "FaceclawEvenHubWebView.h" #import +#import #import #import @@ -7,19 +8,49 @@ @property(nonatomic) WKWebView *webView; @property(nonatomic) UIView *host; @property(nonatomic) UIButton *back; +@property(nonatomic) UIButton *returnButton; @property(nonatomic) NSTimer *timer; @property(nonatomic) NSString *directory; -@property(nonatomic) NSURL *remoteURL; +@property(nonatomic) NSString *script; +/** The app's start page. Only documents from its origin get the bridge. */ +@property(nonatomic) NSURL *startURL; +/** The redirect_uri of a sign-in the app started; the provider's reply is finished inside the app. */ +@property(nonatomic) NSURL *returnURL; +/** The main frame shows a page outside the app's origin (a sign-in page, a followed link). */ +@property(nonatomic) BOOL away; +@property(nonatomic) BOOL committed; @property(nonatomic) BOOL ticking; @property(nonatomic) BOOL destroyed; @end +static NSInteger FCDefaultPort(NSString *scheme) { + scheme = scheme.lowercaseString; + return [scheme isEqual:@"https"] ? 443 : [scheme isEqual:@"http"] ? 80 : 0; +} +static BOOL FCIsWebURL(NSURL *url) { + NSString *scheme = url.scheme.lowercaseString; + return [scheme isEqual:@"http"] || [scheme isEqual:@"https"]; +} +static BOOL FCSameOrigin(NSURL *origin, NSString *scheme, NSString *host, NSInteger port) { + return origin && host.length && [scheme.lowercaseString isEqual:origin.scheme.lowercaseString] && + [host.lowercaseString isEqual:origin.host.lowercaseString] && + (port ?: FCDefaultPort(scheme)) == (origin.port.integerValue ?: FCDefaultPort(origin.scheme)); +} +/** A newer navigation replaced this one, or the policy below cancelled it; the current page is unaffected. */ +static BOOL FCIsInterruption(NSError *error) { + return ([error.domain isEqual:NSURLErrorDomain] && error.code == NSURLErrorCancelled) || + ([error.domain isEqual:@"WebKitErrorDomain"] && error.code == 102); // FrameLoadInterruptedByPolicyChange +} + @implementation FaceclawEvenHubWebView - (void)emit:(NSDictionary *)event { if (!_eventHandler || _destroyed) return; NSData *data = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil]; if (data) _eventHandler([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]); } +- (void)notice:(NSString *)message { + [self emit:@{@"kind": @"notice", @"message": message.length > 4000 ? [message substringToIndex:4000] : message}]; +} - (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script { _directory = directory.stringByStandardizingPath.stringByResolvingSymlinksInPath; NSURLComponents *url = [NSURLComponents new]; @@ -28,14 +59,14 @@ } - (void)startURL:(NSString *)url script:(NSString *)script { NSURL *target = [NSURL URLWithString:url]; - if (!target.host.length || ![@[@"http", @"https"] containsObject:target.scheme.lowercaseString] || target.user || target.password) { + if (!target.host.length || !FCIsWebURL(target) || target.user || target.password) { [self emit:@{@"kind": @"error", @"message": @"Invalid app URL."}]; return; } - _remoteURL = target; [self startRequest:target script:script]; } - (void)startRequest:(NSURL *)url script:(NSString *)script { if (_destroyed) return; + _startURL = url; _script = script; WKWebViewConfiguration *config = [WKWebViewConfiguration new]; // Isolate browser storage by package, retaining it across closing/reopening. // SDK storage also persists independently through the shared TS session. @@ -49,10 +80,9 @@ } [config setURLSchemeHandler:self forURLScheme:@"faceclaw-ehpk"]; [config.userContentController addScriptMessageHandler:self name:@"faceclaw"]; - [config.userContentController addUserScript:[[WKUserScript alloc] initWithSource:script - injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]]; + [self installScript:config.userContentController]; _webView = [[WKWebView alloc] initWithFrame:CGRectZero configuration:config]; - _webView.navigationDelegate = self; + _webView.navigationDelegate = self; _webView.UIDelegate = self; _webView.accessibilityIdentifier = @"evenhub-webview"; UIWindow *window = nil; for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) { @@ -73,13 +103,21 @@ [_back setTitle:@"Back to Faceclaw" forState:UIControlStateNormal]; _back.accessibilityIdentifier = @"evenhub-back"; [_back addTarget:self action:@selector(hideOnPhone) forControlEvents:UIControlEventTouchUpInside]; - [_host addSubview:_back]; _back.translatesAutoresizingMaskIntoConstraints = NO; + // Shown only while the page is outside the app, e.g. an abandoned sign-in. + _returnButton = [UIButton buttonWithType:UIButtonTypeSystem]; + [_returnButton setTitle:@"Return to app" forState:UIControlStateNormal]; + _returnButton.accessibilityIdentifier = @"evenhub-return"; + _returnButton.hidden = YES; + [_returnButton addTarget:self action:@selector(returnToApp) forControlEvents:UIControlEventTouchUpInside]; + UIStackView *bar = [[UIStackView alloc] initWithArrangedSubviews:@[_back, _returnButton]]; + bar.distribution = UIStackViewDistributionFillEqually; + [_host addSubview:bar]; bar.translatesAutoresizingMaskIntoConstraints = NO; [NSLayoutConstraint activateConstraints:@[ - [_back.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor], - [_back.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor], - [_back.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor], - [_back.heightAnchor constraintEqualToConstant:44], - [_webView.topAnchor constraintEqualToAnchor:_back.bottomAnchor], + [bar.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor], + [bar.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor], + [bar.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor], + [bar.heightAnchor constraintEqualToConstant:44], + [_webView.topAnchor constraintEqualToAnchor:bar.bottomAnchor], [_webView.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor], [_webView.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor], [_webView.bottomAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.bottomAnchor] @@ -92,7 +130,7 @@ __weak typeof(self) weakSelf = self; _timer = [NSTimer timerWithTimeInterval:1.0 / 60 repeats:YES block:^(NSTimer *timer) { typeof(self) strongSelf = weakSelf; - if (!strongSelf || strongSelf.destroyed || strongSelf.ticking) return; + if (!strongSelf || strongSelf.destroyed || strongSelf.ticking || strongSelf.away) return; strongSelf.ticking = YES; [strongSelf.webView evaluateJavaScript:@"window.__fcTimerTick && window.__fcTimerTick(); window.__fcRafTick && window.__fcRafTick();" completionHandler:^(id result, NSError *error) { weakSelf.ticking = NO; @@ -101,8 +139,18 @@ [NSRunLoop.mainRunLoop addTimer:_timer forMode:NSRunLoopCommonModes]; [_webView loadRequest:[NSURLRequest requestWithURL:url]]; } +/** The bridge scripts run only in the app's own documents, never in pages it navigates to. */ +- (void)installScript:(WKUserContentController *)controller { + NSData *start = [NSJSONSerialization dataWithJSONObject:_startURL.absoluteString options:NSJSONWritingFragmentsAllowed error:nil]; + NSString *source = [NSString stringWithFormat:@"if ((function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(%@))) {\n%@\n}", + [[NSString alloc] initWithData:start encoding:NSUTF8StringEncoding], _script]; + [controller removeAllUserScripts]; + [controller addUserScript:[[WKUserScript alloc] initWithSource:source + injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]]; +} - (void)evaluate:(NSString *)script { - if (!_destroyed) [_webView evaluateJavaScript:script completionHandler:nil]; + // Host messages are for the app's documents; a page it navigated to must not receive them. + if (!_destroyed && !_away) [_webView evaluateJavaScript:script completionHandler:nil]; } - (void)showOnPhone { if (_destroyed) return; @@ -112,51 +160,152 @@ [_host endEditing:YES]; [_host.superview sendSubviewToBack:_host]; _host.userInteractionEnabled = NO; _host.accessibilityElementsHidden = YES; [self emit:@{@"kind": @"hidden"}]; } +- (void)returnToApp { + if (_destroyed) return; + _returnURL = nil; [_webView loadRequest:[NSURLRequest requestWithURL:_startURL]]; +} - (void)destroy { if (_destroyed) return; _destroyed = YES; [_timer invalidate]; _timer = nil; - [_webView stopLoading]; _webView.navigationDelegate = nil; + [_webView stopLoading]; _webView.navigationDelegate = nil; _webView.UIDelegate = nil; [_webView.configuration.userContentController removeScriptMessageHandlerForName:@"faceclaw"]; [_webView.configuration.userContentController removeAllUserScripts]; - [_host removeFromSuperview]; _host = nil; _back = nil; _webView = nil; _eventHandler = nil; + [_host removeFromSuperview]; _host = nil; _back = nil; _returnButton = nil; _webView = nil; _eventHandler = nil; } -- (BOOL)allowsURL:(NSURL *)url { - if (!_remoteURL) return [url.scheme isEqual:@"faceclaw-ehpk"] && [url.host isEqual:@"app"]; - NSNumber *port = url.port ?: @([url.scheme.lowercaseString isEqual:@"https"] ? 443 : 80); - NSNumber *remotePort = _remoteURL.port ?: @([_remoteURL.scheme.lowercaseString isEqual:@"https"] ? 443 : 80); - return [url.scheme.lowercaseString isEqual:_remoteURL.scheme.lowercaseString] && - [url.host.lowercaseString isEqual:_remoteURL.host.lowercaseString] && [port isEqual:remotePort]; +- (BOOL)isAppURL:(NSURL *)url { + return FCSameOrigin(_startURL, url.scheme, url.host, url.port.integerValue); +} +- (BOOL)isAppFrame:(WKFrameInfo *)frame { + WKSecurityOrigin *origin = frame.securityOrigin; + return frame.isMainFrame && FCSameOrigin(_startURL, origin.protocol, origin.host, origin.port); +} +/** An OAuth authorization request names where the provider replies. */ +- (void)rememberReturnURLFrom:(NSURL *)url { + for (NSURLQueryItem *item in [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO].queryItems) { + NSURL *target = [item.name isEqual:@"redirect_uri"] && item.value ? [NSURL URLWithString:item.value] : nil; + // A redirect_uri on the app's own origin needs no help: the app handles it itself. + if (FCIsWebURL(target) && target.host.length && ![self isAppURL:target]) _returnURL = target; + } +} +- (BOOL)isReturnURL:(NSURL *)url { + // NSURL.path drops a trailing slash, so ".../callback/" and ".../callback" match. + NSString *path = url.path.length ? url.path : @"/", *returnPath = _returnURL.path.length ? _returnURL.path : @"/"; + return FCSameOrigin(_returnURL, url.scheme, url.host, url.port.integerValue) && [path isEqual:returnPath]; +} +/** The app's start page carrying the provider's reply (its query and fragment). */ +- (void)loadAppWithResultOf:(NSURL *)url { + NSURLComponents *result = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO]; + NSURLComponents *app = [NSURLComponents componentsWithURL:_startURL resolvingAgainstBaseURL:NO]; + if (result.percentEncodedQuery.length) app.percentEncodedQuery = app.percentEncodedQuery.length ? + [NSString stringWithFormat:@"%@&%@", app.percentEncodedQuery, result.percentEncodedQuery] : result.percentEncodedQuery; + if (result.percentEncodedFragment) app.percentEncodedFragment = result.percentEncodedFragment; + NSURL *target = app.URL; + _returnURL = nil; + dispatch_async(dispatch_get_main_queue(), ^{ + if (!self.destroyed) [self.webView loadRequest:[NSURLRequest requestWithURL:target]]; + }); } - (void)userContentController:(WKUserContentController *)controller didReceiveScriptMessage:(WKScriptMessage *)message { - // Subframes and navigated third-party documents never receive host access. - NSURL *url = message.frameInfo.request.URL; - if (!message.frameInfo.isMainFrame || ![self allowsURL:url]) return; + // Only the app's own main frame gets host access: not subframes, nor pages the app navigated to. + if (![self isAppFrame:message.frameInfo]) return; if ([message.body isKindOfClass:NSDictionary.class]) [self emit:message.body]; } - (void)webView:(WKWebView *)webView decidePolicyForNavigationAction:(WKNavigationAction *)action decisionHandler:(void (^)(WKNavigationActionPolicy))decisionHandler { NSURL *url = action.request.URL; - BOOL allowed = [self allowsURL:url] && action.targetFrame.isMainFrame; - if (!allowed && action.targetFrame.isMainFrame) - [self emit:@{@"kind": @"error", @"message": @"Navigation outside the app origin is blocked. Open the destination as a separate app URL."}]; - decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel); + NSString *scheme = url.scheme.lowercaseString; + // A new window (target=_blank, window.open): createWebView below opens it outside the app. + if (!action.targetFrame) { decisionHandler(WKNavigationActionPolicyAllow); return; } + // Subframes load as in a browser; they never get the bridge. + if (!action.targetFrame.isMainFrame) { + BOOL allowed = FCIsWebURL(url) || [self isAppURL:url] || [@[@"about", @"data", @"blob"] containsObject:scheme]; + decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel); return; + } + if ([self isAppURL:url]) { _returnURL = nil; decisionHandler(WKNavigationActionPolicyAllow); return; } + if (FCIsWebURL(url) && !_committed && !_directory.length) { + // The app URL redirected before anything loaded: the app lives where it lands. The + // script update reaches the web process ahead of the policy reply. + _startURL = url; [self installScript:webView.configuration.userContentController]; + decisionHandler(WKNavigationActionPolicyAllow); return; + } + if (_returnURL && [self isReturnURL:url]) { + // A sign-in provider replying: finish in the app rather than at the page the redirect_uri names. + decisionHandler(WKNavigationActionPolicyCancel); [self loadAppWithResultOf:url]; return; + } + if (!FCIsWebURL(url)) { + [self notice:[NSString stringWithFormat:@"Blocked navigation to a %@: URL.", scheme ?: @"schemeless"]]; + decisionHandler(WKNavigationActionPolicyCancel); return; + } + // Leaving the app's origin. Such pages never get the bridge; "Return to app" leads back. + if ([self isAppFrame:action.sourceFrame]) [self rememberReturnURLFrom:url]; + decisionHandler(WKNavigationActionPolicyAllow); +} +- (void)webView:(WKWebView *)webView didCommitNavigation:(WKNavigation *)navigation { + _committed = YES; _away = ![self isAppURL:webView.URL]; _returnButton.hidden = !_away; } - (void)webView:(WKWebView *)webView didFinishNavigation:(WKNavigation *)navigation { - [self emit:@{@"kind": @"loaded"}]; + if (!_away) [self emit:@{@"kind": @"loaded"}]; } - (void)webView:(WKWebView *)webView didFailProvisionalNavigation:(WKNavigation *)navigation withError:(NSError *)error { - [self emit:@{@"kind": @"error", @"message": error.localizedDescription}]; + if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}]; } - (void)webView:(WKWebView *)webView didFailNavigation:(WKNavigation *)navigation withError:(NSError *)error { - [self emit:@{@"kind": @"error", @"message": error.localizedDescription}]; + if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}]; } - (void)webViewWebContentProcessDidTerminate:(WKWebView *)webView { - [self emit:@{@"kind": @"error", @"message": @"The app's web process ended. Close and reopen the app."}]; + [self emit:@{@"kind": @"error", @"fatal": @YES, @"message": @"The app's web process ended. Close and reopen the app."}]; +} +/** What to present over, only while the app's phone UI is shown and not mid-transition. */ +- (UIViewController *)presenter { + if (_destroyed || !_host.userInteractionEnabled) return nil; + UIViewController *controller = _host.window.rootViewController; + while (controller.presentedViewController) controller = controller.presentedViewController; + return controller.transitionCoordinator ? nil : controller; +} +- (WKWebView *)webView:(WKWebView *)webView createWebViewWithConfiguration:(WKWebViewConfiguration *)configuration forNavigationAction:(WKNavigationAction *)action windowFeatures:(WKWindowFeatures *)windowFeatures { + NSURL *url = action.request.URL; + UIViewController *presenter = FCIsWebURL(url) ? [self presenter] : nil; + if (presenter) [presenter presentViewController:[[SFSafariViewController alloc] initWithURL:url] animated:YES completion:nil]; + else [self notice:[NSString stringWithFormat:@"Did not open a window for a %@: URL.", url.scheme ?: @"schemeless"]]; + return nil; +} +// JavaScript dialogs. WebKit requires each completion handler to run exactly once, so a +// dialog that cannot be shown (the app is hidden) completes at once with its default. +- (UIAlertController *)dialog:(NSString *)message { + if ([self presenter]) return [UIAlertController alertControllerWithTitle:nil message:message preferredStyle:UIAlertControllerStyleAlert]; + [self notice:[@"Dialog while hidden: " stringByAppendingString:message ?: @""]]; + return nil; +} +- (BOOL)presentDialog:(UIAlertController *)alert { + if (!alert) return NO; + [[self presenter] presentViewController:alert animated:YES completion:nil]; + return alert.presentingViewController != nil; +} +- (void)webView:(WKWebView *)webView runJavaScriptAlertPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(void))completionHandler { + UIAlertController *alert = [self dialog:message]; + [alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(); }]]; + if (![self presentDialog:alert]) completionHandler(); +} +- (void)webView:(WKWebView *)webView runJavaScriptConfirmPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(BOOL))completionHandler { + UIAlertController *alert = [self dialog:message]; + [alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(NO); }]]; + [alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(YES); }]]; + if (![self presentDialog:alert]) completionHandler(NO); +} +- (void)webView:(WKWebView *)webView runJavaScriptTextInputPanelWithPrompt:(NSString *)prompt defaultText:(NSString *)defaultText initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(NSString *))completionHandler { + UIAlertController *alert = [self dialog:prompt]; + [alert addTextFieldWithConfigurationHandler:^(UITextField *field) { field.text = defaultText; }]; + __weak UIAlertController *weakAlert = alert; + [alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(nil); }]]; + [alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { + completionHandler(weakAlert.textFields.firstObject.text ?: @""); + }]]; + if (![self presentDialog:alert]) completionHandler(nil); } - (void)webView:(WKWebView *)webView startURLSchemeTask:(id)task { NSURL *url = task.request.URL; NSString *relative = [url.path stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@"/"]]; NSString *file = [[_directory stringByAppendingPathComponent:relative] stringByStandardizingPath].stringByResolvingSymlinksInPath; - BOOL allowed = !_destroyed && _directory.length && !_remoteURL && [url.host isEqual:@"app"] && + BOOL allowed = !_destroyed && _directory.length && [url.host isEqual:@"app"] && [file hasPrefix:[_directory stringByAppendingString:@"/"]]; NSData *data = allowed ? [NSData dataWithContentsOfFile:file options:NSDataReadingMappedIfSafe error:nil] : nil; NSString *ext = file.pathExtension.lowercaseString; diff --git a/App_Resources/iOS/src/FaceclawGraphics.h b/App_Resources/iOS/src/FaceclawGraphics.h index 1a8eacf7..64614bdf 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.h +++ b/App_Resources/iOS/src/FaceclawGraphics.h @@ -2,6 +2,8 @@ @interface FaceclawGraphics : NSObject + (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green; +/** A glasses icon as white-on-transparent (gray becomes alpha), for the dark header. */ ++ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height; + (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size; + (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height; + (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight; diff --git a/App_Resources/iOS/src/FaceclawGraphics.m b/App_Resources/iOS/src/FaceclawGraphics.m index 0c9e746c..84e7a453 100644 --- a/App_Resources/iOS/src/FaceclawGraphics.m +++ b/App_Resources/iOS/src/FaceclawGraphics.m @@ -76,6 +76,23 @@ CGColorSpaceRelease(space); return result; } ++ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height { + if (width <= 0 || height <= 0 || gray.length != width * height) return nil; + NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4]; + const uint8_t *src = gray.bytes; + uint8_t *dst = rgba.mutableBytes; + // Premultiplied white: every channel equals the alpha. + for (NSInteger i = 0; i < width * height; i++) memset(dst + i * 4, src[i], 4); + CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB(); + CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba); + CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space, + kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault); + UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil; + if (image) CGImageRelease(image); + CGDataProviderRelease(provider); + CGColorSpaceRelease(space); + return result; +} + (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size { if (size <= 0 || size > 1024) return nil; SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]]; diff --git a/App_Resources/iOS/src/FaceclawSocket.h b/App_Resources/iOS/src/FaceclawSocket.h index 84a058c5..8c24e0a1 100644 --- a/App_Resources/iOS/src/FaceclawSocket.h +++ b/App_Resources/iOS/src/FaceclawSocket.h @@ -2,7 +2,10 @@ /** Thread-safe event queue; each NativeScript worker drains it on its own thread. */ @interface FaceclawSocket : NSObject - (instancetype)initWithURL:(NSString *)url; +/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */ +- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue; - (NSString *)takeEvents; - (void)sendText:(NSString *)text; +- (void)sendData:(NSData *)data; - (void)close; @end diff --git a/App_Resources/iOS/src/FaceclawSocket.m b/App_Resources/iOS/src/FaceclawSocket.m index a53e6e6b..701e7a36 100644 --- a/App_Resources/iOS/src/FaceclawSocket.m +++ b/App_Resources/iOS/src/FaceclawSocket.m @@ -8,12 +8,17 @@ @end @implementation FaceclawSocket - (instancetype)initWithURL:(NSString *)url { + return [self initWithURL:url headerName:nil headerValue:nil]; +} +- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue { if ((self = [super init])) { _events = [NSMutableArray array]; NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration]; config.timeoutIntervalForRequest = 20; _session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil]; - _task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]]; + NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]]; + if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName]; + _task = [_session webSocketTaskWithRequest:request]; _task.maximumMessageSize = 8 * 1024 * 1024; [_task resume]; } @@ -25,7 +30,7 @@ _queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding]; if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) { [_events removeAllObjects]; _queuedBytes = 0; - [_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}]; + [_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}]; [self close]; } else [_events addObject:event]; } @@ -56,7 +61,13 @@ [self close]; } - (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error { - if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; + if (error) { + // A refused handshake only says "bad response"; name the HTTP status + // so callers can tell a rejected key (401/403) from a network error. + NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0; + NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription; + [self enqueue:@{@"kind": @"error", @"text": text}]; + } [self close]; } - (void)sendText:(NSString *)text { @@ -65,6 +76,12 @@ if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; } }]; } +- (void)sendData:(NSData *)data { + __weak FaceclawSocket *weakSelf = self; + [_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) { + if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; } + }]; +} - (void)close { @synchronized(self) { if (_ended) return; _ended = YES; diff --git a/App_Resources/iOS/src/FaceclawSpeech.h b/App_Resources/iOS/src/FaceclawSpeech.h index 8f647292..2a29b17b 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.h +++ b/App_Resources/iOS/src/FaceclawSpeech.h @@ -2,6 +2,8 @@ /** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */ @interface FaceclawSpeech : NSObject @property(nonatomic, copy) void (^eventHandler)(NSString *json); +/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */ +@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm); + (NSInteger)authorizationStatus; + (NSInteger)microphoneAuthorizationStatus; + (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion; @@ -9,6 +11,9 @@ /** Returns an actionable error, or an empty string when capture starts. */ - (NSString *)startWithEndpointing:(BOOL)endpointing; - (NSString *)startPhoneWithEndpointing:(BOOL)endpointing; +/** Capture without the recognizer (for a cloud provider): audio goes to + * pcmHandler, and "ended" follows the last chunk. No transcript events. */ +- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone; - (void)acceptPacket:(NSData *)packet; - (void)finish; - (void)cancel; diff --git a/App_Resources/iOS/src/FaceclawSpeech.m b/App_Resources/iOS/src/FaceclawSpeech.m index 5487011e..a93b9676 100644 --- a/App_Resources/iOS/src/FaceclawSpeech.m +++ b/App_Resources/iOS/src/FaceclawSpeech.m @@ -15,6 +15,7 @@ @property(atomic) NSUInteger generation; @property(nonatomic) NSUInteger pendingPackets; @property(atomic) BOOL accepting; +@property(nonatomic) BOOL pcmOutput; @property(nonatomic, strong) AVAudioEngine *phoneEngine; @property(nonatomic) BOOL phoneTapInstalled; @property(nonatomic) BOOL phoneSessionActive; @@ -23,6 +24,23 @@ @property(nonatomic, strong) FaceclawSpeechTranscript *transcript; @property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector; @end +/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial + * audio queue; the converter keeps its filter state between buffers. */ +static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) { + AVAudioFormat *format = converter.outputFormat; + AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32; + AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity]; + __block BOOL supplied = NO; + NSError *error = nil; + AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error + withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) { + // NoDataNow (not EndOfStream) leaves the stream open for the next buffer. + if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; } + supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input; + }]; + if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil; + return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)]; +} @implementation FaceclawSpeech - (instancetype)init { if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL); @@ -46,37 +64,51 @@ if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]); } - (NSString *)startWithEndpointing:(BOOL)endpointing { - return [self startWithEndpointing:endpointing phone:NO]; + return [self startWithEndpointing:endpointing phone:NO pcm:NO]; } - (NSString *)startPhoneWithEndpointing:(BOOL)endpointing { - return [self startWithEndpointing:endpointing phone:YES]; + return [self startWithEndpointing:endpointing phone:YES pcm:NO]; } -- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone { +- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone { + return [self startWithEndpointing:endpointing phone:phone pcm:YES]; +} +- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm { [self cancel]; if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized) return @"Allow Microphone access for Faceclaw in iPhone Settings."; - if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) - return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US"; - self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]]; - if (!self.recognizer.supportsOnDeviceRecognition) - return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language]; - if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly."; + if (!pcm) { + if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized) + return @"Allow Speech Recognition for Faceclaw in iPhone Settings."; + self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]]; + if (!self.recognizer.supportsOnDeviceRecognition) + return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language]; + if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly."; + } if (!phone) { self.decoder = [FaceclawLc3Decoder new]; if (!self.decoder) return @"Could not initialize the glasses audio decoder."; } + self.pcmOutput = pcm; + self.bestText = @""; self.accepting = YES; + self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil; + NSUInteger generation = self.generation; + __weak FaceclawSpeech *weakSelf = self; + if (pcm) { + if (phone) { + NSString *error = [self startPhoneAudio]; + if (error.length) { [self cancel]; return error; } + } + // The PCM consumer reports its own status and has no session limit. + return @""; + } self.recognizer.queue = NSOperationQueue.mainQueue; self.request = [SFSpeechAudioBufferRecognitionRequest new]; self.request.requiresOnDeviceRecognition = YES; self.request.shouldReportPartialResults = YES; self.request.taskHint = SFSpeechRecognitionTaskHintDictation; if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES; - self.bestText = @""; self.accepting = YES; self.transcript = [FaceclawSpeechTranscript new]; - self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil; - NSUInteger generation = self.generation; - __weak FaceclawSpeech *weakSelf = self; self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) { dispatch_async(dispatch_get_main_queue(), ^{ FaceclawSpeech *owner = weakSelf; @@ -124,6 +156,13 @@ AVAudioInputNode *input = self.phoneEngine.inputNode; AVAudioFormat *format = [input outputFormatForBus:0]; if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again."; + AVAudioConverter *converter = nil; + if (self.pcmOutput) { + AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES]; + converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat]; + if (!converter) return @"Could not convert phone microphone audio."; + converter.downmix = YES; + } NSUInteger generation = self.generation; SFSpeechAudioBufferRecognitionRequest *request = self.request; FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector; @@ -141,7 +180,10 @@ memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize); dispatch_async(owner.audioQueue, ^{ @autoreleasepool { if (owner.generation != generation) return; - [request appendAudioPCMBuffer:copy]; + if (converter) { + NSData *pcm = resampleToPcm(converter, copy); + if (pcm.length) [owner deliverPcm:pcm generation:generation]; + } else [request appendAudioPCMBuffer:copy]; double sum = 0; if (copy.floatChannelData) { const float *samples = copy.floatChannelData[0]; @@ -188,6 +230,7 @@ FaceclawLc3Decoder *decoder = self.decoder; SFSpeechAudioBufferRecognitionRequest *request = self.request; FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector; + BOOL pcmOutput = self.pcmOutput; NSData *copy = [packet copy]; dispatch_async(self.audioQueue, ^{ @autoreleasepool { if (self.generation != generation) return; @@ -195,17 +238,26 @@ double rms = 0; BOOL speechEnded = NO; if (pcm.length) { - AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO]; - AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)]; - buffer.frameLength = buffer.frameCapacity; - const int16_t *samples = pcm.bytes; - for (NSUInteger i = 0; i < buffer.frameLength; i++) { - float value = samples[i] / 32768.0f; - buffer.floatChannelData[0][i] = value; rms += value * value; + AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2); + AVAudioPCMBuffer *buffer = nil; + if (!pcmOutput) { + AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO]; + buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames]; + buffer.frameLength = frames; } - rms = sqrt(rms / buffer.frameLength); - if (self.generation == generation) [request appendAudioPCMBuffer:buffer]; - speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength]; + const int16_t *samples = pcm.bytes; + for (NSUInteger i = 0; i < frames; i++) { + float value = samples[i] / 32768.0f; + if (buffer) buffer.floatChannelData[0][i] = value; + rms += value * value; + } + rms = sqrt(rms / frames); + if (self.generation == generation) { + // The decoder's output is already the PCM consumer's format. + if (pcmOutput) [self deliverPcm:pcm generation:generation]; + else [request appendAudioPCMBuffer:buffer]; + } + speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames]; } NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing; dispatch_async(dispatch_get_main_queue(), ^{ @@ -223,9 +275,15 @@ [self stopPhoneAudio]; NSUInteger generation = self.generation; SFSpeechAudioBufferRecognitionRequest *request = self.request; + BOOL pcmOutput = self.pcmOutput; [self emit:@{@"kind":@"finishing"}]; // Drain already received audio before marking the utterance complete. - dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; }); + dispatch_async(self.audioQueue, ^{ + if (self.generation != generation) return; + if (!pcmOutput) { [request endAudio]; return; } + // Drained chunks are already queued for pcmHandler; "ended" follows them. + dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; }); + }); __weak FaceclawSpeech *weakSelf = self; dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{ FaceclawSpeech *owner = weakSelf; @@ -239,9 +297,14 @@ if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}]; if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}]; } +- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation { + dispatch_async(dispatch_get_main_queue(), ^{ + if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm); + }); +} - (void)cancel { self.generation++; - self.accepting = NO; self.pendingPackets = 0; + self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO; [self stopPhoneAudio]; [self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil; self.bestText = @""; self.transcript = nil; self.endpointDetector = nil; diff --git a/App_Resources/iOS/src/module.modulemap b/App_Resources/iOS/src/module.modulemap index 14291369..02935284 100644 --- a/App_Resources/iOS/src/module.modulemap +++ b/App_Resources/iOS/src/module.modulemap @@ -6,6 +6,7 @@ module FaceclawGraphics { header "FaceclawSpeech.h" header "FaceclawLocation.h" header "FaceclawCalendar.h" + header "FaceclawAppleMusic.h" header "FaceclawConfigPort.h" header "FaceclawEvenHubWebView.h" header "FaceclawCrypto.h" diff --git a/CHANGELOG b/CHANGELOG index 1c45f736..8bbbd05c 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,10 +1,57 @@ # Changelog -## 0.8.3 - Future +## 0.8.5 - Future + +### Features + +* Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover +* Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view +* Add an Apple Music integration on iOS + +### Bug fixes + +* Inverting Android notification-icons to indicate selectionis now gamma-correct +* Android notification icons that are full-color pictures rather than shapes no longer show as blank blocks + +## 0.8.4 - 2026-10-17 + +### Features + +* Bottom-bar app switcher switches two a multi-row layout when it runs out of space +* T3 Code client: Add options (worktree, host, etc) when creating threads +* Add configuration options to the glanceboard system card and music player +* Glanceboard compass shows calibration-accuracy indicator +* Status-bar widgets (nightscout, weather, timer) are selectable +* Roam app: Add a date-picker for navigating to a daily page +* Clear notification popups when they're dismissed on the phone, and avoid showing some notifications that are spurious +* Add depth effects to some modals that didn't have them +* Add EvenHub icon to the phone preview screen +* Add Hermes integration +* Add wake-into-last-app setting (changes the default focus after screen wake) +* Nightscout: Add a missed-bolus detection feature + +### Bug fixes + +* iOS: Fix several bugs that would crash the app +* EvenHub: Fix issues that would prevent logging in with the webview on another domain +* EvenHub app localstorage is now partitioned by app +* Phone preview is centered on the phone screen when apps are not maximized +* Fix a bug where album art and track display could get out of sync if you advance twice quickly +* Fix a bug where EvenHub searches with spaces in the search term would fail +* Crashes in a webview renderer can no longer take down the rest of the app +* Fix issues that would prevent sign-in to EvenHub apps that rely on cross-domain navigation +* Fix a bunch of bugs in the Microphones prototype app + +## 0.8.3 - 2026-10-10 * Fix several issues that would cause disconnection and disconnect-looping * Add a client for T3 Code -* Weather app shows an icon for current conditions, and can add the weather to the status bar and the Glanceboard system card (Weather menu > Settings) +* Optional usage statistics +* Add an on-glasses Onboarding app, which prompts you to configure some things +* iOS Notifications have an icon based on ANCS category +* iOS voice input can use cloud transcription providers +* Weather app is more graphical, and can appear on the status bar and glanceboard Glanceboard system card +* Set up an iOS TestFlight (but it may take awhile before Apple approves it) ## 0.8.2 - 2026-10-03 diff --git a/PRIVACY b/PRIVACY index 8b9251c7..fd667185 100644 --- a/PRIVACY +++ b/PRIVACY @@ -1,10 +1,91 @@ # Faceclaw Privacy Policy -Faceclaw is a privacy-conscious app. The developers of Faceclaw don't collect -or store user data. Faceclaw does integrate with some third-party services, -which have their own privacy policies; this document describes what those -services are, what they're used for and what data they collect, and links to -their privacy policies. +Faceclaw's own usage statistics are off unless you opt in. Separately, Apple +collects and shares information when you use an iOS TestFlight build, even +when Faceclaw statistics are set to None. Information you voluntarily send +us, such as feedback, may also identify you. This policy describes these +separate channels and the third-party services you can choose to use. + + +## Usage Statistics + +The first time you connect your glasses, Faceclaw asks whether to share +usage statistics with its developers, to help them fix problems and +improve the app. The choices are None, Minimal and Full, and you can change +your choice at any time in Settings > Privacy > Faceclaw statistics. Until +you choose, it is None. This setting controls only Faceclaw's own statistics. + +With None, Faceclaw sends and stores none of its own usage statistics. This +does not disable Apple's TestFlight reports or services you choose to use. + +With Minimal, Faceclaw keeps counts on your phone and sends them about once a +day to stats.faceclaw.org, a server run by Faceclaw's developers: + + - the Faceclaw version, whether it runs on Android or iOS, and whether it is + an official release or a build someone compiled themselves; + - whether glasses and a ring are paired, and which devices (glasses, ring, + Wear OS watch) were in use each day; + - counts of connection events: connection attempts, disconnections and + their causes, message timeouts and retries, Bluetooth error codes, and + how long connections lasted. + +With Full, it also sends: + + - which kinds of API key are set (for example, that an Anthropic key is + set), but never the keys themselves; + - how many terminal and T3 Code connections are set up, and which of + Faceclaw's built-in fonts is in use; + - the values of settings chosen from fixed options, such as on/off + switches and pick-one lists. Settings you type in, such as addresses, + URLs and names, are never sent. + +Faceclaw's own statistics never include your name or contact details, your location, +Bluetooth device addresses, anything you say or type, or what is shown on +your glasses. Each report carries a random identifier, created when you turn +statistics on, which links your reports to each other but not to you. +Turning statistics off deletes that identifier and everything kept on the +phone, and stops all reports; turning them back on starts a new identifier. +The statistics server doesn't record the IP addresses reports come from. + + +## iOS and TestFlight + +For TestFlight builds, Apple automatically collects crash reports and beta +usage information and shares them with us. This includes installations, +sessions, crashes, app/build and OS versions, device model and identifier, +and language. It is separate from Faceclaw statistics, with no in-app opt-out. + +Joining only through a public TestFlight link hides your name and email from +us, not from Apple. Email invitations can identify you to us. Apple also +collects your device's name (not shared with us) and may use your IP address +to estimate your region for beta availability. + +Feedback is optional. Comments, screenshots and diagnostics submitted +through TestFlight go to Apple and us. Public-link testers can omit their +email in the feedback form. Email sent directly to us reveals your email +even after public-link enrollment, but is not shared with Apple. Check +screenshots and comments for private content; feedback can include device +state such as battery level, free storage and network type. + +Faceclaw has no public iOS API to switch off TestFlight's automatic reporting. +Its statistics setting and iOS tracking permissions do not control it. To +leave, select Stop Testing on Faceclaw's TestFlight information page, then +stop using and uninstall the beta. This does not erase previous reports. +Self-built installations do not use TestFlight reporting. + +For ordinary iOS analytics, use iPhone Settings > Privacy & Security > +Analytics & Improvements, including Share With App Developers. Faceclaw +cannot change these system settings. TestFlight crash reporting applies +regardless of those settings. See https://www.apple.com/legal/privacy/data/en/app-analytics/. + +We use TestFlight reports only to improve Faceclaw, not for advertising or +sharing with third parties. Apple retains feedback for one year and may keep +crash/usage reports until bugs are resolved. For access/deletion requests, +contact us using the developer email in TestFlight; manage Apple's data at +https://privacy.apple.com/. + +Apple's details: https://www.apple.com/legal/privacy/data/en/test-flight/ +and https://testflight.apple.com/. ## Audio @@ -64,7 +145,12 @@ transmitted to Mapbox, and are subject to Mapbox's privacy policy. Mapbox: https://www.mapbox.com/legal/privacy The Weather app will retrieve the user's coarse location, and will use it to -query a third-party weather information provider. +query a third-party weather information provider: the US National Weather +Service, then Open-Meteo if the location is outside the US. Outside the US, it +also asks the phone's built-in geocoder (Google's on most Android phones, +Apple's on iOS) for the name of the place. + + Open-Meteo: https://open-meteo.com/en/terms ## EvenHub diff --git a/app/analytics/analytics-sources.ts b/app/analytics/analytics-sources.ts new file mode 100644 index 00000000..f286d805 --- /dev/null +++ b/app/analytics/analytics-sources.ts @@ -0,0 +1,85 @@ +/** + * The app-wide things analytics reports (the reliability counts come from + * Kotlin, drained in analytics.ts): + * - Minimal: which devices are paired, and which were in use each day; + * - Full: which kinds of API key are set, how many terminal and T3 Code + * connections there are, the UI font, and every fixed-choice setting. + * Never anything a person typed in: no keys, addresses, names or URLs. + */ +import { loadEnvironments } from "../apps/t3code/t3-environments"; +import { loadConnections } from "../apps/terminal/connections"; +import { loadDeviceAddresses } from "../g2/device-addresses"; +import { onGlassesPresenceChanged } from "../g2/glasses-presence"; +import { isPreinstalledFont } from "../graphics/installed-fonts"; +import { getUiFontSelection } from "../graphics/ui-fonts"; +import { getStringSetting } from "../native/settings-store"; +import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; +import { fixedChoiceSettingValues, ringConnectionModeSetting } from "../ui/dashboard-settings"; +import { addInputListener } from "../ui/input-monitor"; +import { flagAnalyticsEvent, registerAnalyticsSnapshot, startAnalytics } from "./analytics"; +import type { Snapshot } from "./analytics-store"; + +/** Settings holding API keys and tokens, reported only as set or not, by kind. */ +const API_KEY_SETTINGS: Record = { + anthropic: "llm.anthropicApiKey", + openai: "voice.openAiApiKey", + elevenlabs: "voice.elevenLabsApiKey", + soniox: "voice.sonioxApiKey", + "whisper-server": "voice.whisperServerApiKey", + mapbox: "maps.mapboxApiKey", + roam: "integrations.roam.apiToken", + nightscout: "integrations.nightscout.apiToken", + "agent-bridge": "assistant.bridgeToken", + evenhub: "integrations.evenhub.token", +}; + +let started = false; + +/** Registers the sources above and starts analytics. Main thread, at boot. */ +export function startAppAnalytics(): void { + if (started) return; + started = true; + registerAnalyticsSnapshot("minimal", deviceSnapshot); + registerAnalyticsSnapshot("full", configurationSnapshot); + + onGlassesPresenceChanged((presence) => { + if (presence.connected) flagAnalyticsEvent("minimal", "device.g2.used"); + }); + addInputListener((event) => { + if (!("source" in event)) return; + if (event.source === "ring") flagAnalyticsEvent("minimal", "device.r1.used"); + else if (event.source === "left-arm" || event.source === "right-arm") flagAnalyticsEvent("minimal", "device.g2.touchpad-used"); + }); + startAnalytics(); +} + +function deviceSnapshot(): Snapshot { + const addresses = loadDeviceAddresses(); + return { + "device.g2.paired": Boolean(addresses.left && addresses.right), + "device.g2.preview-only": isPreviewOnlyMode(), + "device.r1.paired": Boolean(addresses.ring), + "device.r1.connection": ringConnectionModeSetting.get(), + }; +} + +function configurationSnapshot(): Snapshot { + const snapshot: Snapshot = {}; + for (const [kind, storageKey] of Object.entries(API_KEY_SETTINGS)) { + snapshot[`apikey.${kind}`] = getStringSetting(storageKey, "").trim() !== ""; + } + snapshot["config.nightscout-site"] = getStringSetting("integrations.nightscout.siteUrl", "").trim() !== ""; + snapshot["config.terminal-connections"] = loadConnections().length; + snapshot["config.t3code-environments"] = loadEnvironments().length; + + const font = getUiFontSelection(); + snapshot["config.ui-font"] = + font.kind === "bitmap" + ? font.face + : `${isPreinstalledFont(font.file) ? font.file.replace(/\.ttf$/i, "") : "installed"}:${font.size}`; + + for (const [storageKey, value] of Object.entries(fixedChoiceSettingValues())) { + snapshot[`setting.${storageKey}`] = value; + } + return snapshot; +} diff --git a/app/analytics/analytics-store.ts b/app/analytics/analytics-store.ts new file mode 100644 index 00000000..a38cbd45 --- /dev/null +++ b/app/analytics/analytics-store.ts @@ -0,0 +1,410 @@ +/** + * Opt-in analytics, held on the phone between daily uploads to + * stats.faceclaw.org. This is the platform-free core: storage, clock, the + * data-collection setting and the network are passed in (analytics.ts wires + * them up), so the tests can drive it directly. + * + * Data is recorded at a level, and only while the data-collection setting is + * at that level or above: + * - "minimal": the app version and build, which devices are paired and in + * use, and connection-reliability counts; + * - "full": also API-key presence (never the keys) and settings chosen from + * fixed options. + * Turning the setting down discards what was kept above the new level; + * turning it off deletes everything, including the install id, and stops all + * writes and uploads. + * + * Counters accumulate in pending.json (per UTC day). Once a day the pending + * counters and a snapshot of the current state are sealed into an outbox + * report, and every outbox report is posted; each stays until the server + * accepts it, under a report id the server deduplicates on. The report + * format is stats-server/PROTOCOL.md. + */ + +export type AnalyticsLevel = "minimal" | "full"; +export type DataCollectionLevel = "none" | AnalyticsLevel; +export type SnapshotValue = boolean | number | string; +export type Snapshot = Record; + +export interface AppInfo { + version: string; + platform: string; + build: string; +} + +/** Named text files in the analytics directory. */ +export interface AnalyticsStorage { + read(name: string): string | null; + write(name: string, text: string): void; + remove(name: string): void; + list(): string[]; + /** Deletes the directory and everything in it. */ + removeAll(): void; +} + +export interface AnalyticsDeps { + storage: AnalyticsStorage; + /** The data-collection setting, read fresh on every use. */ + level(): DataCollectionLevel; + appInfo(): AppInfo; + now(): number; + randomId(): string; + /** The current state to report at `level` (each level's keys, not cumulative). */ + snapshot(level: AnalyticsLevel): Snapshot; + /** POSTs a report body; resolves to the HTTP status, rejects on network failure. */ + send(body: string): Promise; + log?(message: string): void; +} + +/** UTC day (YYYY-MM-DD) → counter name → count. */ +type Counters = Record>; +type ByLevel = { minimal: T; full: T }; + +interface PendingFile { + format: 1; + /** The app version the counts were made under; a different one seals them. */ + version: string; + counters: ByLevel; +} + +interface OutboxReport { + format: 1; + reportId: string; + createdAt: string; + app: AppInfo; + counters: ByLevel; + snapshot: ByLevel; +} + +interface StateFile { + format: 1; + installId: string; + lastSealAt: number; + /** When to retry after a failed upload; 0 when none failed. */ + retryAt: number; +} + +export const REPORT_SCHEMA_VERSION = 1; +export const DAY_MS = 24 * 60 * 60 * 1000; +export const RETRY_MS = 60 * 60 * 1000; +/** Unsent reports kept while the server is unreachable; the oldest go first. */ +export const MAX_OUTBOX_REPORTS = 30; +/** Distinct counter names per day and level, so a runaway name can't grow the file. */ +export const MAX_NAMES_PER_DAY = 300; +const FLUSH_DELAY_MS = 15_000; + +const PENDING_FILE = "pending.json"; +const STATE_FILE = "state.json"; +const OUTBOX_PREFIX = "outbox-"; + +/** Counter names and snapshot keys the server accepts (stats-server/src/report.ts). */ +const COUNTER_NAME = /^[a-z0-9][a-z0-9_.:-]{0,119}$/; +const SNAPSHOT_KEY = /^[A-Za-z0-9][A-Za-z0-9_.:-]{0,119}$/; +const MAX_SNAPSHOT_STRING = 200; + +export class AnalyticsStore { + private pending: PendingFile | null = null; + private dirty = false; + private flushTimer: ReturnType | null = null; + private uploading = false; + /** Bumped by discardAll, so an upload in flight knows not to write its state back. */ + private generation = 0; + + constructor(private readonly deps: AnalyticsDeps) {} + + /** Whether data at `level` is being kept. */ + allows(level: AnalyticsLevel): boolean { + const setting = this.deps.level(); + return setting === "full" || (setting === "minimal" && level === "minimal"); + } + + /** Adds `amount` to today's count of `name`. */ + count(level: AnalyticsLevel, name: string, amount = 1): void { + if (!this.allows(level) || !(amount > 0)) return; + if (!COUNTER_NAME.test(name)) { + this.log(`analytics: dropping bad counter name ${name}`); + return; + } + const day = this.dayCounters(level); + if (day[name] === undefined && Object.keys(day).length >= MAX_NAMES_PER_DAY) return; + day[name] = (day[name] ?? 0) + Math.round(amount); + this.markDirty(); + } + + /** Records that `name` happened today (a count of 1, however often it happens). */ + flag(level: AnalyticsLevel, name: string): void { + if (!this.allows(level)) return; + if (!COUNTER_NAME.test(name)) { + this.log(`analytics: dropping bad counter name ${name}`); + return; + } + const day = this.dayCounters(level); + if (day[name] !== undefined || Object.keys(day).length >= MAX_NAMES_PER_DAY) return; + day[name] = 1; + this.markDirty(); + } + + /** + * Brings the files in line with the setting: off deletes everything, + * minimal drops what was kept for full. Called at start and whenever the + * setting changes. + */ + applyLevel(): void { + const setting = this.deps.level(); + if (setting === "none") { + this.discardAll(); + return; + } + if (setting === "full") return; + const pending = this.loadPending(); + if (Object.keys(pending.counters.full).length) { + pending.counters.full = {}; + this.writePending(pending); + } + for (const name of this.outboxNames()) { + const report = this.readJson(name) as OutboxReport | null; + if (!report) continue; + if (Object.keys(report.counters.full).length || Object.keys(report.snapshot.full).length) { + report.counters.full = {}; + report.snapshot.full = {}; + this.deps.storage.write(name, JSON.stringify(report)); + } + } + } + + /** Writes pending counts now, if any changed. */ + flush(): void { + if (this.flushTimer !== null) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + if (!this.dirty || !this.pending) return; + this.dirty = false; + if (this.deps.level() === "none") { + this.discardAll(); + return; + } + this.writePending(this.pending); + } + + /** + * The daily upload: once a day, seal the pending counts into a report and + * post every unsent report; after a failure, retry hourly. Does nothing, + * and makes no connection, while the setting is off. + */ + async uploadIfDue(): Promise { + if (this.uploading) return; + if (this.deps.level() === "none") { + this.discardAll(); + return; + } + this.uploading = true; + const generation = this.generation; + try { + const now = this.deps.now(); + const state = this.loadState(); + let sealed = false; + // A clock set backwards shouldn't stall the schedule until it catches up. + if (now - state.lastSealAt >= DAY_MS || now < state.lastSealAt) { + this.seal(); + state.lastSealAt = now; + this.writeState(state); + sealed = true; + } + if (!sealed && !(state.retryAt > 0 && now >= state.retryAt)) return; + + for (const name of this.outboxNames()) { + const report = this.readJson(name) as OutboxReport | null; + if (!report) { + this.deps.storage.remove(name); + continue; + } + let status: number | null; + try { + status = await this.deps.send(this.reportBody(report, state.installId)); + } catch (error) { + this.log(`analytics: upload failed: ${error}`); + status = null; + } + // The setting may have been turned off (and maybe on again) while the + // request was out; then everything this upload knew about is gone. + if (this.deps.level() === "none") { + this.discardAll(); + return; + } + if (this.generation !== generation) return; + if (status !== null && status >= 200 && status < 300) { + this.deps.storage.remove(name); + } else if (status === 400 || status === 413) { + // Rejected as malformed or too large: sending it again won't help. + // (Anything else, like a 404 from a misconfigured server, is retried.) + this.log(`analytics: server rejected report ${report.reportId} (HTTP ${status})`); + this.deps.storage.remove(name); + } else { + state.retryAt = now + RETRY_MS; + this.writeState(state); + return; + } + } + if (state.retryAt !== 0) { + state.retryAt = 0; + this.writeState(state); + } + } finally { + this.uploading = false; + } + } + + /** Deletes everything kept, including the install id. */ + discardAll(): void { + if (this.flushTimer !== null) { + clearTimeout(this.flushTimer); + this.flushTimer = null; + } + this.pending = null; + this.dirty = false; + this.generation++; + this.deps.storage.removeAll(); + } + + /** The JSON body posted for `report`, including only what the current setting allows. */ + reportBody(report: OutboxReport, installId: string): string { + const level: AnalyticsLevel = this.deps.level() === "full" ? "full" : "minimal"; + const counters: Counters = {}; + for (const source of level === "full" ? [report.counters.minimal, report.counters.full] : [report.counters.minimal]) { + for (const [day, counts] of Object.entries(source)) { + const target = (counters[day] ??= {}); + for (const [name, count] of Object.entries(counts)) target[name] = (target[name] ?? 0) + count; + } + } + return JSON.stringify({ + schema: REPORT_SCHEMA_VERSION, + reportId: report.reportId, + installId, + level, + createdAt: report.createdAt, + app: report.app, + counters, + snapshot: level === "full" ? { ...report.snapshot.minimal, ...report.snapshot.full } : report.snapshot.minimal, + }); + } + + private dayCounters(level: AnalyticsLevel): Record { + const day = new Date(this.deps.now()).toISOString().slice(0, 10); + return (this.loadPending().counters[level][day] ??= {}); + } + + private markDirty(): void { + this.dirty = true; + this.flushTimer ??= setTimeout(() => { + this.flushTimer = null; + this.flush(); + }, FLUSH_DELAY_MS); + } + + private loadPending(): PendingFile { + if (this.pending) return this.pending; + const version = this.deps.appInfo().version; + const stored = this.readJson(PENDING_FILE) as PendingFile | null; + if (stored && stored.format === 1 && stored.counters?.minimal && stored.counters?.full) { + this.pending = stored; + if (stored.version !== version && hasCounts(stored)) { + // Counts from before an upgrade go out under the version they were made with. + this.seal(); + } + } + return (this.pending ??= emptyPending(version)); + } + + /** Moves the pending counts, with a snapshot of the current state, into a new outbox report. */ + private seal(): void { + const pending = this.loadPending(); + const app = { ...this.deps.appInfo(), version: pending.version }; + const report: OutboxReport = { + format: 1, + reportId: this.deps.randomId(), + createdAt: new Date(this.deps.now()).toISOString(), + app, + counters: pending.counters, + snapshot: { + minimal: this.takeSnapshot("minimal"), + full: this.allows("full") ? this.takeSnapshot("full") : {}, + }, + }; + this.deps.storage.write(`${OUTBOX_PREFIX}${report.createdAt.replace(/[^0-9]/g, "")}-${report.reportId}.json`, JSON.stringify(report)); + this.pending = emptyPending(this.deps.appInfo().version); + this.dirty = false; + this.writePending(this.pending); + const outbox = this.outboxNames(); + for (const name of outbox.slice(0, Math.max(0, outbox.length - MAX_OUTBOX_REPORTS))) this.deps.storage.remove(name); + } + + private takeSnapshot(level: AnalyticsLevel): Snapshot { + const snapshot: Snapshot = {}; + let entries: [string, SnapshotValue][]; + try { + entries = Object.entries(this.deps.snapshot(level)); + } catch (error) { + this.log(`analytics: ${level} snapshot failed: ${error}`); + return snapshot; + } + for (const [key, value] of entries) { + const ok = + SNAPSHOT_KEY.test(key) && + (typeof value === "boolean" || + (typeof value === "number" && Number.isFinite(value)) || + (typeof value === "string" && value.length <= MAX_SNAPSHOT_STRING)); + if (ok) snapshot[key] = value; + else this.log(`analytics: dropping bad snapshot entry ${key}`); + } + return snapshot; + } + + /** Outbox file names, oldest first (the names start with the seal time). */ + private outboxNames(): string[] { + return this.deps.storage + .list() + .filter((name) => name.startsWith(OUTBOX_PREFIX)) + .sort(); + } + + private loadState(): StateFile { + const stored = this.readJson(STATE_FILE) as StateFile | null; + if (stored && stored.format === 1 && typeof stored.installId === "string") return stored; + const state: StateFile = { format: 1, installId: this.deps.randomId(), lastSealAt: 0, retryAt: 0 }; + this.writeState(state); + return state; + } + + private writeState(state: StateFile): void { + this.deps.storage.write(STATE_FILE, JSON.stringify(state)); + } + + private writePending(pending: PendingFile): void { + this.deps.storage.write(PENDING_FILE, JSON.stringify(pending)); + } + + private readJson(name: string): unknown { + const text = this.deps.storage.read(name); + if (text === null) return null; + try { + return JSON.parse(text); + } catch { + this.log(`analytics: discarding unreadable ${name}`); + this.deps.storage.remove(name); + return null; + } + } + + private log(message: string): void { + this.deps.log?.(message); + } +} + +function emptyPending(version: string): PendingFile { + return { format: 1, version, counters: { minimal: {}, full: {} } }; +} + +function hasCounts(pending: PendingFile): boolean { + return Object.keys(pending.counters.minimal).length > 0 || Object.keys(pending.counters.full).length > 0; +} diff --git a/app/analytics/analytics.ts b/app/analytics/analytics.ts new file mode 100644 index 00000000..c73ca20a --- /dev/null +++ b/app/analytics/analytics.ts @@ -0,0 +1,182 @@ +/** + * The app's analytics API. What is collected, when it's kept and when it's + * sent is described in analytics-store.ts; the snapshot providers (devices, + * API-key presence, settings) are in analytics-snapshots.ts. + * + * Record from the main thread only. The store is created by startAnalytics() + * at boot, which app workers never run, so calls made in a worker are + * dropped. Kotlin-side connection events go through ConnectionCounters + * (drained here) instead. + */ +import { File, Folder, knownFolders, path as fsPath } from "@nativescript/core"; +import { buildKind } from "../native/build-info"; +import { drainConnectionCounters } from "../native/connection-counters"; +import { onSettingsStoreChanged } from "../native/settings-store"; +import { DATA_COLLECTION_KEY, dataCollectionSetting } from "../ui/dashboard-settings"; +import { fetchTextWithUserAgent } from "../util/http"; +import { FACECLAW_VERSION } from "../version"; +import { + AnalyticsStore, + type AnalyticsLevel, + type AnalyticsStorage, + type Snapshot, +} from "./analytics-store"; + +export type { AnalyticsLevel, Snapshot } from "./analytics-store"; + +declare const global: any; + +const REPORT_URL = "https://stats.faceclaw.org/v1/report"; +const REQUEST_TIMEOUT_MS = 30_000; +/** How often to see whether the daily upload is due (it usually isn't). */ +const UPLOAD_CHECK_INTERVAL_MS = 60 * 60 * 1000; +/** The first check, a little after boot or after analytics is turned on. */ +const FIRST_UPLOAD_CHECK_DELAY_MS = 2 * 60 * 1000; +/** How often the Kotlin connection counts are moved into the store. */ +const DRAIN_INTERVAL_MS = 5 * 60 * 1000; + +let store: AnalyticsStore | null = null; +let firstCheckTimer: ReturnType | null = null; +const snapshotProviders: { level: AnalyticsLevel; provide: () => Snapshot }[] = []; + +/** + * Adds `amount` to today's count of `name`, if the data-collection setting + * includes `level`. Names are lowercase, dot-separated (e.g. "g2.connect"). + */ +export function countAnalyticsEvent(level: AnalyticsLevel, name: string, amount = 1): void { + store?.count(level, name, amount); +} + +/** Records that `name` happened today: a count of 1 however often it happens. */ +export function flagAnalyticsEvent(level: AnalyticsLevel, name: string): void { + store?.flag(level, name); +} + +/** + * Adds state to report with each daily upload, when the setting includes + * `level`. Called when the report is sealed; keys must be namespaced like + * "device.g2.paired", and values booleans, numbers or short strings. + */ +export function registerAnalyticsSnapshot(level: AnalyticsLevel, provide: () => Snapshot): void { + snapshotProviders.push({ level, provide }); +} + +/** Starts the store, the daily upload schedule and the counter drain. Main thread, once, at boot. */ +export function startAnalytics(): void { + if (store) return; + const analytics = new AnalyticsStore({ + storage: createStorage(), + level: () => dataCollectionSetting.get(), + appInfo: () => ({ version: FACECLAW_VERSION, platform: global.isIOS ? "ios" : "android", build: buildKind() }), + now: () => Date.now(), + randomId: randomUuid, + snapshot: collectSnapshot, + send: sendReport, + log: (message) => console.log(message), + }); + store = analytics; + analytics.applyLevel(); + + onSettingsStoreChanged((key) => { + if (key !== DATA_COLLECTION_KEY) return; + analytics.applyLevel(); + if (dataCollectionSetting.get() !== "none") scheduleFirstCheck(analytics); + }); + setInterval(() => drainKotlinCounters(analytics), DRAIN_INTERVAL_MS); + setInterval(() => void checkUpload(analytics), UPLOAD_CHECK_INTERVAL_MS); + scheduleFirstCheck(analytics); +} + +function scheduleFirstCheck(analytics: AnalyticsStore): void { + if (firstCheckTimer !== null) clearTimeout(firstCheckTimer); + firstCheckTimer = setTimeout(() => { + firstCheckTimer = null; + void checkUpload(analytics); + }, FIRST_UPLOAD_CHECK_DELAY_MS); +} + +async function checkUpload(analytics: AnalyticsStore): Promise { + drainKotlinCounters(analytics); + try { + await analytics.uploadIfDue(); + } catch (error) { + console.warn(`analytics: upload check failed: ${error}`); + } +} + +function drainKotlinCounters(analytics: AnalyticsStore): void { + let counts: Record; + try { + counts = drainConnectionCounters(); + } catch (error) { + console.warn(`analytics: could not read connection counters: ${error}`); + return; + } + // Drained either way, so counts from while analytics was off don't pile up. + for (const [name, count] of Object.entries(counts)) analytics.count("minimal", name, count); + analytics.flush(); +} + +function collectSnapshot(level: AnalyticsLevel): Snapshot { + const snapshot: Snapshot = {}; + for (const provider of snapshotProviders) { + if (provider.level !== level) continue; + try { + Object.assign(snapshot, provider.provide()); + } catch (error) { + console.warn(`analytics: snapshot provider failed: ${error}`); + } + } + return snapshot; +} + +async function sendReport(body: string): Promise { + let timeout: ReturnType | undefined; + const timedOut = new Promise((_resolve, reject) => { + timeout = setTimeout(() => reject(new Error("timed out")), REQUEST_TIMEOUT_MS); + }); + try { + const response = await Promise.race([ + fetchTextWithUserAgent(REPORT_URL, { method: "POST", headers: { "Content-Type": "application/json" }, body }), + timedOut, + ]); + return response.status; + } finally { + clearTimeout(timeout); + } +} + +/** + * Files in an `analytics` directory: the app's private files directory on + * Android; on iOS, Library rather than Documents, which the Files app shows. + */ +function createStorage(): AnalyticsStorage { + const base = global.isIOS ? knownFolders.ios.library().path : knownFolders.documents().path; + const dir = fsPath.join(base, "analytics"); + const filePath = (name: string) => fsPath.join(dir, name); + return { + read: (name) => (File.exists(filePath(name)) ? File.fromPath(filePath(name)).readTextSync() : null), + write: (name, text) => { + Folder.fromPath(dir); + File.fromPath(filePath(name)).writeTextSync(text); + }, + remove: (name) => { + if (File.exists(filePath(name))) File.fromPath(filePath(name)).removeSync(); + }, + list: () => (Folder.exists(dir) ? Folder.fromPath(dir).getEntitiesSync().map((entity) => entity.name) : []), + removeAll: () => { + if (Folder.exists(dir)) Folder.fromPath(dir).removeSync(); + }, + }; +} + +function randomUuid(): string { + const bytes = new Uint8Array(16); + const crypto = (globalThis as { crypto?: { getRandomValues?: (array: Uint8Array) => Uint8Array } }).crypto; + if (crypto?.getRandomValues) crypto.getRandomValues(bytes); + else for (let index = 0; index < bytes.length; index++) bytes[index] = Math.floor(Math.random() * 256); + bytes[6] = (bytes[6]! & 0x0f) | 0x40; + bytes[8] = (bytes[8]! & 0x3f) | 0x80; + const hex = Array.from(bytes, (byte) => byte.toString(16).padStart(2, "0")).join(""); + return `${hex.slice(0, 8)}-${hex.slice(8, 12)}-${hex.slice(12, 16)}-${hex.slice(16, 20)}-${hex.slice(20)}`; +} diff --git a/app/analytics/privacy-disclosure.ts b/app/analytics/privacy-disclosure.ts new file mode 100644 index 00000000..9cc2ed43 --- /dev/null +++ b/app/analytics/privacy-disclosure.ts @@ -0,0 +1,10 @@ +/** Conditional wording: also shown on self-built iOS apps, where TestFlight does not apply. */ +export const IOS_PRIVACY_NOTICE = [ + "If installed with TestFlight, Apple automatically collects crash logs, installs, sessions, and device/app details and shares them with Faceclaw's developers. This continues even when Faceclaw statistics are None; the app cannot turn it off.", + "Joining only through a public link hides your name and email from us, not from Apple. Email invitations can identify you. Optional feedback can disclose your email, comments, screenshots and diagnostics. Review what you send.", + "iPhone Settings > Privacy & Security > Analytics & Improvements controls ordinary iOS analytics. It does not stop TestFlight crash reports. Faceclaw cannot change Apple's analytics settings.", + "To leave the beta, open Faceclaw in TestFlight and choose Stop Testing, then stop using and uninstall the beta. This does not erase earlier reports. Details: Settings > Privacy > Privacy policy.", +].join("\n\n"); + +export const IOS_PRIVACY_SUMMARY = + "With TestFlight, crash/usage reports continue even with None. Open iOS / TestFlight privacy for details."; diff --git a/app/app.css b/app/app.css index b9ce8ecb..007ace07 100644 --- a/app/app.css +++ b/app/app.css @@ -1,6 +1,7 @@ @import '@nativescript/theme/css/core.css'; @import '@nativescript/theme/css/default.css'; @import './phone-ui/remote-controls.css'; +@import './phone-ui/pairing.css'; Page { background-color: #ffffff; @@ -183,242 +184,6 @@ Button.-primary { margin-bottom: 12; } -/* - * Pairing scan list (phone-ui/pairing-page). Rows are Repeater items: product - * art (or a colourway swatch) on the left, decoded SKU + proximity + serial in - * the middle, a chevron/check on the right. - */ -.pairing-instructions { - color: #555555; - font-size: 14; -} - -.pairing-status { - font-size: 14; - margin-left: 8; -} - -.pairing-section { - font-size: 16; - font-weight: bold; - margin-bottom: 4; -} - -.pairing-empty { - color: #777777; - font-size: 14; - padding: 8 0; -} - -.pairing-row { - background-color: #f4f4f4; - border-color: #dddddd; - border-width: 1; - border-radius: 10; - padding: 10; - margin-bottom: 8; -} - -.pairing-row-selected { - background-color: #e6f0ff; - border-color: #3b6fd6; -} - -.pairing-row-incomplete { - background-color: #fafafa; - border-color: #e6e6e6; -} - -.pairing-art { - vertical-align: center; -} - -.pairing-title { - font-size: 16; - font-weight: bold; - color: #222222; -} - -.pairing-badge { - font-size: 11; - font-weight: bold; - color: #2a5bc4; - background-color: #dbe6fb; - border-radius: 9; - padding: 1 7; -} - -.pairing-variant { - font-size: 13; - font-weight: bold; - color: #333333; -} - -.pairing-proximity { - font-size: 12; - color: #666666; -} - -.pairing-proximity-immediate { - color: #2a5bc4; - font-weight: bold; -} - -.pairing-serial { - font-size: 12; - color: #666666; - font-family: monospace; -} - -.pairing-arms { - font-size: 12; - color: #444444; -} - -.pairing-warning { - font-size: 12; - color: #a33a1f; - margin-top: 4; -} - -.pairing-swatch-label { - font-size: 11; - color: #555555; -} - -.pairing-chevron { - font-size: 22; - color: #444444; - margin-left: 6; -} - -.pairing-selection { - background-color: #eef4ff; - border-radius: 10; - padding: 10; -} - -Button.pairing-link { - background-color: transparent; - color: #2a5bc4; - font-size: 14; - border-width: 0; - /* Buttons keep their Material elevation shadow even with a transparent - * background, which shows up as a stray, downward-offset outline. */ - android-elevation: 0; - android-dynamic-elevation-offset: 0; -} - -/* - * Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row - * / pairing-title / pairing-badge look above; these add the explanation body - * and the already-granted checkmark. - */ -.permissions-description { - font-size: 13; - color: #555555; - margin-top: 2; -} - -.permissions-check { - font-size: 24; - color: #2e7d32; - margin-left: 10; -} - -/* Continue with required permissions still missing: looks disabled but stays - * tappable (the tap raises an are-you-sure prompt instead of being ignored). */ -Button.permissions-primary-dimmed { - opacity: 0.45; -} - -/* - * Dark-mode pairing page. The cards keep their light-mode look above; here - * they go dark-gray, and every text colour set above gets a light counterpart - * so nothing renders dark-on-dark (the instructions) or dark-on-a-dark-card. - * Order matters: the -selected/-incomplete rules must follow .pairing-row so - * they win at equal specificity. - */ -.ns-dark .pairing-instructions { - color: #b0b0b0; -} - -.ns-dark .pairing-empty { - color: #999999; -} - -.ns-dark .pairing-row { - background-color: #2e2e2e; - border-color: #4a4a4a; -} - -.ns-dark .pairing-row-selected { - background-color: #333c4c; - border-color: #5b8def; -} - -.ns-dark .pairing-row-incomplete { - background-color: #282828; - border-color: #3c3c3c; -} - -.ns-dark .pairing-title { - color: #f0f0f0; -} - -.ns-dark .pairing-badge { - color: #a8c4f5; - background-color: #2a3b5e; -} - -.ns-dark .pairing-variant { - color: #cccccc; -} - -.ns-dark .pairing-proximity { - color: #a0a0a0; -} - -.ns-dark .pairing-proximity-immediate { - color: #7da7f5; -} - -.ns-dark .pairing-serial { - color: #a0a0a0; -} - -.ns-dark .pairing-arms { - color: #c0c0c0; -} - -.ns-dark .pairing-warning { - color: #e08a66; -} - -.ns-dark .pairing-swatch-label { - color: #aaaaaa; -} - -.ns-dark .pairing-chevron { - color: #bbbbbb; -} - -.ns-dark .pairing-selection { - background-color: #2e2e2e; -} - -.ns-dark Button.pairing-link { - color: #7da7f5; -} - -.ns-dark .permissions-description { - color: #b0b0b0; -} - -.ns-dark .permissions-check { - color: #7bc47f; -} - /* * Conversation review (phone-ui/conversations-page, conversation-page, * speakers-page). Cards follow the pairing-row look; speaker/emotion chips diff --git a/app/app.ios.css b/app/app.ios.css index 17bcf99b..fbed4e8e 100644 --- a/app/app.ios.css +++ b/app/app.ios.css @@ -1,146 +1,6 @@ @import '@nativescript/theme/css/core.css'; @import './phone-ui/remote-controls.css'; - -.pairing-instructions { - color: #555555; - font-size: 14; -} - -.pairing-status { - font-size: 14; - margin-left: 8; -} - -.pairing-section { - font-size: 16; - font-weight: bold; - margin-bottom: 4; -} - -.pairing-empty { - color: #777777; - font-size: 14; - padding: 8 0; -} - -.pairing-row { - background-color: #f4f4f4; - border-color: #dddddd; - border-width: 1; - border-radius: 10; - padding: 10; - margin-bottom: 8; -} - -.pairing-row-selected { - background-color: #e6f0ff; - border-color: #3b6fd6; -} - -.pairing-row-incomplete { - background-color: #fafafa; - border-color: #e6e6e6; -} - -.pairing-art { - vertical-align: center; -} - -.pairing-title { - font-size: 16; - font-weight: bold; - color: #222222; -} - -.pairing-badge { - font-size: 11; - font-weight: bold; - color: #2a5bc4; - background-color: #dbe6fb; - border-radius: 9; - padding: 1 7; -} - -.pairing-variant { - font-size: 13; - font-weight: bold; - color: #333333; -} - -.pairing-proximity { - font-size: 12; - color: #666666; -} - -.pairing-proximity-immediate { - color: #2a5bc4; - font-weight: bold; -} - -.pairing-serial { - font-size: 12; - color: #666666; - font-family: monospace; -} - -.pairing-arms { - font-size: 12; - color: #444444; -} - -.pairing-warning { - font-size: 12; - color: #a33a1f; - margin-top: 4; -} - -.pairing-swatch-label { - font-size: 11; - color: #555555; -} - -.pairing-chevron { - font-size: 22; - color: #444444; - margin-left: 6; -} - -.pairing-selection { - background-color: #eef4ff; - border-radius: 10; - padding: 10; -} - -Button.pairing-link { - background-color: transparent; - color: #2a5bc4; - font-size: 14; - border-width: 0; -} - -/* - * Permissions screen (phone-ui/permissions-page). Cards reuse the pairing-row - * / pairing-title / pairing-badge look above; these add the explanation body - * and the already-granted checkmark. - */ -.permissions-description { - font-size: 13; - color: #555555; - margin-top: 2; -} - -.permissions-check { - font-size: 24; - color: #2e7d32; - margin-left: 10; -} - -/* Continue with required permissions still missing: looks disabled but stays - * tappable (the tap raises an are-you-sure prompt instead of being ignored). */ -Button.permissions-primary-dimmed { - opacity: 0.45; -} - +@import './phone-ui/pairing.css'; .body-text { font-size: 18; } .h1 { font-size: 28; font-weight: bold; } @@ -175,6 +35,8 @@ Page.main-page { .main-page .action-bar-title { font-size: 18; font-weight: bold; color: #ffffff; } .main-page .action-bar-status { font-size: 14; color: #ffffff; margin-right: 12; } .main-page .action-bar-warning { font-size: 18; padding: 4 6; } +/* Matches Android's 32x32 padded icon; the Image inside is 21x21. */ +.main-page .action-bar-app-icon { width: 32; height: 32; margin-right: 8; } .main-page .overflow-button { background-color: transparent; color: #ffffff; diff --git a/app/apps/ai-chat/voice-draft.ts b/app/apps/ai-chat/voice-draft.ts index f91093fc..d07c4f0b 100644 --- a/app/apps/ai-chat/voice-draft.ts +++ b/app/apps/ai-chat/voice-draft.ts @@ -39,7 +39,7 @@ export class VoiceDraft { this.phase = "listening"; this.status = "Listening... release to send"; this.ownsVoiceActivity = true; - voiceActivity.setActive(true); + voiceActivity.setActive(true, "voice-draft"); this.unsubscribe.push(voiceControlBridge.onTranscript((event) => { if (!this.active || generation !== this.generation) return; if (event.isFinal) { @@ -129,7 +129,7 @@ export class VoiceDraft { } if (this.ownsVoiceActivity) { this.ownsVoiceActivity = false; - voiceActivity.setActive(false); + voiceActivity.setActive(false, "voice-draft"); } this.changed(); } diff --git a/app/apps/all-apps.ts b/app/apps/all-apps.ts index f46c4005..9effebf0 100644 --- a/app/apps/all-apps.ts +++ b/app/apps/all-apps.ts @@ -10,6 +10,7 @@ import musicApp from "./music"; import nightscoutApp from "./nightscout"; import healthApp from "./health"; import transcribeApp from "./transcribe"; +import translateApp from "./translate"; import teleprompterApp from "./teleprompter"; import microphonesApp from "./microphones"; import notificationsApp from "./notifications"; @@ -27,6 +28,7 @@ import developerApp from "./developer"; import evenhubApp from "./evenhub"; import glanceboardApp from "./glanceboard"; import settingsApp from "./settings"; +import onboardingApp from "./onboarding"; /** * Every app, in launcher-grid order (the launcher itself is first but hidden @@ -45,6 +47,7 @@ export const ALL_APPS: readonly AppDefinition[] = [ nightscoutApp, healthApp, transcribeApp, + translateApp, teleprompterApp, microphonesApp, notificationsApp, @@ -62,4 +65,5 @@ export const ALL_APPS: readonly AppDefinition[] = [ evenhubApp, glanceboardApp, settingsApp, + onboardingApp, ]; diff --git a/app/apps/app-definition.ts b/app/apps/app-definition.ts index b2e69a01..b46c809d 100644 --- a/app/apps/app-definition.ts +++ b/app/apps/app-definition.ts @@ -4,7 +4,7 @@ import { type IconName } from "../graphics/icons"; import { type LayerActions } from "../ui/layers"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { WorkerAppHost } from "../ui/shell/worker-window"; -import { shell } from "../ui/shell/shell"; +import { shell, type TrayWidget } from "../ui/shell/shell"; /** * An app, as seen from outside while it isn't running. Each app directory's @@ -96,6 +96,17 @@ export type AppContext = { surfaceId: string, create: (options: InProcessAppOptions) => InProcessWindow, ) => Promise; + /** + * Open a main-thread singleton window in the background (neither + * foregrounded nor focused) unless it is open already; `opened` says this + * call opened it. For windows the shell brings forward itself, such as an + * app widget's (see inProcessTrayWidget). Hosts that leave it out can't. + */ + openInProcessAppInBackground?: ( + windowId: string, + surfaceId: string, + create: (options: InProcessAppOptions) => InProcessWindow, + ) => { window: InProcessWindow; opened: boolean }; /** Get this app's shared worker host, spawning the worker on first use. */ ensureWorkerHost: (createWorker: () => Worker) => WorkerAppHost; /** Submit a window surface's painted frame, as planes (windows created outside launchInProcessApp). */ @@ -120,6 +131,32 @@ export type TextEditorHost = { requestRender: () => void; }; +/** + * A tray widget (see Shell.setTrayIcon) whose window is a main-thread + * singleton app window, opened in the background while the switcher's + * selection is on it; undefined on hosts that can't do that. `show`, if + * given, runs each time the selection moves onto the widget, once the + * window is open, to bring what the widget stands for on screen in it (the + * shell repaints the window after). + */ +export function inProcessTrayWidget( + ctx: AppContext, + windowId: string, + surfaceId: string, + create: (options: InProcessAppOptions) => InProcessWindow, + show?: () => void, +): TrayWidget | undefined { + const open = ctx.openInProcessAppInBackground; + if (!open) return undefined; + return { + openWindow: () => { + const { window, opened } = open(windowId, surfaceId, create); + show?.(); + return { window: window.window, opened }; + }, + }; +} + /** * Launch-or-focus flow shared by the worker-hosted apps: focus the app's * existing window if one is open, otherwise open a window in the app's shared diff --git a/app/apps/blocks/blocks-app.worker.ts b/app/apps/blocks/blocks-app.worker.ts index 2b570197..7ab33372 100644 --- a/app/apps/blocks/blocks-app.worker.ts +++ b/app/apps/blocks/blocks-app.worker.ts @@ -49,7 +49,7 @@ const PANEL_X = 230; /** * Height the layout above is drawn for: a min-height window's viewport under * the standard top bar. A taller viewport (the status bar folded into the - * switcher row, a full-panel display) centers it vertically. + * switcher row, a Tall or Full Screen display) centers it vertically. */ const LAYOUT_HEIGHT = 260; @@ -290,8 +290,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { updateTickTimer(window); } break; + case "resize-window": { + const window = windows.get(message.windowId); + if (!window) break; + window.viewportWidth = message.viewport.width; + window.viewportHeight = message.viewport.height; + window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight }); + window.lastSubmittedFingerprint = ""; + if (window.foreground) renderAndSubmit(window, 0); + break; + } case "navigation-sensors": - case "resize-window": case "text-input": case "tool-call": break; diff --git a/app/apps/calculator/calculator-app.ts b/app/apps/calculator/calculator-app.ts index e71b827a..483ad438 100644 --- a/app/apps/calculator/calculator-app.ts +++ b/app/apps/calculator/calculator-app.ts @@ -116,6 +116,9 @@ export function createCalculatorAppWindow(options: InProcessAppOptions): InProce }, removeSurface: options.removeSurface, onClosed: () => { + // Closing keeps the base layer (clearToBase), so stop its continuous + // listening explicitly; otherwise the mic capture outlives the window. + layer.onRemoved(); toolRegistry.removeAppTools(CALCULATOR_WINDOW_ID); options.onClosed(); }, diff --git a/app/apps/calculator/calculator.ts b/app/apps/calculator/calculator.ts index 1a02dfed..6c7b3263 100644 --- a/app/apps/calculator/calculator.ts +++ b/app/apps/calculator/calculator.ts @@ -16,6 +16,7 @@ import { type Layer, type LayerActions, type LayerContext } from "../../ui/layer import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; import { shell } from "../../ui/shell/shell"; import { voiceControlBridge, type VoiceTranscriptEvent } from "../../native/voice-control"; +import { voiceActivity } from "../../ui/shell/voice-activity"; import { type MathBranchOutcome, MathCoordinator } from "./math/coordinator"; import { type LongRunOffer, type LongRunProgress, offerGlassesPrompt, progressText } from "./math/workload"; import { plain } from "./math/text-renderer"; @@ -56,6 +57,11 @@ export const calculatorDegreesSetting = new ConfigSettingBoolean({ description: "Evaluate sin/cos/tan in degrees instead of radians.", }); +// The calculator's id for shared continuous capture, and its claim on the mic +// while a dictation is armed. +const CONTINUOUS_CLIENT = "calculator"; +const DICTATION_HOLDER = "calculator-dictation"; + // --------------------------------------------------------------------------- // Layer @@ -131,22 +137,22 @@ export class CalculatorLayer implements Layer { if (this.listening === "continuous") { if (!this.continuousActive) { this.continuousActive = true; - void this.actions.startContinuousVoiceCapture(); + void this.actions.startContinuousVoiceCapture(CONTINUOUS_CLIENT); } } else if (this.continuousActive) { this.continuousActive = false; - void this.actions.stopContinuousVoiceCapture(); + void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } private stopListening(): void { if (this.dictating) { - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); } if (this.continuousActive) { this.continuousActive = false; - void this.actions.stopContinuousVoiceCapture(); + void this.actions.stopContinuousVoiceCapture(CONTINUOUS_CLIENT); } } @@ -248,15 +254,25 @@ export class CalculatorLayer implements Layer { beginDictation(): void { if (this.textInput) { void this.textInput(); return; } if (this.dictating) return; - this.dictating = true; + this.setDictating(true); void this.actions.startVoiceCapture(true); this.requestRender(); } + /** + * Arm or disarm dictation, holding the mic claim for it as the voice dialog + * does: the Microphones app and EvenHub mic apps stand down meanwhile (in + * 4-mic mode their stream would otherwise reach the recognizer). + */ + private setDictating(dictating: boolean): void { + this.dictating = dictating; + voiceActivity.setActive(dictating, DICTATION_HOLDER); + } + cancelDictation(): void { if (!this.dictating) return; - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); this.requestRender(); } @@ -277,8 +293,8 @@ export class CalculatorLayer implements Layer { const continuous = this.continuousActive && this.foreground; if (!armed && !continuous) return; if (armed) { - this.dictating = false; void this.actions.stopVoiceCapture(); + this.setDictating(false); } // A standing offer or a run in flight owns the next yes/no said in the // room ("keep going", "never mind", "stop"). diff --git a/app/apps/compass/compass-app.ts b/app/apps/compass/compass-app.ts index 0f546d7f..27cb1303 100644 --- a/app/apps/compass/compass-app.ts +++ b/app/apps/compass/compass-app.ts @@ -23,7 +23,14 @@ import { import { shell } from "../../ui/shell/shell"; import { ensureLocationPermission, hasLocationPermission } from "../../native/location-permissions"; import { isCompassCalibrated, normalizeHeading } from "./calibration"; -import { cardinalDirection, createCompassBackground, drawCompassRose, layoutCompassRose, TICK_HEIGHT } from "./compass-rose"; +import { + cardinalDirection, + createCompassBackground, + drawCompassRose, + layoutCompassRose, + magneticAccuracyDots, + TICK_HEIGHT, +} from "./compass-rose"; import { CompassCalibrationLayer } from "./calibration-layer"; import { getDeclinationAvailability, onDeclinationChanged, refreshDeclination } from "./declination"; import { getNorthReference, resolveHeading, setNorthReference, type NorthReference } from "./heading"; @@ -277,14 +284,7 @@ function frameStatusText(): string { } } -/** - * The firmware's magnetometer calibration readiness (0-3) as filled/empty - * dots, e.g. "Calibration: ●●○" for 2/3. This is the glasses' own sensor - * calibration, not the wearer-fit offset set on the calibration screen. - */ +/** The firmware's magnetometer calibration readiness, e.g. "Calibration: ●●○" for 2/3. */ function magneticAccuracyText(diagnostics: CompassDiagnostics | null): string { - const accuracy = diagnostics && (diagnostics.flags & 0x80) ? diagnostics.magneticAccuracy : -1; - if (accuracy < 0) return "Calibration: --"; - const filled = Math.min(3, accuracy); - return `Calibration: ${"●".repeat(filled)}${"○".repeat(3 - filled)}`; + return `Calibration: ${magneticAccuracyDots(diagnostics) ?? "--"}`; } diff --git a/app/apps/compass/compass-rose.ts b/app/apps/compass/compass-rose.ts index e8410c1a..f32be49f 100644 --- a/app/apps/compass/compass-rose.ts +++ b/app/apps/compass/compass-rose.ts @@ -5,6 +5,7 @@ * own the magnetometer lifecycle and the readout text around it. */ import { GrayImage } from "../../graphics/image"; +import { type CompassDiagnostics } from "../../native/compass-types"; import { normalizeHeading } from "./calibration"; /** @@ -102,6 +103,19 @@ export function cardinalDirection(heading: number): string { return names[Math.round(normalizeHeading(heading) / 45) % names.length]!; } +/** + * The firmware's magnetometer calibration readiness (0-3) as filled/empty + * dots, e.g. "●●○" for 2/3, or null when the firmware doesn't report it. This + * is the glasses' own sensor calibration, not the wearer-fit offset set on the + * calibration screen. + */ +export function magneticAccuracyDots(diagnostics: CompassDiagnostics | null): string | null { + const accuracy = diagnostics && (diagnostics.flags & 0x80) ? diagnostics.magneticAccuracy : -1; + if (accuracy < 0) return null; + const filled = Math.min(3, accuracy); + return "●".repeat(filled) + "○".repeat(3 - filled); +} + interface Point { x: number; y: number; diff --git a/app/apps/developer/microphones-monitor.ts b/app/apps/developer/microphones-monitor.ts index 44100a34..513bc8db 100644 --- a/app/apps/developer/microphones-monitor.ts +++ b/app/apps/developer/microphones-monitor.ts @@ -65,6 +65,14 @@ type ArmStats = { statusAtMs: number | null; }; +/** Drop arrivals older than the rate window. */ +function trimRecent(stats: ArmStats, nowMs: number): void { + const cutoff = nowMs - RATE_WINDOW_MS; + let drop = 0; + while (drop < stats.recent.length && stats.recent[drop]!.wallMs < cutoff) drop++; + if (drop > 0) stats.recent.splice(0, drop); +} + function emptyStats(): ArmStats { return { packets: 0, @@ -169,6 +177,8 @@ export class MicrophonesMonitorLayer implements Layer { this.advanceBuckets(nowMs); stats.packets++; stats.buckets[BUCKETS - 1]!++; + // Trim here too: painting (the other trim) stops while the page is hidden. + trimRecent(stats, nowMs); stats.recent.push({ wallMs: nowMs, arrivalMs, bytes: data.length }); stats.lastArrivalWallMs = nowMs; @@ -268,8 +278,7 @@ export class MicrophonesMonitorLayer implements Layer { const font = getDefaultSmallFont(); const line = font.lineHeight; const stats = this.stats[arm]; - const cutoff = nowMs - RATE_WINDOW_MS; - while (stats.recent.length > 0 && stats.recent[0]!.wallMs < cutoff) stats.recent.shift(); + trimRecent(stats, nowMs); const rate = (stats.recent.length * 1000) / RATE_WINDOW_MS; const byteRate = stats.recent.reduce((sum, arrival) => sum + arrival.bytes, 0) / RATE_WINDOW_MS; let maxGap = 0; diff --git a/app/apps/evenhub/compass-router.ts b/app/apps/evenhub/compass-router.ts index e9a33d46..dd37814d 100644 --- a/app/apps/evenhub/compass-router.ts +++ b/app/apps/evenhub/compass-router.ts @@ -14,7 +14,7 @@ import { isCompassCalibrated } from "../compass/calibration"; import { refreshDeclination } from "../compass/declination"; import { resolveHeading } from "../compass/heading"; -/** Wire shape of the `compass` extension event (faceclaw-extensions `CompassReading`). */ +/** Wire shape of the `compass` extension event (@faceclaw/evenhub-extensions `CompassReading`). */ export type EvenHubCompassReading = { headingDegrees: number; magneticHeadingDegrees: number; diff --git a/app/apps/evenhub/compositor.ts b/app/apps/evenhub/compositor.ts index e154ed41..8518facf 100644 --- a/app/apps/evenhub/compositor.ts +++ b/app/apps/evenhub/compositor.ts @@ -4,7 +4,8 @@ * * Text uses Even's extracted 20px firmware font (via EvenHubFont), measured * and wrapped with @evenrealities/pretext, so glyph widths and line breaks - * match what apps expect. + * match what apps expect. Extended layouts may pick other fonts per container + * (fonts.ts), which wrap with Faceclaw's own text wrapping. * * Known deviations from stock, acceptable for now: * - List rendering is a plain vertical list. A non-zero itemWidth sets the @@ -13,7 +14,7 @@ * - Lists scroll with our menus' animation and bounce (see list-menu.ts). */ import { GrayImage } from "../../graphics/image"; -import { EvenHubFont } from "../../graphics/evenhub-font"; +import { containerFace } from "./fonts"; import { type EvenHubContainer, type EvenHubImageContainer, @@ -47,14 +48,11 @@ function paintTextContainer(image: GrayImage, container: EvenHubTextContainer): // the container's border at full brightness regardless. paintBorder(image, container); const inset = container.borderWidth + container.paddingLength; - EvenHubFont.get().drawTextWrapped( - image, - container.x + inset, - container.y + inset, - Math.max(1, container.width - 2 * inset), - container.content, - textGrey(container.textColor), - ); + const face = containerFace(container.font); + const lines = face.wrap(container.content, Math.max(1, container.width - 2 * inset)); + for (let i = 0; i < lines.length; i++) { + face.drawText(image, container.x + inset, container.y + inset + i * face.lineHeight, lines[i]!, textGrey(container.textColor)); + } } function paintBorder( @@ -94,7 +92,7 @@ function paintImageContainer(image: GrayImage, container: EvenHubImageContainer) function paintListContainer(image: GrayImage, container: EvenHubListContainer, focused: boolean): void { paintBorder(image, container); - paintListItems(image, container, EvenHubFont.get(), focused); + paintListItems(image, container, containerFace(container.font), focused); } /** Bottom-to-top paint order for a page's containers. */ diff --git a/app/apps/evenhub/containers.ts b/app/apps/evenhub/containers.ts index edd3c751..6cb7f629 100644 --- a/app/apps/evenhub/containers.ts +++ b/app/apps/evenhub/containers.ts @@ -5,6 +5,7 @@ * (Container_ID), and numbers sometimes arrive as strings, so all field * reads go through loose key matching (the SDK's pickLoose behavior). */ +import { mergeFontSpecs, parseFontSpec, type EvenHubFontSpec } from "./font-spec"; export type EvenHubTextContainer = { kind: "text"; @@ -24,6 +25,8 @@ export type EvenHubTextContainer = { content: string; /** SDK 0.0.14 textColor: brightness level 0..4; undefined means the device default (4). */ textColor: number | undefined; + /** Extended-layout font (merged with the layout's default); undefined is the stock font. */ + font: EvenHubFontSpec | undefined; }; export type EvenHubImageContainer = { @@ -63,6 +66,8 @@ export type EvenHubListContainer = { selectBorder: boolean; /** Selection is host-local: scroll moves it with no app round-trip. */ selectedIndex: number; + /** Extended-layout font (merged with the layout's default); undefined is the stock font. */ + font: EvenHubFontSpec | undefined; }; export type EvenHubContainer = EvenHubTextContainer | EvenHubImageContainer | EvenHubListContainer; @@ -130,6 +135,17 @@ export function readString(obj: Record, key: string, fallback: return fallback; } +/** + * A 0/1 protobuf flag. Stock apps send numbers; extension layouts may send + * booleans (the extension types allow both), and numeric strings show up in + * the wild. + */ +function readFlag(obj: Record, key: string): boolean { + const value = pickLoose(obj, key); + if (typeof value === "boolean") return value; + return readNumber(obj, key, 0) !== 0; +} + export function asRecord(value: unknown): Record { return value && typeof value === "object" && !Array.isArray(value) ? (value as Record) : {}; } @@ -138,7 +154,12 @@ function readBorderWidth(json: Record): number { return Math.max(0, Math.min(MAX_BORDER_WIDTH, readNumber(json, "borderWidth", 0))); } -function parseTextContainer(json: Record): EvenHubTextContainer { +/** A container's font: its own `font` over the layout-wide default. */ +function readFont(json: Record, layoutFont: EvenHubFontSpec | undefined): EvenHubFontSpec | undefined { + return mergeFontSpecs(layoutFont, parseFontSpec(pickLoose(json, "font"))); +} + +function parseTextContainer(json: Record, layoutFont: EvenHubFontSpec | undefined): EvenHubTextContainer { return { kind: "text", id: readNumber(json, "containerID", 0), @@ -150,11 +171,12 @@ function parseTextContainer(json: Record): EvenHubTextContainer borderWidth: readBorderWidth(json), borderRadius: readNumber(json, "borderRadius", 0), paddingLength: readNumber(json, "paddingLength", 0), - isEventCapture: readNumber(json, "isEventCapture", 0) !== 0, + isEventCapture: readFlag(json, "isEventCapture"), zOrderIndex: readOptionalNumber(json, "zOrderIndex"), - preserve: readNumber(json, "preserve", 0) !== 0, + preserve: readFlag(json, "preserve"), content: readString(json, "content", ""), textColor: readTextBrightness(json), + font: readFont(json, layoutFont), }; } @@ -220,14 +242,14 @@ function parseImageContainer(json: Record): EvenHubImageContain width: readNumber(json, "width", 0), height: readNumber(json, "height", 0), zOrderIndex: readOptionalNumber(json, "zOrderIndex"), - preserve: readNumber(json, "preserve", 0) !== 0, + preserve: readFlag(json, "preserve"), pixels: null, pixelsWidth: 0, pixelsHeight: 0, }; } -function parseListContainer(json: Record): EvenHubListContainer { +function parseListContainer(json: Record, layoutFont: EvenHubFontSpec | undefined): EvenHubListContainer { const itemContainer = asRecord(pickLoose(json, "itemContainer")); const itemNamesRaw = pickLoose(itemContainer, "itemName"); const itemNames = Array.isArray(itemNamesRaw) ? itemNamesRaw.map(String) : []; @@ -242,13 +264,14 @@ function parseListContainer(json: Record): EvenHubListContainer borderWidth: readBorderWidth(json), borderRadius: readNumber(json, "borderRadius", 0), paddingLength: readNumber(json, "paddingLength", 0), - isEventCapture: readNumber(json, "isEventCapture", 0) !== 0, + isEventCapture: readFlag(json, "isEventCapture"), zOrderIndex: readOptionalNumber(json, "zOrderIndex"), - preserve: readNumber(json, "preserve", 0) !== 0, + preserve: readFlag(json, "preserve"), itemNames, itemWidth: readNumber(itemContainer, "itemWidth", 0), - selectBorder: readNumber(itemContainer, "isItemSelectBorderEn", 0) !== 0, + selectBorder: readFlag(itemContainer, "isItemSelectBorderEn"), selectedIndex: 0, + font: readFont(json, layoutFont), }; } @@ -263,9 +286,11 @@ function parseListContainer(json: Record): EvenHubListContainer */ export function parsePage(data: Record): EvenHubPage { const containers: EvenHubContainer[] = []; + // Extended layouts may set a default font for every text and list container. + const layoutFont = parseFontSpec(pickLoose(data, "font")); const lists = pickLoose(data, "listObject"); if (Array.isArray(lists)) { - for (const item of lists) containers.push(parseListContainer(asRecord(item))); + for (const item of lists) containers.push(parseListContainer(asRecord(item), layoutFont)); } const images = pickLoose(data, "imageObject"); if (Array.isArray(images)) { @@ -273,7 +298,7 @@ export function parsePage(data: Record): EvenHubPage { } const texts = pickLoose(data, "textObject"); if (Array.isArray(texts)) { - for (const item of texts) containers.push(parseTextContainer(asRecord(item))); + for (const item of texts) containers.push(parseTextContainer(asRecord(item), layoutFont)); } return { containers, menuItems: parseMenuItems(data) }; } diff --git a/app/apps/evenhub/device-status-source.ts b/app/apps/evenhub/device-status-source.ts new file mode 100644 index 00000000..82423308 --- /dev/null +++ b/app/apps/evenhub/device-status-source.ts @@ -0,0 +1,28 @@ +/** + * The live data behind an EvenHub app's device status: glasses presence (fed + * by the connection controller), the headset battery (from the shell's top + * bar state), and the paired glasses' serial. + */ +import { shell } from "../../ui/shell/shell"; +import { getGlassesPresence, onGlassesPresenceChanged } from "../../g2/glasses-presence"; +import { loadPairedGlassesIdentity } from "../../g2/device-addresses"; +import { type EvenHubDeviceStatusSource } from "./device-status"; + +export const liveDeviceStatusSource: EvenHubDeviceStatusSource = { + read: () => { + const presence = getGlassesPresence(); + return { + serial: loadPairedGlassesIdentity()?.serial ?? null, + connected: presence.connected, + worn: presence.worn, + battery: shell.getBatteryLevels().headset, + charging: presence.charging, + }; + }, + subscribe: (listener) => { + const offs = [onGlassesPresenceChanged(listener), shell.onBatteryLevelsChanged(listener)]; + return () => { + for (const off of offs) off(); + }; + }, +}; diff --git a/app/apps/evenhub/device-status.ts b/app/apps/evenhub/device-status.ts new file mode 100644 index 00000000..e281733c --- /dev/null +++ b/app/apps/evenhub/device-status.ts @@ -0,0 +1,63 @@ +/** + * The stock SDK's DeviceStatus (getGlassesInfo().status and the + * deviceStatusChanged push), built from what Faceclaw actually knows about + * the glasses. Pure, so it can be tested without NativeScript. + */ + +/** Used when no glasses have been paired (preview-only mode). */ +export const PLACEHOLDER_SERIAL = "FACECLAW-G2"; + +/** Wire shape of the SDK's DeviceStatus. Unknown fields are omitted rather than guessed. */ +export type EvenHubDeviceStatus = { + sn: string; + connectType: "connected" | "disconnected"; + isWearing?: boolean; + batteryLevel?: number; + isCharging?: boolean; + isInCase?: boolean; +}; + +export type EvenHubDeviceStatusInput = { + /** The paired glasses' serial number, or null when none are paired. */ + serial: string | null; + connected: boolean; + /** null until the firmware reports a wear state this session. */ + worn: boolean | null; + /** Last known headset battery percentage, or null if never reported. */ + battery: number | null; + charging: boolean; +}; + +/** Where a session reads the device status from (live in the app; fixed in tests). */ +export type EvenHubDeviceStatusSource = { + read(): EvenHubDeviceStatusInput; + /** Call `listener` after anything read() reports may have changed. Returns an unsubscribe function. */ + subscribe(listener: () => void): () => void; +}; + +/** No glasses: what a session reports when it isn't given a live source. */ +export const OFFLINE_DEVICE_STATUS_SOURCE: EvenHubDeviceStatusSource = { + read: () => ({ serial: null, connected: false, worn: null, battery: null, charging: false }), + subscribe: () => () => {}, +}; + +export function buildEvenHubDeviceStatus(input: EvenHubDeviceStatusInput): EvenHubDeviceStatus { + const status: EvenHubDeviceStatus = { + // The SDK only applies a status push whose sn matches getGlassesInfo's, + // so both must come from here. + sn: input.serial?.trim() || PLACEHOLDER_SERIAL, + connectType: input.connected ? "connected" : "disconnected", + isCharging: input.charging, + // The G2 only charges in its case, and Faceclaw has no separate + // in-case report. + isInCase: input.charging, + }; + if (input.worn !== null) status.isWearing = input.worn; + if (input.battery !== null) status.batteryLevel = input.battery; + return status; +} + +export function sameDeviceStatus(a: EvenHubDeviceStatus, b: EvenHubDeviceStatus): boolean { + return a.sn === b.sn && a.connectType === b.connectType && a.isWearing === b.isWearing && + a.batteryLevel === b.batteryLevel && a.isCharging === b.isCharging && a.isInCase === b.isInCase; +} diff --git a/app/apps/evenhub/even-api.ts b/app/apps/evenhub/even-api.ts index 051568f9..08ff4f6c 100644 --- a/app/apps/evenhub/even-api.ts +++ b/app/apps/evenhub/even-api.ts @@ -235,7 +235,7 @@ export class EvenHubApiClient { const token = options.token ?? ""; const headers: Record = { common, - sign: hmacSign(signingParts(method, path, common, query, token, body)), + sign: hmacSign(signingParts(method, path, common, signedQuery(options.query ?? {}), token, body)), "Content-Type": "application/json", }; if (token) headers.token = token; @@ -266,7 +266,7 @@ export function signingParts( body = "", ): string { const parts = [method.toUpperCase(), path, common]; - if (query) parts.push(normalizeQuery(query)); + if (query) parts.push(query); if (token) parts.push(token); if (body) parts.push(body); return parts.sort().join("\n"); @@ -320,19 +320,26 @@ function buildCommon(version: 1 | 3): string { return Object.entries(fields).map(([key, value]) => `${formEncode(key)}=${formEncode(String(value))}`).join("&"); } -function encodePairs(values: Record): string { +/** Query entries sorted by key in code-unit order (Dart's default String sort). */ +function sortedPairs(values: Record): [string, string][] { return Object.entries(values) - .sort(([a], [b]) => a.localeCompare(b)) - .map(([key, value]) => `${formEncode(key)}=${formEncode(String(value))}`) - .join("&"); + .map(([key, value]): [string, string] => [key, String(value)]) + .sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); } -function normalizeQuery(query: string): string { - return query - .split("&") - .filter(Boolean) - .sort((a, b) => a.split("=", 1)[0]!.localeCompare(b.split("=", 1)[0]!)) - .join("&"); +function encodePairs(values: Record): string { + return sortedPairs(values).map(([key, value]) => `${formEncode(key)}=${formEncode(value)}`).join("&"); +} + +/** + * The query as it participates in signing: sorted raw `key=value` pairs, NOT + * URL-encoded (SignatureUtil.calculateSignature in the official 2.2.8 Dart + * image). The server signs the decoded parameters, so signing the encoded + * form only matched when encoding was a no-op (a search for "two words" was + * rejected with HTTP 406). + */ +function signedQuery(values: Record): string { + return sortedPairs(values).map(([key, value]) => `${key}=${value}`).join("&"); } function formEncode(value: string): string { diff --git a/app/apps/evenhub/evenhub-window.ts b/app/apps/evenhub/evenhub-window.ts index 09d399d1..a0937cb5 100644 --- a/app/apps/evenhub/evenhub-window.ts +++ b/app/apps/evenhub/evenhub-window.ts @@ -5,7 +5,7 @@ */ import { GrayImage } from "../../graphics/image"; import { EVENHUB_SCREEN_WIDTH } from "./compositor"; -import { type InputEvent } from "../../ui/gestures"; +import { directionalFallback, isDirectionalInput, type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerContext } from "../../ui/layers"; import { createInProcessWindow, @@ -18,13 +18,20 @@ import { EvenHubSession } from "./session"; import { renderInstalledEvenHubIcon } from "./installed-apps"; class EvenHubAppLayer implements Layer { + /** + * Watch swipes become the usual stock gestures here rather than in the + * stack's fallback, which labels them as ring input: an app reading the + * extension's source field should see that they came from the watch. + */ + readonly acceptsDirectional = true; + constructor(private readonly session: EvenHubSession) {} paint(ctx: LayerContext): GrayImage { const size = ctx.stack.getBaseSize(); - // EvenHub apps lay out for the stock 576-wide surface. In the full-panel - // display mode the window is 640 wide; paint at stock width and centre - // it rather than leaving all the slack on the right. + // EvenHub apps lay out for the stock 576-wide surface. In the Full + // Screen display mode the window is 640 wide; paint at stock width and + // centre it rather than leaving all the slack on the right. if (size.width > EVENHUB_SCREEN_WIDTH) { const content = this.session.paint({ width: EVENHUB_SCREEN_WIDTH, height: size.height }, ctx.stack.isFocused()); const image = new GrayImage(size.width, size.height, 0); @@ -40,7 +47,7 @@ class EvenHubAppLayer implements Layer { // menu intercepts tap-then-hold), which is the guaranteed way out since // EvenHub apps own double-click. Tap-then-hold is still reported to the // app as LONG_PRESS_EVENT — see the handleInput wrapper below. - this.session.handleGesture(event); + this.session.handleGesture(isDirectionalInput(event) ? ({ ...directionalFallback(event), source: "watch" } as InputEvent) : event); } } @@ -50,6 +57,7 @@ export function createEvenHubWindow( session: EvenHubSession, options: InProcessAppOptions, onShowPhone: () => void, + requestShellRender: () => void, ): InProcessWindow { const fallbackIcon = windowIcon("package", session.manifest.name.charAt(0).toUpperCase() || "E"); const created = createInProcessWindow({ @@ -58,11 +66,14 @@ export function createEvenHubWindow( title: session.manifest.name, iconLetter: session.manifest.name.charAt(0).toUpperCase() || "E", closeable: true, + // An icon the app set at runtime (extension setWindowIcon) wins over the + // package's. drawIcon: makeImageWindowIcon( - (size) => renderInstalledEvenHubIcon(session.manifest.packageId, size), + (size) => session.renderWindowIcon(size) ?? renderInstalledEvenHubIcon(session.manifest.packageId, size), fallbackIcon, ), heightMode: "medium", + keepsScreenOn: () => session.keepsScreenOn(), // The window's context menu doubles as the OS contextual menu EvenHub // apps register with `menuObject` (SDK 0.0.14): the app's own entries come // first, then the host entries. Evaluated at open time, so a rebuild that @@ -99,6 +110,14 @@ export function createEvenHubWindow( // The extended layout uses the full 576x452 app area ("max"); stock apps // stay in the 576x288 band ("medium"). setTallCanvas: (tall) => created.setHeightMode(tall ? "max" : "medium"), + setAttention: (attention) => shell.setWindowAttention(windowId, attention), + // Only while the app listens for text: otherwise the shell's Type Into App + // destinations (voice dialog, phone keyboard, watch) would send into a void. + setAcceptsTextInput: (accept) => { + created.window.receiveTextInput = accept ? (text, textOptions) => session.deliverTextInput(text, textOptions) : undefined; + }, + startVoiceInput: () => shell.foregroundWindow()?.windowId === windowId && shell.startVoiceInput(), + iconChanged: requestShellRender, windowId, }); diff --git a/app/apps/evenhub/font-spec.ts b/app/apps/evenhub/font-spec.ts new file mode 100644 index 00000000..a16318b1 --- /dev/null +++ b/app/apps/evenhub/font-spec.ts @@ -0,0 +1,171 @@ +/** + * Font options for EvenHub text and list containers (the extension layout's + * `font: { family, size, weight }`): parsing, inheritance from the layout's + * default, and choosing a face from what is installed. Pure, so it can be + * tested without NativeScript; app/apps/evenhub/fonts.ts loads the result. + * + * A container with no font anywhere keeps the stock firmware font, exactly as + * on stock. Naming any font option switches to Faceclaw's fonts, with the + * defaults below filling the options left out. + */ + +/** A container's font request, after parsing. Absent fields take the defaults. */ +export type EvenHubFontSpec = { + family?: string; + size?: number; + weight?: number; +}; + +export const DEFAULT_FONT_FAMILY = "Roboto"; +/** The stock firmware font's size, so switching family alone keeps text about the same size. */ +export const DEFAULT_FONT_SIZE = 20; +export const DEFAULT_FONT_WEIGHT = 400; +export const MIN_FONT_SIZE = 6; +/** Larger glyphs leave the glyph-cache path (255px limit), so text stops being cheap to send. */ +export const MAX_FONT_SIZE = 128; + +/** + * Parse a `font` object from a layout or container. Undefined when there is + * none; unusable fields are dropped, so `{}` means "Faceclaw's default font". + */ +export function parseFontSpec(raw: unknown): EvenHubFontSpec | undefined { + if (!raw || typeof raw !== "object" || Array.isArray(raw)) return undefined; + const record = raw as Record; + const spec: EvenHubFontSpec = {}; + if (typeof record.family === "string" && record.family.trim()) spec.family = record.family.trim(); + const size = typeof record.size === "string" ? Number(record.size) : record.size; + if (typeof size === "number" && Number.isFinite(size)) { + spec.size = Math.min(MAX_FONT_SIZE, Math.max(MIN_FONT_SIZE, Math.round(size))); + } + const weight = parseWeight(record.weight); + if (weight !== undefined) spec.weight = weight; + return spec; +} + +function parseWeight(raw: unknown): number | undefined { + if (typeof raw === "string") { + const keyword = raw.trim().toLowerCase(); + if (keyword === "normal") return 400; + if (keyword === "bold") return 700; + raw = Number(keyword); + } + if (typeof raw !== "number" || !Number.isFinite(raw)) return undefined; + return Math.min(1000, Math.max(1, Math.round(raw))); +} + +/** A container's font: its own options over the layout default's, field by field. */ +export function mergeFontSpecs( + layoutDefault: EvenHubFontSpec | undefined, + own: EvenHubFontSpec | undefined, +): EvenHubFontSpec | undefined { + if (!layoutDefault) return own; + if (!own) return layoutDefault; + return { ...layoutDefault, ...own }; +} + +/** One loadable face: a TTF file, a Terminus weight, or the stock font. */ +export type FontFaceCandidate = { + /** Family as the font names itself (e.g. "Inter 18pt"). */ + family: string; + /** 100..900. */ + weight: number; + /** The sizes a bitmap face comes in; null for a scalable face. */ + sizes: readonly number[] | null; + monospace: boolean; + /** Identifies the face to the loader. */ + key: string; +}; + +/** + * The family name apps use: an optical-size suffix ("Inter 18pt") is dropped, + * since the bundled Inter is its 18pt cut and apps would ask for "Inter". + */ +export function displayFamily(family: string): string { + return family.replace(/\s+\d+\s*pt$/i, "").trim() || family; +} + +function normalizeFamily(name: string): string { + return name.toLowerCase().replace(/[^a-z0-9]/g, ""); +} + +export function familyMatches(candidate: string, requested: string): boolean { + const wanted = normalizeFamily(requested); + return normalizeFamily(candidate) === wanted || normalizeFamily(displayFamily(candidate)) === wanted; +} + +/** + * Pick the face and pixel size for a spec. An unknown family falls back to + * DEFAULT_FONT_FAMILY; null when even that is missing (the caller then uses + * the stock font). Weight follows CSS's font matching: 400–500 look up to + * 500 first, lighter requests look lighter first, heavier ones heavier + * first. A bitmap face takes its nearest size (the smaller on a tie). + */ +export function selectFontFace( + spec: EvenHubFontSpec, + candidates: readonly FontFaceCandidate[], +): { face: FontFaceCandidate; size: number } | null { + const family = spec.family ?? DEFAULT_FONT_FAMILY; + let faces = candidates.filter((candidate) => familyMatches(candidate.family, family)); + if (faces.length === 0) faces = candidates.filter((candidate) => familyMatches(candidate.family, DEFAULT_FONT_FAMILY)); + if (faces.length === 0) return null; + const desired = spec.weight ?? DEFAULT_FONT_WEIGHT; + const face = [...faces].sort((a, b) => weightRank(a.weight, desired) - weightRank(b.weight, desired))[0]!; + const requestedSize = spec.size ?? DEFAULT_FONT_SIZE; + return { face, size: face.sizes ? nearestSize(face.sizes, requestedSize) : requestedSize }; +} + +/** Lower is better: CSS font-weight matching as a sort key. */ +function weightRank(weight: number, desired: number): number { + const distance = Math.abs(weight - desired); + if (weight === desired) return 0; + if (desired >= 400 && desired <= 500) { + if (weight > desired && weight <= 500) return distance; + if (weight < desired) return 1000 + distance; + return 2000 + distance; + } + const preferLighter = desired < 400; + return (weight < desired) === preferLighter ? distance : 1000 + distance; +} + +function nearestSize(sizes: readonly number[], requested: number): number { + let best = sizes[0]!; + for (const size of sizes) { + const better = Math.abs(size - requested) < Math.abs(best - requested) || + (Math.abs(size - requested) === Math.abs(best - requested) && size < best); + if (better) best = size; + } + return best; +} + +/** A family as the extension's getFonts() reports it. */ +export type FontFamilyInfo = { + family: string; + weights: number[]; + /** Bitmap families' sizes; absent for scalable families. */ + sizes?: number[]; + monospace: boolean; +}; + +/** Group faces into families for getFonts(), sorted by name. */ +export function describeFontFamilies(candidates: readonly FontFaceCandidate[]): FontFamilyInfo[] { + const families = new Map(); + for (const candidate of candidates) { + const name = displayFamily(candidate.family); + const key = normalizeFamily(name); + let info = families.get(key); + if (!info) { + info = { family: name, weights: [], monospace: candidate.monospace }; + families.set(key, info); + } + if (!info.weights.includes(candidate.weight)) info.weights.push(candidate.weight); + if (candidate.sizes) info.sizes = Array.from(new Set([...(info.sizes ?? []), ...candidate.sizes])); + info.monospace = info.monospace && candidate.monospace; + } + return Array.from(families.values()) + .map((info) => ({ + ...info, + weights: [...info.weights].sort((a, b) => a - b), + ...(info.sizes ? { sizes: [...info.sizes].sort((a, b) => a - b) } : {}), + })) + .sort((a, b) => a.family.localeCompare(b.family)); +} diff --git a/app/apps/evenhub/fonts.ts b/app/apps/evenhub/fonts.ts new file mode 100644 index 00000000..52b4236f --- /dev/null +++ b/app/apps/evenhub/fonts.ts @@ -0,0 +1,139 @@ +/** + * Loads the fonts EvenHub text and list containers ask for (see font-spec.ts + * for the matching rules): the stock firmware font, Terminus at its bitmap + * sizes, and every installed TTF/OTF — the bundled Roboto, Inter, Montserrat + * and Roboto Mono, plus any the user installed from the Files app. + */ +import { EvenHubFont } from "../../graphics/evenhub-font"; +import { getTerminusFont, TERMINUS_SIZES } from "../../graphics/bdffont"; +import { TtfFont } from "../../graphics/ttf-font"; +import { listInstalledFonts } from "../../graphics/installed-fonts"; +import { wrapText } from "../../graphics/textwrap"; +import type { GrayImage, UiFont } from "../../graphics/image"; +import { + describeFontFamilies, + familyMatches, + selectFontFace, + type EvenHubFontSpec, + type FontFaceCandidate, + type FontFamilyInfo, +} from "./font-spec"; + +/** What containers draw text with: the stock font's interface, which UiFonts are adapted to. */ +export type EvenHubTextFace = { + readonly lineHeight: number; + measureLine(text: string): number; + wrap(text: string, maxWidth: number): string[]; + drawText(image: GrayImage, x: number, y: number, text: string, value?: number): void; +}; + +class UiFontFace implements EvenHubTextFace { + constructor(private readonly font: UiFont) {} + + get lineHeight(): number { + return this.font.lineHeight; + } + + measureLine(text: string): number { + return this.font.measureText(text); + } + + wrap(text: string, maxWidth: number): string[] { + return wrapText(this.font, text, maxWidth); + } + + drawText(image: GrayImage, x: number, y: number, text: string, value = 255): void { + this.font.drawText(image, x, y, text, value); + } +} + +const STOCK_KEY = "stock"; +const TERMINUS_KEY_PREFIX = "terminus:"; +const TTF_KEY_PREFIX = "ttf:"; + +const BUILT_IN_FACES: readonly FontFaceCandidate[] = [ + // The firmware font: one size, one weight. + { family: "EvenHub", weight: 400, sizes: [20], monospace: false, key: STOCK_KEY }, + { family: "Terminus", weight: 400, sizes: TERMINUS_SIZES, monospace: true, key: `${TERMINUS_KEY_PREFIX}400` }, + { family: "Terminus", weight: 700, sizes: TERMINUS_SIZES, monospace: true, key: `${TERMINUS_KEY_PREFIX}700` }, +]; + +/** A family the listing lacks is re-listed at most this often (the user may have just installed it). */ +const RELIST_INTERVAL_MS = 30_000; + +let faces: FontFaceCandidate[] | null = null; +let listedAtMs = 0; + +function listFaces(): FontFaceCandidate[] { + const installed = listInstalledFonts() + // Upright faces only: there is no italic option yet. + .filter((font) => font.weightOrder < 1000) + .map((font): FontFaceCandidate => ({ + family: font.family, + weight: font.weightOrder, + sizes: null, + monospace: font.monospace, + key: `${TTF_KEY_PREFIX}${font.path}`, + })); + faces = [...BUILT_IN_FACES, ...installed]; + listedAtMs = Date.now(); + return faces; +} + +/** Whether some listed face has the family (no family asks for the default, which is bundled). */ +function hasFamily(available: readonly FontFaceCandidate[], family: string | undefined): boolean { + return !family || available.some((face) => familyMatches(face.family, family)); +} + +/** Resolved faces by spec; null means the stock font (asked for, or nothing loadable). */ +const resolved = new Map(); + +/** The face for a spec, and whether it is in the family asked for (rather than the fallback). */ +function loadFace(spec: EvenHubFontSpec): { face: EvenHubTextFace | null; exact: boolean } { + let available = faces ?? listFaces(); + if (!hasFamily(available, spec.family) && Date.now() - listedAtMs > RELIST_INTERVAL_MS) available = listFaces(); + const exact = hasFamily(available, spec.family); + return { face: openFace(selectFontFace(spec, available)), exact }; +} + +function openFace(choice: { face: FontFaceCandidate; size: number } | null): EvenHubTextFace | null { + if (!choice || choice.face.key === STOCK_KEY) return null; + const { face, size } = choice; + if (face.key.startsWith(TERMINUS_KEY_PREFIX)) { + const font = getTerminusFont(size, face.weight >= 600); + return font ? new UiFontFace(font) : null; + } + const font = TtfFont.load(face.key.slice(TTF_KEY_PREFIX.length), size); + return font ? new UiFontFace(font) : null; +} + +/** + * The face a container draws with: the stock font when it names no font, + * otherwise the closest match to its spec. + */ +export function containerFace(spec: EvenHubFontSpec | undefined): EvenHubTextFace { + if (!spec) return EvenHubFont.get(); + const key = `${spec.family ?? ""}|${spec.size ?? ""}|${spec.weight ?? ""}`; + let face = resolved.get(key); + if (face === undefined) { + const loaded = loadFace(spec); + face = loaded.face; + // A fallback isn't kept, so a family the user installs later gets picked up. + if (loaded.exact) resolved.set(key, face); + } + // Not cached as an object: EvenHubFont.get() changes once firmware fonts are extracted. + return face ?? EvenHubFont.get(); +} + +/** The font families apps can ask for (extension getFonts). Re-lists installed fonts. */ +export function availableFontFamilies(): FontFamilyInfo[] { + return describeFontFamilies(listFaces()); +} + +/** Width of the widest line of `text` and the line height, in the font a container would use. */ +export function measureContainerText(text: string, spec: EvenHubFontSpec | undefined): { width: number; lineHeight: number } { + const face = containerFace(spec); + let width = 0; + for (const line of text.split("\n")) width = Math.max(width, face.measureLine(line)); + return { width: Math.ceil(width), lineHeight: face.lineHeight }; +} diff --git a/app/apps/evenhub/manager.ts b/app/apps/evenhub/manager.ts index 0b12dabe..14b9382f 100644 --- a/app/apps/evenhub/manager.ts +++ b/app/apps/evenhub/manager.ts @@ -22,6 +22,7 @@ import { unpackRuntime } from "./unpack-runtime"; import { type EvenHubPermission } from "./permissions"; import { EvenHubSession } from "./session"; import { createEvenHubWindow } from "./evenhub-window"; +import { liveDeviceStatusSource } from "./device-status-source"; import { createEvenHubWebView, type EvenHubWebView } from "./webview"; import { shell } from "../../ui/shell/shell"; import { GrayImage } from "../../graphics/image"; @@ -126,7 +127,7 @@ async function startApp( ): Promise { const windowId = `evenhub:app:${nextSerial++}`; const session = new EvenHubSession(manifest, distDir, ctx.appendLog, remoteUrl, - payload => ctx.actions.playBuzzerSequence(payload)); + payload => ctx.actions.playBuzzerSequence(payload), liveDeviceStatusSource); const webView = createEvenHubWebView(session); session.attachWebView({ evaluateJs: webView.evaluateJs, @@ -147,7 +148,7 @@ async function startApp( try { await ctx.launchInProcessApp(windowId, `window:${windowId}`, (options) => - createEvenHubWindow(windowId, appId, session, options, () => showOnPhone(windowId)), + createEvenHubWindow(windowId, appId, session, options, () => showOnPhone(windowId), () => ctx.requestShellRender()), ); } catch (error) { session.close(); throw error; } } diff --git a/app/apps/evenhub/mic-router.ts b/app/apps/evenhub/mic-router.ts index fca3e374..76f7fe15 100644 --- a/app/apps/evenhub/mic-router.ts +++ b/app/apps/evenhub/mic-router.ts @@ -17,6 +17,9 @@ import { voiceActivity } from "../../ui/shell/voice-activity"; declare const com: any; +// The router's claim on the shared raw-PCM tap (the Microphones app holds its own). +const RAW_CAPTURE_OWNER = "evenhub-mic-router"; + export type EvenHubMicClient = { readonly windowId: string; isForeground(): boolean; @@ -83,9 +86,9 @@ class EvenHubMicRouter { // eligibility change so it self-heals after an STT preemption. Returns // false if STT currently owns the mic; a later trigger (assistant modal // close, foreground/screen change) re-tries. - voiceControlBridge.startRawCapture({ communicator: activeCommunicator() }); + voiceControlBridge.startRawCapture({ communicator: activeCommunicator(), owner: RAW_CAPTURE_OWNER }); } else { - voiceControlBridge.stopRawCapture(); + voiceControlBridge.stopRawCapture(RAW_CAPTURE_OWNER); } } } diff --git a/app/apps/evenhub/session.ts b/app/apps/evenhub/session.ts index 95957c1d..53289885 100644 --- a/app/apps/evenhub/session.ts +++ b/app/apps/evenhub/session.ts @@ -63,6 +63,16 @@ import { toolRegistry, type ToolResult, type ToolSpec } from "../../assistant/to import { getCurrentLocation } from "../../native/location"; import { LocationTracker, type TrackedLocation } from "../../native/location-tracker"; import { ensureFineLocationPermission } from "../../native/location-permissions"; +import { + buildEvenHubDeviceStatus, + OFFLINE_DEVICE_STATUS_SOURCE, + sameDeviceStatus, + type EvenHubDeviceStatus, + type EvenHubDeviceStatusSource, +} from "./device-status"; +import { parseWindowIcon, WindowIconRenderer } from "./window-icon"; +import { availableFontFamilies, containerFace, measureContainerText } from "./fonts"; +import { parseFontSpec } from "./font-spec"; const UPNG = require("upng-js"); @@ -226,9 +236,15 @@ export const EVENHUB_BRIDGE_INJECT_SCRIPT = ` /** * Injected at document-start (after the stock bridge shim): defines * `window.getFaceclawExtensions`, the single entry point for Faceclaw-only - * capabilities (see the faceclaw-extensions package). It never touches the - * standard SDK or any prototype — everything lives behind this one global, so - * apps stay compatible with the stock Even app (where the global is absent). + * capabilities. Its public contract (the `FaceclawExtensions` interface) is + * the @faceclaw/evenhub-extensions package in evenhub-extensions/. Keep the + * two in step (tests/evenhub-extensions-shim.test.cjs checks the member list). + * When adding members, bump API_VERSION and tag them `@since apiVersion N` in + * the package. + * + * It never touches the standard SDK or any prototype — everything lives behind + * this one global, so apps stay compatible with the stock Even app (where the + * global is absent). * * RPC uses its own id-keyed promise map and `__fcExtResolve`, kept separate * from the stock `__fcResolve` channel. Events arrive via `__fcExtEvent`. @@ -239,13 +255,15 @@ export function buildFaceclawExtensionsScript(versionString: string): string { (function () { if (window.getFaceclawExtensions) return; var VERSION = ${JSON.stringify(versionString)}; + var API_VERSION = 1; var pending = {}; var nextId = 1; window.__fcExtResolve = function (id, ok, value) { var entry = pending[id]; if (!entry) return; delete pending[id]; - (ok ? entry[0] : entry[1])(value); + if (ok) entry[0](value); + else entry[1](new Error(String(value))); }; function call(method, params) { return new Promise(function (resolve, reject) { @@ -260,9 +278,10 @@ export function buildFaceclawExtensionsScript(versionString: string): string { } var listeners = {}; window.__fcExtEvent = function (name, data) { - var arr = listeners[name]; - if (!arr) return; - for (var i = 0; i < arr.slice().length; i++) { + // Iterate a snapshot, so a listener that unsubscribes mid-dispatch + // doesn't make the next one get skipped. + var arr = (listeners[name] || []).slice(); + for (var i = 0; i < arr.length; i++) { try { arr[i](data); } catch (e) { if (window.console) console.error(e); } } }; @@ -275,19 +294,29 @@ export function buildFaceclawExtensionsScript(versionString: string): string { if (i >= 0) arr.splice(i, 1); }; } - // Compass: enable on first listener, disable when the last one leaves. - var compassListeners = []; - function addCompass(cb) { - compassListeners.push(cb); - if (compassListeners.length === 1) send("setCompass", [true]); - var off = on("compass", cb); - return function () { - off(); - var i = compassListeners.indexOf(cb); - if (i >= 0) compassListeners.splice(i, 1); - if (compassListeners.length === 0) send("setCompass", [false]); + // Events the host only produces on request (compass, touch-down, text + // input): it is told to start on the first listener and to stop when the + // last one leaves. Each unsubscribe function works once. + function counted(name, setMethod) { + var count = 0; + return function (cb) { + var off = on(name, cb); + if (++count === 1) send(setMethod, [true]); + var active = true; + return function () { + if (!active) return; + active = false; + off(); + if (--count === 0) send(setMethod, [false]); + }; }; } + function setWindowIcon(icon) { + if (!icon) return call("setWindowIcon", [null]); + // Typed arrays (ImageData.data) don't survive JSON; send a plain array. + var data = Array.prototype.slice.call(icon.data || []); + return call("setWindowIcon", [{ width: icon.width, height: icon.height, data: data }]); + } // Voice-assistant tools: specs go to the host; handlers stay here and run when // the host invokes a tool via __fcExtInvokeTool. var toolHandlers = {}; @@ -318,6 +347,7 @@ export function buildFaceclawExtensionsScript(versionString: string): string { send("setAssistantTools", [specs]); } var extensions = { + apiVersion: API_VERSION, getVersion: function () { return VERSION; }, returnToAppSwitcher: function () { send("returnToAppSwitcher"); }, quit: function () { send("quit"); }, @@ -325,10 +355,18 @@ export function buildFaceclawExtensionsScript(versionString: string): string { getConfiguredApiKeys: function () { return call("getConfiguredApiKeys"); }, requestApiKeyAccess: function (services) { return call("requestApiKeyAccess", [services || []]); }, playBuzzer: function (steps) { return call("playBuzzer", [steps || []]); }, - addCompassListener: addCompass, + addCompassListener: counted("compass", "setCompass"), createLayout: function (layout) { return call("createLayout", [layout || {}]); }, replaceLayout: function (layout) { return call("replaceLayout", [layout || {}]); }, - setAssistantTools: setAssistantTools + setAssistantTools: setAssistantTools, + addTouchDownListener: counted("touchDown", "setTouchDown"), + addTextInputListener: counted("textInput", "setTextInput"), + startVoiceInput: function () { return call("startVoiceInput"); }, + setWindowIcon: setWindowIcon, + setAttention: function (attention) { send("setAttention", [!!attention]); }, + setKeepScreenOn: function (keepOn) { send("setKeepScreenOn", [!!keepOn]); }, + getFonts: function () { return call("getFonts"); }, + measureText: function (text, font) { return call("measureText", [String(text), font || null]); } }; window.getFaceclawExtensions = function () { return extensions; }; })(); @@ -354,6 +392,14 @@ export type EvenHubWindowHooks = { pushOverlay: (layer: Layer) => void; /** Switch the window between the stock 576x288 band and the full 576x452 canvas. */ setTallCanvas: (tall: boolean) => void; + /** Show or clear the attention dot on the window's switcher icon. */ + setAttention: (attention: boolean) => void; + /** Offer (or stop offering) the window as the shell's "Type Into App" destination. */ + setAcceptsTextInput: (accept: boolean) => void; + /** Open the shell's voice dialog aimed at the window; false if it can't open now. */ + startVoiceInput: () => boolean; + /** The app changed its window icon: repaint the switcher. */ + iconChanged: () => void; /** The shell window id (unique per running instance; used for tool registration). */ windowId: string; }; @@ -404,6 +450,21 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH private shellFocused = true; /** API-key services the user has granted this app this session. */ private readonly grantedApiKeys = new Set(); + /** The app listens for touch-down (extension addTouchDownListener). */ + private touchDownRequested = false; + /** The app listens for text input, so the shell offers it as a Type Into App destination. */ + private textInputRequested = false; + /** The app asked for the attention dot on its switcher icon. */ + private attention = false; + /** The app asked the idle screen timeout to wait while it is in the foreground. */ + private keepScreenOnRequested = false; + /** The app's own switcher icon; null draws the package icon. */ + private windowIcon: WindowIconRenderer | null = null; + /** The paired glasses' serial, read once so getGlassesInfo and every status push agree. */ + private glassesSerial: string | null | undefined = undefined; + /** The last deviceStatusChanged sent, and the subscriptions that refresh it. */ + private lastDeviceStatus: EvenHubDeviceStatus | null = null; + private readonly deviceStatusOffs: (() => void)[] = []; private log: (message: string) => void; constructor( @@ -412,6 +473,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH log: (message: string) => void, remoteUrl = "", private readonly sendBuzzerSequence: (payload: Uint8Array) => Promise | void = () => {}, + private readonly deviceStatusSource: EvenHubDeviceStatusSource = OFFLINE_DEVICE_STATUS_SOURCE, ) { this.manifest = manifest; this.distDir = distDir; @@ -425,26 +487,28 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.webViewHandle = handle; } - webViewFailed(message: string): void { + /** + * A WebView load failed. Once the app has loaded, a failed or blocked + * navigation leaves its page running, so only a failure before then (or one + * the host marks `fatal`) replaces the glasses window with the error. + */ + webViewFailed(message: string, fatal = false): void { if (this.closed) return; - this.loadError = message; this.log(`evenhub: ${message}`); this.windowHooks?.requestRender(); + this.log(`evenhub: ${message}`); + if (this.launchContextPushed && !fatal) return; + this.loadError = message; this.windowHooks?.requestRender(); } /** The WebView finished loading the app: push the one-shot launch context. */ webViewLoaded(): void { + if (this.loadError) { this.loadError = ''; this.windowHooks?.requestRender(); } if (this.launchContextPushed) return; this.launchContextPushed = true; // Launched to drive the glasses (from the on-glasses file browser), so // apps that branch on this should take their glasses path. this.pushMessage("evenAppLaunchSource", { launchSource: "glassesMenu" }); - this.pushMessage("deviceStatusChanged", { - sn: "FACECLAW-G2", - connectType: "connected", - isWearing: true, - batteryLevel: 100, - isCharging: false, - isInCase: false, - }); + this.pushDeviceStatus(); + this.deviceStatusOffs.push(this.deviceStatusSource.subscribe(() => this.pushDeviceStatus())); } // ----- glasses window wiring ----- @@ -452,6 +516,10 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH attachWindow(hooks: EvenHubWindowHooks): void { this.windowHooks = hooks; this.shellWindowId = hooks.windowId; + // Requests the app made before the window existed. + if (this.textInputRequested) hooks.setAcceptsTextInput(true); + if (this.attention) hooks.setAttention(true); + if (this.windowIcon) hooks.iconChanged(); } windowClosed(): void { @@ -481,18 +549,29 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH * locally with events only for clicks and boundary scrolls. */ handleGesture(event: InputEvent): void { + // Touch-down has no stock event (an unknown eventType would parse as a + // click), so it goes only to apps that subscribed through the extension. + if (event.type === "ring-press") { + if (this.touchDownRequested) { + this.pushExtEvent("touchDown", { source: extensionInputSource(event.source), timestampMs: event.timestampMs }); + } + return; + } if (!this.page) return; - const capture = this.page ? eventCaptureContainer(this.page) : undefined; + const capture = eventCaptureContainer(this.page); + // The stock eventSource can't tell the watch from the ring, and scrolls + // carry none; the extension field names the device when it is known. + const inputSource = "source" in event && event.source ? extensionInputSource(event.source) : undefined; switch (event.type) { case "click": if (capture?.kind === "list") { - this.emitListEvent(capture, CLICK_EVENT); + this.emitListEvent(capture, CLICK_EVENT, inputSource); } else { - this.emitSysEvent(CLICK_EVENT, gestureSource(event.source)); + this.emitSysEvent(CLICK_EVENT, gestureSource(event.source), inputSource); } break; case "double-click": - this.emitSysEvent(DOUBLE_CLICK_EVENT, gestureSource(event.source)); + this.emitSysEvent(DOUBLE_CLICK_EVENT, gestureSource(event.source), inputSource); break; case "long-press": case "short-then-long-press": @@ -506,9 +585,9 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH // given exclusive use of the gesture. const eventType = event.type === "long-press-release" ? LONG_PRESS_RELEASE_EVENT : LONG_PRESS_EVENT; if (capture?.kind === "list") { - this.emitListEvent(capture, eventType); + this.emitListEvent(capture, eventType, inputSource); } else { - this.emitSysEvent(eventType, gestureSource(event.source)); + this.emitSysEvent(eventType, gestureSource(event.source), inputSource); } break; } @@ -516,9 +595,9 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH case "scroll-down": { const eventType = event.type === "scroll-up" ? SCROLL_TOP_EVENT : SCROLL_BOTTOM_EVENT; if (capture?.kind === "list") { - this.scrollList(capture, eventType); + this.scrollList(capture, eventType, inputSource); } else { - this.emitSysEvent(eventType, 0); + this.emitSysEvent(eventType, 0, inputSource); } break; } @@ -541,11 +620,11 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.pushMessage("evenHubEvent", { type: "menuItemClickEvent", jsonData: { itemID } }); } - private scrollList(list: EvenHubListContainer, eventType: number): void { + private scrollList(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void { // Swipe up selects the previous item, swipe down the next (stock direction). const delta = eventType === SCROLL_TOP_EVENT ? -1 : 1; // At a boundary the selection stays put (the list bounces) and the app hears about it. - if (!stepListSelection(list, EvenHubFont.get(), delta)) this.emitListEvent(list, eventType); + if (!stepListSelection(list, containerFace(list.font), delta)) this.emitListEvent(list, eventType, inputSource); this.windowHooks?.requestRender(); } @@ -810,19 +889,10 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH return ApplicationSettings.getString(this.storageKey(readString(data, "key", "")), ""); case "getUserInfo": return { uid: 0, name: "Faceclaw", avatar: "", country: "" }; - case "getGlassesInfo": - return { - model: "g2", - sn: "FACECLAW-G2", - status: { - sn: "FACECLAW-G2", - connectType: "connected", - isWearing: true, - batteryLevel: 100, - isCharging: false, - isInCase: false, - }, - }; + case "getGlassesInfo": { + const status = this.deviceStatus(); + return { model: "g2", sn: status.sn, status }; + } case "audioControl": return this.audioControl(data); case "getAppLocation": @@ -956,7 +1026,12 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH } private storageKey(key: string): string { - return `evenhub:${this.manifest.packageId}:ls:${key}`; + // The package ID must end at the first ':' so that no other package can + // reach this one's keys (manifest IDs are unvalidated: package "a" asking + // for key "b:ls:c" must not read package "a:ls:b"'s "c"). IDs containing + // ':' or '%' are encoded, so they can't collide with the plain IDs either. + const id = this.manifest.packageId; + return `evenhub:${/[:%]/.test(id) ? encodeURIComponent(id) : id}:ls:${key}`; } // ----- pushes (host -> web) ----- @@ -980,6 +1055,12 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH ); } + /** Log a problem with the app's own calls where its developer will see it: the app's console. */ + private warnApp(message: string): void { + this.log(`evenhub: ${message}`); + this.webViewHandle?.evaluateJs(`window.console && console.warn(${JSON.stringify(`Faceclaw: ${message}`)})`); + } + /** Deliver an extension event to the app (e.g. windowLifecycle). */ private pushExtEvent(name: string, data: unknown): void { this.webViewHandle?.evaluateJs( @@ -991,17 +1072,18 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.pushMessage("evenHubEvent", { type: payloadType, jsonData: elideZeroFields(jsonData) }); } - private emitSysEvent(eventType: number, eventSource: number): void { - this.pushEvenHubEvent("sysEvent", { eventType, eventSource }); + private emitSysEvent(eventType: number, eventSource: number, inputSource?: ExtensionInputSource): void { + this.pushEvenHubEvent("sysEvent", { eventType, eventSource, [INPUT_SOURCE_FIELD]: inputSource }); } - private emitListEvent(list: EvenHubListContainer, eventType: number): void { + private emitListEvent(list: EvenHubListContainer, eventType: number, inputSource?: ExtensionInputSource): void { this.pushEvenHubEvent("listEvent", { containerID: list.id, containerName: list.name, currentSelectItemIndex: list.selectedIndex, currentSelectItemName: list.itemNames[list.selectedIndex] ?? "", eventType, + [INPUT_SOURCE_FIELD]: inputSource, }); } @@ -1017,7 +1099,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH result = await this.dispatchExtension(method, args.slice(1)); } catch (error) { ok = false; - result = String(error); + result = error instanceof Error ? error.message : String(error); this.log(`evenhub: extension call failed: ${error}`); } // id 0 is a fire-and-forget send; nothing is awaiting it. @@ -1050,12 +1132,96 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH case "setAssistantTools": this.setAssistantTools(Array.isArray(params[0]) ? params[0] : []); return null; + case "setTouchDown": + this.touchDownRequested = params[0] === true; + return null; + case "setTextInput": + this.setTextInputRequested(params[0] === true); + return null; + case "startVoiceInput": + // Without a text listener the dialog would have nowhere in the app to send to. + return this.textInputRequested && (this.windowHooks?.startVoiceInput() ?? false); + case "setWindowIcon": + this.setWindowIcon(params[0]); + return null; + case "setAttention": + this.attention = params[0] === true; + this.windowHooks?.setAttention(this.attention); + return null; + case "setKeepScreenOn": + this.keepScreenOnRequested = params[0] === true; + return null; + case "getFonts": + return availableFontFamilies(); + case "measureText": + // No font measures in the stock font, as a container without one draws. + return measureContainerText(String(params[0] ?? ""), parseFontSpec(params[1])); default: this.log(`evenhub: unknown extension method ${method}`); return null; } } + // ----- device status (stock getGlassesInfo / deviceStatusChanged) ----- + + private deviceStatus(): EvenHubDeviceStatus { + const input = this.deviceStatusSource.read(); + if (this.glassesSerial === undefined) this.glassesSerial = input.serial; + return buildEvenHubDeviceStatus({ ...input, serial: this.glassesSerial }); + } + + /** Push deviceStatusChanged if the status differs from the last one sent. */ + private pushDeviceStatus(): void { + if (this.closed) return; + const status = this.deviceStatus(); + if (this.lastDeviceStatus && sameDeviceStatus(status, this.lastDeviceStatus)) return; + this.lastDeviceStatus = status; + this.pushMessage("deviceStatusChanged", status); + } + + // ----- window controls and text input (extensions) ----- + + /** Whether the idle screen timeout should wait while this window is in the foreground. */ + keepsScreenOn(): boolean { + return this.keepScreenOnRequested; + } + + /** The app's own switcher icon at `size`, or null to draw the package icon. */ + renderWindowIcon(size: number): GrayImage | null { + return this.windowIcon?.render(size) ?? null; + } + + private setWindowIcon(raw: unknown): void { + if (raw == null) { + this.windowIcon = null; + } else { + const image = parseWindowIcon(raw); + if (!image) { + throw new Error( + "setWindowIcon: expected {width, height, data} of at most 256x256, with data holding " + + "width*height grayscale or width*height*4 RGBA bytes", + ); + } + this.windowIcon = new WindowIconRenderer(image); + } + this.windowHooks?.iconChanged(); + } + + private setTextInputRequested(accept: boolean): void { + if (this.textInputRequested === accept) return; + this.textInputRequested = accept; + this.windowHooks?.setAcceptsTextInput(accept); + } + + /** + * Text the shell routed to this window: the voice dialog's Type Into App, + * the phone keyboard, the watch, or remote input. Only arrives while the + * app listens (setAcceptsTextInput). + */ + deliverTextInput(text: string, options?: { submit?: boolean }): void { + this.pushExtEvent("textInput", options?.submit === undefined ? { text } : { text, submit: options.submit }); + } + // ----- compass (EvenHubCompassClient) ----- deliverCompass(reading: EvenHubCompassReading): void { @@ -1106,12 +1272,16 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH for (const raw of rawSpecs) { const record = asRecord(raw); const name = readString(record, "name", ""); - if (!name) continue; + const problem = appToolProblem(name, record.parameters); + if (problem) { + this.warnApp(`setAssistantTools: skipped tool ${JSON.stringify(name)}: ${problem}`); + continue; + } const availability = record.availability === "open" ? "open" : "foreground"; specs.push({ name, description: readString(record, "description", ""), - inputSchema: asRecord(record.parameters), + inputSchema: record.parameters == null ? { type: "object", properties: {} } : asRecord(record.parameters), availability, }); } @@ -1202,13 +1372,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH if (steps.length === 0) return; for (let i = 0; i < steps.length; i += CFW_SEQ_MAX) { const chunk = steps.slice(i, i + CFW_SEQ_MAX); - const payload = buildSoundSequencePayload(chunk); - try { - await this.sendBuzzerSequence(payload); - } catch (error) { - this.log(`evenhub: playBuzzer failed: ${error}`); - return; - } + // A send failure rejects the app's promise (handleExtensionCall logs it). + await this.sendBuzzerSequence(buildSoundSequencePayload(chunk)); if (i + CFW_SEQ_MAX < steps.length) await delay(phraseDurationMs(chunk)); } } @@ -1239,6 +1404,7 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH this.pendingToolCalls.clear(); this.locationTracker?.stop(); this.locationTracker = null; + for (const off of this.deviceStatusOffs.splice(0)) off(); if (!this.systemExitSent) { // Best effort: lets the app's teardown handlers run before the webview dies. this.emitSysEvent(SYSTEM_EXIT_EVENT, 0); @@ -1350,6 +1516,35 @@ function configuredApiKeyServices(): string[] { return API_KEY_SERVICES.filter((service) => service.setting.get().trim()).map((service) => service.id); } +/** App assistant-tool names: what every model provider accepts in a tool name. */ +const APP_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/; + +/** + * Why an app's assistant tool can't be offered, or null if it can. Provider + * APIs reject a whole request over one bad tool definition, so offering it + * would break every assistant turn while the app runs. + */ +function appToolProblem(name: string, parameters: unknown): string | null { + if (!APP_TOOL_NAME.test(name)) return "name must be 1-64 letters, digits, underscores or hyphens"; + if (parameters == null) return null; + const schema = parameters as Record; + if (typeof parameters !== "object" || Array.isArray(parameters) || schema.type !== "object") { + return 'parameters must be a JSON Schema with type "object"'; + } + if ("oneOf" in schema || "anyOf" in schema || "allOf" in schema) { + return "parameters can't use oneOf, anyOf or allOf at the top level"; + } + const properties = schema.properties; + if (properties !== undefined && (!properties || typeof properties !== "object" || Array.isArray(properties))) { + return "parameters.properties must be an object"; + } + const required = schema.required; + if (required !== undefined && !(Array.isArray(required) && required.every((key) => typeof key === "string"))) { + return "parameters.required must be an array of property names"; + } + return null; +} + /** Coerce an app-supplied buzzer step list into validated {freq, ms, duty?}. */ function normalizeBuzzerSteps(raw: unknown[]): Step[] { const out: Step[] = []; @@ -1441,6 +1636,27 @@ function luminance(r: number, g: number, b: number): number { return Math.round(0.2126 * r + 0.7152 * g + 0.0722 * b); } +/** + * Input-device names in the extension API: the touchDown event's source, and + * the `faceclawInputSource` field added to stock gesture events. + */ +type ExtensionInputSource = "ring" | "left-touchpad" | "right-touchpad" | "watch"; + +const INPUT_SOURCE_FIELD = "faceclawInputSource"; + +function extensionInputSource(source: InputSource): ExtensionInputSource { + switch (source) { + case "ring": + return "ring"; + case "left-arm": + return "left-touchpad"; + case "right-arm": + return "right-touchpad"; + case "watch": + return "watch"; + } +} + /** * ring/left-arm/right-arm -> PB EventSourceType. The watch is reported to * EvenHub apps as the ring: they only know stock hardware, and the watch's diff --git a/app/apps/evenhub/webview.ios.ts b/app/apps/evenhub/webview.ios.ts index e32ecdbd..c6f9b429 100644 --- a/app/apps/evenhub/webview.ios.ts +++ b/app/apps/evenhub/webview.ios.ts @@ -23,14 +23,19 @@ window.__faceclawEvenHub = { postMessage: function(name, args, id) { export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { const host = FaceclawEvenHubWebView.new() host.packageIdentifier = session.manifest.packageId + // The manager's close callback for the shown phone UI; the native back + // button hides the host itself and reports it as a 'hidden' event. + let onClose: (() => void) | null = null host.eventHandler = (json: string) => { try { const event = JSON.parse(json) if (event.kind === 'call' && typeof event.name === 'string' && typeof event.args === 'string' && Number.isSafeInteger(event.id)) session.handleBridgeCall(event.name, event.args, event.id) else if (event.kind === 'loaded') session.webViewLoaded() - else if (event.kind === 'error') session.webViewFailed(String(event.message)) - else if (event.kind === 'console') console.log(`[evenhub:${session.manifest.name}] ${event.message}`) + else if (event.kind === 'error') session.webViewFailed(String(event.message), event.fatal === true) + else if (event.kind === 'hidden') { const close = onClose; onClose = null; close?.() } + // 'notice': host-side events such as a blocked navigation, logged with the app's console. + else if (event.kind === 'console' || event.kind === 'notice') console.log(`[evenhub:${session.manifest.name}] ${event.message}`) } catch (error) { console.warn(`EvenHub bridge: ${error}`) } } // Defer loading until manager.ts has attached the session/window handles. @@ -41,5 +46,6 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { }, 0) return { native: host, evaluateJs: js => host.evaluate(js), destroy: () => { clearTimeout(loading); host.destroy() }, - showOnPhone: () => host.showOnPhone(), hideOnPhone: () => host.hideOnPhone() } + showOnPhone: close => { onClose = close; host.showOnPhone() }, + hideOnPhone: () => { onClose = null; host.hideOnPhone() } } } diff --git a/app/apps/evenhub/webview.ts b/app/apps/evenhub/webview.ts index 655822ac..b50a9828 100644 --- a/app/apps/evenhub/webview.ts +++ b/app/apps/evenhub/webview.ts @@ -15,6 +15,11 @@ * the offline serving entirely and loads that URL from the network; the shim * is registered as a document-start script instead of being spliced into the * served HTML. + * + * The app can navigate its WebView to other sites, so host access is tied to + * the app's origin: the bridge port is injected only into the app's own frames + * (FaceclawEvenHubJsBridge), and every script the host runs in the WebView is + * guarded by an origin check. */ import { Application, Utils } from "@nativescript/core"; import { EVENHUB_BRIDGE_INJECT_SCRIPT, buildFaceclawExtensionsScript, type EvenHubSession } from "./session"; @@ -34,19 +39,40 @@ export type EvenHubWebView = { native: unknown; }; -/** A distinct (fake, intercepted) https origin per app isolates web storage. */ -function originHost(session: EvenHubSession): string { - const safe = session.manifest.packageId.toLowerCase().replace(/[^a-z0-9.-]/g, "-"); - return `${safe}.evenhub.invalid`; +/** + * Wraps the bridge port (window.__faceclawEvenHubPort, which WebView injects + * only into the app's own frames) as the window.__faceclawEvenHub interface + * the bridge shim and the extensions call. + */ +export const ANDROID_EVENHUB_TRANSPORT_SCRIPT = ` +(function () { + var port = window.__faceclawEvenHubPort; + if (!port || window.__faceclawEvenHub) return; + window.__faceclawEvenHub = { + postMessage: function (name, args, id) { port.postMessage(JSON.stringify(['call', String(name), String(args), id])); }, + wakeTimers: function () { port.postMessage('["wake"]'); } + }; +})(); +`; + +/** A JS expression that is true only in a document from `startUrl`'s origin. */ +export function appOriginCheck(startUrl: string): string { + return `(function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(${JSON.stringify(startUrl)}))`; +} + +/** `js`, run only when the WebView's document is from `startUrl`'s origin. */ +export function guardToAppOrigin(startUrl: string, js: string): string { + return `if (${appOriginCheck(startUrl)}) {\n${js}\n}`; } /** - * The allowed-origin rule for a remote app's document-start script: the - * loaded URL's own origin, so the shim isn't handed to third-party frames. + * A distinct (fake, intercepted) https origin per app. The per-app profile is + * what keeps storage apart; without one, this still separates origin-scoped + * storage, though not cookies scoped to the shared evenhub.invalid domain. */ -function originRule(url: string): string { - const match = /^(https?:\/\/[^/?#]+)/i.exec(url); - return match ? match[1]! : "*"; +function originHost(session: EvenHubSession): string { + const safe = session.manifest.packageId.toLowerCase().replace(/[^a-z0-9.-]/g, "-"); + return `${safe}.evenhub.invalid`; } /** @@ -59,12 +85,20 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { // A mutable wrapper, so the host can re-point the WebView at a recreated // activity (see FaceclawEvenHubWebViewHost). const webView = new com.faceclaw.app.FaceclawEvenHubWebView(new android.content.MutableContextWrapper(context)); + // The app's own WebView profile, so its browser storage (and that of sites it + // navigates to) is separate from every other app's. This must come before + // any other use of the WebView. + if (!com.faceclaw.app.FaceclawEvenHubProfile.apply(webView, session.manifest.packageId)) { + console.warn(`evenhub: this WebView can't separate profiles; ${session.manifest.packageId} shares browser storage with other apps`); + } const settings = webView.getSettings(); settings.setJavaScriptEnabled(true); settings.setDomStorageEnabled(true); settings.setAllowFileAccess(false); settings.setMediaPlaybackRequiresUserGesture(false); + // Set once the renderer is gone: the WebView is destroyed and must not be used. + let gone = false; const listener = new com.faceclaw.app.FaceclawEvenHubListener({ onEvenAppMessage: (handlerName: string, argsJson: string, callId: number) => { session.handleBridgeCall(String(handlerName), String(argsJson), Number(callId)); @@ -72,17 +106,31 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { onPageFinished: () => { session.webViewLoaded(); }, + onRenderProcessGone: (crashed: boolean) => { + gone = true; + session.webViewFailed(crashed + ? "The app's web page crashed. Close and reopen the app." + : "Android stopped the app's web page to free memory. Close and reopen the app.", true); + }, }); const remoteUrl = session.remoteUrl; // Remote apps load from their own server, so nothing is intercepted and the // fake per-app origin is unused. const host = remoteUrl ? "" : originHost(session); - // Stock bridge shim first, then the Faceclaw extensions global (both at - // document-start, before any app JS). - const injectScript = EVENHUB_BRIDGE_INJECT_SCRIPT + buildFaceclawExtensionsScript(`Faceclaw/${FACECLAW_VERSION}`); - const documentStartInstalled = remoteUrl - ? com.faceclaw.app.FaceclawEvenHubDocumentStart.install(webView, injectScript, originRule(remoteUrl)) === true + const startUrl = remoteUrl || `https://${host}/${session.manifest.entrypoint}`; + const originRule: string | null = com.faceclaw.app.FaceclawEvenHubJsBridge.originRule(startUrl); + const appCheck = appOriginCheck(startUrl); + const guarded = (js: string) => guardToAppOrigin(startUrl, js); + // The transport, the stock bridge shim, then the Faceclaw extensions global + // (all at document-start, before any app JS). + const injectScript = guarded(ANDROID_EVENHUB_TRANSPORT_SCRIPT + EVENHUB_BRIDGE_INJECT_SCRIPT + + buildFaceclawExtensionsScript(`Faceclaw/${FACECLAW_VERSION}`)); + const bridgeError: string | null = originRule + ? com.faceclaw.app.FaceclawEvenHubJsBridge.install(webView, originRule, listener) + : `Not a web address: ${startUrl}`; + const documentStartInstalled = remoteUrl && originRule + ? com.faceclaw.app.FaceclawEvenHubDocumentStart.install(webView, injectScript, originRule) === true : false; webView.setWebViewClient( new com.faceclaw.app.FaceclawEvenHubWebViewClient( @@ -95,20 +143,22 @@ export function createEvenHubWebView(session: EvenHubSession): EvenHubWebView { listener, ), ); - webView.addJavascriptInterface(new com.faceclaw.app.FaceclawEvenHubJsBridge(listener), "__faceclawEvenHub"); // Surface the app's console in logcat (tag FaceclawEvenHubConsole). webView.setWebChromeClient(new com.faceclaw.app.FaceclawEvenHubChromeClient()); const nativeHost = com.faceclaw.app.FaceclawEvenHubWebViewHost.getInstance(); - nativeHost.attach(activity, webView); - webView.loadUrl(remoteUrl || `https://${host}/${session.manifest.entrypoint}`); + nativeHost.attach(activity, webView, appCheck); + // Without the bridge the app could only hang waiting for it. + if (bridgeError) session.webViewFailed(String(bridgeError), true); + else webView.loadUrl(startUrl); return { native: webView, evaluateJs: (js: string) => { + if (gone) return; try { - webView.evaluateJavascript(js, null); + webView.evaluateJavascript(guarded(js), null); } catch (error) { console.warn(`evenhub evaluateJs failed: ${error}`); } diff --git a/app/apps/evenhub/window-icon.ts b/app/apps/evenhub/window-icon.ts new file mode 100644 index 00000000..87ff6f5e --- /dev/null +++ b/app/apps/evenhub/window-icon.ts @@ -0,0 +1,119 @@ +/** + * App-supplied window icons (the extension setWindowIcon): parse the pixel + * buffer an app sends and scale it to the sizes the app switcher draws at. + * Pure, so it can be tested without NativeScript. + */ +import { GrayImage } from "../../graphics/image"; + +/** Larger icons are refused: they only cost bridge traffic, since switcher icons are ~32px. */ +export const MAX_WINDOW_ICON_DIMENSION = 256; + +/** + * Parse `{width, height, data}` from the bridge into a grayscale image. + * `data` is width*height brightness bytes, or width*height*4 RGBA bytes (an + * ImageData), which are reduced to luminance times alpha. Arrays arrive as + * plain arrays or as JSON-ified typed arrays ({"0": n, ...}). Returns null for + * anything malformed. + */ +export function parseWindowIcon(raw: unknown): GrayImage | null { + if (!raw || typeof raw !== "object") return null; + const record = raw as Record; + const width = Number(record.width); + const height = Number(record.height); + if (!Number.isInteger(width) || !Number.isInteger(height)) return null; + if (width < 1 || height < 1 || width > MAX_WINDOW_ICON_DIMENSION || height > MAX_WINDOW_ICON_DIMENSION) return null; + const data = toNumberArray(record.data); + if (!data) return null; + const area = width * height; + const image = new GrayImage(width, height, 0); + if (data.length === area) { + for (let i = 0; i < area; i++) image.pixels[i] = clampByte(data[i]!); + } else if (data.length === area * 4) { + for (let i = 0; i < area; i++) { + const o = i * 4; + const luminance = 0.2126 * data[o]! + 0.7152 * data[o + 1]! + 0.0722 * data[o + 2]!; + image.pixels[i] = clampByte((luminance * data[o + 3]!) / 255); + } + } else { + return null; + } + return image; +} + +/** + * Scale an icon to fit a size x size box, keeping its aspect ratio, with an + * area-averaging filter. Results are cached per size, since the switcher + * repaints often. + */ +export class WindowIconRenderer { + private readonly scaled = new Map(); + + constructor(private readonly source: GrayImage) {} + + render(size: number): GrayImage { + const box = Math.max(1, Math.round(size)); + let image = this.scaled.get(box); + if (!image) { + image = scaleToFit(this.source, box); + this.scaled.set(box, image); + } + return image; + } +} + +function scaleToFit(source: GrayImage, box: number): GrayImage { + const scale = Math.min(box / source.width, box / source.height); + const width = Math.max(1, Math.round(source.width * scale)); + const height = Math.max(1, Math.round(source.height * scale)); + if (width === source.width && height === source.height) return source; + const out = new GrayImage(width, height, 0); + for (let y = 0; y < height; y++) { + const y0 = (y * source.height) / height; + const y1 = ((y + 1) * source.height) / height; + for (let x = 0; x < width; x++) { + const x0 = (x * source.width) / width; + const x1 = ((x + 1) * source.width) / width; + out.pixels[y * width + x] = clampByte(areaAverage(source, x0, y0, x1, y1)); + } + } + return out; +} + +/** Mean of the source over the rectangle [x0,x1) x [y0,y1), weighting partly covered pixels. */ +function areaAverage(source: GrayImage, x0: number, y0: number, x1: number, y1: number): number { + let sum = 0; + let weight = 0; + for (let sy = Math.floor(y0); sy < Math.min(source.height, Math.ceil(y1)); sy++) { + const wy = Math.min(y1, sy + 1) - Math.max(y0, sy); + for (let sx = Math.floor(x0); sx < Math.min(source.width, Math.ceil(x1)); sx++) { + const w = wy * (Math.min(x1, sx + 1) - Math.max(x0, sx)); + sum += source.pixels[sy * source.width + sx]! * w; + weight += w; + } + } + return weight > 0 ? sum / weight : 0; +} + +function toNumberArray(raw: unknown): number[] | null { + let values: unknown[]; + if (Array.isArray(raw)) { + values = raw; + } else if (raw && typeof raw === "object") { + const record = raw as Record; + values = []; + for (let i = 0; String(i) in record; i++) values.push(record[String(i)]); + } else { + return null; + } + const out: number[] = []; + for (const entry of values) { + const value = Number(entry); + if (!Number.isFinite(value)) return null; + out.push(value); + } + return out; +} + +function clampByte(value: number): number { + return Math.max(0, Math.min(255, Math.round(value))); +} diff --git a/app/apps/freecell/freecell-app.worker.ts b/app/apps/freecell/freecell-app.worker.ts index fb939857..bd611a44 100644 --- a/app/apps/freecell/freecell-app.worker.ts +++ b/app/apps/freecell/freecell-app.worker.ts @@ -205,9 +205,18 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { if (window.foreground) renderAndSubmit(window, 0); break; } + case "resize-window": { + const window = windows.get(message.windowId); + if (!window) break; + window.viewportWidth = message.viewport.width; + window.viewportHeight = message.viewport.height; + window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight }); + window.lastSubmittedFingerprint = ""; + if (window.foreground) renderAndSubmit(window, 0); + break; + } case "input-focus": case "navigation-sensors": - case "resize-window": case "screen": case "text-input": case "tool-call": diff --git a/app/apps/glanceboard/configuration.ts b/app/apps/glanceboard/configuration.ts index 36c40f5a..fcb485df 100644 --- a/app/apps/glanceboard/configuration.ts +++ b/app/apps/glanceboard/configuration.ts @@ -15,6 +15,11 @@ export function isTerminalWidgetConfigured(): boolean { return isGlanceWidgetConfigured("terminal"); } +/** Whether the system card can show: it is in a slot and the Glanceboard is enabled. */ +export function isSystemCardWidgetInUse(): boolean { + return getBooleanSetting("glanceboard.enabled", false) && isGlanceWidgetConfigured("system-card"); +} + /** * Whether the T3 Code widget can show: it is in a slot and the Glanceboard is * enabled (the T3 worker holds its server connections open only then). diff --git a/app/apps/glanceboard/glanceboard-app.ts b/app/apps/glanceboard/glanceboard-app.ts index 303b02d7..8afaf474 100644 --- a/app/apps/glanceboard/glanceboard-app.ts +++ b/app/apps/glanceboard/glanceboard-app.ts @@ -5,7 +5,7 @@ import { enumSettingMenuItem, onAnySettingChanged, toggleSettingMenuItem } from import { uiFontOverridePickerMenuItem } from "../../ui/font-picker"; import { type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerContext } from "../../ui/layers"; -import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, type MenuItem } from "../../ui/menu"; +import { drawSelectionHighlight, isMenuItemDisabled, MenuLayer, openModalMenu, submenuItem, type MenuItem } from "../../ui/menu"; import { Menu } from "../../ui/menu-core"; import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; @@ -27,7 +27,8 @@ import { type GlanceSlotChoice, } from "./glanceboard-settings"; import { resolveGlanceRegions, slotDividers } from "./layout"; -import { glanceWidgetSpans } from "./widgets"; +import { type GlanceWidgetDefinition } from "./widget"; +import { findGlanceWidget, glanceWidgetSpans } from "./widgets"; export const GLANCEBOARD_WINDOW_ID = "glanceboard"; export const GLANCEBOARD_SURFACE_ID = "window:glanceboard"; @@ -300,10 +301,14 @@ function openSlotPicker(ctx: LayerContext, slotIndex: number): void { openModalMenu(ctx, setting.label, items, Math.max(0, setting.values.indexOf(current))); } -/** Settings: layout, sleep gestures, duration, slot lines, widget font, and stereo depth. */ +/** + * Settings: layout, sleep gestures, duration, slot lines, widget font, and + * stereo depth, then a row per widget on the board for its own settings. + */ function openGlanceSettings(ctx: LayerContext): void { - const items: MenuItem[] = [ - enumSettingMenuItem(glanceLayoutSetting), + const items = (): MenuItem[] => [ + // The layout decides which slots (and so which widgets) are on the board. + enumSettingMenuItem(glanceLayoutSetting, { onChange: () => layer.setItems(items()) }), enumSettingMenuItem(glanceTapDurationSetting), toggleSettingMenuItem(glanceShowOnLongPressSetting), toggleSettingMenuItem(glanceShowOnHeadTiltSetting), @@ -314,8 +319,31 @@ function openGlanceSettings(ctx: LayerContext): void { "Typeface for the Glanceboard's widgets. Same as UI font follows the Font setting in Settings > Customization.", }), enumSettingMenuItem(glanceDepthSetting), + ...widgetSettingsMenuItems(), ]; - ctx.stack.push(new MenuLayer("Glanceboard settings", items, pageMenuLayout(ctx.stack.getBaseSize().width))); + const layer = new MenuLayer("Glanceboard settings", items(), pageMenuLayout(ctx.stack.getBaseSize().width)); + ctx.stack.push(layer); +} + +/** + * One row per widget on the board, in slot order: a submenu of the widget's + * settings, or a disabled row for a widget that has none. + */ +function widgetSettingsMenuItems(): MenuItem[] { + const widgets = new Set(); + for (const setting of glanceSlotSettings()) { + const widget = findGlanceWidget(setting.get()); + if (widget) widgets.add(widget); + } + return Array.from(widgets, (widget): MenuItem => { + const settingsMenuItems = widget.settingsMenuItems; + if (!settingsMenuItems) { + return { label: widget.label, description: `${widget.label} has no settings.`, disabled: true, onSelect: () => {} }; + } + return submenuItem(widget.label, (ctx) => { + ctx.stack.push(new MenuLayer(`${widget.label} settings`, settingsMenuItems(), pageMenuLayout(ctx.stack.getBaseSize().width))); + }, { description: `Settings for the ${widget.label} widget.` }); + }); } /** The live board in the window; any click or double-click returns to the list. */ diff --git a/app/apps/glanceboard/widget.ts b/app/apps/glanceboard/widget.ts index 03c6e4b6..1d274c68 100644 --- a/app/apps/glanceboard/widget.ts +++ b/app/apps/glanceboard/widget.ts @@ -1,4 +1,5 @@ import { type GrayImage } from "../../graphics/image"; +import { type MenuItem } from "../../ui/menu"; /** * A Glanceboard widget: a small self-contained view that paints into a slot @@ -39,5 +40,11 @@ export type GlanceWidgetDefinition = { * gets the taller canvas; the divider between the slots is not drawn. */ tall?: boolean; + /** + * The rows of this widget's submenu in the Glanceboard app's Settings. A + * widget without settings still gets a row there (disabled) while it is + * on the board. + */ + settingsMenuItems?: () => MenuItem[]; create: () => GlanceWidget; }; diff --git a/app/apps/glanceboard/widgets/compass-widget.ts b/app/apps/glanceboard/widgets/compass-widget.ts index adf0eda8..8d3deff8 100644 --- a/app/apps/glanceboard/widgets/compass-widget.ts +++ b/app/apps/glanceboard/widgets/compass-widget.ts @@ -1,8 +1,15 @@ import { type GrayImage } from "../../../graphics/image"; import { hasLocationPermission } from "../../../native/location-permissions"; -import { addCompassListener, COMPASS_CHANGED, setCompassEnabled } from "../../../native/compass"; +import { addCompassListener, COMPASS_CHANGED, setCompassEnabled, type CompassDiagnostics } from "../../../native/compass"; import { isCompassCalibrated, normalizeHeading } from "../../compass/calibration"; -import { cardinalDirection, createCompassBackground, drawCompassRose, layoutCompassRose, TICK_HEIGHT } from "../../compass/compass-rose"; +import { + cardinalDirection, + createCompassBackground, + drawCompassRose, + layoutCompassRose, + magneticAccuracyDots, + TICK_HEIGHT, +} from "../../compass/compass-rose"; import { getDeclinationAvailability, onDeclinationChanged, refreshDeclination } from "../../compass/declination"; import { getNorthReference, resolveHeading } from "../../compass/heading"; import { glanceFont } from "../glance-font"; @@ -14,16 +21,21 @@ const TOP_PAD = 4; const BOTTOM_PAD = 4; const EDGE_PAD = 8; const MAX_ROSE_RADIUS = 60; +/** Inset of the top-right magnetometer calibration dots from the card edges. */ +const ACCURACY_TOP_PAD = TOP_PAD; +const ACCURACY_RIGHT_PAD = EDGE_PAD; /** * The Compass app's rose at card size: heading readout above, tilted rose * below. Turns the magnetometer on for as long as the board shows it (the * Compass app keeps its own hold, so the two never switch each other off). * Never prompts for location: true north is used when a declination is - * already known, otherwise the readout says it is magnetic. + * already known, otherwise the readout says it is magnetic. The firmware's + * magnetometer calibration shows as the Compass app's dots, without a label. */ export class CompassWidget implements GlanceWidget { private rawHeading: number | null = null; + private diagnostics: CompassDiagnostics | null = null; private unsubscribe: (() => void) | null = null; private unsubscribeDeclination: (() => void) | null = null; private enabled = false; @@ -33,6 +45,7 @@ export class CompassWidget implements GlanceWidget { this.unsubscribe = addCompassListener((event) => { if (event.command === COMPASS_CHANGED && event.headingDegrees >= 0) { this.rawHeading = normalizeHeading(event.headingDegrees); + this.diagnostics = event.diagnostics ?? null; requestRender(); } }); @@ -96,5 +109,9 @@ export class CompassWidget implements GlanceWidget { if (status) { image.drawText(small, Math.round(cx - small.measureText(status) / 2), y + medium.lineHeight + 2, status, 125); } + const accuracyDots = magneticAccuracyDots(this.diagnostics); + if (accuracyDots) { + image.drawText(small, width - ACCURACY_RIGHT_PAD - small.measureText(accuracyDots), ACCURACY_TOP_PAD, accuracyDots, 175); + } } } diff --git a/app/apps/glanceboard/widgets/index.ts b/app/apps/glanceboard/widgets/index.ts index 5a7e0ce5..b07f7b7c 100644 --- a/app/apps/glanceboard/widgets/index.ts +++ b/app/apps/glanceboard/widgets/index.ts @@ -1,22 +1,22 @@ import { type GlanceWidgetDefinition, type GlanceWidgetId } from "../widget"; import { CalendarWidget } from "./calendar-widget"; import { CompassWidget } from "./compass-widget"; -import { MusicWidget } from "./music-widget"; +import { MusicWidget, musicSettingsMenuItems } from "./music-widget"; import { NightscoutWidget } from "./nightscout-widget"; -import { SystemCardWidget } from "./system-card"; +import { SystemCardWidget, systemCardSettingsMenuItems } from "./system-card"; import { T3CodeWidget } from "./t3code-widget"; import { TerminalWidget } from "./terminal-widget"; import { HealthWidget } from "./health-widget"; /** Every widget a board slot can hold. */ export const GLANCE_WIDGETS: readonly GlanceWidgetDefinition[] = [ - { id: "system-card", label: "System card", create: () => new SystemCardWidget() }, + { id: "system-card", label: "System card", settingsMenuItems: systemCardSettingsMenuItems, create: () => new SystemCardWidget() }, { id: "calendar", label: "Calendar", tall: true, create: () => new CalendarWidget() }, { id: "terminal", label: "Terminal", tall: true, create: () => new TerminalWidget() }, { id: "t3code", label: "T3 Code", tall: true, create: () => new T3CodeWidget() }, { id: "nightscout", label: "Nightscout", create: () => new NightscoutWidget() }, { id: "compass", label: "Compass", create: () => new CompassWidget() }, - { id: "music", label: "Music", create: () => new MusicWidget() }, + { id: "music", label: "Music", settingsMenuItems: musicSettingsMenuItems, create: () => new MusicWidget() }, { id: "health", label: "Health", create: () => new HealthWidget() }, ]; diff --git a/app/apps/glanceboard/widgets/music-widget.ts b/app/apps/glanceboard/widgets/music-widget.ts index f94418e3..ecb23a0c 100644 --- a/app/apps/glanceboard/widgets/music-widget.ts +++ b/app/apps/glanceboard/widgets/music-widget.ts @@ -2,6 +2,8 @@ import { GrayImage } from "../../../graphics/image"; import { renderIcon } from "../../../graphics/icons"; import { truncateText } from "../../../graphics/textwrap"; import { mediaControllerBridge, type MediaControllerState } from "../../../native/media-controller"; +import { ConfigSettingBoolean, onAnySettingChanged, toggleSettingMenuItem } from "../../../ui/dashboard-settings"; +import { type MenuItem } from "../../../ui/menu"; import { clamp } from "../../../util/numeric-util"; import { glanceFont } from "../glance-font"; import { type GlanceWidget } from "../widget"; @@ -36,20 +38,37 @@ function idleNoteImage(): GrayImage | null { return note; } const ART_X = PAD; +const ART_TEXT_GAP = 12; const PROGRESS_BAR_HEIGHT = 5; +export const musicShowAlbumArtSetting = new ConfigSettingBoolean({ + id: "glanceboard-music-show-album-art", + label: "Show album art", + storageKey: "glanceboard.music.showAlbumArt", + defaultValue: true, + description: + "Show the album art beside the track details. When off, the title, artist, album and progress bar use the full width of the card.", +}); + +/** The music card's submenu in Glanceboard > Settings. */ +export function musicSettingsMenuItems(): MenuItem[] { + return [toggleSettingMenuItem(musicShowAlbumArtSetting)]; +} + /** - * Now playing: album art, title/artist/album and a progress bar for the - * active Android media session, the Music app's header at card size. The - * media bridge runs for the whole session; the widget subscribes and ticks - * once a second only while something is playing. + * Now playing: album art (unless turned off), title/artist/album and a + * progress bar for the active media session (Apple Music on iOS), the Music + * app's header at card size. The media bridge runs for the whole session; the + * widget subscribes and ticks once a second only while something is playing. */ export class MusicWidget implements GlanceWidget { private media: MediaControllerState = mediaControllerBridge.snapshot(); private unsubscribe: (() => void) | null = null; + private unsubscribeSettings: (() => void) | null = null; private progressTimer: ReturnType | null = null; private art: GrayImage | null = null; - private artKey = ""; + /** The artVersion `art` was fetched for; -1 = not fetched yet. */ + private artVersion = -1; start(requestRender: () => void): void { this.unsubscribe = mediaControllerBridge.onStateChange((state) => { @@ -61,11 +80,14 @@ export class MusicWidget implements GlanceWidget { } requestRender(); }); + this.unsubscribeSettings = onAnySettingChanged(requestRender); } stop(): void { this.unsubscribe?.(); this.unsubscribe = null; + this.unsubscribeSettings?.(); + this.unsubscribeSettings = null; this.stopProgressTimer(); } @@ -83,7 +105,7 @@ export class MusicWidget implements GlanceWidget { if (!media.accessEnabled) { // Actionable, so it stays as text. image.drawText(small, PAD, PAD, "Music", 150); - image.drawText(small, PAD, PAD + small.lineHeight + 6, "Notification access needed", 140); + image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140); return; } // Nothing playing: a dim note, centred, and nothing else. @@ -92,10 +114,12 @@ export class MusicWidget implements GlanceWidget { return; } + // Without the art, the text and progress bar keep its rows and take its columns. const artY = Math.max(PAD, ((image.height - ART_SIZE) / 2) | 0); - this.drawArt(image, media, artY); + const showArt = musicShowAlbumArtSetting.get(); + if (showArt) this.drawArt(image, media, artY); - const textX = ART_X + ART_SIZE + 12; + const textX = showArt ? ART_X + ART_SIZE + ART_TEXT_GAP : PAD; const textWidth = image.width - textX - PAD; let y = artY; image.drawText(medium, textX, y, truncateText(medium, media.title || "Unknown title", textWidth), 230); @@ -129,10 +153,8 @@ export class MusicWidget implements GlanceWidget { } private drawArt(image: GrayImage, media: MediaControllerState, artY: number): void { - const key = `${media.packageName}|${media.title}|${media.album}`; - // Retry while null: players often publish metadata before the bitmap. - if (key !== this.artKey || this.art === null) { - this.artKey = key; + if (media.artVersion !== this.artVersion) { + this.artVersion = media.artVersion; this.art = mediaControllerBridge.getAlbumArt(ART_SIZE); } if (this.art) { diff --git a/app/apps/glanceboard/widgets/system-card.ts b/app/apps/glanceboard/widgets/system-card.ts index b497a434..e823cb5e 100644 --- a/app/apps/glanceboard/widgets/system-card.ts +++ b/app/apps/glanceboard/widgets/system-card.ts @@ -1,25 +1,30 @@ import { BATTERY_ICON_WIDTH, DENSE_BATTERY_BLOCK_HEIGHT, + denseBatteriesWidth, drawBattery, drawDenseBatteries, type BatteryDevice, } from "../../../graphics/battery"; import { type GrayImage, type UiFont } from "../../../graphics/image"; -import { onAndroidNotificationPosted, readActiveNotificationIcons } from "../../../native/notification-icons"; +import { ensureLocationPermission, hasLocationPermission } from "../../../native/location-permissions"; +import { onAndroidNotificationsChanged, readActiveNotificationIcons } from "../../../native/notification-icons"; import { readPhoneBatteryState } from "../../../native/phone-battery"; import { weatherBridge } from "../../../native/weather"; import { formatClockDate, formatClockTime } from "../../../ui/clock-format"; import { batteryDisplayModeSetting, batteryIndicatorVisible, + ConfigSettingBoolean, glassesBatteryVisibilitySetting, onAnySettingChanged, phoneBatteryVisibilitySetting, ringBatteryVisibilitySetting, + toggleSettingMenuItem, watchBatteryVisibilitySetting, type BatteryIndicatorVisibility, } from "../../../ui/dashboard-settings"; +import { type MenuItem } from "../../../ui/menu"; import { shell } from "../../../ui/shell/shell"; import { noteStaleDataUsed, renderPassAllowsStaleData } from "../../../util/render-freshness"; import { truncateText } from "../../../graphics/textwrap"; @@ -36,20 +41,113 @@ const NOTIFICATION_ICON_SIZE = 24; const NOTIFICATION_ICON_GAP = 4; const BATTERY_ITEM_GAP = 12; const BATTERY_LABEL_VALUE = 150; -/** Gap between the battery block and the weather reading below it. */ -const WEATHER_GAP = 6; +/** Gap between the battery block and the weather reading stacked after it. */ +const SIDE_BLOCK_GAP = 6; +/** Rows between the time and the date. */ +const CLOCK_LINE_GAP = 2; +/** Rows between the clock and the timers, and between the timers and the notification icons. */ +const TIMERS_GAP = 4; +const TIMER_NAME_GAP = 8; type BatteryItem = { label: string; device: BatteryDevice; percent: number; charging: boolean }; -/** A block drawn against the card's right edge: its left edge and its rows. */ -type RightBlock = { left: number; top: number; bottom: number }; +type Rect = { x: number; y: number; width: number; height: number }; +/** A side-column block, measured so it can be placed against either edge before it is drawn. */ +type SideBlock = { width: number; height: number; draw: (image: GrayImage, left: number, top: number) => void }; +/** The main column's horizontal extent on some rows. */ +type Span = { left: number; right: number }; + +export const systemCardShowTimeSetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-show-time", + label: "Time", + storageKey: "glanceboard.systemCard.showTime", + defaultValue: true, + description: "Show the clock on the system card.", +}); + +export const systemCardShowDateSetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-show-date", + label: "Date", + storageKey: "glanceboard.systemCard.showDate", + defaultValue: true, + description: "Show the date on the system card, under the time.", +}); + +export const systemCardShowBatterySetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-show-battery", + label: "Battery", + storageKey: "glanceboard.systemCard.showBattery", + defaultValue: true, + description: + "Show battery indicators on the system card. Their style and which devices appear follow Settings > Customization > Battery indicators.", +}); + +export const systemCardShowNotificationsSetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-show-notifications", + label: "Notifications", + storageKey: "glanceboard.systemCard.showNotifications", + defaultValue: true, + description: "Show the icons of the phone's notifications along the edge of the system card.", +}); + +export const systemCardFlipHorizontalSetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-flip-horizontal", + label: "Flip layout horizontally", + storageKey: "glanceboard.systemCard.flipHorizontal", + defaultValue: false, + description: + "Mirror the system card left to right: time, date, timers and notifications on the right, batteries and weather on the left.", +}); + +export const systemCardFlipVerticalSetting = new ConfigSettingBoolean({ + id: "glanceboard-system-card-flip-vertical", + label: "Flip layout vertically", + storageKey: "glanceboard.systemCard.flipVertical", + defaultValue: false, + description: + "Mirror the system card top to bottom: notifications along the top; time, date, batteries and weather along the bottom.", +}); + +/** + * The system card's submenu in Glanceboard > Settings. Weather is the + * Weather app's "Show on system card" setting under a shorter label. + */ +export function systemCardSettingsMenuItems(): MenuItem[] { + return [ + toggleSettingMenuItem(systemCardShowTimeSetting), + toggleSettingMenuItem(systemCardShowDateSetting), + toggleSettingMenuItem(systemCardShowBatterySetting), + toggleSettingMenuItem(weatherShowOnSystemCardSetting, { + label: "Weather", + onChange: (_ctx, enabled) => { + if (enabled) requestWeatherLocationAccess(); + }, + }), + toggleSettingMenuItem(systemCardShowNotificationsSetting), + toggleSettingMenuItem(systemCardFlipHorizontalSetting), + toggleSettingMenuItem(systemCardFlipVerticalSetting), + ]; +} + +/** + * Weather switched on without location access: ask for it now, as the + * Weather app's own toggle does, since the background refresh has no screen + * to prompt from. + */ +function requestWeatherLocationAccess(): void { + if (hasLocationPermission()) return; + void ensureLocationPermission().then((granted) => { + if (granted) void weatherBridge.refreshNow(); + }).catch((error) => console.warn(`Weather permission: ${error}`)); +} /** * Date and time, the phone's notification icons, the phone/watch/G2/R1 * battery indicators (the Wear OS watch's only while one is reachable), the - * current weather (when the Weather app's "Show on system card" is on), and - * the Timers app's countdowns: the top bar's contents plus timers, laid out - * for a card. Battery style and per-device visibility - * follow Settings > Customization > Battery indicators, so the card agrees with + * current weather, and the Timers app's countdowns: the top bar's contents + * plus timers, laid out for a card. Every part but the timers can be turned + * off, and the layout mirrored either way, in the card's submenu of + * Glanceboard > Settings. Battery style and per-device visibility follow + * Settings > Customization > Battery indicators, so the card agrees with * the bar. Countdowns tick once a second only while one is running. */ export class SystemCardWidget implements GlanceWidget { @@ -64,7 +162,7 @@ export class SystemCardWidget implements GlanceWidget { start(requestRender: () => void): void { this.requestRender = requestRender; this.scheduleMinuteTick(); - this.unsubscribeNotifications = onAndroidNotificationPosted(() => this.requestRender?.()); + this.unsubscribeNotifications = onAndroidNotificationsChanged(() => this.requestRender?.()); this.unsubscribeSettings = onAnySettingChanged(() => this.requestRender?.()); this.unsubscribeTimers = timerEngine.onChange(() => { this.syncSecondTick(); @@ -120,35 +218,93 @@ export class SystemCardWidget implements GlanceWidget { const medium = glanceFont.medium(); const small = glanceFont.small(); const now = new Date(); + const { width, height } = image; - // Clock and date down the left; batteries take the right edge, with the - // weather right-aligned beneath them. Text on the left stops short of - // whichever of those shares its rows. - const battery = drawBatteryBlock(image, small, image.width - PAD, PAD); - const weatherTop = battery.bottom > battery.top ? battery.bottom + WEATHER_GAP : PAD; - const weather = drawWeatherBlock(image, medium, image.width - PAD, weatherTop, now.getTime()); - const textRight = (top: number, bottom: number): number => - weather && weather.top < bottom && weather.bottom > top ? Math.min(battery.left, weather.left) : battery.left; - const textWidth = Math.max(0, textRight(PAD, PAD + large.lineHeight) - BATTERY_ITEM_GAP - PAD); + // Unflipped, the time and date run down the left from the top, the + // batteries and then the weather stack down the right edge, and the + // notification icons take the bottom row, with the timers in the band + // between the date and the icons. The flips mirror where each block sits + // (positions below are given for the unflipped card), not what is inside + // it: the time stays above the date, and timers read soonest first. + const flipX = systemCardFlipHorizontalSetting.get(); + const flipY = systemCardFlipVerticalSetting.get(); + const placeX = (x: number, w: number): number => (flipX ? width - x - w : x); + const placeY = (y: number, h: number): number => (flipY ? height - y - h : y); + + const sideBlocks: SideBlock[] = []; + const batteryItems = systemCardShowBatterySetting.get() ? collectBatteryItems() : []; + if (batteryItems.length) sideBlocks.push(batteryBlock(small, batteryItems)); + const weather = weatherShowOnSystemCardSetting.get() ? weatherBlock(medium, now.getTime()) : null; + if (weather) sideBlocks.push(weather); + const side: Rect[] = []; + let sideOffset = PAD; + for (const block of sideBlocks) { + const rect = { + x: placeX(width - PAD - block.width, block.width), + y: placeY(sideOffset, block.height), + width: block.width, + height: block.height, + }; + block.draw(image, rect.x, rect.y); + side.push(rect); + sideOffset += block.height + SIDE_BLOCK_GAP; + } + // Text in the main column stops short of whichever side blocks share its + // rows, and hugs the card's outer edge. + const span = (top: number, bottom: number): Span => { + let left = PAD; + let right = width - PAD; + for (const rect of side) { + if (rect.y >= bottom || rect.y + rect.height <= top) continue; + if (flipX) left = Math.max(left, rect.x + rect.width + BATTERY_ITEM_GAP); + else right = Math.min(right, rect.x - BATTERY_ITEM_GAP); + } + return { left, right }; + }; + const lineX = ({ left, right }: Span, lineWidth: number): number => (flipX ? right - lineWidth : left); + + const showTime = systemCardShowTimeSetting.get(); + const showDate = systemCardShowDateSetting.get(); const timeText = formatClockTime(now); - const timeFont = large.measureText(timeText) <= textWidth ? large : medium; - let y = PAD; - image.drawText(timeFont, PAD, y, timeText, 230); - y += timeFont.lineHeight + 2; - image.drawText(medium, PAD, y, formatClockDate(now), 170); - y += medium.lineHeight + 4; + const dateText = formatClockDate(now); + const clockHeight = (timeFont: UiFont): number => + (showTime ? timeFont.lineHeight : 0) + (showTime && showDate ? CLOCK_LINE_GAP : 0) + (showDate ? medium.lineHeight : 0); + // The time drops to the medium font when the large one would run into a side block. + let timeFont = large; + if (showTime) { + const top = placeY(PAD, clockHeight(large)); + const { left, right } = span(top, top + large.lineHeight); + if (large.measureText(timeText) > right - left) timeFont = medium; + } + const clockSize = clockHeight(timeFont); + let y = placeY(PAD, clockSize); + if (showTime) { + image.drawText(timeFont, lineX(span(y, y + timeFont.lineHeight), timeFont.measureText(timeText)), y, timeText, 230); + y += timeFont.lineHeight + CLOCK_LINE_GAP; + } + if (showDate) { + image.drawText(medium, lineX(span(y, y + medium.lineHeight), medium.measureText(dateText)), y, dateText, 170); + } - // Notification icons along the bottom, as many as fit; the Timers app's - // countdowns take the band between the date and the icons. - const iconY = image.height - PAD - NOTIFICATION_ICON_SIZE; - const timersWidth = Math.max(0, textRight(y, iconY - 4) - BATTERY_ITEM_GAP - PAD); - drawTimers(image, small, PAD, y, timersWidth, iconY - 4, now.getTime()); - const maxIcons = Math.max(0, ((image.width - 2 * PAD + NOTIFICATION_ICON_GAP) / (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP)) | 0); - if (maxIcons > 0) { + const showNotifications = systemCardShowNotificationsSetting.get(); + const bandTop = clockSize > 0 ? PAD + clockSize + TIMERS_GAP : PAD; + const bandBottom = height - PAD - (showNotifications ? NOTIFICATION_ICON_SIZE + TIMERS_GAP : 0); + drawTimers(image, small, { + top: placeY(bandTop, bandBottom - bandTop), + height: bandBottom - bandTop, + fromBottom: flipY, + alignRight: flipX, + span, + }, now.getTime()); + + // Notification icons, as many as fit, the first nearest the main column's edge. + const maxIcons = Math.max(0, ((width - 2 * PAD + NOTIFICATION_ICON_GAP) / (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP)) | 0); + if (showNotifications && maxIcons > 0) { const { icons, stale } = readActiveNotificationIcons(maxIcons, renderPassAllowsStaleData()); if (stale) noteStaleDataUsed(); + const iconY = placeY(height - PAD - NOTIFICATION_ICON_SIZE, NOTIFICATION_ICON_SIZE); for (let index = 0; index < icons.length; index++) { - image.drawImage(icons[index]!, PAD + index * (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP), iconY); + image.drawImage(icons[index]!, placeX(PAD + index * (NOTIFICATION_ICON_SIZE + NOTIFICATION_ICON_GAP), NOTIFICATION_ICON_SIZE), iconY); } } } @@ -180,59 +336,62 @@ function collectBatteryItems(): BatteryItem[] { } /** - * The battery indicators right-aligned at `right`, in the configured style. - * Side-by-side styles put the label beside the gauge (or percentage) on one - * line; stacked styles centre the label above it; the dense style swaps - * labels for device icons and stacks two indicators per column, as in the - * top bar. Returns the block's extent (zero-height at `right` when nothing - * is shown). + * The battery indicators in the configured style. Side-by-side styles put + * the label beside the gauge (or percentage) on one line; stacked styles + * centre the label above it; the dense style swaps labels for device icons + * and stacks two indicators per column, as in the top bar. */ -function drawBatteryBlock(image: GrayImage, font: UiFont, right: number, top: number): RightBlock { - const items = collectBatteryItems(); - if (!items.length) return { left: right, top, bottom: top }; +function batteryBlock(font: UiFont, items: readonly BatteryItem[]): SideBlock { const mode = batteryDisplayModeSetting.get(); if (mode === "dense") { - return { left: drawDenseBatteries(image, items, right, top), top, bottom: top + DENSE_BATTERY_BLOCK_HEIGHT }; + const width = denseBatteriesWidth(items); + return { + width, + height: DENSE_BATTERY_BLOCK_HEIGHT, + draw: (image, left, top) => { + drawDenseBatteries(image, items, left + width, top); + }, + }; } const percentage = mode === "percentage" || mode === "stacked-percentage"; const stacked = mode === "stacked" || mode === "stacked-percentage"; const gaugeHeight = drawBattery(0, false).height; const valueHeight = percentage ? font.lineHeight : Math.max(font.lineHeight, gaugeHeight); - let x = right; - for (let index = items.length - 1; index >= 0; index--) { - const item = items[index]!; - const percentText = `${item.percent}%`; + const cells = items.map((item) => { const labelWidth = font.measureText(item.label); - const valueWidth = percentage ? font.measureText(percentText) : BATTERY_ICON_WIDTH; - if (stacked) { - const itemWidth = Math.max(labelWidth, valueWidth); - x -= itemWidth; - image.drawText(font, x + (((itemWidth - labelWidth) / 2) | 0), top, item.label, BATTERY_LABEL_VALUE); - drawBatteryValue(image, font, x + (((itemWidth - valueWidth) / 2) | 0), top + font.lineHeight + 2, item, percentage, gaugeHeight); - } else { - x -= labelWidth + 5 + valueWidth; - image.drawText(font, x, top, item.label, BATTERY_LABEL_VALUE); - drawBatteryValue(image, font, x + labelWidth + 5, top, item, percentage, gaugeHeight); - } - x -= BATTERY_ITEM_GAP; - } - return { left: x + BATTERY_ITEM_GAP, top, bottom: top + (stacked ? font.lineHeight + 2 : 0) + valueHeight }; + const valueWidth = percentage ? font.measureText(`${item.percent}%`) : BATTERY_ICON_WIDTH; + return { item, labelWidth, valueWidth, width: stacked ? Math.max(labelWidth, valueWidth) : labelWidth + 5 + valueWidth }; + }); + return { + width: cells.reduce((total, cell) => total + cell.width, 0) + BATTERY_ITEM_GAP * (cells.length - 1), + height: (stacked ? font.lineHeight + 2 : 0) + valueHeight, + draw: (image, left, top) => { + let x = left; + for (const { item, labelWidth, valueWidth, width } of cells) { + if (stacked) { + image.drawText(font, x + (((width - labelWidth) / 2) | 0), top, item.label, BATTERY_LABEL_VALUE); + drawBatteryValue(image, font, x + (((width - valueWidth) / 2) | 0), top + font.lineHeight + 2, item, percentage, gaugeHeight); + } else { + image.drawText(font, x, top, item.label, BATTERY_LABEL_VALUE); + drawBatteryValue(image, font, x + labelWidth + 5, top, item, percentage, gaugeHeight); + } + x += width + BATTERY_ITEM_GAP; + } + }, + }; } -/** - * The current weather's condition icon and temperature, right-aligned at - * `right` from `top`, while the system-card weather setting is on and a - * fresh reading exists. Returns the block's extent, or null when not drawn. - */ -function drawWeatherBlock(image: GrayImage, font: UiFont, right: number, top: number, nowMs: number): RightBlock | null { - if (!weatherShowOnSystemCardSetting.get()) return null; +/** The current weather's condition icon and temperature, or null without a fresh reading. */ +function weatherBlock(font: UiFont, nowMs: number): SideBlock | null { const reading = weatherReading(weatherBridge.snapshot(), nowMs); if (!reading) return null; const iconSize = font.lineHeight + 4; const height = Math.max(iconSize, font.lineHeight); - const left = right - Math.ceil(measureWeatherReading(reading, font, iconSize)); - drawWeatherReading(image, reading, font, iconSize, left, top, height, 200); - return { left, top, bottom: top + height }; + return { + width: Math.ceil(measureWeatherReading(reading, font, iconSize)), + height, + draw: (image, left, top) => drawWeatherReading(image, reading, font, iconSize, left, top, height, 200), + }; } function drawBatteryValue( @@ -262,26 +421,36 @@ function drawBatteryValue( /** * The Timers app's countdowns, soonest first: remaining time (or "Done" once * rung, inverted so it is noticed; "paused" for a paused one) and the - * timer's name, one per line while they fit between `top` and `bottom`. + * timer's name, one per line, as many as fit in the band. The lines sit + * against the band's top (or its bottom), each as wide as the main column + * allows on its rows and hugging its left (or right) edge. */ -function drawTimers(image: GrayImage, font: UiFont, x: number, top: number, width: number, bottom: number, nowMs: number): void { - const timers = sortTimers(timerEngine.timers(), nowMs); +function drawTimers( + image: GrayImage, + font: UiFont, + band: { top: number; height: number; fromBottom: boolean; alignRight: boolean; span: (top: number, bottom: number) => Span }, + nowMs: number, +): void { const step = lineStep(font); - let y = top; + const fit = band.height < font.lineHeight ? 0 : 1 + Math.floor((band.height - font.lineHeight) / step); + const timers = sortTimers(timerEngine.timers(), nowMs).slice(0, fit); + if (!timers.length) return; + let y = band.fromBottom ? band.top + band.height - (timers.length - 1) * step - font.lineHeight : band.top; for (const timer of timers) { - if (y + font.lineHeight > bottom) break; + const { left, right } = band.span(y, y + font.lineHeight); const phase = timerPhase(timer); const clock = phase === "rung" ? "Done" : formatCountdown(timerRemainingMs(timer, nowMs)); - const clockWidth = font.measureText(clock); + const nameOffset = font.measureText(clock) + TIMER_NAME_GAP; + const fullName = phase === "paused" ? `${timerDisplayName(timer)} (paused)` : timerDisplayName(timer); + const name = truncateText(font, fullName, Math.max(0, right - left - nameOffset)); + const x = band.alignRight ? right - nameOffset - font.measureText(name) : left; if (phase === "rung") { - image.fillRect(x - 2, y - 1, clockWidth + 4, font.lineHeight + 2, 230); + image.fillRect(x - 2, y - 1, nameOffset - TIMER_NAME_GAP + 4, font.lineHeight + 2, 230); image.drawText(font, x, y, clock, 1); } else { image.drawText(font, x, y, clock, phase === "paused" ? 120 : 200); } - const nameX = x + clockWidth + 8; - const name = phase === "paused" ? `${timerDisplayName(timer)} (paused)` : timerDisplayName(timer); - image.drawText(font, nameX, y, truncateText(font, name, Math.max(0, x + width - nameX)), 160); + image.drawText(font, x + nameOffset, y, name, 160); y += step; } } diff --git a/app/apps/glanceboard/widgets/terminal-widget.ts b/app/apps/glanceboard/widgets/terminal-widget.ts index e3e55728..c6601214 100644 --- a/app/apps/glanceboard/widgets/terminal-widget.ts +++ b/app/apps/glanceboard/widgets/terminal-widget.ts @@ -24,6 +24,10 @@ const BOTTOM_MARGIN = 2; * The indicator animates on the hub's cadence while any session is active; * each row carries its own activity deadline, so nothing needs the worker * once the snapshot is in. + * + * The worker sorts sessions most-recently-updated first; the widget keeps the + * order it first sees after the board opens so rows don't jump around while + * it is showing. Sessions that appear later go to the end of their host. */ export class TerminalWidget implements GlanceWidget { private snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined; @@ -31,12 +35,17 @@ export class TerminalWidget implements GlanceWidget { private requestRender: (() => void) | null = null; private phase = 0; private animationTimer: ReturnType | null = null; + /** Row key -> display rank, fixed from the first snapshot seen since start. */ + private readonly rowOrder = new Map(); start(requestRender: () => void): void { this.requestRender = requestRender; + this.rowOrder.clear(); this.snapshot = readWorkerState(TERMINAL_SESSIONS_STATE_KEY) as TerminalSessionsSnapshot | undefined; + this.rankNewRows(); this.unsubscribe = onWorkerStateChanged(TERMINAL_SESSIONS_STATE_KEY, (snapshot) => { this.snapshot = snapshot as TerminalSessionsSnapshot; + this.rankNewRows(); this.syncAnimation(); requestRender(); }); @@ -51,6 +60,15 @@ export class TerminalWidget implements GlanceWidget { this.animationTimer = null; } + /** Give rows not seen yet the next ranks, in the snapshot's (recency) order. */ + private rankNewRows(): void { + for (const host of this.snapshot?.hosts ?? []) { + for (const row of host.sessions) { + if (!this.rowOrder.has(row.key)) this.rowOrder.set(row.key, this.rowOrder.size); + } + } + } + private anyActive(nowMs = Date.now()): boolean { return Boolean(this.snapshot?.hosts.some((host) => host.sessions.some((row) => isSessionRowActive(row, nowMs)))); } @@ -107,7 +125,10 @@ export class TerminalWidget implements GlanceWidget { image.drawText(font, PAD, y, truncateText(font, host.name, textWidth), 110); y += step; } - for (const row of host.sessions) { + const rows = host.sessions + .slice() + .sort((a, b) => (this.rowOrder.get(a.key) ?? 0) - (this.rowOrder.get(b.key) ?? 0)); + for (const row of rows) { if (!fits(y)) return; drawSessionRow(image, font, { x: GUTTER_X, diff --git a/app/apps/ios-availability.ts b/app/apps/ios-availability.ts index c2f062c3..21f0dbf3 100644 --- a/app/apps/ios-availability.ts +++ b/app/apps/ios-availability.ts @@ -1,7 +1,5 @@ /** Native services still outside the current iOS app-screen milestone. */ const unavailable: Record = { - music: 'Media-player integration is not available on iOS yet.', - transcribe: 'Voice capture and transcription are not available on iOS yet.', microphones: 'Glasses microphone capture is not available on iOS yet.', } export function iosAppUnavailableReason(appId: string): string | null { diff --git a/app/apps/microphones/caption-clock.ts b/app/apps/microphones/caption-clock.ts new file mode 100644 index 00000000..ceb2e653 --- /dev/null +++ b/app/apps/microphones/caption-clock.ts @@ -0,0 +1,44 @@ +// Utterance times come from the caption engine's sample clock, which only +// advances with audio fed to it. Audio that never reaches the engine (beam +// filtered, paused for the voice dialog, lost to a stall) would otherwise +// shift every later line earlier, so the clock is re-anchored to wall time +// (arrival time, so ordinary BLE jitter is tolerated) whenever the two drift +// this far apart, in either direction: missing audio puts wall time ahead, +// and a backlog delivered late in a burst puts it behind once the stream +// catches up. +export const CAPTION_CLOCK_DRIFT_MS = 300; +const MAX_ANCHORS = 1000; + +/** Maps a caption engine's clock (ms of audio fed to it) to wall time. */ +export class CaptionClock { + private fedSamples = 0; + // Engine-clock times with the wall time each corresponds to, in order. + private readonly anchors: Array<{ engineMs: number; wallMs: number }> = []; + + constructor( + private readonly sampleRate: number, + private readonly startWallMs: number, + ) {} + + /** `bytes` of 16-bit mono PCM were fed to the engine, arriving at `nowMs`. */ + noteFed(bytes: number, nowMs: number): void { + const samples = Math.floor(bytes / 2); + const engineMs = (this.fedSamples * 1000) / this.sampleRate; + this.fedSamples += samples; + // The chunk arrived as it ended; its first sample is that much older. + const wallMs = nowMs - (samples * 1000) / this.sampleRate; + const last = this.anchors[this.anchors.length - 1]; + if (last && Math.abs(wallMs - (last.wallMs + engineMs - last.engineMs)) <= CAPTION_CLOCK_DRIFT_MS) return; + this.anchors.push({ engineMs, wallMs }); + if (this.anchors.length > MAX_ANCHORS) this.anchors.shift(); + } + + /** Wall time of a moment on the engine's clock. */ + wallMs(engineMs: number): number { + for (let i = this.anchors.length - 1; i >= 0; i--) { + const anchor = this.anchors[i]!; + if (anchor.engineMs <= engineMs) return anchor.wallMs + engineMs - anchor.engineMs; + } + return this.startWallMs + engineMs; + } +} diff --git a/app/apps/microphones/captions-layer.ts b/app/apps/microphones/captions-layer.ts index c27807b9..01d87f54 100644 --- a/app/apps/microphones/captions-layer.ts +++ b/app/apps/microphones/captions-layer.ts @@ -15,12 +15,15 @@ export class CaptionsLayer implements Layer { private state: MicSessionState = micSession.getState(); private unsubscribe: (() => void) | null = null; private scrollback = 0; - private lastLineCount = 0; + private lastLine: CaptionLine | null = null; start(requestRender: () => void): void { this.unsubscribe = micSession.onState((state) => { - if (state.captionLines.length !== this.lastLineCount) { - this.lastLineCount = state.captionLines.length; + // Watch the newest line itself: once the session's line cap is reached + // the count stops changing even though new speech keeps arriving. + const last = state.captionLines[state.captionLines.length - 1] ?? null; + if (last !== this.lastLine) { + this.lastLine = last; this.scrollback = 0; } this.state = state; @@ -36,7 +39,7 @@ export class CaptionsLayer implements Layer { image.drawText(font, 12, 4, "Captions", 220); if (!state.captionsActive) { - image.drawText(font, 12, 26, "Captions are off (enable in the Microphones menu).", 130); + image.drawText(font, 12, 26, state.captionsNote, 130); } const rows = this.buildRows(font, width - 36, state.captionLines); diff --git a/app/apps/microphones/conversation-qa.ts b/app/apps/microphones/conversation-qa.ts index 40f464ee..c3300c2e 100644 --- a/app/apps/microphones/conversation-qa.ts +++ b/app/apps/microphones/conversation-qa.ts @@ -16,6 +16,10 @@ declare const com: any; // Qwen runs with a 4096-token context; leave room for the answer. const MAX_CONTEXT_CHARS = 7000; +// The speaker roster grows with every voice ever heard; cap it and the +// conversation list so they can't crowd the transcript excerpts out. +const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2; +const EXCERPTS_HEADING = "# Transcript excerpts\n"; const MAX_KEYWORDS = 6; const SEGMENTS_PER_KEYWORD = 30; const RECENT_SEGMENT_LIMIT = 120; @@ -156,31 +160,57 @@ function buildContext(question: string): { context: string; note: string } { } } - const segments = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt); + let header = + `# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` + + `# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` + + `${sessionLines.join("\n") || "(none in range)"}\n\n`; + let truncated = false; + if (header.length > MAX_HEADER_CHARS) { + header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`; + truncated = true; + } + + const sessionHeading = (sessionId: number) => + `\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`; + const segmentText = (segment: SegmentRow) => { + const speaker = + segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker"; + const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : ""; + const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`; + return segment.translation ? `${line}\n (translated: ${segment.translation})` : line; + }; + + // Fill the remaining budget newest-first, so an overflowing history drops + // its oldest excerpts rather than its most recent ones. + const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt); + let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length; + const segments: SegmentRow[] = []; + let runSession = -1; + for (let index = sorted.length - 1; index >= 0; index--) { + const segment = sorted[index]!; + let cost = segmentText(segment).length + 1; + if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1; + if (cost > budget) { + truncated = true; + break; + } + budget -= cost; + runSession = segment.sessionId; + segments.push(segment); + } + segments.reverse(); + const excerptLines: string[] = []; let currentSession = -1; for (const segment of segments) { if (segment.sessionId !== currentSession) { currentSession = segment.sessionId; - excerptLines.push(`\n## ${sessionTitles.get(currentSession) ?? `Conversation #${currentSession}`}`); + excerptLines.push(sessionHeading(currentSession)); } - const speaker = - segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker"; - const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : ""; - excerptLines.push(`[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`); - if (segment.translation) excerptLines.push(` (translated: ${segment.translation})`); + excerptLines.push(segmentText(segment)); } - let context = - `# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` + - `# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` + - `${sessionLines.join("\n") || "(none in range)"}\n\n` + - `# Transcript excerpts\n${excerptLines.join("\n") || "(no matching lines)"}`; - let truncated = false; - if (context.length > MAX_CONTEXT_CHARS) { - context = context.slice(0, MAX_CONTEXT_CHARS); - truncated = true; - } + const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`; const note = `${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` + `${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` + diff --git a/app/apps/microphones/dsp.ts b/app/apps/microphones/dsp.ts index 9c683d28..34a58045 100644 --- a/app/apps/microphones/dsp.ts +++ b/app/apps/microphones/dsp.ts @@ -286,6 +286,12 @@ export function angleInArc(azimuth: number, arcCenter: number, arcHalfWidth: num export class TempleAudioPipeline { private readonly beamformer: DelaySumBeamformer; private readonly anc = new InterMicNoiseCanceller(); + // Steering delays reach back before the start of each frame. Carrying the + // previous frame's last samples keeps those real; zero-filling them (as a + // standalone beam() does) dips the output at every frame boundary. + private readonly carry: number; + private frontTail: Float32Array; + private rearTail: Float32Array; enableANC = true; constructor( @@ -294,19 +300,38 @@ export class TempleAudioPipeline { geometry: MicGeometry = G2_DEFAULT_GEOMETRY, ) { this.beamformer = new DelaySumBeamformer(sampleRate, geometry); + const { front, rear } = geometry.positions[side]; + const spacing = Math.hypot(front.x - rear.x, front.y - rear.y, front.z - rear.z); + this.carry = Math.ceil((spacing / geometry.speedOfSound) * sampleRate) + 1; + this.frontTail = new Float32Array(this.carry); + this.rearTail = new Float32Array(this.carry); } /** * Process one frame's deinterleaved front/rear channels toward a * device-frame target, returning the enhanced mono beam for this temple. + * Frames must be consecutive: each one continues the previous. */ process(front: Float32Array, rear: Float32Array, target: Bearing): Float32Array { - const beam = this.beamformer.beam(this.side, front, rear, target); + const n = Math.min(front.length, rear.length); + const extendedFront = this.withTail(this.frontTail, front, n); + const extendedRear = this.withTail(this.rearTail, rear, n); + this.frontTail = extendedFront.slice(n); + this.rearTail = extendedRear.slice(n); + const beam = this.beamformer + .beam(this.side, extendedFront, extendedRear, target) + .slice(this.carry); if (!this.enableANC) return beam; // Null-steer reference: the difference channel carries off-axis noise. - const n = Math.min(front.length, rear.length); const reference = new Float32Array(n); for (let i = 0; i < n; i++) reference[i] = front[i]! - rear[i]!; return this.anc.process(beam, reference); } + + private withTail(tail: Float32Array, frame: Float32Array, n: number): Float32Array { + const out = new Float32Array(tail.length + n); + out.set(tail); + out.set(frame.subarray(0, n), tail.length); + return out; + } } diff --git a/app/apps/microphones/index.ts b/app/apps/microphones/index.ts index f17d5c30..c75ef469 100644 --- a/app/apps/microphones/index.ts +++ b/app/apps/microphones/index.ts @@ -17,7 +17,9 @@ const microphonesApp: AppDefinition = { title: "Microphones", icon: "mic", boot: () => { - // Prune expired captions/recordings per the retention settings. + // Close out conversations and recordings a dead process left open, then + // prune expired captions/recordings per the retention settings. + micSession.recoverInterruptedConversations(); void micSession.applyRetentionSweep(); }, launch: async (ctx) => { diff --git a/app/apps/microphones/mic-control.ts b/app/apps/microphones/mic-control.ts index fbb6a7b6..3ac8ddca 100644 --- a/app/apps/microphones/mic-control.ts +++ b/app/apps/microphones/mic-control.ts @@ -110,6 +110,7 @@ class MicArrayController { * doesn't spend BLE bandwidth. */ apply(config: MicConfig): boolean { + if (config.armHardware && !micControlSupported()) return false; const leftActive = templeActive(config, "left"); const rightActive = templeActive(config, "right"); if (leftActive || rightActive) { @@ -124,6 +125,17 @@ class MicArrayController { return true; } + /** + * Re-send CONFIGURE to one temple whose stream went quiet (a lost write, or + * a temple that restarted on its own), leaving the other temple's stream + * untouched. + */ + reconfigureTemple(config: MicConfig, side: MicSide): boolean { + if (!templeActive(config, side)) return false; + if (config.armHardware && !micControlSupported()) return false; + return this.send("configure", config, `configure-${side}`, side === "right", side === "left"); + } + query(): boolean { return this.send("query", undefined, "query"); } @@ -148,11 +160,11 @@ class MicArrayController { ): boolean { const communicator = activeCommunicator(); if (!communicator) return false; - this.ensureStatusProxy(communicator); try { + this.ensureStatusProxy(communicator); const record = encodeMicControl(op, config); - communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple); - return true; + // False when the session wasn't ready and nothing was queued. + return communicator.sendFaceclawMicControl(toJavaBytes(record), label, rightTemple, leftTemple) !== false; } catch (error) { console.warn(`mic control ${label} failed: ${error}`); return false; diff --git a/app/apps/microphones/mic-models.ts b/app/apps/microphones/mic-models.ts index 0d2ff693..75db76d7 100644 --- a/app/apps/microphones/mic-models.ts +++ b/app/apps/microphones/mic-models.ts @@ -75,6 +75,9 @@ export type MicModelState = { }; const downloaders = new Map(); +// Each download's listener is referenced only from Java while it runs; hold +// it here so it isn't GC'd before its callbacks. +const downloadListeners = new Map(); const progress = new Map(); const stateListeners = new Set<(id: MicModelId, state: MicModelState) => void>(); @@ -142,6 +145,7 @@ function notifyStateChanged(id: MicModelId): void { export function startMicModelDownload(id: MicModelId): void { if (!global.isAndroid || downloaders.has(id) || isMicModelReady(id)) return; + progress.delete(id); downloadNextFile(id); notifyStateChanged(id); } @@ -151,6 +155,7 @@ function downloadNextFile(id: MicModelId): void { const nextFile = definition.files.find((file) => !isFilePresent(id, file)); if (!nextFile) { downloaders.delete(id); + downloadListeners.delete(id); notifyStateChanged(id); return; } @@ -168,18 +173,28 @@ function downloadNextFile(id: MicModelId): void { onError: (message: string) => { console.error(`Mic model download failed (${id}/${nextFile.name}): ${message}`); downloaders.delete(id); + downloadListeners.delete(id); notifyStateChanged(id); }, }); - const downloader = new com.faceclaw.app.FaceclawModelDownloader( - `${definition.baseUrl}${nextFile.remoteName}`, - filePath(id, nextFile), - nextFile.sha256, - nextFile.sizeBytes, - listener, - ); - downloaders.set(id, downloader); - downloader.start(); + downloadListeners.set(id, listener); + try { + const downloader = new com.faceclaw.app.FaceclawModelDownloader( + `${definition.baseUrl}${nextFile.remoteName}`, + filePath(id, nextFile), + nextFile.sha256, + nextFile.sizeBytes, + listener, + ); + downloaders.set(id, downloader); + downloader.start(); + } catch (error) { + // Without this, a dead entry would pin the model at "downloading". + console.error(`Mic model download failed to start (${id}/${nextFile.name}): ${String(error)}`); + downloaders.delete(id); + downloadListeners.delete(id); + notifyStateChanged(id); + } } export function cancelMicModelDownload(id: MicModelId): void { @@ -187,6 +202,7 @@ export function cancelMicModelDownload(id: MicModelId): void { if (!downloader) return; downloader.cancel(); downloaders.delete(id); + downloadListeners.delete(id); notifyStateChanged(id); } diff --git a/app/apps/microphones/mic-session.ts b/app/apps/microphones/mic-session.ts index f6ee7a54..5139f09b 100644 --- a/app/apps/microphones/mic-session.ts +++ b/app/apps/microphones/mic-session.ts @@ -16,6 +16,7 @@ import { type HeadOrientation, } from "./dsp"; import { micArrayController, micControlSupported, loadMicConfig, saveMicConfig } from "./mic-control"; +import { CaptionClock } from "./caption-clock"; import { micModelPath } from "./mic-models"; import { MIC_CHANNEL_KEYS, @@ -67,6 +68,7 @@ import { formatRelativeTime } from "../../util/date-util"; import { deviceLanguage, identifyLanguage, translateText } from "./translate"; declare const com: any; +declare const java: any; /** * The Microphones session: owns mic capture while the app window is open, @@ -83,6 +85,11 @@ declare const com: any; * mics as raw PCM 'SM' frames; per-temple delay-and-sum beamforming plus * NLMS noise cancellation run on the phone, and the two temple bearings * are fused for the radar. + * + * Neither path can assume an earlier start still holds: the mic enable + * lives in the glasses session (an arm drop or EvenHub resume clears it), + * and other mic users preempt the shared tap. A periodic check brings + * capture up when it isn't and re-acquires it when audio stops arriving. */ const SAMPLE_RATE = 16000; @@ -95,6 +102,37 @@ const RECORDING_AAC_BITRATE = 32_000; const ENCOUNTER_GAP_MS = 15 * 60 * 1000; const ENCOUNTER_CARD_TTL_MS = 12_000; const ENCOUNTER_CARD_MAX_ACTION_ITEMS = 2; +// A temple that has sent no frame (or no bearing) for this long no longer +// counts: its stream stopped, its mics were toggled off, or its arm dropped. +const TEMPLE_STALE_MS = 1000; +// An enabled temple silent this long while forwarding holds gets its config +// re-sent; it is also the time a temple gets to start streaming after each +// CONFIGURE. +const TEMPLE_SILENT_MS = 5000; +// The fused direction of arrival only gates audio while it is this fresh: a +// direction nobody reports any more (Direction detection turned off, the +// temples gone quiet) must not keep muting the captions. +const DOA_STALE_MS = TEMPLE_STALE_MS; +// The caption feed moves to the other temple only once that temple has +// clearly out-heard the current one for a while. Each switch splices in +// audio on another temple's clock, so flapping near the threshold glitches it. +const FEED_SWITCH_RATIO = 1.5; +const FEED_SWITCH_HOLD_MS = 500; +// How often capture is checked: brought up if it isn't, re-acquired if +// audio stopped arriving. +const CAPTURE_CHECK_MS = 3000; +// A caption engine that dies on its own is restarted on the same +// conversation, with growing delays, this many times before captions give up. +const CAPTION_MAX_RESTARTS = 3; +const CAPTION_RESTART_BASE_MS = 2000; +// How long a stopping caption engine gets to deliver its trailing utterance +// before its conversation is closed without it. +const CAPTION_STOP_TIMEOUT_MS = 20_000; +const COMPASS_OWNER = "microphones"; +// This app's claim on the shared raw-PCM tap (stock path). +const RAW_CAPTURE_OWNER = "microphones"; +const WAITING_TEXT = "Waiting for the glasses microphone..."; +const STALLED_TEXT = "No audio from the glasses; reconnecting the mic..."; export type CaptionLine = { speakerId: number | null; @@ -133,22 +171,59 @@ export type MicSessionState = { /** Human-readable description of the active noise-cancellation engine. */ ancEngine: string; captionsActive: boolean; + /** Why captions aren't running, for the captions view ("" while they are). */ + captionsNote: string; captionLines: CaptionLine[]; speakerDots: SpeakerDot[]; }; type StateListener = (state: MicSessionState) => void; +/** One saved caption conversation: its store row and its running tone average. */ +type ConversationRecord = { rowId: number; sentimentSum: number; sentimentCount: number }; + +/** + * One caption-engine instance. Its utterance times are on its own sample + * clock (audio fed to it), converted to wall time by `clock` and to + * recording time with the recording offset taken at its start; an engine + * restarted after a failure is a new run on the same conversation. + */ +type CaptionRun = { + engine: any; + // The Java listener proxy must stay referenced or it gets GC'd. + listener: any; + audioBaseMs: number; + clock: CaptionClock; + conversation: ConversationRecord | null; + /** The app session (start() count) it belongs to. */ + generation: number; + stopTimer: ReturnType | null; + /** Runs once the stopped engine has delivered its trailing utterance. */ + afterStop: (() => void) | null; +}; + +// Boot-time recovery runs once per process. +let recoveryDone = false; + class MicSession { private readonly stateListeners = new Set(); private running = false; private mode: "off" | "stock" | "extended" = "off"; private statusText = "Idle"; + // Whether the current mode's capture path is up; until it is, the capture + // check keeps retrying. + private captureUp = false; + // Capture is up but audio isn't arriving (the check is re-acquiring it). + private captureStalled = false; + // Consecutive checks that found the stock raw tap dead and restarted it. + private stockRestarts = 0; // Radar state. private levels: number[] = []; private levelLabels: string[] = []; private doaDeviceDeg: number | null = null; + // When doaDeviceDeg was last updated from a reported bearing. + private doaAtMs = 0; private ssr = 0; private headingDeg: number | null = null; private beamWorldDeg = 0; @@ -158,13 +233,20 @@ class MicSession { // Captions state. private captionLines: CaptionLine[] = []; - private captionEngine: any = null; - private captionListenerProxy: any = null; + // Captions are on for this app session: an engine is running, or being + // restarted after a failure. private captionsActive = false; - private engineStartWallMs = 0; - private sessionRowId = -1; - private sentimentSum = 0; - private sentimentCount = 0; + private captionRun: CaptionRun | null = null; + // Runs told to stop that haven't reported back; each holds its + // conversation open until its trailing utterance is in. + private readonly stoppingRuns = new Set(); + private conversation: ConversationRecord | null = null; + private captionFailures = 0; + private captionRestartTimer: ReturnType | null = null; + // Why captions gave up after repeated engine failures ("" when they haven't). + private captionsFailure = ""; + // Bumped per start(), so a stopping run's late line isn't shown in a later session. + private sessionGeneration = 0; private conversationLang = "und"; // Recording state. @@ -181,7 +263,13 @@ class MicSession { private temples = new Map(); private templeAngles: { left: number | null; right: number | null } = { left: null, right: null }; private templeSsr: { left: number; right: number } = { left: 0, right: 0 }; + private templeAngleAtMs: Record = { left: 0, right: 0 }; + private templeFrameAtMs: Record = { left: 0, right: 0 }; + // When each temple was last sent CONFIGURE, for its start-up grace period. + private templeArmedAtMs: Record = { left: 0, right: 0 }; private captionFeedSide: MicSide = "left"; + // When the other temple first clearly out-heard the caption feed's (0 = it hasn't). + private feedSwitchSinceMs = 0; // Cached at capture start so the ~80 Hz packet path never re-reads settings. private activeConfig: MicConfig = loadMicConfig(); // Hot-path flags mirrored from settings so the packet path avoids a JNI @@ -202,6 +290,7 @@ class MicSession { private speechActive = false; private renderTimer: ReturnType | null = null; + private captureCheckTimer: ReturnType | null = null; onState(listener: StateListener): () => void { this.stateListeners.add(listener); @@ -232,6 +321,7 @@ class MicSession { ancOn: ancEnabledSetting.get(), ancEngine: this.describeAncEngine(), captionsActive: this.captionsActive, + captionsNote: this.captionsNote(), captionLines: [...this.captionLines], speakerDots: [...(this.speakerDots = pruneDots(this.speakerDots, Date.now()))], }; @@ -244,6 +334,16 @@ class MicSession { : this.suppressorEngine; } + private captionsNote(): string { + if (this.captionsActive) return ""; + if (this.captionsFailure) { + return `Captions stopped: ${this.captionsFailure}. Turn Captions back on in the menu to retry.`; + } + // Wanted, but waiting for the microphone to come up. + if (this.running && captionsEnabledSetting.get()) return this.statusText; + return "Captions are off (enable in the Microphones menu)."; + } + private notify(): void { const state = this.getState(); this.stateListeners.forEach((listener) => listener(state)); @@ -254,41 +354,26 @@ class MicSession { start(): void { if (this.running || !global.isAndroid) return; this.running = true; + this.sessionGeneration += 1; this.captionLines = []; this.speakerDots = []; - this.sentimentSum = 0; - this.sentimentCount = 0; + this.captionsFailure = ""; this.conversationLang = "und"; this.activeConfig = loadMicConfig(); this.refreshCachedFlags(); - this.offSettings = onSettingsStoreChanged(() => this.refreshCachedFlags()); + this.mode = this.desiredMode(); + this.offSettings = onSettingsStoreChanged(() => this.handleSettingsChanged()); this.offSpeakerChanges = speakerRegistry.onChanged(() => this.refreshSpeakerIdentities()); this.offVoiceActivity = voiceActivity.subscribe((active) => this.handleVoiceActivity(active)); - const extended = micControlSupported() && this.activeConfig.armHardware; - this.mode = extended ? "extended" : "stock"; this.startSuppressor(); this.startCompass(); - if (captionsEnabledSetting.get()) { - this.startCaptions(); - } - if (saveRecordingsSetting.get() && recordingsRetentionSetting.get() !== "none") { - this.startRecording(); - } - const ok = extended ? this.startExtended() : this.startStock(); - if (!ok) { - this.statusText = "Microphone unavailable (glasses not connected?)"; - this.running = false; - this.stopCaptions(); - this.stopRecording(false); - this.stopCompass(); - this.mode = "off"; - this.notify(); - return; - } - this.statusText = extended ? "Listening (4-mic array)" : "Listening"; + // Captions start once capture is up (bringUpCapture), so a session + // waiting on the glasses doesn't open an empty conversation. + this.bringUpCapture(); // Levels/DOA update at packet rate; coalesce UI renders to 5 Hz. this.renderTimer = setInterval(() => this.notify(), 200); + this.captureCheckTimer = setInterval(() => this.checkCapture(), CAPTURE_CHECK_MS); this.notify(); } @@ -299,31 +384,124 @@ class MicSession { clearInterval(this.renderTimer); this.renderTimer = null; } - if (this.mode === "extended") { - this.stopExtended(); - } else { - this.stopStock(); + if (this.captureCheckTimer) { + clearInterval(this.captureCheckTimer); + this.captureCheckTimer = null; } + this.stopCapture(); this.offSettings?.(); this.offSettings = null; this.offSpeakerChanges?.(); this.offSpeakerChanges = null; this.offVoiceActivity?.(); this.offVoiceActivity = null; + // Closes the recording now; the conversation closes once the engine's + // trailing utterance is in. this.stopCaptions(); - this.stopRecording(true); this.stopCompass(); this.suppressor = null; this.speechActive = false; - this.finishSessionRow(); this.mode = "off"; this.statusText = "Idle"; + this.captureStalled = false; this.levels = []; - this.doaDeviceDeg = null; + this.clearDirection(); this.notify(); void this.applyRetentionSweep(); } + private listeningText(): string { + return this.mode === "extended" ? "Listening (4-mic array)" : "Listening"; + } + + private desiredMode(): "stock" | "extended" { + return micControlSupported() && this.activeConfig.armHardware ? "extended" : "stock"; + } + + /** + * Start the current mode's capture path. Until it is up the session stays + * running and the capture check retries; captions begin once it is. + */ + private bringUpCapture(): boolean { + // The voice dialog or Transcribe owns the mic; take it back when it ends. + if (voiceActivity.isActive() || !this.startCapture()) { + this.captureUp = false; + this.statusText = WAITING_TEXT; + return false; + } + this.captureUp = true; + this.captureStalled = false; + this.statusText = this.listeningText(); + if (captionsEnabledSetting.get() && !this.captionsActive && !this.captionsFailure) { + this.startCaptions(); + } + return true; + } + + /** Start the current mode's capture path, undoing any partial start on failure. */ + private startCapture(): boolean { + try { + if (this.mode === "extended" ? this.startExtended() : this.startStock()) return true; + } catch (error) { + console.warn(`mic capture start failed: ${error}`); + } + this.stopCapture(); + return false; + } + + private stopCapture(): void { + this.captureUp = false; + if (this.mode === "extended") { + this.stopExtended(); + } else { + this.stopStock(); + } + } + + /** + * Persist a Microphone setup change. While running it takes effect live: + * toggling 4-mic capture swaps the capture path, and any other change is + * re-sent to the streaming temples. + */ + updateConfig(mutate: (config: MicConfig) => MicConfig): void { + const config = mutate(loadMicConfig()); + const directionChanged = config.beamform !== this.activeConfig.beamform; + const modeBefore = this.mode; + // Before saving: the save's settings event re-checks the capture mode. + this.activeConfig = config; + saveMicConfig(config); + if (this.running) { + if (directionChanged) this.clearDirection(); + this.syncCaptureMode(); + if ( + this.mode === modeBefore && + this.mode === "extended" && + this.captureUp && + !voiceActivity.isActive() + ) { + this.armExtended(); + } + } + this.notify(); + } + + private handleSettingsChanged(): void { + this.refreshCachedFlags(); + // The developer gate on mic control can change under a running session. + this.syncCaptureMode(); + } + + /** Swap capture paths when 4-mic capture (or its developer gate) changed. */ + private syncCaptureMode(): void { + if (!this.running) return; + const mode = this.desiredMode(); + if (mode === this.mode) return; + this.stopCapture(); + this.mode = mode; + this.clearDirection(); + this.bringUpCapture(); + } + private refreshCachedFlags(): void { this.beamFilterOn = beamFilterSetting.get(); const wasOn = this.ancOn; @@ -358,15 +536,16 @@ class MicSession { } /** - * The assistant modal owns the single G2 mic while open. Pause our capture - * for its duration and re-acquire when it closes — the stock raw tap is - * preempted by the bridge and must be re-requested; the extended SM stream - * must stop so the assistant's stock-format session can run at all. + * Speech capture (the assistant modal, a dictation draft, Transcribe) owns + * the single G2 mic while active. Pause our capture for its duration and + * re-acquire when it ends — the stock raw tap is preempted by the bridge + * and must be re-requested; the extended SM stream must stop so the + * stock-format capture can run at all. */ private handleVoiceActivity(active: boolean): void { if (!this.running) return; if (active) { - if (this.mode === "extended") { + if (this.mode === "extended" && this.captureUp) { micArrayController.stopStreaming(); try { this.nativeCommunicator()?.stopG2AudioForwarding(); @@ -376,17 +555,90 @@ class MicSession { } return; } - const communicator = this.nativeCommunicator(); - if (!communicator) return; - if (this.mode === "extended") { - if (this.forwardingProxy && communicator.startG2AudioForwarding(this.forwardingProxy)) { - micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 }); + // Take the mic back now rather than at the next check. + this.checkCapture(); + } + + /** + * Bring capture up if it isn't, and re-acquire it if audio stopped + * arriving. Runs every CAPTURE_CHECK_MS and when speech capture ends. + */ + private checkCapture(): void { + if (!this.running || voiceActivity.isActive()) return; + try { + if (!this.captureUp) { + // Retrying while disconnected would only log skipped sends. + if (this.nativeCommunicator()?.isSessionReady()) this.bringUpCapture(); + return; } - } else { - voiceControlBridge.startRawCapture({ communicator }); + const healthy = this.mode === "extended" ? this.checkExtendedCapture() : this.checkStockCapture(); + this.setCaptureStalled(!healthy); + } catch (error) { + console.warn(`mic capture check failed: ${error}`); } } + private setCaptureStalled(stalled: boolean): void { + if (stalled === this.captureStalled) return; + this.captureStalled = stalled; + this.statusText = stalled ? STALLED_TEXT : this.listeningText(); + } + + /** + * The stock raw tap dies with the glasses session, and an STT capture + * preempts it. Restart it whenever it isn't live. Returns false (stalled) + * while disconnected or once restarts keep failing. + */ + private checkStockCapture(): boolean { + if (voiceControlBridge.isRawCaptureLive()) { + this.stockRestarts = 0; + return true; + } + const communicator = this.nativeCommunicator(); + if (!communicator?.isSessionReady()) return false; + voiceControlBridge.startRawCapture({ communicator, owner: RAW_CAPTURE_OWNER }); + // One restart is routine (after the voice dialog, or a reconnect); only + // report a stall once restarts keep failing. + this.stockRestarts += 1; + return this.stockRestarts < 2; + } + + /** + * The extended stream's forwarding lives in the glasses session: an arm + * drop or EvenHub resume clears it, so re-establish it and re-send the + * config. With forwarding in place, each enabled temple must actually be + * streaming: a lost CONFIGURE or a temple that restarted on its own looks + * like silence, so a temple quiet past its grace period is reconfigured. + * Returns whether audio is arriving from every enabled temple. + */ + private checkExtendedCapture(): boolean { + const communicator = this.nativeCommunicator(); + if (!communicator?.isSessionReady()) return false; + if (!communicator.isAudioCaptureActive()) { + if (!this.forwardingProxy || !communicator.startG2AudioForwarding(this.forwardingProxy)) return false; + // Routine after the voice dialog or a reconnect; the frame check below + // reports a stall if the temples don't start streaming. + return this.armExtended() && !this.captureStalled; + } + const now = Date.now(); + const wire = wireConfig(this.activeConfig); + let healthy = true; + for (const side of ["left", "right"] as const) { + if (!templeActive(this.activeConfig, side)) continue; + if (now - this.templeFrameAtMs[side] < TEMPLE_SILENT_MS) continue; + if (now - this.templeArmedAtMs[side] < TEMPLE_SILENT_MS) { + // Just (re)configured: give it time to start streaming, without + // calling it healthy if it was already stalled. + if (this.captureStalled) healthy = false; + continue; + } + healthy = false; + this.templeArmedAtMs[side] = now; + micArrayController.reconfigureTemple(wire, side); + } + return healthy; + } + private startSuppressor(): void { try { this.suppressor = new com.faceclaw.app.FaceclawNoiseSuppressor(SAMPLE_RATE); @@ -455,11 +707,39 @@ class MicSession { ); } + /** Record a newly reported direction of arrival (device frame). */ + private updateDirection(deviceDeg: number, nowMs: number): void { + this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, deviceDeg, DOA_SMOOTHING); + this.doaAtMs = nowMs; + if (this.beamLocked && this.doaDeviceDeg !== null) { + this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg); + } + } + + private clearDirection(): void { + this.doaDeviceDeg = null; + this.doaAtMs = 0; + this.ssr = 0; + this.templeAngles = { left: null, right: null }; + this.templeSsr = { left: 0, right: 0 }; + this.templeAngleAtMs = { left: 0, right: 0 }; + } + + /** + * Beam filter: whether to drop audio because it arrives from outside the + * beam. Only a current direction can gate; with none, audio passes. + */ + private outsideBeam(): boolean { + if (!this.beamFilterOn || this.ssr <= 0 || this.doaDeviceDeg === null) return false; + if (Date.now() - this.doaAtMs >= DOA_STALE_MS) return false; + return !this.inBeam(this.doaDeviceDeg); + } + // ---- compass (world-frame radar + beam-follow while head turns) ---- private startCompass(): void { try { - setCompassEnabled(true); + setCompassEnabled(true, COMPASS_OWNER); this.offCompass = addCompassListener((event) => { if (event.headingDegrees >= 0) { this.headingDeg = event.headingDegrees; @@ -474,7 +754,7 @@ class MicSession { this.offCompass?.(); this.offCompass = null; try { - setCompassEnabled(false); + setCompassEnabled(false, COMPASS_OWNER); } catch { // Disconnected; the stream is already gone. } @@ -484,19 +764,18 @@ class MicSession { private startStock(): boolean { const communicator = this.nativeCommunicator(); - if (!communicator) return false; + // The tap only records intent without a session; wait for one instead. + if (!communicator?.isSessionReady()) return false; this.levels = [0]; this.levelLabels = ["Glasses mic"]; - const started = voiceControlBridge.startRawCapture({ communicator }); + this.stockRestarts = 0; + const started = voiceControlBridge.startRawCapture({ communicator, owner: RAW_CAPTURE_OWNER }); if (!started) return false; this.offRawPcm = voiceControlBridge.onRawPcm((pcm) => this.handleStockPcm(pcm)); this.offFrameMeta = voiceControlBridge.onFrameMeta((meta) => { this.ssr = meta.ssr; if (meta.angleDegrees !== 0 || meta.ssr !== 0) { - this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, meta.angleDegrees, DOA_SMOOTHING); - if (this.beamLocked && this.doaDeviceDeg !== null) { - this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg); - } + this.updateDirection(meta.angleDegrees, Date.now()); } this.trackLiveSpeaker(); }); @@ -508,18 +787,13 @@ class MicSession { this.offRawPcm = null; this.offFrameMeta?.(); this.offFrameMeta = null; - voiceControlBridge.stopRawCapture(); + voiceControlBridge.stopRawCapture(RAW_CAPTURE_OWNER); } private handleStockPcm(pcm: Uint8Array): void { const level = pcmLevel(pcm); this.levels[0] = Math.max(level, (this.levels[0] ?? 0) * LEVEL_DECAY); - const gated = - this.beamFilterOn && - this.doaDeviceDeg !== null && - this.ssr > 0 && - !this.inBeam(this.doaDeviceDeg); - if (gated) return; + if (this.outsideBeam()) return; this.feedPcm(pcm); } @@ -527,7 +801,7 @@ class MicSession { private startExtended(): boolean { const communicator = this.nativeCommunicator(); - if (!communicator) return false; + if (!communicator?.isSessionReady()) return false; // Each active temple streams its full front+rear stereo pair; the four // mics are split, metered, and selected here on the host. this.levelLabels = MIC_CHANNEL_KEYS.map(micChannelLabel); @@ -542,11 +816,14 @@ class MicSession { }, }); if (!communicator.startG2AudioForwarding(this.forwardingProxy)) return false; - if (!micArrayController.apply({ ...this.activeConfig, channelMask: 0b11 })) { - communicator.stopG2AudioForwarding(); - return false; - } - return true; + return this.armExtended(); + } + + /** Send the config to the temples, starting each one's grace period to begin streaming. */ + private armExtended(): boolean { + const now = Date.now(); + this.templeArmedAtMs = { left: now, right: now }; + return micArrayController.apply(wireConfig(this.activeConfig)); } /** @@ -555,14 +832,7 @@ class MicSession { * captions; a temple with both mics off has its stream stopped entirely. */ setMicChannelEnabled(key: MicChannelKey, enabled: boolean): void { - const config = loadMicConfig(); - config.hostMics = { ...config.hostMics, [key]: enabled }; - saveMicConfig(config); - this.activeConfig = config; - if (this.running && this.mode === "extended") { - micArrayController.apply({ ...config, channelMask: 0b11 }); - } - this.notify(); + this.updateConfig((config) => ({ ...config, hostMics: { ...config.hostMics, [key]: enabled } })); } private stopExtended(): void { @@ -576,6 +846,16 @@ class MicSession { this.forwardingProxy = null; this.temples.clear(); this.templeAngles = { left: null, right: null }; + this.templeSsr = { left: 0, right: 0 }; + this.templeAngleAtMs = { left: 0, right: 0 }; + this.templeFrameAtMs = { left: 0, right: 0 }; + this.templeArmedAtMs = { left: 0, right: 0 }; + this.feedSwitchSinceMs = 0; + } + + /** A temple's last bearing, or null once it has gone quiet. */ + private freshTempleAngle(side: MicSide, nowMs: number): number | null { + return nowMs - this.templeAngleAtMs[side] < TEMPLE_STALE_MS ? this.templeAngles[side] : null; } private handleExtendedPacket(data: Uint8Array, arm: string): void { @@ -588,6 +868,8 @@ class MicSession { const side: MicSide = arm === "L" ? "left" : "right"; const channels = splitInterleavedPcm16(frame); if (!channels || channels.length === 0) return; + const now = Date.now(); + this.templeFrameAtMs[side] = now; // Meter every arriving channel — a host-disabled mic still shows its // live level (marked off in the UI), it just doesn't feed anything. @@ -606,8 +888,10 @@ class MicSession { if (frame.angleDegrees !== 0 || frame.ssr !== 0) { this.templeAngles[side] = frame.angleDegrees; this.templeSsr[side] = frame.ssr; - const left = this.templeAngles.left; - const right = this.templeAngles.right; + this.templeAngleAtMs[side] = now; + // A temple that stopped reporting must not keep steering the fusion. + const left = this.freshTempleAngle("left", now); + const right = this.freshTempleAngle("right", now); let fusedDeg: number; if (left !== null && right !== null) { const fused = fuseBearings( @@ -620,11 +904,8 @@ class MicSession { } else { fusedDeg = (left ?? right)!; } - this.doaDeviceDeg = smoothAngleDeg(this.doaDeviceDeg, fusedDeg, DOA_SMOOTHING); - this.ssr = Math.max(this.templeSsr.left, this.templeSsr.right); - if (this.beamLocked && this.doaDeviceDeg !== null) { - this.beamWorldDeg = this.toWorldDeg(this.doaDeviceDeg); - } + this.ssr = Math.max(left === null ? 0 : this.templeSsr.left, right === null ? 0 : this.templeSsr.right); + this.updateDirection(fusedDeg, now); this.trackLiveSpeaker(); } @@ -650,32 +931,52 @@ class MicSession { } private deliverExtendedAudio(side: MicSide, samples: Float32Array): void { - // Feed captions from one temple only; prefer the one hearing more - // signal, among the temples that still have an enabled microphone. - const leftUsable = templeActive(this.activeConfig, "left"); - const rightUsable = templeActive(this.activeConfig, "right"); - let stronger: MicSide = this.templeSsr.right > this.templeSsr.left * 1.5 ? "right" : "left"; - if (stronger === "right" && !rightUsable) stronger = "left"; - if (stronger === "left" && !leftUsable) stronger = "right"; - if (strongerChanged(this.captionFeedSide, stronger)) { - this.captionFeedSide = stronger; - } + // Feed captions from one temple only. + this.updateCaptionFeedSide(Date.now()); if (side !== this.captionFeedSide) return; - if ( - this.beamFilterOn && - this.doaDeviceDeg !== null && - this.ssr > 0 && - !this.inBeam(this.doaDeviceDeg) - ) { + if (this.outsideBeam()) return; + this.feedPcm(floatToPcm16(samples)); + } + + /** + * Pick the temple that feeds captions: the one hearing more signal, among + * the temples with an enabled microphone that are still streaming. A + * switch needs the other temple ahead by FEED_SWITCH_RATIO for + * FEED_SWITCH_HOLD_MS; losing the current temple switches at once. + */ + private updateCaptionFeedSide(nowMs: number): void { + const usable = (side: MicSide) => + templeActive(this.activeConfig, side) && nowMs - this.templeFrameAtMs[side] < TEMPLE_STALE_MS; + const heard = (side: MicSide) => (this.freshTempleAngle(side, nowMs) === null ? 0 : this.templeSsr[side]); + const current = this.captionFeedSide; + const other: MicSide = current === "left" ? "right" : "left"; + if (!usable(other)) { + this.feedSwitchSinceMs = 0; return; } - this.feedPcm(floatToPcm16(samples)); + if (!usable(current)) { + this.captionFeedSide = other; + this.feedSwitchSinceMs = 0; + return; + } + if (heard(other) <= heard(current) * FEED_SWITCH_RATIO) { + this.feedSwitchSinceMs = 0; + return; + } + if (this.feedSwitchSinceMs === 0) { + this.feedSwitchSinceMs = nowMs; + } else if (nowMs - this.feedSwitchSinceMs >= FEED_SWITCH_HOLD_MS) { + this.captionFeedSide = other; + this.feedSwitchSinceMs = 0; + } } // ---- shared audio consumers ---- private feedPcm(pcm: Uint8Array): void { - if (!this.captionEngine && !this.recorder) return; + const run = this.captionRun; + const engine = run?.engine; + if (!engine && !this.recorder) return; let javaBytes = toJavaBytes(pcm); if (this.ancOn && this.suppressor) { try { @@ -684,9 +985,10 @@ class MicSession { console.warn(`noise suppression failed: ${error}`); } } - if (this.captionEngine) { + if (run) { try { - this.captionEngine.acceptPcm(javaBytes); + run.engine.acceptPcm(javaBytes); + run.clock.noteFed(Number(javaBytes.length), Date.now()); } catch (error) { console.warn(`caption feed failed: ${error}`); } @@ -702,61 +1004,219 @@ class MicSession { // ---- captions ---- + /** Whether captions are on, as the menu toggle shows it (off after they gave up). */ + captionsEnabled(): boolean { + return captionsEnabledSetting.get() && !this.captionsFailure; + } + setCaptionsEnabled(enabled: boolean): void { captionsEnabledSetting.set(enabled); - if (!this.running) return; - if (enabled && !this.captionsActive) { - this.startCaptions(); - if (saveRecordingsSetting.get() && !this.recorder) this.startRecording(); - } else if (!enabled && this.captionsActive) { - this.stopCaptions(); - this.finishSessionRow(); + this.captionsFailure = ""; + if (this.running) { + if (enabled && !this.captionsActive) { + // While capture is still coming up, bringUpCapture starts them. + if (this.captureUp) this.startCaptions(); + } else if (!enabled && this.captionsActive) { + this.stopCaptions(); + } } this.notify(); } + /** Begin a captioned conversation: the engine, then its store row and recording. */ private startCaptions(): void { if (this.captionsActive) return; - const engine = new com.faceclaw.app.FaceclawCaptionEngine(); - this.captionListenerProxy = new com.faceclaw.app.FaceclawCaptionEngineListener({ - onUtterance: (text: string, embedding: any, startMs: number, endMs: number, peakRms: number) => { - this.handleUtterance(String(text), embedding, Number(startMs), Number(endMs), Number(peakRms)); - }, - onSpeechStart: (_startMs: number) => { - this.speechActive = true; - }, - onStatus: (status: string) => { - this.statusText = String(status); - }, - }); - engine.setListener(this.captionListenerProxy); - // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) - // that only ever uses Moonshine; unaffected by the two-model on-device - // Whisper addition in native/voice-control.ts's push-to-talk path. - engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); - engine.setSpeakerModelPath(micModelPath("speaker-embedding")); - engine.start(); - this.captionEngine = engine; + const run = this.createCaptionRun(); + if (!run) { + this.captionsFailure = "the caption engine could not start"; + return; + } this.captionsActive = true; - this.engineStartWallMs = Date.now(); + this.captionsFailure = ""; + this.captionFailures = 0; this.encounterShown.clear(); - if (saveCaptionsSetting.get() && captionsRetentionSetting.get() !== "none") { - this.sessionRowId = Number( - this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`), - ); + this.conversation = this.openConversation(); + run.conversation = this.conversation; + this.captionRun = run; + if (this.conversation && this.shouldRecord()) this.startRecording(this.conversation); + } + + /** Start a caption engine on the current conversation, or null if it can't start. */ + private createCaptionRun(): CaptionRun | null { + let engine: any = null; + try { + engine = new com.faceclaw.app.FaceclawCaptionEngine(); + const run: CaptionRun = { + engine, + listener: null, + // A restarted engine's clock starts at zero partway into the recording. + audioBaseMs: this.recorder ? Number(this.recorder.getDurationMs()) : 0, + clock: new CaptionClock(SAMPLE_RATE, Date.now()), + conversation: this.conversation, + generation: this.sessionGeneration, + stopTimer: null, + afterStop: null, + }; + run.listener = new com.faceclaw.app.FaceclawCaptionEngineListener({ + onUtterance: (text: string, embedding: any, startMs: number, endMs: number, _peakRms: number) => { + // Never let an exception escape into the engine's Java callback. + try { + this.handleUtterance(run, String(text), embedding, Number(startMs), Number(endMs)); + } catch (error) { + console.warn(`caption utterance failed: ${error}`); + } + }, + onSpeechStart: (_startMs: number) => { + if (run === this.captionRun) this.speechActive = true; + }, + onStatus: (status: string) => { + // Capture problems outrank the engine's own progress messages. + if (run === this.captionRun && this.captureUp && !this.captureStalled) { + this.statusText = String(status); + } + }, + onStopped: (error: string | null) => { + try { + this.handleRunStopped(run, error === null || error === undefined ? null : String(error)); + } catch (stopError) { + console.warn(`caption engine stop handling failed: ${stopError}`); + } + }, + }); + engine.setListener(run.listener); + // Live captions are a separate sherpa-onnx pipeline (FaceclawCaptionEngine) + // that only ever uses Moonshine; unaffected by the two-model on-device + // Whisper addition in native/voice-control.ts's push-to-talk path. + engine.setAsrModelDir(isAsrModelReady("moonshine") ? moonshineModelDir() : null); + engine.setSpeakerModelPath(micModelPath("speaker-embedding")); + engine.start(); + return run; + } catch (error) { + console.warn(`caption engine start failed: ${error}`); + try { + engine?.stop(); + } catch { + // It never started. + } + return null; } } + private handleRunStopped(run: CaptionRun, error: string | null): void { + if (run === this.captionRun) { + // The live engine ended on its own (its models failed to load, or the + // worker died): restart it on the same conversation. + this.captionRun = null; + this.speechActive = false; + this.scheduleCaptionRestart(error ?? "the caption engine stopped"); + return; + } + this.settleRun(run); + } + + private scheduleCaptionRestart(reason: string): void { + this.captionFailures += 1; + if (this.captionFailures > CAPTION_MAX_RESTARTS) { + console.warn(`captions giving up after ${this.captionFailures} failures: ${reason}`); + this.stopCaptions(); + this.captionsFailure = reason; + this.notify(); + return; + } + this.statusText = `Captions failed (${reason}); restarting...`; + this.captionRestartTimer = setTimeout(() => { + this.captionRestartTimer = null; + if (!this.running || !this.captionsActive || this.captionRun) return; + const run = this.createCaptionRun(); + if (run) { + this.captionRun = run; + } else { + this.scheduleCaptionRestart("the caption engine could not restart"); + } + }, CAPTION_RESTART_BASE_MS * this.captionFailures); + this.notify(); + } + + private clearCaptionRestart(): void { + if (this.captionRestartTimer) { + clearTimeout(this.captionRestartTimer); + this.captionRestartTimer = null; + } + } + + /** + * End the captioned conversation. The recording holds every byte the + * engine was fed, so it closes now; the store row stays open until the + * engine has delivered its trailing utterance (or a timeout passes). + */ private stopCaptions(): void { if (!this.captionsActive) return; + this.captionsActive = false; + this.speechActive = false; + this.clearCaptionRestart(); + const endedAtMs = Date.now(); + const run = this.captionRun; + this.captionRun = null; + const conversation = this.conversation; + this.conversation = null; + this.stopRecording(true, conversation); + if (run) { + this.stopRun(run, () => this.finishConversation(conversation, endedAtMs)); + } else { + this.finishConversation(conversation, endedAtMs); + } + } + + private stopRun(run: CaptionRun, afterStop: () => void): void { + run.afterStop = afterStop; + this.stoppingRuns.add(run); + run.stopTimer = setTimeout(() => { + console.warn("caption engine did not report stopping; closing its conversation anyway"); + this.settleRun(run); + }, CAPTION_STOP_TIMEOUT_MS); try { - this.captionEngine?.stop(); + run.engine.requestStop(); } catch (error) { console.warn(`caption engine stop failed: ${error}`); + this.settleRun(run); } - this.captionEngine = null; - this.captionListenerProxy = null; - this.captionsActive = false; + } + + /** A stopping run is done (reported back, or timed out): close what it held open. */ + private settleRun(run: CaptionRun): void { + if (!this.stoppingRuns.delete(run)) return; + if (run.stopTimer) clearTimeout(run.stopTimer); + run.stopTimer = null; + const afterStop = run.afterStop; + run.afterStop = null; + afterStop?.(); + } + + /** The store row for a new conversation, when captions are being saved. */ + private openConversation(): ConversationRecord | null { + if (!saveCaptionsSetting.get() || captionsRetentionSetting.get() === "none") return null; + try { + const rowId = Number( + this.store().startSession(Date.now(), `Conversation ${new Date().toLocaleString()}`), + ); + return rowId >= 0 ? { rowId, sentimentSum: 0, sentimentCount: 0 } : null; + } catch (error) { + console.warn(`caption session start failed: ${error}`); + return null; + } + } + + private finishConversation(conversation: ConversationRecord | null, endedAtMs: number): void { + if (!conversation) return; + const avg = conversation.sentimentCount > 0 ? conversation.sentimentSum / conversation.sentimentCount : 0; + try { + this.store().endSession(conversation.rowId, endedAtMs, "", "", avg); + } catch (error) { + console.warn(`session finish failed: ${error}`); + } + // Recap/action-items/facts per speaker, generated in the background with + // the on-phone model (no-op when it isn't downloaded). + generateSessionInsights(conversation.rowId); } /** The next captioned utterance enrolls (or refreshes) the wearer's voice. */ @@ -882,25 +1342,43 @@ class MicSession { } private handleUtterance( + run: CaptionRun, text: string, embedding: any, startMs: number, endMs: number, - _peakRms: number, ): void { if (!text.trim() && !embedding) return; + const live = run === this.captionRun; + // A stopping run's trailing line still shows if it belongs to this app + // session; it is saved to its own conversation either way. + const shown = this.running && run.generation === this.sessionGeneration; + if (live) this.captionFailures = 0; const embeddingFloats = embedding ? floatArrayToFloat32(embedding) : null; - if (this.wearerEnrollmentArmed && embeddingFloats) { + if (live && this.wearerEnrollmentArmed && embeddingFloats) { this.wearerEnrollmentArmed = false; - const wearer = speakerRegistry.enrollWearer(embeddingFloats); - this.statusText = `Enrolled your voice as "${wearer.name}"`; + try { + const wearer = speakerRegistry.enrollWearer(embeddingFloats); + this.statusText = `Enrolled your voice as "${wearer.name}"`; + } catch (error) { + console.warn(`wearer enrollment failed: ${error}`); + this.statusText = "Voice enrollment failed"; + } } - let match = embeddingFloats ? speakerRegistry.assign(embeddingFloats) : null; - if (match) this.maybeShowEncounterCard(match); + let match: SpeakerMatch | null = null; + if (embeddingFloats) { + try { + match = speakerRegistry.assign(embeddingFloats); + } catch (error) { + // Still caption the line, just unattributed. + console.warn(`speaker assignment failed: ${error}`); + } + } + if (match && shown) this.maybeShowEncounterCard(match); // Spoken introductions name the auto-numbered speakers: "I'm Alice" // renames the speaker of the line; "this is Bob" pends the name for the // next different still-generic voice. User-set names are never touched. - if (match && text.trim()) { + if (match && shown && text.trim()) { const now = Date.now(); const pending = this.pendingIntroduction; if (pending && pendingApplies(pending, match.profile.id, match.profile.name, now)) { @@ -918,11 +1396,12 @@ class MicSession { if (refreshed) match = { ...match, profile: refreshed }; } const isWearer = match?.profile.isWearer ?? false; - const doaAtUtterance = this.doaDeviceDeg; + const doaAtUtterance = shown ? this.doaDeviceDeg : null; const analyzed = text.trim() ? analyzeLine(text) : null; - if (analyzed) { - this.sentimentSum += analyzed.score; - this.sentimentCount += 1; + const conversation = run.conversation; + if (analyzed && conversation) { + conversation.sentimentSum += analyzed.score; + conversation.sentimentCount += 1; } const line: CaptionLine = { @@ -935,47 +1414,53 @@ class MicSession { lang: "", emotion: analyzed?.emotion ?? "", sentiment: analyzed?.score ?? 0, - atMs: this.engineStartWallMs + startMs, + atMs: run.clock.wallMs(startMs), }; - this.captionLines.push(line); - if (this.captionLines.length > CAPTION_LINE_LIMIT) { - this.captionLines.splice(0, this.captionLines.length - CAPTION_LINE_LIMIT); - } + if (shown) { + this.captionLines.push(line); + if (this.captionLines.length > CAPTION_LINE_LIMIT) { + this.captionLines.splice(0, this.captionLines.length - CAPTION_LINE_LIMIT); + } - // Radar dot at the utterance's direction (world frame, so it stays put - // as the head turns and follows the person when they move). - this.speechActive = false; - clearSpeaking(this.speakerDots); - if (match && doaAtUtterance !== null) { - this.speakerDots = placeDotForUtterance( - this.speakerDots, - { speakerId: match.profile.id, name: match.profile.name, color: match.profile.color }, - this.toWorldDeg(doaAtUtterance), - Date.now(), - ); + // Radar dot at the utterance's direction (world frame, so it stays put + // as the head turns and follows the person when they move). + this.speechActive = false; + clearSpeaking(this.speakerDots); + if (match && doaAtUtterance !== null) { + this.speakerDots = placeDotForUtterance( + this.speakerDots, + { speakerId: match.profile.id, name: match.profile.name, color: match.profile.color }, + this.toWorldDeg(doaAtUtterance), + Date.now(), + ); + } } let segmentId = -1; - if (this.sessionRowId >= 0 && line.text) { - segmentId = Number( - this.store().insertSegment( - JSON.stringify({ - sessionId: this.sessionRowId, - speakerId: match?.profile.id, - startedAt: this.engineStartWallMs + startMs, - endedAt: this.engineStartWallMs + endMs, - audioOffsetMs: startMs, - text: line.text, - sentiment: line.sentiment, - emotion: line.emotion, - searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "", - angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance), - embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined, - }), - ), - ); + if (conversation && line.text) { + try { + segmentId = Number( + this.store().insertSegment( + JSON.stringify({ + sessionId: conversation.rowId, + speakerId: match?.profile.id, + startedAt: run.clock.wallMs(startMs), + endedAt: run.clock.wallMs(endMs), + audioOffsetMs: run.audioBaseMs + startMs, + text: line.text, + sentiment: line.sentiment, + emotion: line.emotion, + searchMeta: analyzed ? searchableMetadata(analyzed.emotion, analyzed.score) : "", + angle: doaAtUtterance === null ? undefined : Math.round(doaAtUtterance), + embeddingBase64: embeddingFloats ? encodeEmbeddingBase64(embeddingFloats) : undefined, + }), + ), + ); + } catch (error) { + console.warn(`caption segment save failed: ${error}`); + } } - this.notify(); + if (shown) this.notify(); // Language identification + translation are async; the UI line and the // stored segment update when they resolve. @@ -1024,10 +1509,19 @@ class MicSession { // ---- recording ---- - private startRecording(): void { + /** + * Recordings belong to a saved caption conversation: retention and the + * phone UI find audio only through its session row, so a file recorded + * without one would never be shown or expired. + */ + private shouldRecord(): boolean { + return saveRecordingsSetting.get() && recordingsRetentionSetting.get() !== "none"; + } + + private startRecording(conversation: ConversationRecord): void { try { - const context = Utils.android.getApplicationContext(); - const dir = `${context.getExternalFilesDir(null).getAbsolutePath()}/mic-recordings`; + const dir = recordingsDir(); + if (!dir) throw new Error("external storage unavailable"); const stamp = new Date().toISOString().replace(/[:.]/g, "-"); this.recordingPath = `${dir}/conversation-${stamp}.wav`; this.recorder = new com.faceclaw.app.FaceclawWavRecorder(this.recordingPath, SAMPLE_RATE, 1); @@ -1035,46 +1529,34 @@ class MicSession { console.warn(`recording start failed: ${error}`); this.recorder = null; this.recordingPath = ""; + return; + } + // Link the file at once: if the process dies mid-recording, the row + // still names it, and boot recovery repairs and keeps it. + try { + this.store().setSessionAudio(conversation.rowId, this.recordingPath, "wav"); + } catch (error) { + console.warn(`recording link failed: ${error}`); } } - private stopRecording(keep: boolean): void { + private stopRecording(keep: boolean, conversation: ConversationRecord | null): void { const recorder = this.recorder; this.recorder = null; if (!recorder) return; + const wavPath = this.recordingPath; try { const bytes = Number(recorder.finish()); - if (!keep || bytes < SAMPLE_RATE) { - // Under half a second of audio isn't worth keeping. + if (!keep || bytes < SAMPLE_RATE || !conversation) { + // Under half a second of audio isn't worth keeping, and audio with no + // session row to own it would be unreachable and never expire. recorder.discard(); + if (conversation) this.store().setSessionAudio(conversation.rowId, null, null); return; } - const wavPath = this.recordingPath; - const m4aPath = wavPath.replace(/\.wav$/, ".m4a"); - const sessionRowId = this.sessionRowId; const store = this.store(); - if (sessionRowId >= 0) { - store.setSessionAudio(sessionRowId, wavPath, "wav"); - } - // Transcode in the background; swap the session's audio to the M4A on - // success (the transcoder deletes the WAV). - const transcoder = new com.faceclaw.app.FaceclawAudioTranscoder( - wavPath, - m4aPath, - RECORDING_AAC_BITRATE, - true, - new com.faceclaw.app.FaceclawAudioTranscoder.Listener({ - onDone: (outputPath: string, _outputBytes: number) => { - if (sessionRowId >= 0) { - store.setSessionAudio(sessionRowId, String(outputPath), "aac"); - } - }, - onError: (message: string) => { - console.warn(`recording transcode failed: ${message}`); - }, - }), - ); - transcoder.start(); + store.setSessionAudio(conversation.rowId, wavPath, "wav"); + transcodeRecording(store, conversation.rowId, wavPath); } catch (error) { console.warn(`recording stop failed: ${error}`); } @@ -1087,21 +1569,6 @@ class MicSession { return com.faceclaw.app.FaceclawConversationStore.getInstance(context); } - private finishSessionRow(): void { - if (this.sessionRowId < 0) return; - const avg = this.sentimentCount > 0 ? this.sentimentSum / this.sentimentCount : 0; - const finishedSessionId = this.sessionRowId; - try { - this.store().endSession(finishedSessionId, Date.now(), "", "", avg); - } catch (error) { - console.warn(`session finish failed: ${error}`); - } - this.sessionRowId = -1; - // Recap/action-items/facts per speaker, generated in the background with - // the on-phone model (no-op when it isn't downloaded). - generateSessionInsights(finishedSessionId); - } - async applyRetentionSweep(): Promise { if (!global.isAndroid || this.running) return; try { @@ -1118,6 +1585,71 @@ class MicSession { } } + /** + * Repair what a dead process left behind, once per process at boot: + * conversations it never closed, recording files no session row knows + * about, and recordings never transcoded (header patched to the audio on + * disk, then transcoded). The steps fail independently, and the last one + * works from what the store still holds as WAV rather than from what this + * boot closed, so whatever a failure leaves is picked up by a later boot. + */ + recoverInterruptedConversations(): void { + if (!global.isAndroid || recoveryDone || this.running) return; + recoveryDone = true; + let store: any; + try { + store = this.store(); + } catch (error) { + console.warn(`mic recovery failed: ${error}`); + return; + } + let closed: number[] = []; + try { + const rows = JSON.parse( + String(store.closeInterruptedSessions(JSON.stringify(this.openConversationRows()))), + ) as Array<{ id: number }>; + closed = rows.map((row) => Number(row.id)); + } catch (error) { + console.warn(`mic recovery: closing conversations failed: ${error}`); + } + for (const rowId of closed) generateSessionInsights(rowId); + // Before any transcode below starts writing, so its output isn't swept. + let removed = 0; + try { + const dir = recordingsDir(); + if (dir) removed = Number(store.deleteUnreferencedRecordings(dir)); + } catch (error) { + console.warn(`mic recovery: orphan sweep failed: ${error}`); + } + // The ones just closed, plus any whose transcode died with an earlier + // process; patching an already-finished header changes nothing. + let untranscoded: Array<{ id: number; audioPath: string }> = []; + try { + untranscoded = JSON.parse(String(store.listUntranscodedSessions())) as typeof untranscoded; + } catch (error) { + console.warn(`mic recovery: listing recordings failed: ${error}`); + } + for (const session of untranscoded) { + recoverRecording(store, Number(session.id), String(session.audioPath)); + } + if (closed.length > 0 || removed > 0 || untranscoded.length > 0) { + console.log( + `mic recovery: closed ${closed.length} conversation(s), removed ${removed} orphaned recording(s), ` + + `finishing ${untranscoded.length} recording(s)`, + ); + } + } + + /** Session rows legitimately still open: the live conversation and ones awaiting a trailing line. */ + private openConversationRows(): number[] { + const rows = new Set(); + if (this.conversation) rows.add(this.conversation.rowId); + for (const run of this.stoppingRuns) { + if (run.conversation) rows.add(run.conversation.rowId); + } + return Array.from(rows); + } + private nativeCommunicator(): any { try { return com.faceclaw.app.FaceclawBleCommunicator.getActive(); @@ -1129,6 +1661,14 @@ class MicSession { // ---- helpers ---- +/** + * The config as sent to the temples: each active temple always streams its + * full front+rear pair, and per-mic selection happens host-side. + */ +function wireConfig(config: MicConfig): MicConfig { + return { ...config, channelMask: 0b11 }; +} + function pcmLevel(pcm: Uint8Array): number { let sumSquares = 0; const count = pcm.length / 2; @@ -1178,13 +1718,81 @@ function encodeEmbeddingBase64(embedding: Float32Array): string { return encodeBase64(new Uint8Array(embedding.buffer, embedding.byteOffset, embedding.byteLength)); } -function strongerChanged(current: MicSide, candidate: MicSide): boolean { - return current !== candidate; -} - function moonshineModelDir(): string { const context = Utils.android.getApplicationContext(); return `${context.getFilesDir().getAbsolutePath()}/faceclaw-voice-asr/sherpa-onnx-moonshine-base-en-quantized-2026-02-27`; } +/** Where recordings live, or null while external storage is unavailable. */ +function recordingsDir(): string | null { + const context = Utils.android.getApplicationContext(); + const base = context.getExternalFilesDir(null); + return base ? `${base.getAbsolutePath()}/mic-recordings` : null; +} + +function deleteFile(path: string): void { + try { + new java.io.File(path).delete(); + } catch (error) { + console.warn(`could not delete ${path}: ${error}`); + } +} + +// Transcoder listeners are referenced only from Java while a transcode runs; +// hold them here so they aren't GC'd before their callback. +const pendingTranscodes = new Set(); + +/** + * Transcode a finished WAV recording to M4A in the background and move its + * session onto it. The WAV is deleted only once the row names the M4A, so a + * failure at any step leaves the session pointing at a file that exists. + */ +function transcodeRecording(store: any, rowId: number, wavPath: string): void { + const m4aPath = wavPath.replace(/\.wav$/, ".m4a"); + const listener = new com.faceclaw.app.FaceclawAudioTranscoder.Listener({ + onDone: (outputPath: string, _outputBytes: number) => { + pendingTranscodes.delete(listener); + try { + store.setSessionAudio(rowId, String(outputPath), "aac"); + } catch (error) { + console.warn(`recording audio update failed: ${error}`); + deleteFile(String(outputPath)); + return; + } + deleteFile(wavPath); + }, + onError: (message: string) => { + pendingTranscodes.delete(listener); + console.warn(`recording transcode failed: ${message}`); + }, + }); + pendingTranscodes.add(listener); + try { + // deleteSourceOnSuccess false: the WAV goes only after the row update above. + new com.faceclaw.app.FaceclawAudioTranscoder(wavPath, m4aPath, RECORDING_AAC_BITRATE, false, listener).start(); + } catch (error) { + pendingTranscodes.delete(listener); + console.warn(`recording transcode start failed: ${error}`); + } +} + +/** + * Finish a recording left as WAV (its process died writing or transcoding + * it): patch the header to the audio on disk, then transcode it as a normal + * stop would. Too short (or gone) and it is dropped from the session instead. + */ +function recoverRecording(store: any, rowId: number, wavPath: string): void { + try { + const bytes = Number(com.faceclaw.app.FaceclawWavRecorder.repairHeader(wavPath, SAMPLE_RATE, 1)); + if (bytes < SAMPLE_RATE) { + deleteFile(wavPath); + store.setSessionAudio(rowId, null, null); + return; + } + transcodeRecording(store, rowId, wavPath); + } catch (error) { + console.warn(`recording recovery failed (${wavPath}): ${error}`); + } +} + export const micSession = new MicSession(); diff --git a/app/apps/microphones/microphones-app.ts b/app/apps/microphones/microphones-app.ts index c8f6a6ec..a520da1d 100644 --- a/app/apps/microphones/microphones-app.ts +++ b/app/apps/microphones/microphones-app.ts @@ -22,7 +22,7 @@ import { isAsrModelReady, startAsrModelDownload, asrModelState } from "../../nat import { CaptionsLayer } from "./captions-layer"; import { LevelsLayer } from "./levels-layer"; import { RadarLayer } from "./radar-layer"; -import { loadMicConfig, micArrayController, micControlSupported, saveMicConfig } from "./mic-control"; +import { loadMicConfig, micArrayController, micControlSupported } from "./mic-control"; import { isMicModelReady, micModelState, startMicModelDownload } from "./mic-models"; import { micChannelLabel, type MicChannelKey, type MicConfig } from "./mic-protocol"; import { micSession } from "./mic-session"; @@ -72,13 +72,9 @@ const MENU_LAYOUT = { opaque: true, }; -/** Persist a config change and re-apply it live when the array is streaming. */ +/** Persist a config change; the running session applies it live. */ function updateMicConfig(mutate: (config: MicConfig) => MicConfig): void { - const updated = mutate(loadMicConfig()); - saveMicConfig(updated); - if (micSession.getState().mode === "extended") { - micArrayController.apply(updated); - } + micSession.updateConfig(mutate); } function micChannelToggleItem( @@ -457,8 +453,9 @@ export function createMicrophonesAppWindow(options: InProcessAppOptions): InProc }), configToggleItem( "Captions", - () => captionsEnabledSetting.get(), - () => micSession.setCaptionsEnabled(!captionsEnabledSetting.get()), + // Shows off after repeated engine failures, so turning it on retries. + () => micSession.captionsEnabled(), + () => micSession.setCaptionsEnabled(!micSession.captionsEnabled()), undefined, captionsEnabledSetting.description, ), diff --git a/app/apps/microphones/name-guess.ts b/app/apps/microphones/name-guess.ts index 0b38c76b..1db24241 100644 --- a/app/apps/microphones/name-guess.ts +++ b/app/apps/microphones/name-guess.ts @@ -55,7 +55,9 @@ const SELF_PATTERNS = [ new RegExp(`\\b[Mm]y name is ${NAME}`), new RegExp(`\\b[Ii](?:'| a)m ${NAME}`), new RegExp(`\\b[Cc]all me ${NAME}`), - new RegExp(`\\b[Tt]his is ${NAME} speaking\\b`), + // Phone-style: "this is Dana calling about..." is the speaker, not a + // third party being introduced. + new RegExp(`\\b[Tt]his is ${NAME} (?:[Ss]peaking|[Cc]alling)\\b`), ]; const THIRD_PARTY_PATTERNS = [ diff --git a/app/apps/microphones/rediarize.ts b/app/apps/microphones/rediarize.ts index 49423181..3a2bfc4e 100644 --- a/app/apps/microphones/rediarize.ts +++ b/app/apps/microphones/rediarize.ts @@ -35,6 +35,10 @@ function store(): any { return com.faceclaw.app.FaceclawConversationStore.getInstance(context); } +// Each run's listener is referenced only from Java while the diarizer works; +// hold it here so it isn't GC'd before its callbacks. +const activeListeners = new Set(); + export function rediarizeSession( sessionId: number, options: { numSpeakers?: number; threshold?: number }, @@ -54,6 +58,7 @@ export function rediarizeSession( const listener = new com.faceclaw.app.FaceclawDiarizer.Listener({ onProgress: (progress: number) => callbacks.onProgress(Number(progress)), onDone: (turnsJson: string) => { + activeListeners.delete(listener); try { const turns = JSON.parse(String(turnsJson)) as Turn[]; callbacks.onDone(applyTurns(sessionId, turns)); @@ -61,17 +66,26 @@ export function rediarizeSession( callbacks.onError(`Applying diarization failed: ${String(error)}`); } }, - onError: (message: string) => callbacks.onError(String(message)), + onError: (message: string) => { + activeListeners.delete(listener); + callbacks.onError(String(message)); + }, }); - const diarizer = new com.faceclaw.app.FaceclawDiarizer( - segmentationModel, - embeddingModel, - audioPath, - options.numSpeakers ?? -1, - options.threshold ?? 0.5, - listener, - ); - diarizer.start(); + activeListeners.add(listener); + try { + const diarizer = new com.faceclaw.app.FaceclawDiarizer( + segmentationModel, + embeddingModel, + audioPath, + options.numSpeakers ?? -1, + options.threshold ?? 0.5, + listener, + ); + diarizer.start(); + } catch (error) { + activeListeners.delete(listener); + callbacks.onError(`Re-diarization failed to start: ${String(error)}`); + } } type StoredSegment = { @@ -140,6 +154,8 @@ function applyTurns(sessionId: number, turns: Turn[]): RediarizeResult { } else { const id = Number(store().createSpeaker(nextAutoName(), "#90A4AE", false, encodeEmbedding(centroid))); speakerRegistry.reload(); + // A failed insert returns -1; leave that cluster's segments as they were. + if (id < 0) continue; clusterSpeaker.set(cluster, id); newSpeakers += 1; } diff --git a/app/apps/microphones/speakers.ts b/app/apps/microphones/speakers.ts index 7d4a7189..5975ea24 100644 --- a/app/apps/microphones/speakers.ts +++ b/app/apps/microphones/speakers.ts @@ -139,8 +139,9 @@ export class SpeakerRegistry { })); this.loaded = true; } catch (error) { + // Keep the last good cache: emptying it would make the next utterance + // enroll an already-known voice as a duplicate "Speaker N". console.error(`speaker reload failed: ${String(error)}`); - this.profiles = []; } } @@ -194,7 +195,8 @@ export class SpeakerRegistry { const id = Number(store().createSpeaker(name, color, false, encodeEmbedding(embedding))); this.reload(); this.notifyChanged(); - const profile = this.byId(id)!; + const profile = this.byId(id); + if (!profile) throw new Error(`speaker "${name}" was not created (id ${id})`); return { profile, similarity: best?.similarity ?? 0, classification: "new", previousLastHeardAt: 0 }; } @@ -213,12 +215,14 @@ export class SpeakerRegistry { store().updateSpeakerEmbedding(existing.id, encodeEmbedding(embedding), CENTROID_UPDATE_CAP); this.reload(); this.notifyChanged(); - return this.byId(existing.id)!; + return this.byId(existing.id) ?? existing; } const id = Number(store().createSpeaker("Me", "#64B5F6", true, encodeEmbedding(embedding))); this.reload(); this.notifyChanged(); - return this.byId(id)!; + const profile = this.byId(id); + if (!profile) throw new Error(`wearer profile was not created (id ${id})`); + return profile; } /** Is this utterance the wearer's own voice? Used to gate command recognition. */ diff --git a/app/apps/microphones/translate.ts b/app/apps/microphones/translate.ts index d27bbfa4..25fee989 100644 --- a/app/apps/microphones/translate.ts +++ b/app/apps/microphones/translate.ts @@ -18,7 +18,10 @@ const downloadedPairs = new Set(); export function deviceLanguage(): string { if (!global.isAndroid) return "en"; try { - return String(java.util.Locale.getDefault().getLanguage() || "en"); + // toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit + // uses; getLanguage() returns the legacy "iw", "in", "ji". + const tag = String(java.util.Locale.getDefault().toLanguageTag() || ""); + return tag.split("-")[0] || "en"; } catch { return "en"; } @@ -95,11 +98,18 @@ export function translateText(text: string, sourceLang: string, targetLang: stri if (!translator) return Promise.resolve(""); const key = `${sourceLang}>${targetLang}`; return new Promise((resolve) => { + // Also runs from the download listener, where a throw would escape into + // the Play Services callback and leave this promise unsettled. const runTranslate = () => { - translator - .translate(text) - .addOnSuccessListener(successListener((result: any) => resolve(String(result)))) - .addOnFailureListener(failureListener(() => resolve(""))); + try { + translator + .translate(text) + .addOnSuccessListener(successListener((result: any) => resolve(String(result)))) + .addOnFailureListener(failureListener(() => resolve(""))); + } catch (error) { + console.warn(`translation failed (${key}): ${error}`); + resolve(""); + } }; if (downloadedPairs.has(key)) { runTranslate(); diff --git a/app/apps/minesweeper/minesweeper-app.worker.ts b/app/apps/minesweeper/minesweeper-app.worker.ts index a4320d83..3cd49ee9 100644 --- a/app/apps/minesweeper/minesweeper-app.worker.ts +++ b/app/apps/minesweeper/minesweeper-app.worker.ts @@ -250,8 +250,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { syncClock(window); } break; + case "resize-window": { + const window = windows.get(message.windowId); + if (!window) break; + window.viewportWidth = message.viewport.width; + window.viewportHeight = message.viewport.height; + window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight }); + window.lastSubmittedFingerprint = ""; + if (window.foreground) renderAndSubmit(window, 0); + break; + } case "navigation-sensors": - case "resize-window": case "text-input": case "tool-call": break; diff --git a/app/apps/music/music-app.ts b/app/apps/music/music-app.ts index 5d7e24ff..f35a1130 100644 --- a/app/apps/music/music-app.ts +++ b/app/apps/music/music-app.ts @@ -46,10 +46,11 @@ type FocusColumn = "actions" | "playlist"; /** * Music controller app: metadata + album art + transport controls for the - * active Android media session (any player that publishes one), plus the - * player's queue when it exposes one (scroll to a track, click to jump). - * With no session, the front page lists the apps whose library can be - * browsed (click one to open its MediaBrowseLayer). + * active Android media session (any player that publishes one) or, on iOS, + * the Apple Music system player, plus the player's queue when it exposes one + * (scroll to a track, click to jump). With no session, the front page lists + * the apps whose library can be browsed (click one to open its + * MediaBrowseLayer). */ class MusicAppLayer implements Layer { // Watch swipes skip tracks: right = next, left = previous. @@ -83,7 +84,8 @@ class MusicAppLayer implements Layer { */ private libraryAppsFresh = false; private art: GrayImage | null = null; - private artKey = ""; + /** The artVersion `art` was fetched for; -1 = not fetched yet. */ + private artVersion = -1; isPlaylistFocused(): boolean { return this.focusColumn === "playlist"; @@ -96,15 +98,11 @@ class MusicAppLayer implements Layer { const media = mediaControllerBridge.snapshot(); if (!media.accessEnabled) { - const lines = wrapText( - font, - "Notification access is required to control Android media sessions. Tap to open settings.", - width - 48, - ); + const lines = wrapText(font, mediaControllerBridge.accessPrompt, width - 48); for (let index = 0; index < lines.length; index++) { image.drawText(font, 24, 16 + index * lineStep(font), lines[index]!, 180); } - image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings`, 110); + image.drawText(font, 20, height - 16, `${GESTURE_CLICK} ${mediaControllerBridge.accessAction}`, 110); return image; } @@ -112,10 +110,12 @@ class MusicAppLayer implements Layer { const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh); this.libraryAppsFresh = true; this.libraryMenu.setItems(apps); - image.drawText(font, 24, 16, "No active media session.", 180); - const prompt = apps.length - ? "Browse a music app's library, or start playback on the phone." - : "Start playback in another app on the phone."; + image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180); + const prompt = global.isIOS + ? "Browse your library, or start playback on the phone." + : apps.length + ? "Browse a music app's library, or start playback on the phone." + : "Start playback in another app on the phone."; let y = 22 + lineStep(font); for (const line of wrapText(font, prompt, width - 48)) { image.drawText(font, 24, y, line, 150); @@ -228,7 +228,7 @@ class MusicAppLayer implements Layer { } if (!media.accessEnabled) { if (event.type === "click") { - mediaControllerBridge.openNotificationAccessSettings(); + void mediaControllerBridge.requestAccess(); } else if (event.type === "double-click") { ctx.stack.pop(); } @@ -267,8 +267,8 @@ class MusicAppLayer implements Layer { const item = this.queueMenu.selectedItem; if (item) { await mediaControllerBridge.skipToQueueItem(item.id); - // The player rebuilds the queue with the chosen track at index 0. - this.queueMenu.select(0); + // Android players rebuild the queue with the chosen track at index 0. + if (mediaControllerBridge.skipRebuildsQueue) this.queueMenu.select(0); } return; } @@ -371,12 +371,12 @@ class MusicAppLayer implements Layer { } private drawArt(image: GrayImage, media: MediaControllerState): void { - const key = `${media.packageName}|${media.title}|${media.album}`; - // Retry while null: players (e.g. Spotify) often publish metadata before - // the art bitmap is downloaded, then re-emit the same strings with the - // bitmap attached. The no-art fetch path is a cheap near-no-op. - if (key !== this.artKey || this.art === null) { - this.artKey = key; + // Keyed on artVersion, not the strings: players (e.g. Spotify) publish + // metadata before the art bitmap is downloaded, or with the previous + // track's bitmap after a quick double skip, then re-emit the same + // strings with the right bitmap attached. + if (media.artVersion !== this.artVersion) { + this.artVersion = media.artVersion; this.art = mediaControllerBridge.getAlbumArt(ART_SIZE); } if (this.art) { @@ -496,7 +496,8 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin iconLetter: "M", icon: "music", closeable: true, - menuItems: () => [{ + // Music settings choose among Android media apps; iOS has only Apple Music. + menuItems: () => global.isIOS ? [] : [{ label: "Music settings", onSelect: (ctx) => { ctx.stack.pop(); diff --git a/app/apps/navigate/navigate-app.worker.ts b/app/apps/navigate/navigate-app.worker.ts index db34f057..b79b09b1 100644 --- a/app/apps/navigate/navigate-app.worker.ts +++ b/app/apps/navigate/navigate-app.worker.ts @@ -33,8 +33,7 @@ import { navigateDestinationAddressDraftSetting, navigateDestinationNameDraftSetting, navigateRememberRecentSetting, - navigateDisplayModeSetting, - navigateVerticalPositionSetting, + navigateDisplayLayoutSetting, mapboxApiKeySetting, enumSettingMenuItem, type ConfigSettingString, @@ -935,10 +934,7 @@ function windowMenuItems(): MenuItem[] { }, }); } - items.push( - enumSettingMenuItem(navigateDisplayModeSetting), - enumSettingMenuItem(navigateVerticalPositionSetting), - ); + items.push(enumSettingMenuItem(navigateDisplayLayoutSetting)); items.push(submenuItem("Saved destinations", (ctx) => { ctx.stack.push(new WindowMenuLayer("Saved destinations", savedDestinationMenuItems())); })); diff --git a/app/apps/nightscout/index.ts b/app/apps/nightscout/index.ts index 6f5f59ad..87e3c960 100644 --- a/app/apps/nightscout/index.ts +++ b/app/apps/nightscout/index.ts @@ -1,11 +1,18 @@ -import { type AppDefinition } from "../app-definition"; +import { inProcessTrayWidget, type AppDefinition } from "../app-definition"; import { createNightscoutAppWindow, startNightscoutTrayIcon, NIGHTSCOUT_SURFACE_ID, NIGHTSCOUT_WINDOW_ID } from "./nightscout-app"; +import { startMissedBolusMonitor } from "./missed-bolus-monitor"; const nightscoutApp: AppDefinition = { appId: "nightscout", title: "Nightscout", icon: "nightscout", - boot: () => startNightscoutTrayIcon(), + boot: (ctx) => { + // Selected in the switcher, the tray readout shows the app's window. + startNightscoutTrayIcon( + inProcessTrayWidget(ctx, NIGHTSCOUT_WINDOW_ID, NIGHTSCOUT_SURFACE_ID, createNightscoutAppWindow), + ); + startMissedBolusMonitor(ctx); + }, launch: (ctx) => ctx.launchInProcessApp(NIGHTSCOUT_WINDOW_ID, NIGHTSCOUT_SURFACE_ID, createNightscoutAppWindow), }; diff --git a/app/apps/nightscout/missed-bolus-detector.ts b/app/apps/nightscout/missed-bolus-detector.ts new file mode 100644 index 00000000..37a8a985 --- /dev/null +++ b/app/apps/nightscout/missed-bolus-detector.ts @@ -0,0 +1,336 @@ +/** + * Missed-bolus detector: given a recent window of CGM readings and + * treatments, decide whether glucose is rising the way an unannounced meal + * does, with no meal bolus, manual bolus or carb entry to account for it. + * + * Deliberately self-contained (no imports; plain data in, plain data out) so + * it can move to the server side of Nightscout unchanged, and so it can be + * iterated on against recorded history with + * scripts/nightscout-missed-bolus-replay.cjs. The Faceclaw side (when to run + * it, cooldown, the glasses notification) is in missed-bolus-monitor.ts. + * + * Glucose is mg/dL throughout: Nightscout stores sgv in mg/dL whatever the + * site's display units. + */ + +export type GlucoseReading = { timeMs: number; mgdl: number }; + +/** + * - meal: a meal/snack/wizard bolus, or any bolus entered together with carbs. + * - manual: a bolus explicitly marked user-initiated (e.g. Loop's `automatic: false`). + * - automatic: a bolus the loop issued (SMB, automatic bolus). + * - ambiguous: not marked either way. OpenAPS on a Medtronic pump uploads its + * SMBs as plain "Correction Bolus" records, indistinguishable from a manual + * correction except by size; `ambiguousBolusMinUnits` decides. + */ +export type BolusKind = "meal" | "manual" | "automatic" | "ambiguous"; +export type BolusEvent = { timeMs: number; units: number; kind: BolusKind }; +export type CarbEntry = { timeMs: number; grams: number }; + +export type MissedBolusInput = { + /** The time to classify as of. Readings and treatments after it are ignored. */ + nowMs: number; + glucose: readonly GlucoseReading[]; + boluses: readonly BolusEvent[]; + carbs: readonly CarbEntry[]; +}; + +export type MissedBolusOptions = { + /** The latest reading must be at most this old, or there is nothing current to judge. */ + maxReadingAgeMs: number; + /** How far back from the latest reading to look for where the rise started. */ + lookbackMs: number; + /** A gap between readings longer than this ends the window (the CGM dropped out). */ + maxGapMs: number; + /** Recent span the rate of rise is fitted over (least squares). */ + slopeWindowMs: number; + /** Fewest readings the rate fit needs. */ + minSlopeReadings: number; + /** Rise from the window's lowest reading to the latest. */ + minRiseMgdl: number; + /** The latest reading must be at least this high. */ + minGlucoseMgdl: number; + /** Rises from below this are taken as recovering from a low (rescue carbs go unbolused). */ + hypoNadirMgdl: number; + /** The rise must have lasted at least this long, to rule out sensor noise. */ + minRiseDurationMs: number; + /** Rate of rise over the slope window, mg/dL per minute. */ + minRateMgdlPerMin: number; + /** A bolus or carb entry this long before the rise began still accounts for it (pre-bolusing). */ + preBolusWindowMs: number; + /** Smallest meal or manual bolus that accounts for a rise. */ + minManualBolusUnits: number; + /** Smallest unmarked bolus taken as manual rather than an SMB. */ + ambiguousBolusMinUnits: number; + /** Whether a carb entry with no bolus accounts for a rise (the user evidently knows they ate). */ + carbsCountAsAnnouncement: boolean; +}; + +export const DEFAULT_MISSED_BOLUS_OPTIONS: Readonly = { + maxReadingAgeMs: 12 * 60_000, + lookbackMs: 90 * 60_000, + maxGapMs: 16 * 60_000, + slopeWindowMs: 20 * 60_000, + minSlopeReadings: 3, + minRiseMgdl: 40, + minGlucoseMgdl: 140, + hypoNadirMgdl: 70, + minRiseDurationMs: 20 * 60_000, + minRateMgdlPerMin: 1.5, + preBolusWindowMs: 60 * 60_000, + minManualBolusUnits: 0.5, + ambiguousBolusMinUnits: 1.5, + carbsCountAsAnnouncement: true, +}; + +export type GlucoseRise = { + /** When the rise started: the lowest reading in the window (the latest, if tied). */ + onsetMs: number; + nadirMgdl: number; + latestMs: number; + latestMgdl: number; + riseMgdl: number; + /** Least-squares rate over the slope window, mg/dL per minute. */ + rateMgdlPerMin: number; +}; + +export type MissedBolusClassification = "missed-bolus" | "no-alert" | "insufficient-data"; + +export type MissedBolusResult = { + classification: MissedBolusClassification; + /** One-line explanation, for logs and the replay tool. */ + reason: string; + /** The rise that was judged, when there was enough data to measure one. */ + rise: GlucoseRise | null; +}; + +export function detectMissedBolus( + input: MissedBolusInput, + overrides: Partial = {}, +): MissedBolusResult { + const options: MissedBolusOptions = { ...DEFAULT_MISSED_BOLUS_OPTIONS, ...overrides }; + const { nowMs } = input; + const readings = usableReadings(input.glucose, nowMs); + const latest = readings[readings.length - 1]; + if (!latest) return insufficient("no glucose readings"); + if (nowMs - latest.timeMs > options.maxReadingAgeMs) { + return insufficient(`latest reading is ${formatMinutes(nowMs - latest.timeMs)} old`); + } + + // The contiguous run of readings ending at the latest, within the lookback. + const windowStartMs = latest.timeMs - options.lookbackMs; + let first = readings.length - 1; + while (first > 0) { + const previous = readings[first - 1]!; + if (previous.timeMs < windowStartMs || readings[first]!.timeMs - previous.timeMs > options.maxGapMs) break; + first--; + } + const window = readings.slice(first); + let nadir = window[0]!; + for (const reading of window) { + if (reading.mgdl <= nadir.mgdl) nadir = reading; + } + + const slopeStartMs = Math.max(nadir.timeMs, latest.timeMs - options.slopeWindowMs); + const slopeReadings = window.filter((reading) => reading.timeMs >= slopeStartMs); + const rise: GlucoseRise = { + onsetMs: nadir.timeMs, + nadirMgdl: nadir.mgdl, + latestMs: latest.timeMs, + latestMgdl: latest.mgdl, + riseMgdl: latest.mgdl - nadir.mgdl, + rateMgdlPerMin: slopeReadings.length >= 2 ? slopePerMinute(slopeReadings) : 0, + }; + const noAlert = (reason: string): MissedBolusResult => ({ classification: "no-alert", reason, rise }); + + if (rise.riseMgdl < options.minRiseMgdl) { + return noAlert(`rise of ${rise.riseMgdl} mg/dL is under ${options.minRiseMgdl}`); + } + if (latest.mgdl < options.minGlucoseMgdl) { + return noAlert(`glucose ${latest.mgdl} mg/dL is under ${options.minGlucoseMgdl}`); + } + if (nadir.mgdl < options.hypoNadirMgdl) { + return noAlert(`rising from a low of ${nadir.mgdl} mg/dL`); + } + if (latest.timeMs - nadir.timeMs < options.minRiseDurationMs) { + return noAlert(`rise has lasted only ${formatMinutes(latest.timeMs - nadir.timeMs)}`); + } + if (slopeReadings.length < options.minSlopeReadings) { + return { classification: "insufficient-data", reason: `only ${slopeReadings.length} readings to fit a rate`, rise }; + } + if (rise.rateMgdlPerMin < options.minRateMgdlPerMin) { + return noAlert(`rising ${formatRate(rise.rateMgdlPerMin)}, under ${formatRate(options.minRateMgdlPerMin)}`); + } + + const coverStartMs = nadir.timeMs - options.preBolusWindowMs; + const inCoverWindow = (timeMs: number) => timeMs >= coverStartMs && timeMs <= nowMs; + const bolus = latestMatching(input.boluses, (event) => inCoverWindow(event.timeMs) && accountsForRise(event, options)); + if (bolus) { + return noAlert(`${formatUnits(bolus.units)} U ${bolus.kind} bolus ${formatMinutes(nowMs - bolus.timeMs)} ago`); + } + if (options.carbsCountAsAnnouncement) { + const carbs = latestMatching(input.carbs, (entry) => inCoverWindow(entry.timeMs) && entry.grams > 0); + if (carbs) return noAlert(`${carbs.grams} g carbs entered ${formatMinutes(nowMs - carbs.timeMs)} ago`); + } + return { + classification: "missed-bolus", + reason: `up ${rise.riseMgdl} mg/dL in ${formatMinutes(latest.timeMs - nadir.timeMs)}, rising ${formatRate(rise.rateMgdlPerMin)}, no bolus`, + rise, + }; +} + +function insufficient(reason: string): MissedBolusResult { + return { classification: "insufficient-data", reason, rise: null }; +} + +function accountsForRise(bolus: BolusEvent, options: MissedBolusOptions): boolean { + switch (bolus.kind) { + case "meal": + case "manual": + return bolus.units >= options.minManualBolusUnits; + case "ambiguous": + return bolus.units >= Math.max(options.minManualBolusUnits, options.ambiguousBolusMinUnits); + case "automatic": + return false; + } +} + +/** + * Finite, in-range readings at or before now, oldest first, one per + * timestamp. Dexcom reports 39 for "LOW" and smaller values as error codes. + */ +function usableReadings(glucose: readonly GlucoseReading[], nowMs: number): GlucoseReading[] { + const sorted = glucose + .filter((reading) => Number.isFinite(reading.timeMs) && Number.isFinite(reading.mgdl) + && reading.mgdl >= 39 && reading.mgdl <= 600 && reading.timeMs <= nowMs) + .slice() + .sort((a, b) => a.timeMs - b.timeMs); + return sorted.filter((reading, index) => index === 0 || reading.timeMs !== sorted[index - 1]!.timeMs); +} + +/** Least-squares slope through the readings, in mg/dL per minute. */ +function slopePerMinute(readings: readonly GlucoseReading[]): number { + const t0 = readings[0]!.timeMs; + const n = readings.length; + let sumX = 0, sumY = 0, sumXY = 0, sumXX = 0; + for (const reading of readings) { + const x = (reading.timeMs - t0) / 60_000; + sumX += x; + sumY += reading.mgdl; + sumXY += x * reading.mgdl; + sumXX += x * x; + } + const denominator = n * sumXX - sumX * sumX; + return denominator > 0 ? (n * sumXY - sumX * sumY) / denominator : 0; +} + +function latestMatching(events: readonly T[], predicate: (event: T) => boolean): T | null { + let found: T | null = null; + for (const event of events) { + if (predicate(event) && (!found || event.timeMs > found.timeMs)) found = event; + } + return found; +} + +function formatMinutes(ms: number): string { + return `${Math.round(ms / 60_000)} min`; +} + +function formatRate(mgdlPerMin: number): string { + return `${mgdlPerMin.toFixed(1)} mg/dL/min`; +} + +function formatUnits(units: number): string { + return `${Math.round(units * 100) / 100}`; +} + +// --------------------------------------------------------------------------- +// Nightscout record normalization, shared by the Faceclaw bridge and a +// server-side port. + +/** The fields of a Nightscout treatment record the detector reads. */ +export type NightscoutTreatmentRecord = { + eventType?: unknown; + created_at?: unknown; + timestamp?: unknown; + insulin?: unknown; + carbs?: unknown; + automatic?: unknown; + isSMB?: unknown; + type?: unknown; + wizard?: unknown; +}; + +/** The fields of a Nightscout entry record the detector reads. */ +export type NightscoutEntryRecord = { date?: unknown; sgv?: unknown }; + +const MEAL_BOLUS_EVENT_TYPES = new Set(["Meal Bolus", "Snack Bolus", "Bolus Wizard", "Combo Bolus"]); + +/** + * How a treatment's insulin was given. Uploaders mark automatic boluses + * differently: Loop sets `automatic`, AndroidAPS `isSMB`, Trio/OpenAPS + * `type: "SMB"` or eventType "SMB". Meal boluses carry carbs or a wizard + * record, or a meal-type eventType. + */ +export function bolusKindFromTreatment(treatment: NightscoutTreatmentRecord): BolusKind { + const eventType = typeof treatment.eventType === "string" ? treatment.eventType : ""; + if (treatment.automatic === true || treatment.isSMB === true || treatment.type === "SMB" || eventType === "SMB") { + return "automatic"; + } + if (MEAL_BOLUS_EVENT_TYPES.has(eventType) || positiveNumber(treatment.carbs) !== null + || (typeof treatment.wizard === "object" && treatment.wizard !== null)) { + return "meal"; + } + if (treatment.automatic === false || treatment.isSMB === false) return "manual"; + return "ambiguous"; +} + +/** Boluses and carb entries from raw Nightscout treatment records, oldest first. */ +export function normalizeNightscoutTreatments(treatments: readonly NightscoutTreatmentRecord[]): { + boluses: BolusEvent[]; + carbs: CarbEntry[]; +} { + const boluses: BolusEvent[] = []; + const carbs: CarbEntry[] = []; + for (const treatment of treatments) { + const timeMs = treatmentTimeMs(treatment); + if (timeMs === null) continue; + const units = positiveNumber(treatment.insulin); + if (units !== null) boluses.push({ timeMs, units, kind: bolusKindFromTreatment(treatment) }); + const grams = positiveNumber(treatment.carbs); + if (grams !== null) carbs.push({ timeMs, grams }); + } + boluses.sort((a, b) => a.timeMs - b.timeMs); + carbs.sort((a, b) => a.timeMs - b.timeMs); + return { boluses, carbs }; +} + +/** Glucose readings from raw Nightscout sgv entries, oldest first. */ +export function normalizeNightscoutEntries(entries: readonly NightscoutEntryRecord[]): GlucoseReading[] { + const readings: GlucoseReading[] = []; + for (const entry of entries) { + const timeMs = Number(entry.date); + const mgdl = Number(entry.sgv); + if (Number.isFinite(timeMs) && timeMs > 0 && Number.isFinite(mgdl)) readings.push({ timeMs, mgdl }); + } + return readings.sort((a, b) => a.timeMs - b.timeMs); +} + +function treatmentTimeMs(treatment: NightscoutTreatmentRecord): number | null { + for (const value of [treatment.created_at, treatment.timestamp]) { + if (typeof value === "number" && Number.isFinite(value) && value > 0) return value; + if (typeof value === "string" && value.trim()) { + const numeric = Number(value); + const parsed = Number.isFinite(numeric) ? numeric : Date.parse(value); + if (Number.isFinite(parsed) && parsed > 0) return parsed; + } + } + return null; +} + +function positiveNumber(value: unknown): number | null { + if (typeof value !== "number" && typeof value !== "string") return null; + if (typeof value === "string" && !value.trim()) return null; + const numeric = Number(value); + return Number.isFinite(numeric) && numeric > 0 ? numeric : null; +} diff --git a/app/apps/nightscout/missed-bolus-monitor.ts b/app/apps/nightscout/missed-bolus-monitor.ts new file mode 100644 index 00000000..1a64641b --- /dev/null +++ b/app/apps/nightscout/missed-bolus-monitor.ts @@ -0,0 +1,73 @@ +import { type AppContext } from "../app-definition"; +import { nightscoutBridge, type NightscoutState } from "../../native/nightscout-bridge"; +import { getStringSetting, setStringSetting } from "../../native/settings-store"; +import { nightscoutMissedBolusAlertsSetting, nightscoutMissedBolusCooldownSetting } from "../../ui/dashboard-settings"; +import { shell } from "../../ui/shell/shell"; +import { detectMissedBolus, type MissedBolusInput } from "./missed-bolus-detector"; +import { MissedBolusNotificationLayer } from "./missed-bolus-notification"; + +/** When the last alert was shown, persisted so a restart doesn't reset the cooldown. */ +const LAST_ALERT_STORAGE_KEY = "integrations.nightscout.missedBolusLastAlertMs"; + +let started = false; +let lastInputKey = ""; +let lastSummary = ""; + +/** + * Runs the missed-bolus detector each time a Nightscout sync brings new + * readings or treatments, and shows a glasses notification when it fires + * outside the cooldown. Shell-lifetime, started from the app's boot hook. + */ +export function startMissedBolusMonitor(ctx: AppContext): void { + if (started) return; + started = true; + nightscoutBridge.onStateChange((state) => checkForMissedBolus(state, ctx)); +} + +function checkForMissedBolus(state: NightscoutState, ctx: AppContext): void { + if (!nightscoutMissedBolusAlertsSetting.get() || !state.available) return; + const key = inputKey(state); + if (key === lastInputKey) return; + lastInputKey = key; + + const nowMs = Date.now(); + const result = detectMissedBolus(detectorInput(state, nowMs)); + const summary = `${result.classification}: ${result.reason}`; + if (summary !== lastSummary) { + lastSummary = summary; + console.log(`missed-bolus detector: ${summary}`); + } + if (result.classification !== "missed-bolus" || !result.rise) return; + + const lastAlertMs = Number(getStringSetting(LAST_ALERT_STORAGE_KEY, "0")) || 0; + const cooldownMs = Number(nightscoutMissedBolusCooldownSetting.get()) * 60_000; + // A last-alert time in the future means the clock moved back; don't let it block alerts. + if (lastAlertMs <= nowMs && nowMs - lastAlertMs < cooldownMs) return; + setStringSetting(LAST_ALERT_STORAGE_KEY, String(nowMs)); + ctx.appendLog(`missed-bolus alert: ${result.reason}`); + const rise = result.rise; + shell.postAppNotification({ + key: `nightscout-missed-bolus:${nowMs}`, + createLayer: (close) => new MissedBolusNotificationLayer(rise, nowMs, close, () => { + void ctx.launchApp("nightscout"); + }), + }); +} + +/** Changes whenever a sync brings a new reading or a treatment the detector reads. */ +function inputKey(state: NightscoutState): string { + return [ + state.latest?.timestampMs ?? 0, + state.boluses.map((bolus) => bolus.timestampMs).join(","), + state.carbs.map((entry) => entry.timestampMs).join(","), + ].join("|"); +} + +function detectorInput(state: NightscoutState, nowMs: number): MissedBolusInput { + return { + nowMs, + glucose: state.history.map((point) => ({ timeMs: point.timestampMs, mgdl: point.sgv })), + boluses: state.boluses.map((bolus) => ({ timeMs: bolus.timestampMs, units: bolus.insulin, kind: bolus.kind })), + carbs: state.carbs.map((entry) => ({ timeMs: entry.timestampMs, grams: entry.carbs })), + }; +} diff --git a/app/apps/nightscout/missed-bolus-notification.ts b/app/apps/nightscout/missed-bolus-notification.ts new file mode 100644 index 00000000..3b09ecf0 --- /dev/null +++ b/app/apps/nightscout/missed-bolus-notification.ts @@ -0,0 +1,111 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GrayImage } from "../../graphics/image"; +import { renderIcon } from "../../graphics/icons"; +import { wrapText } from "../../graphics/textwrap"; +import { formatRelativeTime } from "../../util/date-util"; +import { nightscoutBridge } from "../../native/nightscout-bridge"; +import { type InputEvent } from "../../ui/gestures"; +import { type Layer, type LayerContext } from "../../ui/layers"; +import { lineStep } from "../../ui/metrics"; +import { + NOTIFICATION_CONTENT_X, + createNotificationActionMenu, + drawNotificationHeader, + notificationActionMenuBox, + notificationContentWidth, +} from "../../ui/notifications"; +import { type GlucoseRise } from "./missed-bolus-detector"; +import { + drawDirectionIndicator, + drawNightscoutGraph, + drawNightscoutValueStrikeThrough, + formatDelta, + isNightscoutPointStale, +} from "./nightscout"; + +const ICON_SIZE = 24; +const GRAPH_GAP = 6; +const GRAPH_BOTTOM_MARGIN = 12; +const MIN_GRAPH_HEIGHT = 32; + +type ActionItem = { label: string; run: () => void }; + +/** + * The missed-bolus alert, laid out like an Android notification's detail view + * (header, text, right-hand action column), with the live glucose readout and + * 2-hour graph below the text. + */ +export class MissedBolusNotificationLayer implements Layer { + private readonly menu = createNotificationActionMenu(); + + constructor( + private readonly rise: GlucoseRise, + private readonly postedAtMs: number, + private readonly close: () => void, + openNightscout: () => void, + ) { + this.menu.setItems([ + { label: "Dismiss", run: close }, + { label: "Open Nightscout", run: () => { + close(); + openNightscout(); + } }, + ]); + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const step = lineStep(font); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + const contentX = NOTIFICATION_CONTENT_X; + const contentWidth = notificationContentWidth(width); + const nowMs = Date.now(); + + let y = drawNotificationHeader(image, font, renderIcon("nightscout", ICON_SIZE), `Nightscout ${formatRelativeTime(this.postedAtMs)}`); + image.drawText(font, contentX, y, "Missed bolus?", 230); + y += step; + const minutes = Math.round((this.rise.latestMs - this.rise.onsetMs) / 60_000); + const body = `Glucose rose ${this.rise.riseMgdl} mg/dL in ${minutes} min with no bolus or carbs entered.`; + for (const line of wrapText(font, body, contentWidth)) { + image.drawText(font, contentX, y, line, 190); + y += step; + } + + // Live readout and graph: the modal repaints with each shell render, so + // these follow new readings (and a bolus, once entered) while it is up. + const nightscout = nightscoutBridge.snapshot(); + const latest = nightscout.latest; + if (latest) { + const value = `${latest.sgv}`; + image.drawText(font, contentX, y, value, 230); + if (isNightscoutPointStale(latest, nowMs)) { + drawNightscoutValueStrikeThrough(image, contentX, y + (font.lineHeight >> 1), font.measureText(value)); + } + let x = contentX + font.measureText(`${value} `); + x += drawDirectionIndicator(image, font, x, y, nightscout.direction, 200); + x += font.measureText(" "); + const iob = nightscout.iob === null ? "" : ` IOB ${nightscout.iob.toFixed(2)}`; + image.drawText(font, x, y, `${formatDelta(nightscout.delta)}${iob}`, 180); + y += step; + } + const graphTop = y + GRAPH_GAP; + const graphHeight = height - GRAPH_BOTTOM_MARGIN - graphTop; + if (graphHeight >= MIN_GRAPH_HEIGHT) { + drawNightscoutGraph(image, { x: contentX, y: graphTop, width: contentWidth, height: graphHeight }, nightscout, nowMs, font); + } + + this.menu.paint(image, notificationActionMenuBox(width, height), ctx.stack.isFocused()); + return image; + } + + async handleInput(event: InputEvent): Promise { + if (event.type === "double-click") { + this.close(); + } else if (event.type === "click") { + this.menu.selectedItem?.run(); + } else if (event.type === "scroll-up" || event.type === "scroll-down") { + await this.menu.handleInput(event); + } + } +} diff --git a/app/apps/nightscout/nightscout-app.ts b/app/apps/nightscout/nightscout-app.ts index 2f6c536e..6458541a 100644 --- a/app/apps/nightscout/nightscout-app.ts +++ b/app/apps/nightscout/nightscout-app.ts @@ -10,7 +10,7 @@ import { } from "../../ui/shell/in-process-window"; import { evaluateNightscoutAlerts } from "./nightscout-alerts"; import { loadNightscoutThresholds, nightscoutAlwaysShowInTopBarSetting, onAnySettingChanged } from "../../ui/dashboard-settings"; -import { shell } from "../../ui/shell/shell"; +import { shell, type TrayWidget } from "../../ui/shell/shell"; export const NIGHTSCOUT_WINDOW_ID = "nightscout"; export const NIGHTSCOUT_SURFACE_ID = "window:nightscout"; @@ -23,11 +23,12 @@ const TRAY_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000; const windowRenders = new Set<() => void>(); let trayStarted = false; +let trayWidget: TrayWidget | undefined; let trayTick: ReturnType | null = null; function syncNightscout(): void { const visible = windowRenders.size > 0 || nightscoutAlwaysShowInTopBarSetting.get(); - shell.setTrayIcon(TRAY_ICON_ID, visible ? buildNightscoutTrayIcon(nightscoutBridge.snapshot()) : null); + shell.setTrayIcon(TRAY_ICON_ID, visible ? buildNightscoutTrayIcon(nightscoutBridge.snapshot()) : null, trayWidget); for (const render of windowRenders) render(); // Age warnings and stale glucose must advance even during a stalled fetch. if (visible && trayTick === null) trayTick = setInterval(syncNightscout, 60_000); @@ -37,8 +38,12 @@ function syncNightscout(): void { } } -/** Shell-lifetime subscription so the tray can be enabled before any window opens. */ -export function startNightscoutTrayIcon(): void { +/** + * Shell-lifetime subscription so the tray can be enabled before any window + * opens; `widget` makes it selectable in the switcher. + */ +export function startNightscoutTrayIcon(widget?: TrayWidget): void { + trayWidget ??= widget; if (trayStarted) return; trayStarted = true; nightscoutBridge.onStateChange(syncNightscout); diff --git a/app/apps/nightscout/nightscout.ts b/app/apps/nightscout/nightscout.ts index 675a07f8..8266a31d 100644 --- a/app/apps/nightscout/nightscout.ts +++ b/app/apps/nightscout/nightscout.ts @@ -13,6 +13,8 @@ import { nightscoutBatteryLowSetting, nightscoutMaxLoopAgeSetting, nightscoutAlwaysShowInTopBarSetting, + nightscoutMissedBolusAlertsSetting, + nightscoutMissedBolusCooldownSetting, toggleSettingMenuItem, nightscoutApiTokenSetting, nightscoutSiteUrlSetting, @@ -28,6 +30,7 @@ export const NIGHTSCOUT_STALE_MS = 15 * 60 * 1000; const NIGHTSCOUT_GRAPH_WINDOW_MS = 2 * 60 * 60 * 1000; const NIGHTSCOUT_GRAPH_TIME_QUANTUM_MS = 60 * 1000; +/** Draws the trend indicator; returns its width. */ export function drawDirectionIndicator( image: GrayImage, font: UiFont, @@ -35,16 +38,10 @@ export function drawDirectionIndicator( y: number, direction: string, shade: number, -): void { - const label = directionGlyphLabel(font, direction); - if (label) { - image.drawText(font, x, y, label, shade); - return; - } - - if (direction) { - image.drawText(font, x, y, truncateLine(direction, 10), shade); - } +): number { + const label = directionGlyphLabel(font, direction) || (direction ? truncateLine(direction, 10) : ""); + if (label) image.drawText(font, x, y, label, shade); + return font.measureText(label); } /** @@ -425,6 +422,8 @@ export function nightscoutMenuItems(): MenuItem[] { textSettingMenuItem(nightscoutCartridgeLowSetting), textSettingMenuItem(nightscoutBatteryLowSetting), textSettingMenuItem(nightscoutMaxLoopAgeSetting), + toggleSettingMenuItem(nightscoutMissedBolusAlertsSetting), + textSettingMenuItem(nightscoutMissedBolusCooldownSetting), ]); }, }, diff --git a/app/apps/notifications/index.ts b/app/apps/notifications/index.ts index a91538a5..6f9cd5ed 100644 --- a/app/apps/notifications/index.ts +++ b/app/apps/notifications/index.ts @@ -1,4 +1,4 @@ -import { isNotificationListenerEnabled, requestNotificationListenerAccess } from "../../native/notification-access"; +import { notificationListenerState, requestNotificationListenerAccess } from "../../native/notification-access"; import { type AppDefinition } from "../app-definition"; import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_WINDOW_ID } from "./notifications-app"; @@ -7,10 +7,11 @@ const notificationsApp: AppDefinition = { title: "Notifications", icon: "bell", launch: (ctx) => { - // Without notification-listener access the tray reads as empty; prompt - // the user on the phone so the on-glasses "grant permission" message is - // actionable. The app still opens to show that message. - if (!isNotificationListenerEnabled()) { + // Without notification-listener access (or with a listener Android won't + // rebind) the tray reads as empty; open the access screen on the phone so + // the on-glasses message is actionable. The app still opens to show it. + const listenerState = notificationListenerState(); + if (listenerState === "no-access" || listenerState === "stuck") { requestNotificationListenerAccess(); } return ctx.launchInProcessApp(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow); diff --git a/app/apps/notifications/notifications-app.ts b/app/apps/notifications/notifications-app.ts index a7f0bf75..65719b3f 100644 --- a/app/apps/notifications/notifications-app.ts +++ b/app/apps/notifications/notifications-app.ts @@ -1,6 +1,6 @@ import { NotificationsListLayer } from "../../ui/notifications"; import { NotificationFilterLayer } from "../../ui/notification-filter"; -import { onAndroidNotificationPosted } from "../../native/notification-icons"; +import { onAndroidNotificationsChanged } from "../../native/notification-icons"; import { createInProcessWindow, YieldAtRootLayer, @@ -18,7 +18,7 @@ export const NOTIFICATIONS_SURFACE_ID = "window:notifications"; */ export function createNotificationsAppWindow(options: InProcessAppOptions): InProcessWindow { // Newly posted notifications repaint the list while the window is open. - let offNotificationPosted: (() => void) | null = null; + let offNotificationsChanged: (() => void) | null = null; const created = createInProcessWindow({ appId: "notifications", windowId: NOTIFICATIONS_WINDOW_ID, @@ -39,10 +39,10 @@ export function createNotificationsAppWindow(options: InProcessAppOptions): InPr setSurfaceVisible: options.setSurfaceVisible, removeSurface: options.removeSurface, onClosed: () => { - offNotificationPosted?.(); + offNotificationsChanged?.(); options.onClosed(); }, }); - offNotificationPosted = onAndroidNotificationPosted(() => created.requestRender()); + offNotificationsChanged = onAndroidNotificationsChanged(() => created.requestRender()); return created; } diff --git a/app/apps/onboarding/data-collection-page.ts b/app/apps/onboarding/data-collection-page.ts new file mode 100644 index 00000000..ad52e32b --- /dev/null +++ b/app/apps/onboarding/data-collection-page.ts @@ -0,0 +1,76 @@ +import type { GrayImage } from "../../graphics/image"; +import { truncateText } from "../../graphics/textwrap"; +import { dataCollectionSetting, dataCollectionSummary, type DataCollectionLevel } from "../../ui/dashboard-settings"; +import type { InputEvent } from "../../ui/gestures"; +import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, listRowHeight } from "../../ui/metrics"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const CONTENT_GAP = 10; +const BOTTOM_MARGIN = 8; +const LABEL_GAP = 24; + +/** None / Minimal / Full, each with a one-line summary; a tap saves the selected level. */ +export class DataCollectionPage implements OnboardingPage { + readonly immersive = true; + /** Width of the level-name column, measured each paint. */ + private labelColumnWidth = 0; + private readonly menu = new Menu({ + items: dataCollectionSetting.values, + wrap: false, + rowGap: 2, + highlight: { radius: 8 }, + getHeight: () => listRowHeight(onboardingFont("body")), + draw: (args) => this.drawRow(args), + }); + + enter(): void { + this.menu.select(dataCollectionSetting.values.indexOf(dataCollectionSetting.get())); + } + + paint(image: GrayImage, focused: boolean): void { + const top = drawPageHeading( + image, + "Data Sharing", + (global.isIOS ? "If using TestFlight, crash/usage reports cannot be disabled here. " : "") + + "Choose how much data you want to share. Change later in Settings > Privacy.", + 3, + ) + CONTENT_GAP; + const body = onboardingFont("body"); + this.labelColumnWidth = + Math.max(...dataCollectionSetting.values.map((level) => body.measureText(dataCollectionSetting.displayValue(level)))) + + LABEL_GAP; + this.menu.paint( + image, + { x: PAGE_X - 10, y: top, width: image.width - 2 * (PAGE_X - 10), height: image.height - top - BOTTOM_MARGIN }, + focused, + ); + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + switch (event.type) { + case "scroll-up": + case "scroll-down": + void this.menu.handleInput(event); + return; + case "click": { + const level = this.menu.selectedItem; + if (level !== null) dataCollectionSetting.set(level); + nav.next(); + return; + } + default: + return; + } + } + + private drawRow({ image, item: level, x, y, width, height, selected }: MenuDrawArgs): void { + const body = onboardingFont("body"); + const detail = onboardingFont("detail"); + image.drawText(body, x + 10, y + LIST_ROW_TEXT_INSET, dataCollectionSetting.displayValue(level), selected ? 255 : 200); + const summaryX = x + 10 + this.labelColumnWidth; + const summary = truncateText(detail, dataCollectionSummary(level), x + width - 10 - summaryX); + image.drawText(detail, summaryX, y + Math.round((height - detail.lineHeight) / 2), summary, selected ? 210 : 150); + } +} diff --git a/app/apps/onboarding/depth-page.ts b/app/apps/onboarding/depth-page.ts new file mode 100644 index 00000000..ad3b1c1d --- /dev/null +++ b/app/apps/onboarding/depth-page.ts @@ -0,0 +1,71 @@ +import type { GrayImage } from "../../graphics/image"; +import { appSwitcherPositionSetting, uiDepthSetting, type UiDepth } from "../../ui/dashboard-settings"; +import type { InputEvent } from "../../ui/gestures"; +import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, listRowHeight } from "../../ui/metrics"; +import { layoutHasDepth } from "./layout-page"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const CONTENT_GAP = 10; +const BOTTOM_MARGIN = 8; +const COLUMN_WIDTH = 220; + +/** + * The whole display's stereo depth, after a layout that has one (Bottom or + * Popup). Not immersive, so the page sits in the chosen layout, and the + * depth is saved as the selection moves: the shell shifts everything with + * it, which is the only way to see the difference. + */ +export class DepthPage implements OnboardingPage { + private readonly menu = new Menu({ + items: uiDepthSetting.values, + wrap: false, + rowGap: 2, + highlight: { radius: 8 }, + getHeight: () => listRowHeight(onboardingFont("body")), + draw: (args) => this.drawRow(args), + }); + + applies(): boolean { + return layoutHasDepth(appSwitcherPositionSetting.get()); + } + + enter(): void { + this.menu.select(uiDepthSetting.values.indexOf(uiDepthSetting.get())); + } + + paint(image: GrayImage, focused: boolean): void { + const top = drawPageHeading( + image, + "Depth", + "Swipe to move the display nearer or farther away, then tap to continue. Change later in Settings > Customization.", + ) + CONTENT_GAP; + this.menu.paint( + image, + { x: PAGE_X - 10, y: top, width: COLUMN_WIDTH, height: image.height - top - BOTTOM_MARGIN }, + focused, + ); + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + switch (event.type) { + case "scroll-up": + case "scroll-down": { + void this.menu.handleInput(event); + const depth = this.menu.selectedItem; + if (depth !== null) uiDepthSetting.set(depth); + return; + } + case "click": + nav.next(); + return; + default: + return; + } + } + + private drawRow({ image, item: depth, x, y, selected }: MenuDrawArgs): void { + image.drawText(onboardingFont("body"), x + 10, y + LIST_ROW_TEXT_INSET, uiDepthSetting.displayValue(depth), selected ? 255 : 190); + } +} diff --git a/app/apps/onboarding/display-mode-page.ts b/app/apps/onboarding/display-mode-page.ts new file mode 100644 index 00000000..5f449478 --- /dev/null +++ b/app/apps/onboarding/display-mode-page.ts @@ -0,0 +1,95 @@ +import type { GrayImage } from "../../graphics/image"; +import { wrapText } from "../../graphics/textwrap"; +import { displayLayoutSetting, type DisplayLayout } from "../../ui/dashboard-settings"; +import type { InputEvent } from "../../ui/gestures"; +import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, listRowHeight } from "../../ui/metrics"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const CONTENT_GAP = 10; +const BOTTOM_MARGIN = 8; +const COLUMN_GAP = 16; + +const DESCRIPTIONS = { + band: "The stock 288px band, at this height in your view, leaving the rest of it clear.", + tall: "Windows use the display's full height.", + fullscreen: "Full height, and the display's whole width. Depth is turned off, since windows reach its edges.", +}; + +/** + * The display mode: how much of the display windows use (a band at one of + * five heights, Tall, or Full Screen where the layout allows it), with a + * line about the selected one. Not immersive, and the mode is saved as the + * selection moves, so this window itself shows each mode as it really is + * (the dashboard controller moves or resizes it, and every other window). + */ +export class DisplayModePage implements OnboardingPage { + private readonly menu = new Menu({ + wrap: false, + rowGap: 2, + highlight: { radius: 8 }, + getHeight: () => listRowHeight(onboardingFont("body")), + draw: (args) => this.drawRow(args), + }); + + enter(): void { + // Full Screen comes and goes with the layout; where the layout doesn't + // allow it, a saved Full Screen shows as Tall, as the shell does. + const values = displayLayoutSetting.values.filter((value) => !displayLayoutSetting.isDisabled(value)); + const index = values.indexOf(displayLayoutSetting.get()); + this.menu.setItems(values, index >= 0 ? index : values.indexOf("tall")); + } + + paint(image: GrayImage, focused: boolean): void { + const top = drawPageHeading( + image, + "Display mode", + "Swipe to choose how much of the display windows use; it changes as you go. Tap to continue.", + ) + CONTENT_GAP; + const body = onboardingFont("body"); + // TTF widths are fractional; the menu's row images need whole pixels. + const columnWidth = Math.ceil(20 + Math.max(...this.menu.items.map((value) => body.measureText(displayLayoutSetting.displayValue(value))))); + const menuBox = { x: PAGE_X - 10, y: top, width: columnWidth, height: image.height - top - BOTTOM_MARGIN }; + this.menu.paint(image, menuBox, focused); + const layout = this.menu.selectedItem; + if (layout === null) return; + const detail = onboardingFont("detail"); + const left = menuBox.x + columnWidth + COLUMN_GAP; + let y = top + LIST_ROW_TEXT_INSET; + for (const line of wrapText(detail, description(layout), image.width - PAGE_X - left)) { + if (y + detail.lineHeight > image.height - BOTTOM_MARGIN) break; + image.drawText(detail, left, y, line, 170); + y += detail.lineHeight; + } + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + switch (event.type) { + case "scroll-up": + case "scroll-down": + void this.menu.handleInput(event); + this.apply(); + return; + case "click": + this.apply(); + nav.next(); + return; + default: + return; + } + } + + private apply(): void { + const layout = this.menu.selectedItem; + if (layout !== null && layout !== displayLayoutSetting.get()) displayLayoutSetting.set(layout); + } + + private drawRow({ image, item: layout, x, y, selected }: MenuDrawArgs): void { + image.drawText(onboardingFont("body"), x + 10, y + LIST_ROW_TEXT_INSET, displayLayoutSetting.displayValue(layout), selected ? 255 : 190); + } +} + +function description(layout: DisplayLayout): string { + return layout === "tall" || layout === "fullscreen" ? DESCRIPTIONS[layout] : DESCRIPTIONS.band; +} diff --git a/app/apps/onboarding/index.ts b/app/apps/onboarding/index.ts new file mode 100644 index 00000000..727eac24 --- /dev/null +++ b/app/apps/onboarding/index.ts @@ -0,0 +1,18 @@ +import { type AppDefinition } from "../app-definition"; +import { createOnboardingAppWindow, ONBOARDING_SURFACE_ID, ONBOARDING_WINDOW_ID } from "./onboarding-app"; +import { ONBOARDING_APP_ID } from "./onboarding-progress"; + +/** + * First-run setup on the glasses: welcome, text size, data-collection + * preference, layout (app switcher position), depth for layouts that have + * one, and display mode. The connection controllers open it on their own when it is + * due (claimGlassesOnboardingAutoLaunch); the launcher entry reruns it. + */ +const onboardingApp: AppDefinition = { + appId: ONBOARDING_APP_ID, + title: "Onboarding", + icon: "sparkles", + launch: (ctx) => ctx.launchInProcessApp(ONBOARDING_WINDOW_ID, ONBOARDING_SURFACE_ID, createOnboardingAppWindow), +}; + +export default onboardingApp; diff --git a/app/apps/onboarding/ios-privacy-page.ts b/app/apps/onboarding/ios-privacy-page.ts new file mode 100644 index 00000000..dd3b94c6 --- /dev/null +++ b/app/apps/onboarding/ios-privacy-page.ts @@ -0,0 +1,43 @@ +import { IOS_PRIVACY_NOTICE } from "../../analytics/privacy-disclosure"; +import type { GrayImage } from "../../graphics/image"; +import { wrapText } from "../../graphics/textwrap"; +import type { InputEvent } from "../../ui/gestures"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +/** Paginate the notice so no disclosure is silently clipped on a short glasses viewport. */ +export class IosPrivacyPage implements OnboardingPage { + readonly immersive = true; + private page = 0; + private pageCount = 1; + + enter(): void { this.page = 0; } + + paint(image: GrayImage): void { + const top = drawPageHeading(image, "iOS / TestFlight privacy", "") + 8; + const font = onboardingFont("detail"); + const lines = IOS_PRIVACY_NOTICE.split("\n\n").flatMap((paragraph, index) => [ + ...(index ? [""] : []), ...wrapText(font, paragraph, image.width - 2 * PAGE_X), + ]); + const perPage = Math.max(1, Math.floor((image.height - top - font.lineHeight - 20) / font.lineHeight)); + this.pageCount = Math.ceil(lines.length / perPage); + this.page = Math.min(this.page, this.pageCount - 1); + lines.slice(this.page * perPage, (this.page + 1) * perPage).forEach((line, index) => { + image.drawText(font, PAGE_X, top + index * font.lineHeight, line, 200); + }); + const action = this.page + 1 < this.pageCount ? "Tap for more" : "Tap to choose statistics"; + image.drawText(font, PAGE_X, image.height - font.lineHeight - 8, + `${this.page + 1}/${this.pageCount} - ${action}`, 230); + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + if (event.type === "click") { + if (this.page + 1 < this.pageCount) this.page++; + else nav.next(); + } else if (event.type === "scroll-down" || event.type === "swipe-up") { + this.page = Math.min(this.page + 1, this.pageCount - 1); + } else if (event.type === "scroll-up" || event.type === "swipe-down") { + this.page = Math.max(0, this.page - 1); + } + } +} diff --git a/app/apps/onboarding/layout-diagram.ts b/app/apps/onboarding/layout-diagram.ts new file mode 100644 index 00000000..3ade4108 --- /dev/null +++ b/app/apps/onboarding/layout-diagram.ts @@ -0,0 +1,177 @@ +/** + * Schematic diagrams of the shell's layouts for the onboarding's Layout + * page: to scale in real-display pixels, simplified, and drawn brighter + * than the real chrome, which they shrink to a few pixels. + */ +import { G2_LENS_WIDTH, type GrayImage } from "../../graphics/image"; +import { statusBarPositionSetting, type AppSwitcherPosition } from "../../ui/dashboard-settings"; +import { SIDEBAR_WIDTH, SWITCHER_ROW_HEIGHT, MIN_WINDOW_HEIGHT, TOP_BAR_HEIGHT } from "../../ui/shell/geometry"; + +export const AREA_OUTLINE = 30; +const CHROME_LINE = 100; +const WINDOW_FRAME = 140; +const ICON = 160; +const SELECTED_ICON = 230; +const STATUS = 140; +const CONTENT = 110; +const CONTENT_TITLE = 170; +/** Window content under a popup switcher, which dims it. */ +const DIM = 0.5; + +/** Draws in real-display pixels, scaled into an image at (x, y). */ +export class Diagram { + constructor( + private readonly image: GrayImage, + private readonly x: number, + private readonly y: number, + private readonly scale: number, + ) {} + + private box(x: number, y: number, width: number, height: number) { + const left = this.x + Math.round(x * this.scale), top = this.y + Math.round(y * this.scale); + return { + x: left, + y: top, + width: Math.max(1, this.x + Math.round((x + width) * this.scale) - left), + height: Math.max(1, this.y + Math.round((y + height) * this.scale) - top), + }; + } + + fill(x: number, y: number, width: number, height: number, value: number, radius = 0): void { + const box = this.box(x, y, width, height); + if (radius) this.image.fillRoundedRect(box.x, box.y, box.width, box.height, value, Math.max(1, Math.round(radius * this.scale))); + else this.image.fillRect(box.x, box.y, box.width, box.height, value); + } + + outline(x: number, y: number, width: number, height: number, value: number, radius = 0): void { + const box = this.box(x, y, width, height); + this.image.drawRoundedRect(box.x, box.y, box.width, box.height, value, Math.max(0, Math.round(radius * this.scale))); + } + + /** A one-pixel line along a real-pixel row (horizontal) or column. */ + hline(x: number, y: number, width: number, value: number): void { + const box = this.box(x, y, width, 1); + this.image.fillRect(box.x, box.y, box.width, 1, value); + } + + vline(x: number, y: number, height: number, value: number): void { + const box = this.box(x, y, 1, height); + this.image.fillRect(box.x, box.y, 1, box.height, value); + } +} + +/** + * The foreground window's band, in real-display rows: its top and height + * (header and content; a bottom switcher row hangs under it), and whether + * it spans the panel's whole width (Full Screen; never beside a side strip). + */ +export type DiagramBand = { top: number; height: number; fullWidth: boolean }; + +/** + * A layout as the real chrome lays it out (geometry.ts, chrome-layer.ts): + * the window band with its status bar and frame, and the switcher with a + * few icons, the first selected. A popup switcher shows only with + * `popupOpen`, dimming the window as it does. Status bar placement follows + * the current setting. + */ +export function drawLayout(d: Diagram, position: AppSwitcherPosition, band: DiagramBand, popupOpen: boolean): void { + const statusAtBottom = statusBarPositionSetting.get() === "bottom"; + const { top, height } = band; + switch (position) { + case "left": + case "right": { + const stripLeft = position === "left" ? 0 : G2_LENS_WIDTH - SIDEBAR_WIDTH; + const contentLeft = position === "left" ? SIDEBAR_WIDTH : 0; + const windowWidth = G2_LENS_WIDTH - SIDEBAR_WIDTH; + drawStatusBar(d, contentLeft, top, windowWidth, TOP_BAR_HEIGHT - 1); + drawContent(d, contentLeft, top + TOP_BAR_HEIGHT, windowWidth, height - TOP_BAR_HEIGHT, 1); + // The strip keeps to the 288px band's height, even beside taller windows. + const separator = position === "left" ? SIDEBAR_WIDTH - 1 : stripLeft; + d.vline(separator, top + TOP_BAR_HEIGHT, MIN_WINDOW_HEIGHT - TOP_BAR_HEIGHT, CHROME_LINE); + // One column of icons against the separator. + const column = position === "left" ? SIDEBAR_WIDTH - 36 : stripLeft; + for (let slot = 0; slot < 4; slot++) drawIcon(d, column + 2, top + TOP_BAR_HEIGHT + 10 + slot * 40, slot === 0); + return; + } + case "bottom": { + const { left, width, framed } = windowSpan(band); + // With the status bar in the row, the window's header is just the frame's top. + const header = !statusAtBottom ? TOP_BAR_HEIGHT : framed ? 1 : 0; + if (!statusAtBottom) drawStatusBar(d, left, top, width, framed ? null : TOP_BAR_HEIGHT - 1); + drawContent(d, left, top + header, width, height - header, 1); + if (framed) d.outline(left - 1, top + header - 1, width + 2, height - header + 2, WINDOW_FRAME, 8); + else d.hline(left, top + height, width, CHROME_LINE); + // The first cell starts at the frame's side, or inset like a scrolled row's (chrome-layer switcherLayout). + const cellsLeft = framed ? left - 1 : left + 8; + for (let slot = 0; slot < 4; slot++) drawIcon(d, cellsLeft + 2 + slot * 40, top + height + 2, slot === 0); + if (statusAtBottom) drawStatusItems(d, left + width, top + height + 4); + return; + } + case "popup": { + const { left, width, framed } = windowSpan(band); + const edge = framed ? 1 : 0; + const statusTop = statusAtBottom ? top + height - TOP_BAR_HEIGHT : top; + const contentTop = statusAtBottom ? top + edge : top + TOP_BAR_HEIGHT; + const contentHeight = height - TOP_BAR_HEIGHT - edge; + drawStatusBar(d, left, statusTop, width, framed ? null : statusAtBottom ? 0 : TOP_BAR_HEIGHT - 1); + drawContent(d, left, contentTop, width, contentHeight, popupOpen ? DIM : 1); + if (framed) d.outline(left - 1, contentTop - 1, width + 2, contentHeight + 2, WINDOW_FRAME, 8); + if (!popupOpen) return; + // The popup: a row of icons over the selected window's title, centred on the content. + const box = { width: 272, height: 98 }; + const boxLeft = left + (width - box.width) / 2; + const boxTop = contentTop + (contentHeight - box.height) / 2; + d.fill(boxLeft, boxTop, box.width, box.height, 0); + d.outline(boxLeft, boxTop, box.width, box.height, WINDOW_FRAME); + const iconsLeft = boxLeft + (box.width - (3 * 48 - 4)) / 2; + for (let slot = 0; slot < 3; slot++) drawIcon(d, iconsLeft + slot * 48 + 6, boxTop + 18, slot === 0); + d.fill(boxLeft + box.width / 2 - 50, boxTop + 70, 100, 12, STATUS, 6); + return; + } + } +} + +/** A bottom row or popup layout's window: centred at 576 wide and framed, or the panel's width without a frame. */ +function windowSpan(band: DiagramBand): { left: number; width: number; framed: boolean } { + return band.fullWidth + ? { left: 0, width: G2_LENS_WIDTH, framed: false } + : { left: SIDEBAR_WIDTH / 2, width: G2_LENS_WIDTH - SIDEBAR_WIDTH, framed: true }; +} + +/** + * A status bar's 28 rows from `y`; `divider` is the row (within them) of its + * line toward the window, if it draws one rather than meeting a frame. + */ +function drawStatusBar(d: Diagram, x: number, y: number, width: number, divider: number | null): void { + if (divider !== null) d.hline(x, y + divider, width, CHROME_LINE); + drawStatusItems(d, x + width, y); +} + +/** Notification icons, a battery and the clock, ending at `right`, in a status bar's 28 rows from `top`. */ +function drawStatusItems(d: Diagram, right: number, top: number): void { + d.fill(right - 64, top + 7, 56, 14, STATUS, 4); + d.outline(right - 108, top + 8, 32, 12, STATUS, 2); + d.outline(right - 148, top + 4, 20, 20, CHROME_LINE, 4); + d.outline(right - 176, top + 4, 20, 20, CHROME_LINE, 4); +} + +/** A switcher icon (32px); the selected one filled, as the switcher's selection tab is. */ +function drawIcon(d: Diagram, x: number, y: number, selected: boolean): void { + if (selected) d.fill(x, y, 32, 32, SELECTED_ICON, 6); + else d.outline(x, y, 32, 32, ICON, 6); +} + +/** Stand-in window content: a title and as many list rows as fit, the first selected. */ +function drawContent(d: Diagram, x: number, y: number, width: number, height: number, brightness: number): void { + const rowWidths = [width - 60, 300, 360, 240]; + d.fill(x + 18, y + 14, 150, 18, Math.round(CONTENT_TITLE * brightness), 6); + d.outline(x + 8, y + 44, width - 16, 34, Math.round(CONTENT * brightness), 8); + for (let row = 0; ; row++) { + const rowTop = y + 52 + row * 44; + if (rowTop + 18 > y + height - 8) break; + d.fill(x + 18, rowTop, Math.min(rowWidths[row % rowWidths.length]!, width - 36), 18, Math.round(CONTENT * brightness), 6); + } +} + +/** Real-display rows the Layout page's diagrams cover: a 288px band and a bottom switcher row under it. */ +export const LAYOUT_AREA_HEIGHT = MIN_WINDOW_HEIGHT + SWITCHER_ROW_HEIGHT; diff --git a/app/apps/onboarding/layout-page.ts b/app/apps/onboarding/layout-page.ts new file mode 100644 index 00000000..0d2eae6c --- /dev/null +++ b/app/apps/onboarding/layout-page.ts @@ -0,0 +1,114 @@ +import { G2_LENS_WIDTH, type GrayImage, type UiFont } from "../../graphics/image"; +import { appSwitcherPositionSetting, type AppSwitcherPosition } from "../../ui/dashboard-settings"; +import type { InputEvent } from "../../ui/gestures"; +import { MENU_HIGHLIGHT_FILL, MENU_HIGHLIGHT_STROKE } from "../../ui/menu-core"; +import { MIN_WINDOW_HEIGHT } from "../../ui/shell/geometry"; +import { AREA_OUTLINE, Diagram, drawLayout, LAYOUT_AREA_HEIGHT } from "./layout-diagram"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const CONTENT_GAP = 10; +const BOTTOM_MARGIN = 8; +/** Diagram width as a fraction of the display's: the most a diagram is drawn at. */ +const DIAGRAM_SCALE = 0.4; +/** Padding inside a cell's selection box, round the diagram and its caption. */ +const CELL_PADDING = 8; +const CAPTION_GAP = 4; +const COLUMN_GAP = 16; +const ROW_GAP = 6; +const DEPTH_SUBTITLE = "Customizable Depth"; +/** Grid order, in rows of two: the layouts with depth on top. */ +const LAYOUT_ORDER: readonly AppSwitcherPosition[] = ["bottom", "popup", "left", "right"]; +/** Selected on the first visit, and marked. */ +const RECOMMENDED_LAYOUT: AppSwitcherPosition = "bottom"; + +/** Whether a layout leaves the side margins the display's stereo depth shifts into (geometry.ts uiDepth). */ +export function layoutHasDepth(position: AppSwitcherPosition): boolean { + return position === "bottom" || position === "popup"; +} + +/** + * The app switcher position, as a 2×2 grid of captioned diagrams of each + * layout, Bottom (recommended, and selected at first) and Popup on top; + * swipes move through them in reading order, a tap saves the selected one. + * The page is immersive, so the diagrams get 40% of the display's width + * each. Choosing a layout reflows open windows (the dashboard controller + * watches the setting). + */ +export class LayoutPage implements OnboardingPage { + readonly immersive = true; + /** Kept across visits, so going back returns to the last choice. */ + private selected = LAYOUT_ORDER.indexOf(RECOMMENDED_LAYOUT); + + paint(image: GrayImage, focused: boolean): void { + const top = drawPageHeading( + image, + "Layout", + "Swipe to choose where the app switcher goes, then tap to continue.", + ) + CONTENT_GAP; + const title = onboardingFont("body"); + const subtitle = onboardingFont("detail"); + const captionHeight = CAPTION_GAP + title.lineHeight + subtitle.lineHeight; + const available = image.height - top - BOTTOM_MARGIN; + // Short of DIAGRAM_SCALE only where two rows of cells wouldn't fit. + const scale = Math.min( + DIAGRAM_SCALE, + ((available - ROW_GAP) / 2 - 2 * CELL_PADDING - captionHeight) / LAYOUT_AREA_HEIGHT, + ((image.width - COLUMN_GAP) / 2 - 2 * CELL_PADDING) / G2_LENS_WIDTH, + ); + const diagramWidth = Math.round(G2_LENS_WIDTH * scale); + const diagramHeight = Math.round(LAYOUT_AREA_HEIGHT * scale); + const cellWidth = diagramWidth + 2 * CELL_PADDING; + const cellHeight = diagramHeight + 2 * CELL_PADDING + captionHeight; + const left = Math.round((image.width - 2 * cellWidth - COLUMN_GAP) / 2); + const gridTop = top + Math.max(0, Math.round((available - 2 * cellHeight - ROW_GAP) / 2)); + LAYOUT_ORDER.forEach((position, index) => { + const x = left + (index % 2) * (cellWidth + COLUMN_GAP); + const y = gridTop + Math.floor(index / 2) * (cellHeight + ROW_GAP); + const selected = index === this.selected; + if (selected) { + if (focused) image.fillRoundedRect(x, y, cellWidth, cellHeight, MENU_HIGHLIGHT_FILL, 8); + image.drawRoundedRect(x, y, cellWidth, cellHeight, MENU_HIGHLIGHT_STROKE, 8); + } + drawLayoutDiagram(new Diagram(image, x + CELL_PADDING, y + CELL_PADDING, scale), position); + const centre = (text: string, font: UiFont, textY: number, value: number) => + image.drawText(font, x + Math.round((cellWidth - font.measureText(text)) / 2), textY, text, value); + const captionTop = y + CELL_PADDING + diagramHeight + CAPTION_GAP; + const name = appSwitcherPositionSetting.displayValue(position); + const caption = position === RECOMMENDED_LAYOUT ? `${name} (Recommended)` : name; + centre(caption, title, captionTop, selected ? 255 : 190); + if (layoutHasDepth(position)) { + centre(DEPTH_SUBTITLE, subtitle, captionTop + title.lineHeight, selected ? 210 : 150); + } + }); + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + const last = LAYOUT_ORDER.length - 1; + switch (event.type) { + case "scroll-up": + this.selected = Math.max(0, this.selected - 1); + return; + case "scroll-down": + this.selected = Math.min(last, this.selected + 1); + return; + case "click": + appSwitcherPositionSetting.set(LAYOUT_ORDER[this.selected]!); + nav.next(); + return; + default: + return; + } + } +} + +/** + * A layout's diagram: the 288px band, with the popup switcher open. Only a + * bottom switcher row takes rows under the band; without one, the band sits + * centred in their place, as the band and row share the screen's slack. + */ +function drawLayoutDiagram(d: Diagram, position: AppSwitcherPosition): void { + d.outline(0, 0, G2_LENS_WIDTH, LAYOUT_AREA_HEIGHT, AREA_OUTLINE, 12); + const top = position === "bottom" ? 0 : (LAYOUT_AREA_HEIGHT - MIN_WINDOW_HEIGHT) / 2; + drawLayout(d, position, { top, height: MIN_WINDOW_HEIGHT, fullWidth: false }, true); +} diff --git a/app/apps/onboarding/onboarding-app.ts b/app/apps/onboarding/onboarding-app.ts new file mode 100644 index 00000000..66a7000d --- /dev/null +++ b/app/apps/onboarding/onboarding-app.ts @@ -0,0 +1,126 @@ +import { GrayImage } from "../../graphics/image"; +import type { InputEvent } from "../../ui/gestures"; +import type { Layer, LayerActions, LayerContext } from "../../ui/layers"; +import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; +import { shell } from "../../ui/shell/shell"; +import { findSoundEffect, playSoundEffect } from "../../ui/sound-effects"; +import { DataCollectionPage } from "./data-collection-page"; +import { DepthPage } from "./depth-page"; +import { DisplayModePage } from "./display-mode-page"; +import { IosPrivacyPage } from "./ios-privacy-page"; +import { LayoutPage } from "./layout-page"; +import { markGlassesOnboardingFinished, ONBOARDING_APP_ID } from "./onboarding-progress"; +import type { OnboardingPage } from "./onboarding-page"; +import { TextSizePage } from "./text-size-page"; +import { WelcomePage } from "./welcome-page"; + +export const ONBOARDING_WINDOW_ID = "onboarding"; +export const ONBOARDING_SURFACE_ID = "window:onboarding"; + +/** + * The on-glasses onboarding: a fixed sequence of pages, opened automatically + * on the first connection after phone-side setup and after each upgrade + * (see onboarding-progress.ts), and from the launcher any time. Finishing + * the last page records this version as onboarded and closes the window; + * closing it early leaves onboarding due. + */ +export function createOnboardingAppWindow(options: InProcessAppOptions): InProcessWindow { + const pages: OnboardingPage[] = [ + new WelcomePage(), new TextSizePage(), + ...(global.isIOS ? [new IosPrivacyPage()] : []), + new DataCollectionPage(), new LayoutPage(), new DepthPage(), new DisplayModePage(), + ]; + const app = createInProcessWindow({ + appId: ONBOARDING_APP_ID, + windowId: ONBOARDING_WINDOW_ID, + title: "Onboarding", + iconLetter: "On", + icon: "sparkles", + closeable: true, + immersive: pages[0]!.immersive, + actions: options.actions, + baseLayer: new OnboardingLayer( + pages, + // Only called on page changes, by when `app` is set. + (immersive) => app.setImmersive(immersive), + () => { + markGlassesOnboardingFinished(); + shell.closeWindow(ONBOARDING_WINDOW_ID); + }, + ), + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + reconfigureSurface: options.reconfigureSurface, + onClosed: options.onClosed, + }); + playWelcomeSound(options.actions); + return app; +} + +/** The celebratory jingle that greets the welcome page (formerly played on the first connection). */ +function playWelcomeSound(actions: LayerActions): void { + const effect = findSoundEffect("questcomplete"); + if (!effect) return; + playSoundEffect( + effect, + (payload) => actions.playBuzzerSequence(payload), + (ms) => new Promise((resolve) => setTimeout(resolve, ms)), + ).catch((error) => console.warn(`onboarding welcome sound failed: ${error}`)); +} + +/** + * Hosts the current page, immersive or not as it asks. Double-click steps + * back a page, or leaves the app from the first one (the standard yield to + * the app switcher); every other gesture goes to the page. Pages that don't + * apply (Depth beside a side-strip app switcher) are skipped either way. + */ +export class OnboardingLayer implements Layer { + private index = 0; + + constructor( + private readonly pages: readonly OnboardingPage[], + private readonly setImmersive: (immersive: boolean) => void, + private readonly finish: () => void, + ) { + pages[0]!.enter?.(); + } + + paint(ctx: LayerContext): GrayImage { + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + this.pages[this.index]!.paint(image, ctx.stack.isFocused()); + return image; + } + + async handleInput(event: InputEvent): Promise { + if (event.type === "double-click") { + const previous = this.applicableIndex(this.index - 1, -1); + if (previous === null) shell.yieldFocusToSidebar(); + else this.show(previous); + return; + } + await this.pages[this.index]!.handleInput(event, { next: () => this.next() }); + } + + private next(): void { + const next = this.applicableIndex(this.index + 1, 1); + if (next === null) this.finish(); + else this.show(next); + } + + /** The first page from `index` on in direction `step` that applies, if any. */ + private applicableIndex(index: number, step: 1 | -1): number | null { + for (; index >= 0 && index < this.pages.length; index += step) { + if (this.pages[index]!.applies?.() ?? true) return index; + } + return null; + } + + private show(index: number): void { + this.index = index; + const page = this.pages[index]!; + page.enter?.(); + this.setImmersive(page.immersive ?? false); + } +} diff --git a/app/apps/onboarding/onboarding-fonts.ts b/app/apps/onboarding/onboarding-fonts.ts new file mode 100644 index 00000000..e74ac400 --- /dev/null +++ b/app/apps/onboarding/onboarding-fonts.ts @@ -0,0 +1,38 @@ +/** + * The onboarding's own fonts: bundled Roboto at fixed sizes, deliberately + * not the user's UI font, so the pages are readable before (and while) the + * text size is being chosen. Bitmap Terminus stands in where TTF rendering + * is unavailable. + */ +import { getFont, getTerminusFont } from "../../graphics/bdffont"; +import type { UiFont } from "../../graphics/image"; +import { ensurePreinstalledFonts, installedFontPath } from "../../graphics/installed-fonts"; +import { TtfFont } from "../../graphics/ttf-font"; + +export type OnboardingFontRole = "hero" | "title" | "body" | "detail"; + +const ROLE_FONTS: Record UiFont }> = { + /** The welcome page's "Faceclaw". */ + hero: { file: "Roboto-Regular.ttf", size: 44, fallback: () => getFont("terminus32") }, + /** Page titles. */ + title: { file: "Roboto-Regular.ttf", size: 26, fallback: () => getFont("terminus24") }, + /** Prompts and choices. */ + body: { file: "Roboto-Regular.ttf", size: 20, fallback: () => getTerminusFont(20, false) ?? getFont("terminus16") }, + /** Explanations under titles and beside choices. */ + detail: { file: "Roboto-Regular.ttf", size: 16, fallback: () => getFont("terminus16") }, +}; + +let fontsEnsured = false; + +export function onboardingFont(role: OnboardingFontRole): UiFont { + if (!fontsEnsured) { + fontsEnsured = true; + try { + ensurePreinstalledFonts(); // onboarding may be the first thing to draw TTF text + } catch (error) { + console.warn(`onboarding font preinstall failed: ${error}`); + } + } + const spec = ROLE_FONTS[role]; + return TtfFont.load(installedFontPath(spec.file), spec.size) ?? spec.fallback(); +} diff --git a/app/apps/onboarding/onboarding-page.ts b/app/apps/onboarding/onboarding-page.ts new file mode 100644 index 00000000..5081854d --- /dev/null +++ b/app/apps/onboarding/onboarding-page.ts @@ -0,0 +1,56 @@ +import type { GrayImage } from "../../graphics/image"; +import { wrapText } from "../../graphics/textwrap"; +import type { InputEvent } from "../../ui/gestures"; +import { onboardingFont } from "./onboarding-fonts"; + +/** Page-to-flow navigation, handed to a page's input handler. */ +export type OnboardingNav = { + /** Go to the next page, or finish onboarding from the last one. */ + next: () => void; +}; + +/** + * One onboarding step. The flow owns the page list, back navigation + * (double-click) and finishing; a page paints itself into the window's + * viewport and handles every other gesture. + */ +export interface OnboardingPage { + /** + * Show the page immersive (full-screen): the whole 640×480 panel, with + * the app switcher and status bar hidden. Otherwise it is an ordinary + * window in the chosen layout. + */ + readonly immersive?: boolean; + /** Whether the page belongs in the flow right now; skipped both ways when false. */ + applies?(): boolean; + /** The page became current, going forward or back. */ + enter?(): void; + paint(image: GrayImage, focused: boolean): void; + handleInput(event: InputEvent, nav: OnboardingNav): void | Promise; +} + +/** Left margin and title position shared by the pages (the UI-wide (18, 10) title spot). */ +export const PAGE_X = 18; +export const TITLE_Y = 10; + +/** + * Draw a page's title and its wrapped explanation (at most `maxLines` + * lines) in the onboarding fonts. Returns the y just below the explanation. + */ +export function drawPageHeading( + image: GrayImage, + title: string, + explanation: string, + maxLines = 2, +): number { + const titleFont = onboardingFont("title"); + const detailFont = onboardingFont("detail"); + image.drawText(titleFont, PAGE_X, TITLE_Y, title, 230); + let y = TITLE_Y + titleFont.lineHeight + 2; + const lines = wrapText(detailFont, explanation, image.width - 2 * PAGE_X).slice(0, maxLines); + for (const line of lines) { + image.drawText(detailFont, PAGE_X, y, line, 160); + y += detailFont.lineHeight; + } + return y; +} diff --git a/app/apps/onboarding/onboarding-progress.ts b/app/apps/onboarding/onboarding-progress.ts new file mode 100644 index 00000000..7c897ff8 --- /dev/null +++ b/app/apps/onboarding/onboarding-progress.ts @@ -0,0 +1,34 @@ +/** + * Whether the on-glasses onboarding is due: it is recorded as finished per + * Faceclaw version, so it runs on the first connection after phone-side + * setup and again after any upgrade. The key shares the phone onboarding's + * "onboarding." prefix so scripts/resetOnboarding.sh clears it too. + */ +import { getStringSetting, setStringSetting } from "../../native/settings-store"; +import { FACECLAW_VERSION } from "../../version"; + +export const ONBOARDING_APP_ID = "onboarding"; + +const FINISHED_VERSION_KEY = "onboarding.glassesFinishedVersion"; + +export function isGlassesOnboardingDue(): boolean { + return getStringSetting(FINISHED_VERSION_KEY, "") !== FACECLAW_VERSION; +} + +export function markGlassesOnboardingFinished(): void { + setStringSetting(FINISHED_VERSION_KEY, FACECLAW_VERSION); +} + +let autoLaunchChecked = false; + +/** + * Asked by the connection controllers on each rendered frame (the session + * is warm by then, so the onboarding's welcome jingle won't be dropped). + * True at most once per run, and only when onboarding is due: closing it + * unfinished brings it back on the next app start, not on every reconnect. + */ +export function claimGlassesOnboardingAutoLaunch(): boolean { + if (autoLaunchChecked) return false; + autoLaunchChecked = true; + return isGlassesOnboardingDue(); +} diff --git a/app/apps/onboarding/text-size-page.ts b/app/apps/onboarding/text-size-page.ts new file mode 100644 index 00000000..46498f45 --- /dev/null +++ b/app/apps/onboarding/text-size-page.ts @@ -0,0 +1,174 @@ +import type { GrayImage, UiFont } from "../../graphics/image"; +import { installedFontPath } from "../../graphics/installed-fonts"; +import { wrapText } from "../../graphics/textwrap"; +import { TtfFont } from "../../graphics/ttf-font"; +import { + fontSelectionLabel, + getDefaultSmallFont, + getUiFontSelection, + setUiFontSelection, + uiFontSizeAllowed, +} from "../../graphics/ui-fonts"; +import { FONT_SIZE_CHOICES } from "../../ui/font-picker"; +import type { InputEvent } from "../../ui/gestures"; +import { Menu, type MenuBox, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, listRowHeight } from "../../ui/metrics"; +import { onboardingFont } from "./onboarding-fonts"; +import { drawPageHeading, PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const CONTENT_GAP = 10; +const BOTTOM_MARGIN = 8; +const COLUMN_GAP = 12; +const PREVIEW_PADDING = 10; +const ROW_GAP = 2; +/** Between a size and its "Recommended" note. */ +const NOTE_GAP = 8; +/** The sizes offered here; Settings has the full range. */ +const SIZE_CHOICES = FONT_SIZE_CHOICES.filter((size) => size >= 12 && size <= 18); +/** Selected on the first visit, and marked. */ +const RECOMMENDED_SIZE = 16; +const RECOMMENDED_NOTE = "Recommended"; +/** Drawn while they fit, so smaller sizes visibly show more words. */ +const PREVIEW_PARAGRAPHS = [ + "This is how text will look on your glasses.", + "Smaller sizes fit more words on the display at once, while larger sizes are easier to read at a " + + "glance or on the move. Most people find the recommended size a good balance between the two.", +]; + +/** The UI font's face, as far as this page is concerned: resizable (TTF) or not (bitmap). */ +type Face = + | { kind: "sized"; file: string; path: string; sizes: number[] } + | { kind: "fixed"; label: string }; + +/** + * The UI font's size, 12 to 18px (16 recommended, and selected at first), + * beside a preview of text in the selected size. The face and weight stay + * as they are (they live in the Settings font picker); a bitmap face has + * only one size, so the page just says so. The size is saved on tap, not + * while scrolling. + */ +export class TextSizePage implements OnboardingPage { + readonly immersive = true; + private face: Face = { kind: "fixed", label: "Terminus" }; + private readonly sizeMenu = new Menu({ + wrap: false, + rowGap: ROW_GAP, + highlight: { radius: 8 }, + getHeight: () => listRowHeight(onboardingFont("body")), + draw: (args) => this.drawSizeRow(args), + }); + + enter(): void { + // Re-read on every visit: the face may have changed in Settings meanwhile. + const selection = getUiFontSelection(); + if (selection.kind === "ttf") { + const path = installedFontPath(selection.file); + const sizes = SIZE_CHOICES.filter((size) => uiFontSizeAllowed(path, size)); + if (sizes.length) { + this.face = { kind: "sized", file: selection.file, path, sizes }; + // The recommended size the first time; on a return, what was left selected. + this.sizeMenu.setItems(sizes, nearestIndex(sizes, this.sizeMenu.selectedItem ?? RECOMMENDED_SIZE)); + return; + } + } + // An unloadable TTF selection renders as Terminus, so call it that. + this.face = { kind: "fixed", label: selection.kind === "bitmap" ? fontSelectionLabel(selection) : "Terminus" }; + } + + paint(image: GrayImage, focused: boolean): void { + if (this.face.kind === "fixed") { + const top = drawPageHeading( + image, + "Text size", + `Your UI font, ${this.face.label}, comes in one size. To choose a size, pick a different font in ` + + "Settings > Customization. Tap to continue.", + 3, + ) + CONTENT_GAP; + drawPreview(image, getDefaultSmallFont(), { + x: PAGE_X - 10, + y: top, + width: image.width - 2 * (PAGE_X - 10), + height: image.height - top - BOTTOM_MARGIN, + }); + return; + } + const top = drawPageHeading( + image, + "Text size", + "Swipe to choose a size, then tap to continue. The typeface and weight can be changed in Settings.", + ) + CONTENT_GAP; + const height = image.height - top - BOTTOM_MARGIN; + const body = onboardingFont("body"); + // TTF widths are fractional; the menu's row images need whole pixels. + const columnWidth = Math.ceil(20 + Math.max(...this.face.sizes.map((size) => body.measureText(sizeLabel(size)))) + + NOTE_GAP + onboardingFont("detail").measureText(RECOMMENDED_NOTE)); + const menuBox = { x: PAGE_X - 10, y: top, width: columnWidth, height }; + this.sizeMenu.paint(image, menuBox, focused); + const previewX = menuBox.x + menuBox.width + COLUMN_GAP; + const size = this.sizeMenu.selectedItem; + const font = (size !== null ? TtfFont.load(this.face.path, size) : null) ?? getDefaultSmallFont(); + drawPreview(image, font, { x: previewX, y: top, width: image.width - (PAGE_X - 10) - previewX, height }); + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + switch (event.type) { + case "scroll-up": + case "scroll-down": + if (this.face.kind === "sized") void this.sizeMenu.handleInput(event); + return; + case "click": + this.save(); + nav.next(); + return; + default: + return; + } + } + + private save(): void { + const size = this.sizeMenu.selectedItem; + if (this.face.kind !== "sized" || size === null) return; + const current = getUiFontSelection(); + if (current.kind === "ttf" && current.file === this.face.file && current.size === size) return; + setUiFontSelection({ kind: "ttf", file: this.face.file, size }); + } + + private drawSizeRow({ image, item: size, x, y, height, selected }: MenuDrawArgs): void { + const body = onboardingFont("body"); + const label = sizeLabel(size); + image.drawText(body, x + 10, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 190); + if (size !== RECOMMENDED_SIZE) return; + const detail = onboardingFont("detail"); + const noteY = y + Math.round((height - detail.lineHeight) / 2); + image.drawText(detail, x + 10 + body.measureText(label) + NOTE_GAP, noteY, RECOMMENDED_NOTE, selected ? 210 : 150); + } +} + +function sizeLabel(size: number): string { + return `${size} px`; +} + +/** A framed sample in `font`: paragraphs of text, as many whole lines as fit. */ +function drawPreview(image: GrayImage, font: UiFont, box: MenuBox): void { + image.drawRoundedRect(box.x, box.y, box.width, box.height, 60, 8); + const left = box.x + PREVIEW_PADDING; + const width = box.width - 2 * PREVIEW_PADDING; + const bottom = box.y + box.height - PREVIEW_PADDING; + let y = box.y + PREVIEW_PADDING; + for (const [index, paragraph] of PREVIEW_PARAGRAPHS.entries()) { + if (index > 0) y += font.lineHeight >> 1; + for (const line of wrapText(font, paragraph, width)) { + if (y + font.lineHeight > bottom) return; + image.drawText(font, left, y, line, 220); + y += font.lineHeight; + } + } +} + +function nearestIndex(values: readonly number[], target: number): number { + let best = 0; + for (let index = 1; index < values.length; index++) { + if (Math.abs(values[index]! - target) < Math.abs(values[best]! - target)) best = index; + } + return best; +} diff --git a/app/apps/onboarding/welcome-page.ts b/app/apps/onboarding/welcome-page.ts new file mode 100644 index 00000000..4d9c62b4 --- /dev/null +++ b/app/apps/onboarding/welcome-page.ts @@ -0,0 +1,53 @@ +import type { GrayImage, UiFont } from "../../graphics/image"; +import { getFullLogo } from "../../graphics/logo"; +import { wrapText } from "../../graphics/textwrap"; +import type { InputEvent } from "../../ui/gestures"; +import { FACECLAW_VERSION } from "../../version"; +import { onboardingFont } from "./onboarding-fonts"; +import { PAGE_X, type OnboardingNav, type OnboardingPage } from "./onboarding-page"; + +const LOGO_MAX_HEIGHT = 180; +const LOGO_MARGIN = 12; +const LOGO_TEXT_GAP = 28; +const PROMPT_GAP = 22; +const PROMPT = "Tap the ring or touchpad to continue"; + +type TextLine = { font: UiFont; text: string; value: number; spaceBefore: number }; + +/** The splash: logo beside the name, version and how to go on, centred as one group. */ +export class WelcomePage implements OnboardingPage { + readonly immersive = true; + + paint(image: GrayImage): void { + const { width, height } = image; + const logo = getFullLogo(Math.min(LOGO_MAX_HEIGHT, height - 2 * LOGO_MARGIN)); + const logoWidth = logo ? logo.width + LOGO_TEXT_GAP : 0; + const body = onboardingFont("body"); + const lines: TextLine[] = [ + { font: onboardingFont("hero"), text: "Faceclaw", value: 240, spaceBefore: 0 }, + { font: body, text: `Version ${FACECLAW_VERSION}`, value: 150, spaceBefore: 0 }, + ...wrapText(body, PROMPT, width - 2 * PAGE_X - logoWidth).map((text, index) => ({ + font: body, + text, + value: 215, + spaceBefore: index === 0 ? PROMPT_GAP : 0, + })), + ]; + const textWidth = Math.max(...lines.map((line) => line.font.measureText(line.text))); + const textHeight = lines.reduce((total, line) => total + line.spaceBefore + line.font.lineHeight, 0); + const left = Math.max(PAGE_X, Math.round((width - logoWidth - textWidth) / 2)); + if (logo) { + image.bitBlt(logo, left, Math.round((height - logo.height) / 2), { transparentZero: true }); + } + let y = Math.round((height - textHeight) / 2); + for (const line of lines) { + y += line.spaceBefore; + image.drawText(line.font, left + logoWidth, y, line.text, line.value); + y += line.font.lineHeight; + } + } + + handleInput(event: InputEvent, nav: OnboardingNav): void { + if (event.type === "click") nav.next(); + } +} diff --git a/app/apps/pinball/pinball-app.worker.ts b/app/apps/pinball/pinball-app.worker.ts index 9881d602..8ff5a1bb 100644 --- a/app/apps/pinball/pinball-app.worker.ts +++ b/app/apps/pinball/pinball-app.worker.ts @@ -66,7 +66,7 @@ const PANEL_X = 310; /** * Height the table and panel are drawn for: a min-height window's viewport * under the standard top bar. A taller viewport (the status bar folded into - * the switcher row, a full-panel display) centers the layout vertically. + * the switcher row, a Tall or Full Screen display) centers the layout vertically. */ const LAYOUT_HEIGHT = 260; @@ -411,8 +411,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { syncTickTimer(window); } break; + case "resize-window": { + const window = windows.get(message.windowId); + if (!window) break; + window.viewportWidth = message.viewport.width; + window.viewportHeight = message.viewport.height; + window.menu?.resize({ width: window.viewportWidth, height: window.viewportHeight }); + window.lastSubmittedFingerprint = ""; + if (window.foreground) renderAndSubmit(window, 0); + break; + } case "navigation-sensors": - case "resize-window": case "text-input": case "tool-call": break; diff --git a/app/apps/roam/roam-api.ts b/app/apps/roam/roam-api.ts index ef33022b..21b4d881 100644 --- a/app/apps/roam/roam-api.ts +++ b/app/apps/roam/roam-api.ts @@ -37,6 +37,15 @@ export function dailyPageUid(date: Date): string { return `${month}-${day}-${date.getFullYear()}`; } +/** The date a daily-note page uid names, or null for any other uid. */ +export function dailyPageDate(uid: string): Date | null { + const match = /^(\d{2})-(\d{2})-(\d{4})$/.exec(uid); + if (!match) return null; + const date = new Date(Number(match[3]), Number(match[1]) - 1, Number(match[2])); + // Rejects out-of-range fields such as 02-31, which Date would roll over. + return dailyPageUid(date) === uid ? date : null; +} + /** Daily-note page title for a date, e.g. "August 3rd, 2026". */ export function dailyPageTitle(date: Date): string { const months = [ diff --git a/app/apps/roam/roam-app.worker.ts b/app/apps/roam/roam-app.worker.ts index 5575154b..a06be8bc 100644 --- a/app/apps/roam/roam-app.worker.ts +++ b/app/apps/roam/roam-app.worker.ts @@ -2,7 +2,8 @@ * Roam app worker. Shows a Roam Research page (today's daily note by default) * as a selectable outline: ring scroll moves the selection, click toggles a * todo or follows a [[page]] link, double-click walks back through visited - * pages then yields focus. Edits (check/uncheck, add todo, edit block) go + * pages then yields focus. The context menu jumps to today's or a picked + * date's daily note. Edits (check/uncheck, add todo, edit block) go * through the Roam backend API and are also exposed as assistant tools. * The graph name and API token are configured here too, from the context * menu, via the phone app's text editor. @@ -19,6 +20,7 @@ import { getActiveDisplay } from "../../native/active-display"; import { onSettingsStoreChanged } from "../../native/settings-store"; import type { MenuItem } from "../../ui/menu"; import { WindowMenu } from "../../ui/window-menu"; +import { DatePickerLayer } from "../../ui/date-picker"; import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-window"; import type { ToolResult, ToolSpec } from "../../assistant/tool-registry"; import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures"; @@ -32,6 +34,7 @@ import { docNodePageLink, type DocNode } from "../../ui/document/document-model" import { createBlock, createPage, + dailyPageDate, dailyPageTitle, dailyPageUid, fetchPageByUid, @@ -251,9 +254,17 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { ); break; } + case "resize-window": + if (!window || window.windowId !== message.windowId) break; + window.viewportWidth = message.viewport.width; + window.viewportHeight = message.viewport.height; + window.view.resize(message.viewport.width - DOC_MARGIN * 2, message.viewport.height - HEADER_HEIGHT - BOTTOM_MARGIN); + window.menu?.resize(message.viewport); + window.lastSubmittedFingerprint = ""; + render(); + break; case "input-focus": case "navigation-sensors": - case "resize-window": break; } }; @@ -272,9 +283,12 @@ function inferForeground(focused: boolean): void { // --------------------------------------------------------------------------- // Page loading +function dailyRef(date: Date): PageRef { + return { uid: dailyPageUid(date), title: dailyPageTitle(date) }; +} + function todayRef(): PageRef { - const now = new Date(); - return { uid: dailyPageUid(now), title: dailyPageTitle(now) }; + return dailyRef(new Date()); } async function loadPage(ref: PageRef, options: { resetStack?: boolean; pushCurrent?: boolean } = {}): Promise { @@ -303,8 +317,8 @@ async function loadPage(ref: PageRef, options: { resetStack?: boolean; pushCurre if (page) { currentPage = page; pageMissing = false; - } else if (uid === dailyPageUid(new Date())) { - // Today's daily note isn't created until something is written to it. + } else if (dailyPageDate(uid)) { + // A daily note isn't created until something is written to it. currentPage = { uid, title: ref.title, children: [] }; pageMissing = true; } else { @@ -363,6 +377,22 @@ function menuItems(): MenuItem[] { }, }); } + items.push({ + label: "Go to date", + onSelect: (ctx) => { + ctx.stack.pop(); + ctx.stack.push(new DatePickerLayer({ + title: "Go to date", + initial: (currentRef?.uid ? dailyPageDate(currentRef.uid) : null) ?? new Date(), + confirmLabel: "open", + onPick: (date) => { + const ref = dailyRef(date); + if (ref.uid === currentRef?.uid) void reloadCurrentPage(); + else void loadPage(ref, { pushCurrent: true }); + }, + })); + }, + }); items.push({ label: "Refresh", onSelect: (ctx) => { @@ -507,7 +537,7 @@ async function toggleTodo(node: DocNode): Promise { } } -/** Create today's daily page if it only exists locally. */ +/** Create the shown daily page if it only exists locally. */ async function ensureCurrentPageExists(): Promise { if (!currentPage || !pageMissing) return; await createPage(currentPage.title, currentPage.uid); diff --git a/app/apps/t3code/t3-client.ts b/app/apps/t3code/t3-client.ts index 776475ed..b977a038 100644 --- a/app/apps/t3code/t3-client.ts +++ b/app/apps/t3code/t3-client.ts @@ -15,7 +15,8 @@ import { webSocketUrl, type T3Fetch, } from "./t3-auth"; -import { ShellModel, ThreadDetailModel, type T3ServerProvider } from "./t3-model"; +import type { T3ServerProvider } from "./t3-launch"; +import { ShellModel, ThreadDetailModel } from "./t3-model"; import { T3RpcConnection, T3RpcError, type RpcSocketFactory, type T3Subscription } from "./t3-rpc"; /** A paired environment as stored on the phone. */ @@ -79,6 +80,8 @@ export class T3EnvironmentClient { readonly shell = new ShellModel(); providers: T3ServerProvider[] = []; settings: Record | null = null; + /** The folder of threads without a project (its project is the "Scratch" one). */ + scratchWorkspaceRoot: string | null = null; private phaseValue: EnvironmentPhase = "idle"; private statusValue = ""; private connection: T3RpcConnection | null = null; @@ -200,6 +203,11 @@ export class T3EnvironmentClient { return String(result?.threadId ?? payload.threadId); } + /** Any other unary RPC (vcs.listRefs, projects.readFile, ...). */ + call(method: string, payload: Record): Promise { + return this.requireConnection().call(method, payload); + } + private requireConnection(): T3RpcConnection { if (this.phaseValue !== "connected" || !this.connection) { throw new T3RpcError("disconnected", `Not connected to ${this.config.label || "the server"}.`); @@ -265,6 +273,7 @@ export class T3EnvironmentClient { } this.providers = Array.isArray(config.providers) ? config.providers : []; this.settings = config.settings ?? null; + this.scratchWorkspaceRoot = typeof config.scratchWorkspaceRoot === "string" ? config.scratchWorkspaceRoot : null; if (typeof config.environment?.label === "string" && config.environment.label && config.environment.label !== this.config.label) { this.config = { ...this.config, label: config.environment.label }; } diff --git a/app/apps/t3code/t3-launch.ts b/app/apps/t3code/t3-launch.ts new file mode 100644 index 00000000..49de6783 --- /dev/null +++ b/app/apps/t3code/t3-launch.ts @@ -0,0 +1,516 @@ +/** + * New-thread options for orchestration.launchThread, resolved the way T3 + * Code's own new-task flow does it (apps/mobile/src/features/threads/ + * new-task-flow-provider.tsx, lib/modelOptions.ts, thread-settings-options.ts, + * new-task-context-presentation.ts; packages/shared/src/projectSettings.ts + * and model.ts): which models, model options, access modes, workspaces and + * branches to offer, their defaults, and the launch payload. + * + * Defaults follow T3's project-settings chain: the project's override, then + * the environment's setting, then (for the workspace mode) the repository's + * checked-in t3.json, then the built-in value. The user's explicit picks live + * in a LaunchDraft; resolveLaunch() layers them over those defaults. + * + * Pure (no I/O), so it runs under the Node tests; the worker owns the draft, + * the branch list and t3.json reads. + */ +import type { T3ModelOptionSelection, T3ModelSelection, T3ProjectShell } from "./t3-model"; +import { uuidv4 } from "./t3-model"; + +// --------------------------------------------------------------------------- +// Wire shapes (subset) + +export type T3OptionChoice = { id: string; label: string; description?: string; isDefault?: boolean }; + +/** One per-model option a provider offers (reasoning effort, fast mode, ...). */ +export type T3OptionDescriptor = + | { + id: string; + label: string; + description?: string; + type: "select"; + options: T3OptionChoice[]; + currentValue?: string; + /** Values applied by rewriting the prompt (Claude's "ultrathink"), not offered as picks. */ + promptInjectedValues?: string[]; + } + | { id: string; label: string; description?: string; type: "boolean"; currentValue?: boolean }; + +export type T3ProviderModel = { + slug: string; + name?: string; + isDefault?: boolean; + isLegacy?: boolean; + capabilities?: { optionDescriptors?: T3OptionDescriptor[] } | null; +}; + +export type T3ServerProvider = { + instanceId: string; + displayName?: string; + driver?: string; + enabled?: boolean; + installed?: boolean; + availability?: string; + auth?: { status?: string }; + supportedRuntimeModes?: string[]; + models?: T3ProviderModel[]; +}; + +/** A branch from vcs.listRefs. */ +export type T3VcsRef = { + name: string; + isRemote?: boolean; + remoteName?: string; + current: boolean; + isDefault: boolean; + /** Set when the branch is checked out in a worktree (the project root counts). */ + worktreePath: string | null; +}; + +export type T3RefList = { + refs: T3VcsRef[]; + isRepo: boolean; + hasPrimaryRemote: boolean; + nextCursor?: number | null; + totalCount?: number; +}; + +/** The settings a repository's t3.json can supply. */ +export type T3ProjectFileDefaults = { defaultThreadEnvMode?: WorkspaceMode }; + +export const T3_PROJECT_FILE_NAME = "t3.json"; + +export function parseProjectFile(contents: string): T3ProjectFileDefaults | null { + try { + const parsed = JSON.parse(contents); + if (!parsed || typeof parsed !== "object") return null; + const mode = parsed.defaultThreadEnvMode; + return mode === "local" || mode === "worktree" ? { defaultThreadEnvMode: mode } : {}; + } catch { + return null; + } +} + +// --------------------------------------------------------------------------- +// Project-scoped settings + +type Settings = Record | null | undefined; + +/** + * The project's overrides. Until the server has folded them into + * settings.projectSettingsOverrides, the project's own legacy model and + * workspace fields count as overrides too. + */ +function projectOverrides(settings: Settings, project: T3ProjectShell): Record { + const stored = (settings?.projectSettingsOverrides?.[project.id] ?? {}) as Record; + if (settings?.projectSettingsFolded) return stored; + return { + ...(project.defaultModelSelection != null ? { defaultModelSelection: project.defaultModelSelection } : {}), + ...(project.defaultThreadEnvMode != null ? { defaultThreadEnvMode: project.defaultThreadEnvMode } : {}), + ...stored, + }; +} + +/** A project-scoped setting: the project's override (even null), else the environment's. */ +function projectSetting(settings: Settings, project: T3ProjectShell, key: string): unknown { + const override = projectOverrides(settings, project)[key]; + return override !== undefined ? override : settings?.[key]; +} + +/** The project that is the environment's Scratch folder (threads without a project). */ +export function isScratchProject(project: T3ProjectShell, scratchWorkspaceRoot: string | null | undefined): boolean { + if (!scratchWorkspaceRoot || !project.workspaceRoot) return false; + const normalize = (path: string) => path.replace(/[\\/]+$/, ""); + return normalize(project.workspaceRoot) === normalize(scratchWorkspaceRoot); +} + +/** + * The same repository among another environment's projects (moving a new + * thread to another computer): repository identity, else the checkout's + * folder name, else the title. + */ +export function matchingProject(projects: Iterable, target: T3ProjectShell): T3ProjectShell | null { + const list = [...projects]; + const key = target.repositoryIdentity?.canonicalKey ?? null; + if (key) { + const match = list.find((project) => project.repositoryIdentity?.canonicalKey === key); + if (match) return match; + } + const folder = baseName(target.workspaceRoot); + return ( + (folder ? list.find((project) => baseName(project.workspaceRoot) === folder) : undefined) ?? + list.find((project) => project.title === target.title) ?? + null + ); +} + +function baseName(path: string | undefined): string { + return (path ?? "").replace(/[\\/]+$/, "").split(/[\\/]/).pop() ?? ""; +} + +// --------------------------------------------------------------------------- +// Providers and models + +function providerUsable(provider: T3ServerProvider): boolean { + return ( + provider.enabled !== false && + provider.installed !== false && + provider.auth?.status !== "unauthenticated" && + provider.availability !== "unavailable" + ); +} + +function findProvider(providers: readonly T3ServerProvider[], instanceId: string | null | undefined): T3ServerProvider | undefined { + return instanceId ? providers.find((provider) => provider.instanceId === instanceId) : undefined; +} + +function findModel(providers: readonly T3ServerProvider[], selection: T3ModelSelection | null | undefined): T3ProviderModel | undefined { + return findProvider(providers, selection?.instanceId)?.models?.find((model) => model.slug === selection?.model); +} + +const DRIVER_NAMES: Record = { codex: "Codex", claudeAgent: "Claude", pi: "Pi" }; + +/** "Claude", "Codex", or the provider's configured name. */ +export function providerDisplayName(providers: readonly T3ServerProvider[], instanceId: string | null | undefined): string { + if (!instanceId) return ""; + const provider = findProvider(providers, instanceId); + const driver = provider?.driver ?? ""; + return provider?.displayName || DRIVER_NAMES[driver] || driver || instanceId; +} + +/** The model's display name ("Claude Opus 5"), else its slug. */ +export function modelDisplayName(providers: readonly T3ServerProvider[], selection: T3ModelSelection | null | undefined): string { + if (!selection) return ""; + return findModel(providers, selection)?.name || selection.model; +} + +export type ModelChoiceGroup = { + instanceId: string; + label: string; + /** Current models first, then legacy ones. */ + models: Array<{ selection: T3ModelSelection; label: string; isLegacy: boolean }>; +}; + +/** Every model on a usable provider, grouped by provider. */ +export function modelChoices(providers: readonly T3ServerProvider[]): ModelChoiceGroup[] { + const groups: ModelChoiceGroup[] = []; + for (const provider of providers) { + if (!providerUsable(provider)) continue; + const models = (provider.models ?? []) + .map((model) => ({ + selection: { instanceId: provider.instanceId, model: model.slug }, + label: model.name || model.slug, + isLegacy: model.isLegacy === true, + })) + .sort((a, b) => Number(a.isLegacy) - Number(b.isLegacy)); + if (models.length) groups.push({ instanceId: provider.instanceId, label: providerDisplayName(providers, provider.instanceId), models }); + } + return groups; +} + +/** + * Model for a new thread, the way T3's mobile app defaults it: the project's + * default (project override, then environment), then the first usable + * provider's default model, then any current model. Legacy models are never + * picked implicitly. + */ +export function defaultModelSelection( + settings: Settings, + providers: readonly T3ServerProvider[], + project: T3ProjectShell, +): T3ModelSelection | null { + const usable = (selection: unknown): selection is T3ModelSelection => { + const candidate = selection as T3ModelSelection | null | undefined; + if (!candidate?.instanceId || !candidate.model) return false; + const provider = findProvider(providers, candidate.instanceId); + if (!provider) return providers.length === 0; + return providerUsable(provider) && findModel(providers, candidate)?.isLegacy !== true; + }; + const override = projectOverrides(settings, project).defaultModelSelection; + for (const candidate of [override, settings?.defaultModelSelection]) if (usable(candidate)) return candidate; + for (const pickDefault of [true, false]) { + for (const provider of providers) { + if (!providerUsable(provider)) continue; + const model = (provider.models ?? []).find((entry) => !entry.isLegacy && (!pickDefault || entry.isDefault)); + if (model) return { instanceId: provider.instanceId, model: model.slug }; + } + } + return null; +} + +// --------------------------------------------------------------------------- +// Per-model options (effort, fast mode, ...) + +/** Effort values T3 mobile leaves out of its picker (a workflow trigger, not a reasoning level). */ +const HIDDEN_CHOICE_IDS = new Set(["ultracode"]); + +export function modelOptionDescriptors(providers: readonly T3ServerProvider[], selection: T3ModelSelection | null | undefined): T3OptionDescriptor[] { + return findModel(providers, selection)?.capabilities?.optionDescriptors ?? []; +} + +function selectionOptions(selection: T3ModelSelection | null | undefined): T3ModelOptionSelection[] { + return Array.isArray(selection?.options) ? selection!.options! : []; +} + +function validChoice(descriptor: Extract, value: unknown): string | undefined { + if (typeof value !== "string" || !value) return undefined; + if (!descriptor.options.length) return value; + if (descriptor.promptInjectedValues?.includes(value)) return undefined; + return descriptor.options.some((option) => option.id === value) ? value : undefined; +} + +/** The option's effective value: the selection's pick, else the provider's current value, else its default. */ +export function optionValue(descriptor: T3OptionDescriptor, selection: T3ModelSelection | null | undefined): string | boolean | undefined { + const picked = selectionOptions(selection).find((option) => option.id === descriptor.id)?.value; + if (descriptor.type === "boolean") { + if (typeof picked === "boolean") return picked; + return descriptor.currentValue; + } + return ( + validChoice(descriptor, picked) ?? + validChoice(descriptor, descriptor.currentValue) ?? + descriptor.options.find((option) => option.isDefault)?.id + ); +} + +export function optionValueLabel(descriptor: T3OptionDescriptor, value: string | boolean | undefined): string { + if (descriptor.type === "boolean") return value ? "On" : "Off"; + return descriptor.options.find((option) => option.id === value)?.label ?? (typeof value === "string" ? value : "Default"); +} + +/** The choices to offer for a select option. */ +export function selectableChoices(descriptor: Extract): T3OptionChoice[] { + const injected = new Set(descriptor.promptInjectedValues ?? []); + return descriptor.options.filter((option) => !injected.has(option.id) && !HIDDEN_CHOICE_IDS.has(option.id)); +} + +/** The selection with one option picked. */ +export function withModelOption(selection: T3ModelSelection, id: string, value: string | boolean): T3ModelSelection { + return { ...selection, options: [...selectionOptions(selection).filter((option) => option.id !== id), { id, value }] }; +} + +/** + * The selection as sent: only explicitly picked options whose values the + * model still accepts (T3's normalizeSelectionOptions); none at all leaves + * the provider's defaults in charge. + */ +export function dispatchModelSelection(providers: readonly T3ServerProvider[], selection: T3ModelSelection): T3ModelSelection { + const bare = { instanceId: selection.instanceId, model: selection.model }; + const picked = selectionOptions(selection); + if (!picked.length) return bare; + const model = findModel(providers, selection); + if (!model?.capabilities) return { ...bare, options: picked }; + const options: T3ModelOptionSelection[] = []; + for (const descriptor of model.capabilities.optionDescriptors ?? []) { + if (!picked.some((option) => option.id === descriptor.id)) continue; + const value = optionValue(descriptor, selection); + if (typeof value === "string" || typeof value === "boolean") options.push({ id: descriptor.id, value }); + } + return options.length ? { ...bare, options } : bare; +} + +// --------------------------------------------------------------------------- +// Access (runtime mode) + +export const RUNTIME_MODE_CHOICES: ReadonlyArray<{ mode: string; label: string; description: string }> = [ + { mode: "approval-required", label: "Supervised", description: "Ask before commands and file changes." }, + { mode: "auto-accept-edits", label: "Auto-accept edits", description: "Auto-approve edits, ask before other actions." }, + { mode: "auto", label: "Auto", description: "Supported providers approve routine actions; others still ask." }, + { mode: "full-access", label: "Full access", description: "Allow commands and edits without prompts." }, +]; + +const DEFAULT_RUNTIME_MODE = "full-access"; + +/** The access modes the provider supports (all of them when it doesn't say). */ +export function runtimeModeChoices(provider: T3ServerProvider | null | undefined): typeof RUNTIME_MODE_CHOICES { + const supported = provider?.supportedRuntimeModes; + if (!supported?.length) return RUNTIME_MODE_CHOICES; + const filtered = RUNTIME_MODE_CHOICES.filter((choice) => supported.includes(choice.mode)); + return filtered.length ? filtered : RUNTIME_MODE_CHOICES; +} + +export function runtimeModeLabel(mode: string): string { + return RUNTIME_MODE_CHOICES.find((choice) => choice.mode === mode)?.label ?? mode; +} + +// --------------------------------------------------------------------------- +// Workspace and branch + +export type WorkspaceMode = "local" | "worktree"; + +/** Project override, then environment setting, then the repo's t3.json, then "local". */ +export function defaultWorkspaceMode(settings: Settings, project: T3ProjectShell, projectFile: T3ProjectFileDefaults | null | undefined): WorkspaceMode { + const value = projectSetting(settings, project, "defaultThreadEnvMode"); + if (value === "local" || value === "worktree") return value; + return projectFile?.defaultThreadEnvMode ?? "local"; +} + +function defaultStartFromOrigin(settings: Settings, project: T3ProjectShell): boolean { + const value = projectSetting(settings, project, "newWorktreesStartFromOrigin"); + return typeof value === "boolean" ? value : true; +} + +/** The checked-out branch's worktree, when it isn't the project checkout itself. */ +function separateWorktree(ref: T3VcsRef, project: T3ProjectShell): string | null { + return ref.worktreePath && ref.worktreePath !== project.workspaceRoot ? ref.worktreePath : null; +} + +/** A short tag for a branch in the picker. */ +export function refBadge(ref: T3VcsRef, project: T3ProjectShell): string { + if (ref.current) return "current"; + if (separateWorktree(ref, project)) return "worktree"; + if (ref.isRemote) return "remote"; + if (ref.isDefault) return "default"; + return ""; +} + +/** + * Picking this branch for a thread in the project checkout means switching + * that checkout to it first (vcs.switchRef), as T3 mobile does. Branches + * already checked out somewhere run where they are. + */ +export function needsCheckout(ref: T3VcsRef, mode: WorkspaceMode): boolean { + return mode === "local" && !ref.current && !ref.worktreePath; +} + +// --------------------------------------------------------------------------- +// Draft → launch + +/** The user's explicit picks; null fields take the resolved default. */ +export type LaunchDraft = { + model: T3ModelSelection | null; + runtimeMode: string | null; + workspaceMode: WorkspaceMode | null; + /** Local: the branch the thread runs on; worktree: the base branch. */ + branch: string | null; + /** Local: run in this existing worktree (the picked branch is checked out there). */ + worktreePath: string | null; + startFromOrigin: boolean | null; +}; + +export function emptyLaunchDraft(): LaunchDraft { + return { model: null, runtimeMode: null, workspaceMode: null, branch: null, worktreePath: null, startFromOrigin: null }; +} + +export type LaunchContext = { + settings: Settings; + providers: readonly T3ServerProvider[]; + project: T3ProjectShell; + scratchWorkspaceRoot?: string | null; + /** The repo's t3.json defaults: undefined while unread, null when absent. */ + projectFile?: T3ProjectFileDefaults | null; + /** vcs.listRefs for the project, or null until loaded. */ + refs: T3RefList | null; +}; + +export type ResolvedLaunch = { + model: T3ModelSelection | null; + runtimeMode: string; + /** False for the Scratch project and for folders that aren't repositories. */ + canChooseWorkspace: boolean; + workspaceMode: WorkspaceMode; + branch: string | null; + worktreePath: string | null; + startFromOrigin: boolean; + /** Why the thread can't start yet (nothing to run it with, no base branch). */ + blocker: string | null; +}; + +export function resolveLaunch(draft: LaunchDraft, context: LaunchContext): ResolvedLaunch { + const { settings, providers, project, refs } = context; + const model = draft.model ?? defaultModelSelection(settings, providers, project); + const choices = runtimeModeChoices(findProvider(providers, model?.instanceId)); + const wantedMode = draft.runtimeMode ?? (projectSetting(settings, project, "defaultRuntimeMode") as string | null | undefined) ?? DEFAULT_RUNTIME_MODE; + const runtimeMode = choices.some((choice) => choice.mode === wantedMode) ? wantedMode : choices[0]!.mode; + const canChooseWorkspace = !isScratchProject(project, context.scratchWorkspaceRoot) && refs?.isRepo !== false; + const workspaceMode: WorkspaceMode = canChooseWorkspace ? (draft.workspaceMode ?? defaultWorkspaceMode(settings, project, context.projectFile)) : "local"; + const startFromOrigin = draft.startFromOrigin ?? defaultStartFromOrigin(settings, project); + const localRefs = (refs?.refs ?? []).filter((ref) => !ref.isRemote); + let branch: string | null; + let worktreePath: string | null = null; + if (workspaceMode === "worktree") { + // The default branch may only exist as origin/, so search every ref for it. + branch = draft.branch ?? refs?.refs.find((ref) => ref.isDefault)?.name ?? localRefs.find((ref) => ref.current)?.name ?? null; + } else if (draft.branch) { + branch = draft.branch; + worktreePath = draft.worktreePath; + } else { + const current = localRefs.find((ref) => ref.current); + branch = current?.name ?? null; + worktreePath = current ? separateWorktree(current, project) : null; + } + let blocker: string | null = null; + if (!model) blocker = "No usable agent on that computer: sign in to Claude Code, Codex or another provider in T3 Code first."; + else if (workspaceMode === "worktree" && !branch) blocker = refs ? "Choose a base branch for the worktree." : "Still loading branches."; + return { model, runtimeMode, canChooseWorkspace, workspaceMode, branch, worktreePath, startFromOrigin, blocker }; +} + +/** "Current checkout", "Existing worktree" or "New worktree". */ +export function workspaceLabel(resolved: Pick): string { + if (resolved.workspaceMode === "worktree") return "New worktree"; + return resolved.worktreePath ? "Existing worktree" : "Current checkout"; +} + +/** The branch row: "main", or a worktree's base "From origin/main". */ +export function branchLabel(resolved: Pick, hasPrimaryRemote: boolean): string { + if (!resolved.branch) return resolved.workspaceMode === "worktree" ? "Choose branch" : "Unknown"; + if (resolved.workspaceMode === "local") return resolved.branch; + const fromOrigin = resolved.startFromOrigin && hasPrimaryRemote && !resolved.branch.startsWith("origin/"); + return `From ${fromOrigin ? `origin/${resolved.branch}` : resolved.branch}`; +} + +/** The draft after picking a branch (or worktree base) in the given mode. */ +export function pickBranch(draft: LaunchDraft, ref: T3VcsRef, mode: WorkspaceMode, project: T3ProjectShell): LaunchDraft { + return { ...draft, branch: ref.name, worktreePath: mode === "local" ? separateWorktree(ref, project) : null }; +} + +/** + * The draft after switching workspace mode. A new worktree keeps an explicitly + * picked base; the project checkout goes back to whatever is checked out. + */ +export function pickWorkspaceMode(draft: LaunchDraft, mode: WorkspaceMode): LaunchDraft { + return mode === "local" + ? { ...draft, workspaceMode: mode, branch: null, worktreePath: null } + : { ...draft, workspaceMode: mode, worktreePath: null }; +} + +/** Where the server runs the thread (OrchestrationV2 workspaceStrategy). */ +export function workspaceStrategy(resolved: ResolvedLaunch): Record { + if (resolved.workspaceMode === "worktree") { + return { type: "worktree", baseRef: resolved.branch, ...(resolved.startFromOrigin ? { startFromOrigin: true } : {}) }; + } + if (resolved.worktreePath) { + return { type: "existing_worktree", worktreePath: resolved.worktreePath, ...(resolved.branch ? { branch: resolved.branch } : {}) }; + } + return { type: "root", ...(resolved.branch ? { branch: resolved.branch } : {}) }; +} + +/** First line of a prompt as a provisional title (the server generates a better one). */ +export function provisionalTitle(text: string): string { + const collapsed = text.replace(/\s+/g, " ").trim(); + if (!collapsed) return "New thread"; + return collapsed.length > 72 ? `${collapsed.slice(0, 71).trimEnd()}…` : collapsed; +} + +/** The orchestration.launchThread payload. Throws when resolveLaunch reported a blocker. */ +export function launchThreadPayload( + project: T3ProjectShell, + text: string, + resolved: ResolvedLaunch, + providers: readonly T3ServerProvider[], +): Record { + if (resolved.blocker || !resolved.model) throw new Error(resolved.blocker ?? "No model selected."); + return { + commandId: uuidv4(), + creationSource: "mobile", + threadId: uuidv4(), + projectId: project.id, + title: provisionalTitle(text), + generateTitle: true, + modelSelection: dispatchModelSelection(providers, resolved.model), + runtimeMode: resolved.runtimeMode, + interactionMode: "default", + workspaceStrategy: workspaceStrategy(resolved), + initialMessage: { messageId: uuidv4(), text, attachments: [] }, + }; +} diff --git a/app/apps/t3code/t3-model.ts b/app/apps/t3code/t3-model.ts index 63ecab1e..9d356797 100644 --- a/app/apps/t3code/t3-model.ts +++ b/app/apps/t3code/t3-model.ts @@ -3,6 +3,7 @@ * transport and rendering: the shell (projects + thread list) reducer, the * per-thread detail reducer, thread status / ordering for the glasses list, * one-line summaries of timeline items, and the dispatch-command builders. + * (New-thread options and the launch payload are in t3-launch.ts.) * * Mirrors T3's own client runtime (packages/client-runtime/src/state/ * shellReducer.ts, orchestrationV2Projection.ts, models.ts; apps/mobile's @@ -18,10 +19,16 @@ export type T3ProjectShell = { id: string; title: string; workspaceRoot?: string; + repositoryIdentity?: { canonicalKey?: string | null } | null; + /** Legacy per-project defaults (superseded by settings.projectSettingsOverrides once folded). */ defaultModelSelection?: T3ModelSelection | null; + defaultThreadEnvMode?: string | null; }; -export type T3ModelSelection = { instanceId: string; model: string; options?: unknown[] }; +/** A provider model option picked for a run ("effort" = "high", "fastMode" = true). */ +export type T3ModelOptionSelection = { id: string; value: string | boolean }; + +export type T3ModelSelection = { instanceId: string; model: string; options?: T3ModelOptionSelection[] }; export type T3ThreadShell = { id: string; @@ -839,86 +846,3 @@ export function approvalOptions(item: T3TurnItem): T3ApprovalOption[] { export function optionAnswer(option: { label: string; value?: string }): string { return option.value ?? option.label; } - -// --------------------------------------------------------------------------- -// New threads - -export type T3ServerProvider = { - instanceId: string; - displayName?: string; - driver?: string; - enabled?: boolean; - installed?: boolean; - auth?: { status?: string }; - models?: Array<{ slug: string; name?: string; isDefault?: boolean; isLegacy?: boolean }>; -}; - -function providerUsable(provider: T3ServerProvider): boolean { - return provider.enabled !== false && provider.installed !== false && provider.auth?.status !== "unauthenticated"; -} - -export function providerDisplayName(providers: readonly T3ServerProvider[], instanceId: string | null | undefined): string { - if (!instanceId) return ""; - const provider = providers.find((candidate) => candidate.instanceId === instanceId); - return provider?.displayName || provider?.driver || instanceId; -} - -/** - * Model for a new thread, the way T3's mobile app defaults it: project - * override, environment default, the project's legacy default, then the - * first usable provider's default model, then any usable model. - */ -export function defaultModelSelection( - settings: Record | null, - providers: readonly T3ServerProvider[], - project: T3ProjectShell, -): T3ModelSelection | null { - const usable = (selection: T3ModelSelection | null | undefined): selection is T3ModelSelection => { - if (!selection?.instanceId || !selection.model) return false; - const provider = providers.find((candidate) => candidate.instanceId === selection.instanceId); - return provider ? providerUsable(provider) : providers.length === 0; - }; - const candidates = [ - settings?.projectSettingsOverrides?.[project.id]?.defaultModelSelection, - settings?.defaultModelSelection, - project.defaultModelSelection, - ]; - for (const candidate of candidates) if (usable(candidate)) return candidate; - for (const pickDefault of [true, false]) { - for (const provider of providers) { - if (!providerUsable(provider)) continue; - const model = (provider.models ?? []).find((entry) => !entry.isLegacy && (!pickDefault || entry.isDefault)); - if (model) return { instanceId: provider.instanceId, model: model.slug }; - } - } - return null; -} - -/** First line of a prompt as a provisional title (the server generates a better one). */ -export function provisionalTitle(text: string): string { - const collapsed = oneLine(text); - if (!collapsed) return "New thread"; - return collapsed.length > 72 ? `${collapsed.slice(0, 71).trimEnd()}…` : collapsed; -} - -export function launchThreadPayload( - project: T3ProjectShell, - text: string, - modelSelection: T3ModelSelection, - settings: Record | null, -): Record { - const runtimeMode = settings?.projectSettingsOverrides?.[project.id]?.defaultRuntimeMode ?? settings?.defaultRuntimeMode ?? "full-access"; - return { - commandId: uuidv4(), - creationSource: "mobile", - threadId: uuidv4(), - projectId: project.id, - title: provisionalTitle(text), - generateTitle: true, - modelSelection, - runtimeMode, - interactionMode: "default", - workspaceStrategy: { type: "root" }, - initialMessage: { messageId: uuidv4(), text, attachments: [] }, - }; -} diff --git a/app/apps/t3code/t3code-app.worker.ts b/app/apps/t3code/t3code-app.worker.ts index b4db75f4..ec2697fc 100644 --- a/app/apps/t3code/t3code-app.worker.ts +++ b/app/apps/t3code/t3code-app.worker.ts @@ -15,7 +15,9 @@ * chosen from the menu). Tap-then-hold has Stop, Settle, etc. * - environments / pair: manage paired servers; pairing takes the link from * `t3 pair`, typed into the phone app's text editor. - * - compose: the first message of a new thread in a chosen project. + * - compose: a new thread's options (host, project, model and its options, + * access mode, workspace/worktree, branch), then its first message by + * voice or phone keyboard. * Double-click backs out a level, and from the list yields focus. * * Connections (one T3EnvironmentClient per paired environment) live while the @@ -39,7 +41,7 @@ import { openSocket } from "../../native/socket"; import { fetchWithUserAgent } from "../../util/http"; import { drawSubmenuIndicator, submenuItem, type MenuItem } from "../../ui/menu"; import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; -import { lineStep, listRowHeight } from "../../ui/metrics"; +import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; import { WindowMenu, WindowMenuLayer } from "../../ui/window-menu"; import type { WorkerAppMessage, WorkerAppReply } from "../../ui/shell/worker-window"; import type { ToolResult, ToolSpec } from "../../assistant/tool-registry"; @@ -57,17 +59,14 @@ import { buildGlanceSnapshot, glanceSummary, T3_GLANCE_STATE_KEY } from "./t3-gl import { approvalOptions, approvalResponseCommand, - defaultModelSelection, describeRequest, formatAge, interruptCommand, isThreadSettled, isThreadUnread, - launchThreadPayload, messageDispatchCommand, optionAnswer, plainMarkdown, - providerDisplayName, questionAnswerCommand, sectionThreads, settleCommand, @@ -78,10 +77,43 @@ import { timelineEntries, visitCommand, type PendingRequest, + type T3ModelSelection, type T3ProjectShell, type T3ThreadShell, type T3ThreadStatus, } from "./t3-model"; +import { + branchLabel, + emptyLaunchDraft, + isScratchProject, + launchThreadPayload, + matchingProject, + modelChoices, + modelDisplayName, + modelOptionDescriptors, + needsCheckout, + optionValue, + optionValueLabel, + parseProjectFile, + pickBranch, + pickWorkspaceMode, + providerDisplayName, + refBadge, + resolveLaunch, + runtimeModeChoices, + runtimeModeLabel, + selectableChoices, + T3_PROJECT_FILE_NAME, + withModelOption, + workspaceLabel, + type LaunchDraft, + type ResolvedLaunch, + type T3OptionDescriptor, + type T3ProjectFileDefaults, + type T3RefList, + type T3VcsRef, + type WorkspaceMode, +} from "./t3-launch"; import { parsePairingInput, baseUrlLabel } from "./t3-pairing"; import type { RpcSocketFactory } from "./t3-rpc"; import { TranscriptLayout } from "./t3-transcript"; @@ -118,7 +150,7 @@ type Screen = | { kind: "thread"; envId: string; threadId: string } | { kind: "environments" } | { kind: "pair"; repairId: string | null } - | { kind: "compose"; envId: string; projectId: string }; + | { kind: "compose" }; type AppWindow = { windowId: string; @@ -157,9 +189,10 @@ type ThreadView = { type ListItem = { key: string; - kind: "heading" | "thread" | "action"; + /** "field": a new-thread option, its name in a column and `right` as its value. */ + kind: "heading" | "thread" | "action" | "field"; label: string; - /** Dim text at the right edge (project, age, status). */ + /** Dim text at the right edge (project, age, status); a field's value. */ right?: string; /** Status marker drawn left of a thread title. */ marker?: string; @@ -184,6 +217,25 @@ let noticeTimer: ReturnType | null = null; let busyText = ""; let screenError = ""; +/** The new thread being set up on the compose screen. */ +type ComposeState = { + envId: string; + projectId: string; + draft: LaunchDraft; + /** vcs.listRefs for the project: null while loading, or when it failed (refsError). */ + refs: T3RefList | null; + refsError: string; + /** The repo's t3.json defaults: undefined while reading, null when it has none. */ + projectFile: T3ProjectFileDefaults | null | undefined; + /** Settles when the branch list and t3.json reads in flight finish. */ + loading: Promise; +}; + +let compose: ComposeState | null = null; +let composeSelectionKey: string | null = null; +/** Where field values start on the compose screen (set per paint from the field names). */ +let composeValueColumn = 0; + const clients = new Map(); const clientUnsubscribers = new Map void>(); /** The label last saved for each environment. */ @@ -276,7 +328,8 @@ const T3_TOOLS: ToolSpec[] = [ }, { name: "start_thread", - description: "Start a new T3 Code thread in a project (found by matching its name) with a first message for the agent.", + description: + "Start a new T3 Code thread in a project (found by matching its name) with a first message for the agent. Uses the project's defaults for model, access mode and workspace.", inputSchema: { type: "object", properties: { @@ -339,6 +392,7 @@ global.onmessage = (event: { data: WorkerAppMessage }) => { if (window.renderTimer) clearTimeout(window.renderTimer); window = null; closeThreadView(); + compose = null; screen = { kind: "list" }; // The Glanceboard widget may still want the connections. if (!hasT3BackgroundWork()) stopAllClients(); @@ -582,10 +636,11 @@ function noteAttention(client: T3EnvironmentClient): void { function setScreen(next: Screen): void { leaveScreenSideEffects(); if (next.kind !== "thread") closeThreadView(); + if (next.kind !== "compose") compose = null; screen = next; busyText = ""; screenError = ""; - if (next.kind === "list" || next.kind === "environments") listMenu?.select(0); + if (next.kind === "list" || next.kind === "environments" || next.kind === "compose") listMenu?.select(0); if (window) post({ type: "set-title", windowId: window.windowId, title: next.kind === "thread" ? threadTitle() : APP_TITLE }); updateListRefreshTimer(); } @@ -689,10 +744,10 @@ function showNotice(text: string): void { * From inside another menu's item the old menu is still closing (WindowMenu * ignores open() until its input handling finishes), so wait a tick. */ -function openWindowMenu(items: MenuItem[], title: string): void { +function openWindowMenu(items: MenuItem[], title: string, selectedIndex?: number): void { const open = () => { if (!window) return; - windowMenu(window).open(items, title); + windowMenu(window).open(items, title, selectedIndex); renderNow(); }; if (window?.menu?.isOpen()) setTimeout(open, 0); @@ -732,6 +787,7 @@ function handleInput(win: AppWindow, event: InputEvent, frameId: number): void { switch (screen.kind) { case "list": case "environments": + case "compose": handleListInput(event, frameId); return; case "thread": @@ -744,13 +800,6 @@ function handleInput(win: AppWindow, event: InputEvent, frameId: number): void { return; } break; - case "compose": - if (event.type === "click" && !busyText) { - post({ type: "start-voice-input", windowId: win.windowId }); - frameTimings.finishFrame(frameId, "discarded: t3code opened voice input"); - return; - } - break; } frameTimings.finishFrame(frameId, "discarded: t3code ignored input"); } @@ -761,14 +810,26 @@ function handleListInput(event: InputEvent, frameId: number): void { return; } const menu = list(); - const items = screen.kind === "environments" ? environmentItems() : listItems(); + const items = currentListItems(); menu.setItems(items, selectionIndex(items)); // Item actions are synchronous, so the move or action has applied when this returns. menu.handleInput(event).catch((error) => console.error("t3code list action failed", error)); - if (event.type !== "click") listSelectionKey = menu.selectedItem?.key ?? null; + if (event.type !== "click") setSelectedKey(menu.selectedItem?.key ?? null); renderNow(frameId); } +/** The rows of whichever list-shaped screen is showing. */ +function currentListItems(): ListItem[] { + switch (screen.kind) { + case "environments": + return environmentItems(); + case "compose": + return composeItems(); + default: + return listItems(); + } +} + function handleThreadInput(event: InputEvent, frameId: number): void { const view = threadView; if (!view) { @@ -825,7 +886,7 @@ function handleTextInput(text: string): void { render(); return; case "compose": - void launchNewThread(screen.envId, screen.projectId, text); + void launchNewThread(text); return; case "thread": { const view = threadView; @@ -1081,8 +1142,7 @@ function projectMenuItems(): MenuItem[] { label: multiEnv ? `${project.title} · ${environmentName(client)}` : project.title, onSelect: (ctx) => { ctx.stack.clearToBase(); - setScreen({ kind: "compose", envId: client.config.id, projectId: project.id }); - if (window) post({ type: "start-voice-input", windowId: window.windowId }); + beginCompose(client, project); }, })); } @@ -1156,37 +1216,432 @@ async function pairFromDraft(): Promise { // --------------------------------------------------------------------------- // New threads -function composeTarget(): { client: T3EnvironmentClient; project: T3ProjectShell } | null { - if (screen.kind !== "compose") return null; - const client = clients.get(screen.envId); - const project = client?.shell.projects.get(screen.projectId); - return client && project ? { client, project } : null; +/** Most branches a picker lists (vcs.listRefs pages past this aren't fetched). */ +const BRANCH_LIST_LIMIT = 100; + +type ComposeTarget = { client: T3EnvironmentClient; project: T3ProjectShell; state: ComposeState }; + +function composeTarget(): ComposeTarget | null { + if (screen.kind !== "compose" || !compose) return null; + const client = clients.get(compose.envId); + const project = client?.shell.projects.get(compose.projectId); + return client && project ? { client, project, state: compose } : null; } -async function launchNewThread(envId: string, projectId: string, text: string): Promise { - const client = clients.get(envId); - const project = client?.shell.projects.get(projectId); - if (!client || !project) return; - const selection = defaultModelSelection(client.settings, client.providers, project); - if (!selection) { - screenError = "No usable agent on that computer: sign in to Claude Code, Codex or another provider in T3 Code first."; +/** Show the new-thread options for a project, keeping whatever `draft` picks still apply there. */ +function beginCompose(client: T3EnvironmentClient, project: T3ProjectShell, draft: LaunchDraft = emptyLaunchDraft()): void { + const state: ComposeState = { + envId: client.config.id, + projectId: project.id, + draft, + refs: null, + refsError: "", + projectFile: undefined, + loading: Promise.resolve(), + }; + if (screen.kind === "compose") { + // Another host or project: keep the selection on the row that changed it. + compose = state; + screenError = ""; + } else { + compose = state; + composeSelectionKey = null; + setScreen({ kind: "compose" }); + } + loadComposeContext(state); + render(); +} + +/** Read the project's branches and t3.json (defaults and the branch picker). */ +function loadComposeContext(state: ComposeState): void { + const client = clients.get(state.envId); + const project = client?.shell.projects.get(state.projectId); + if (!client || !project?.workspaceRoot || isScratchProject(project, client.scratchWorkspaceRoot)) { + state.projectFile = null; + return; + } + state.refsError = ""; + const refs = fetchRefs(client, project, true).then( + (refList) => { + if (compose !== state) return; + state.refs = refList; + render(); + }, + (error) => { + if (compose !== state) return; + state.refsError = errorMessage(error); + render(); + }, + ); + const projectFile = + state.projectFile !== undefined + ? Promise.resolve() + : readProjectFile(client, project).then((file) => { + if (compose !== state) return; + state.projectFile = file; + render(); + }); + state.loading = Promise.all([refs, projectFile]).then(() => {}); +} + +function fetchRefs(client: T3EnvironmentClient, project: T3ProjectShell, refresh: boolean): Promise { + // refresh: the cached list's "current" flag can lag a checkout made outside T3 by minutes. + return client.call("vcs.listRefs", { cwd: project.workspaceRoot, limit: BRANCH_LIST_LIMIT, ...(refresh ? { refresh: true } : {}) }); +} + +/** The repo's t3.json defaults; null when it has none or it can't be read. */ +async function readProjectFile(client: T3EnvironmentClient, project: T3ProjectShell): Promise { + try { + const result = await client.call<{ contents?: unknown; truncated?: boolean }>("projects.readFile", { + cwd: project.workspaceRoot, + relativePath: T3_PROJECT_FILE_NAME, + }); + return typeof result?.contents === "string" && !result.truncated ? parseProjectFile(result.contents) : null; + } catch { + return null; + } +} + +function composeResolved({ client, project, state }: ComposeTarget): ResolvedLaunch { + return resolveLaunch(state.draft, { + settings: client.settings, + providers: client.providers, + project, + scratchWorkspaceRoot: client.scratchWorkspaceRoot, + projectFile: state.projectFile, + refs: state.refs, + }); +} + +/** A new-thread option row; `action` opens its picker (or toggles it). */ +function field(key: string, label: string, value: string, action?: () => void, submenu = true): ListItem { + return { + key: `field:${key}`, + kind: "field", + label, + right: value, + submenu: Boolean(action) && submenu, + onSelect: action + ? () => { + if (!busyText) action(); + } + : undefined, + }; +} + +function composeItems(): ListItem[] { + const target = composeTarget(); + if (!target) return []; + const { client, project, state } = target; + const resolved = composeResolved(target); + const model = resolved.model; + const items: ListItem[] = [ + { + key: "start", + kind: "action", + label: "Say what to do…", + right: "or type it on the phone", + onSelect: () => { + if (!busyText && window) post({ type: "start-voice-input", windowId: window.windowId }); + }, + }, + field("host", "Host", environmentName(client), readyClients().length > 1 ? () => openHostPicker(target) : undefined), + field("project", "Project", project.title, () => openProjectPicker(target)), + field( + "model", + "Model", + model ? modelRowValue(client, model) : "None available", + modelChoices(client.providers).length ? () => openModelPicker(target, resolved) : undefined, + ), + ]; + if (model) { + for (const descriptor of modelOptionDescriptors(client.providers, model)) { + const value = optionValue(descriptor, model); + const shown = optionValueLabel(descriptor, value); + if (descriptor.type === "boolean") { + items.push(field(`option:${descriptor.id}`, descriptor.label, shown, () => setModelOption(target, resolved, descriptor.id, !value), false)); + } else if (selectableChoices(descriptor).length) { + items.push(field(`option:${descriptor.id}`, descriptor.label, shown, () => openOptionPicker(target, resolved, descriptor))); + } + } + } + const accessChoices = runtimeModeChoices(client.providers.find((provider) => provider.instanceId === model?.instanceId)); + items.push(field("access", "Access", runtimeModeLabel(resolved.runtimeMode), accessChoices.length > 1 ? () => openAccessPicker(target, resolved) : undefined)); + if (!resolved.canChooseWorkspace) { + const why = isScratchProject(project, client.scratchWorkspaceRoot) ? "Own folder (no project)" : "Project folder (not a git repo)"; + items.push(field("workspace", "Workspace", why)); + return items; + } + items.push(field("workspace", "Workspace", workspaceLabel(resolved), () => openWorkspacePicker(target, resolved))); + const branchName = resolved.workspaceMode === "worktree" ? "Base" : "Branch"; + if (state.refsError) { + items.push(field("branch", branchName, "Couldn't load; retry", () => { + loadComposeContext(state); + render(); + }, false)); + } else if (!state.refs && !resolved.branch) { + items.push(field("branch", branchName, "Loading…")); + } else { + items.push(field("branch", branchName, branchLabel(resolved, Boolean(state.refs?.hasPrimaryRemote)), state.refs ? () => openBranchPicker(target, resolved) : undefined)); + } + if (resolved.workspaceMode === "worktree" && state.refs?.hasPrimaryRemote) { + items.push(field("origin", "From origin", resolved.startFromOrigin ? "On" : "Off", () => { + state.draft = { ...state.draft, startFromOrigin: !resolved.startFromOrigin }; + }, false)); + } + return items; +} + +/** "Claude Opus 5", or "Codex · GPT-5.4" when the model's name doesn't say whose it is. */ +function modelRowValue(client: T3EnvironmentClient, model: T3ModelSelection): string { + const provider = providerDisplayName(client.providers, model.instanceId); + const name = modelDisplayName(client.providers, model); + return name.toLowerCase().startsWith(provider.toLowerCase()) ? name : `${provider} · ${name}`; +} + +/** A picker row, " *" marking the current value, with an optional dim tag at the right. */ +function choiceItem(label: string, onSelect: MenuItem["onSelect"], options: { current?: boolean; right?: string } = {}): MenuItem { + const text = options.current ? `${label} *` : label; + const right = options.right; + return { + label: text, + onSelect, + render: right + ? ({ image, x, y, width, selected, disabled }) => { + const f = font(); + const rightWidth = f.measureText(right); + image.drawText(f, x, y + LIST_ROW_TEXT_INSET, truncateText(f, text, width - rightWidth - 12), disabled ? 70 : selected ? 255 : 200); + image.drawText(f, x + width - rightWidth - 2, y + LIST_ROW_TEXT_INSET, right, selected ? 200 : 120); + } + : undefined, + }; +} + +/** Open a picker on its current row. */ +function openChoices(items: MenuItem[], title: string, currentIndex: number): void { + openWindowMenu(items, title, currentIndex >= 0 ? currentIndex : undefined); +} + +function openHostPicker({ client, project }: ComposeTarget): void { + const hosts = readyClients(); + const items = hosts.map((host) => + choiceItem( + environmentName(host), + (ctx) => { + ctx.stack.pop(); + if (host === client) return; + // The same repository there if it has it, else its most recently used project. + const target = matchingProject(host.shell.projects.values(), project) ?? projectChoices().find((choice) => choice.client === host)?.project; + if (!target) { + showNotice(`${environmentName(host)} has no projects.`); + return; + } + beginCompose(host, target); + }, + { current: host === client }, + ), + ); + openChoices(items, "Host", hosts.indexOf(client)); +} + +function openProjectPicker({ client, project, state }: ComposeTarget): void { + const projects = projectChoices() + .filter((choice) => choice.client === client) + .map((choice) => choice.project); + const items = projects.map((candidate) => + choiceItem( + candidate.title, + (ctx) => { + ctx.stack.pop(); + if (candidate.id === project.id) return; + // The model and access picks carry over; workspace and branch belong to the old repo. + beginCompose(client, candidate, { ...emptyLaunchDraft(), model: state.draft.model, runtimeMode: state.draft.runtimeMode }); + }, + { current: candidate.id === project.id }, + ), + ); + openChoices(items, "Project", projects.findIndex((candidate) => candidate.id === project.id)); +} + +function openModelPicker({ client, state }: ComposeTarget, resolved: ResolvedLaunch): void { + const current = resolved.model; + const isCurrent = (instanceId: string, model: string) => current?.instanceId === instanceId && current.model === model; + const groups = modelChoices(client.providers); + const modelItems = (group: (typeof groups)[number]) => + group.models.map((choice) => + choiceItem( + choice.label, + (ctx) => { + ctx.stack.clearToBase(); + // A different model starts from its own option defaults. + if (!isCurrent(choice.selection.instanceId, choice.selection.model)) state.draft = { ...state.draft, model: choice.selection }; + }, + { current: isCurrent(choice.selection.instanceId, choice.selection.model), right: choice.isLegacy ? "legacy" : undefined }, + ), + ); + const currentModelIndex = (group: (typeof groups)[number]) => group.models.findIndex((choice) => isCurrent(choice.selection.instanceId, choice.selection.model)); + if (groups.length === 1) { + openChoices(modelItems(groups[0]!), `${groups[0]!.label} model`, currentModelIndex(groups[0]!)); + return; + } + const items = groups.map((group) => + submenuItem(group.instanceId === current?.instanceId ? `${group.label} *` : group.label, (ctx) => + ctx.stack.push(new WindowMenuLayer(`${group.label} model`, modelItems(group)).selectItem(Math.max(0, currentModelIndex(group)))), + ), + ); + openChoices(items, "Model", groups.findIndex((group) => group.instanceId === current?.instanceId)); +} + +function setModelOption({ state }: ComposeTarget, resolved: ResolvedLaunch, id: string, value: string | boolean): void { + if (!resolved.model) return; + state.draft = { ...state.draft, model: withModelOption(resolved.model, id, value) }; +} + +function openOptionPicker(target: ComposeTarget, resolved: ResolvedLaunch, descriptor: Extract): void { + const value = optionValue(descriptor, resolved.model); + const choices = selectableChoices(descriptor); + const items = choices.map((choice) => + choiceItem( + choice.label, + (ctx) => { + ctx.stack.pop(); + setModelOption(target, resolved, descriptor.id, choice.id); + }, + { current: choice.id === value }, + ), + ); + openChoices(items, descriptor.label, choices.findIndex((choice) => choice.id === value)); +} + +function openAccessPicker({ client, state }: ComposeTarget, resolved: ResolvedLaunch): void { + const choices = runtimeModeChoices(client.providers.find((provider) => provider.instanceId === resolved.model?.instanceId)); + const items = choices.map((choice) => + choiceItem( + choice.label, + (ctx) => { + ctx.stack.pop(); + state.draft = { ...state.draft, runtimeMode: choice.mode }; + }, + { current: choice.mode === resolved.runtimeMode }, + ), + ); + openChoices(items, "Access", choices.findIndex((choice) => choice.mode === resolved.runtimeMode)); +} + +function openWorkspacePicker({ state }: ComposeTarget, resolved: ResolvedLaunch): void { + const modes: Array<{ mode: WorkspaceMode; label: string }> = [ + { mode: "local", label: "Current checkout" }, + { mode: "worktree", label: "New worktree" }, + ]; + const items = modes.map(({ mode, label }) => + choiceItem( + label, + (ctx) => { + ctx.stack.pop(); + if (mode !== resolved.workspaceMode || state.draft.worktreePath) state.draft = pickWorkspaceMode(state.draft, mode); + }, + { current: mode === resolved.workspaceMode && !resolved.worktreePath }, + ), + ); + openChoices(items, "Workspace", modes.findIndex(({ mode }) => mode === resolved.workspaceMode)); +} + +function openBranchPicker(target: ComposeTarget, resolved: ResolvedLaunch): void { + const { project, state } = target; + const mode = resolved.workspaceMode; + // A thread in the checkout runs on a local branch; a worktree can start from a remote one too. + const refs = (state.refs?.refs ?? []).filter((ref) => mode === "worktree" || !ref.isRemote); + const items: MenuItem[] = refs.map((ref) => + choiceItem( + ref.name, + (ctx) => { + ctx.stack.pop(); + void selectBranch(target, ref, mode); + }, + { current: ref.name === resolved.branch, right: needsCheckout(ref, mode) ? "check out" : refBadge(ref, project) || undefined }, + ), + ); + const hidden = (state.refs?.totalCount ?? 0) - (state.refs?.refs.length ?? 0); + if (state.refs?.nextCursor != null && hidden > 0) items.push({ label: `(${hidden} more not listed)`, disabled: true, onSelect: () => {} }); + if (!items.length) items.push({ label: "No branches", disabled: true, onSelect: () => {} }); + openChoices(items, mode === "worktree" ? "New worktree from" : "Run on branch", refs.findIndex((ref) => ref.name === resolved.branch)); +} + +/** + * Use a branch. In the project checkout, a branch that isn't checked out + * anywhere is checked out first (vcs.switchRef), as T3's own app does; + * one checked out in another worktree runs there. + */ +async function selectBranch({ client, project, state }: ComposeTarget, ref: T3VcsRef, mode: WorkspaceMode): Promise { + if (!needsCheckout(ref, mode)) { + state.draft = pickBranch(state.draft, ref, mode, project); render(); return; } + busyText = `Checking out ${ref.name}…`; + screenError = ""; + render(); + try { + const result = await client.call<{ refName?: string | null }>("vcs.switchRef", { cwd: project.workspaceRoot, refName: ref.name }); + if (compose !== state) return; + state.draft = pickBranch(state.draft, { ...ref, name: result?.refName || ref.name, current: true }, mode, project); + busyText = ""; + showNotice(`Checked out ${result?.refName || ref.name}`); + loadComposeContext(state); + } catch (error) { + if (compose !== state) return; + busyText = ""; + screenError = `Couldn't check out ${ref.name}: ${errorMessage(error)}`; + render(); + } +} + +async function launchNewThread(text: string): Promise { + const target = composeTarget(); + if (!target) return; + if (busyText) { + showNotice(`Not started: still ${busyText.replace(/…$/, "").toLowerCase()}.`); + return; + } + const { client, project, state } = target; busyText = "Starting thread…"; screenError = ""; render(); try { - const threadId = await client.launchThread(launchThreadPayload(project, text, selection, client.settings)); - if (screen.kind === "compose") openThread(envId, threadId); + // Defaults can depend on the branch list and t3.json. + await state.loading; + if (compose !== state) return; + const resolved = composeResolved(target); + if (resolved.blocker) throw new Error(resolved.blocker); + const threadId = await client.launchThread(launchThreadPayload(project, text, resolved, client.providers)); + if (compose === state) openThread(client.config.id, threadId); render(); } catch (error) { + if (compose !== state) return; busyText = ""; screenError = errorMessage(error); render(); } } +/** The project's own defaults for a new thread (the assistant tool has no screen to pick on). */ +async function defaultLaunch(client: T3EnvironmentClient, project: T3ProjectShell): Promise { + const scratch = isScratchProject(project, client.scratchWorkspaceRoot); + const usesRepo = Boolean(project.workspaceRoot) && !scratch; + const [projectFile, refs] = await Promise.all([ + usesRepo ? readProjectFile(client, project) : Promise.resolve(null), + usesRepo ? fetchRefs(client, project, false).catch(() => null) : Promise.resolve(null), + ]); + return resolveLaunch(emptyLaunchDraft(), { + settings: client.settings, + providers: client.providers, + project, + scratchWorkspaceRoot: client.scratchWorkspaceRoot, + projectFile, + refs, + }); +} + // --------------------------------------------------------------------------- // List items @@ -1208,11 +1663,17 @@ function list(): Menu { /** The selection index for the remembered key (selection follows its thread as the list reorders). */ function selectionIndex(items: readonly ListItem[]): number | undefined { - if (!listSelectionKey) return undefined; - const index = items.findIndex((item) => item.key === listSelectionKey); + const key = screen.kind === "compose" ? composeSelectionKey : listSelectionKey; + if (!key) return undefined; + const index = items.findIndex((item) => item.key === key); return index >= 0 ? index : undefined; } +function setSelectedKey(key: string | null): void { + if (screen.kind === "compose") composeSelectionKey = key; + else listSelectionKey = key; +} + function threadItem(client: T3EnvironmentClient, thread: T3ThreadShell, multiEnv: boolean, nowMs: number): ListItem { const { marker, value } = threadMarker(threadStatus(thread), { settled: isThreadSettled(thread), unread: isThreadUnread(thread) }); const project = client.shell.projects.get(thread.projectId)?.title ?? ""; @@ -1365,6 +1826,15 @@ function drawListRow({ image, item, x, y, width, height, selected }: MenuDrawArg image.drawText(f, x + 4, textY, truncateText(f, item.label, width - 8), selected ? 255 : 140); return; } + if (item.kind === "field") { + const submenuWidth = item.submenu ? 16 : 0; + const valueX = x + 6 + composeValueColumn; + image.drawText(f, x + 6, textY, truncateText(f, item.label, composeValueColumn - 8), selected ? 230 : 140); + const valueValue = selected ? 255 : item.onSelect ? 210 : 150; + image.drawText(f, valueX, textY, truncateText(f, item.right ?? "", Math.max(10, x + width - valueX - submenuWidth - 6)), valueValue); + if (item.submenu) drawSubmenuIndicator(image, f, x, y, width, height, selected ? 255 : 200); + return; + } const value = selected ? 255 : 200; const markerWidth = item.kind === "thread" ? 18 : 0; if (item.marker) image.drawText(f, x + 6, textY, item.marker, selected ? 255 : (item.markerValue ?? value)); @@ -1580,20 +2050,33 @@ function paintPair(win: AppWindow): GrayImage { function paintCompose(win: AppWindow): GrayImage { const target = composeTarget(); if (!target) return paintMessageScreen(win, "New thread", [{ text: "That project is no longer available.", value: 170 }], ""); - const { client, project } = target; - const selection = defaultModelSelection(client.settings, client.providers, project); - const agent = selection ? `${providerDisplayName(client.providers, selection.instanceId)} · ${selection.model}` : "No usable agent"; - return paintMessageScreen( - win, - `New thread: ${project.title}`, - [ - { text: `Agent: ${agent}`, value: 150 }, - { text: "Say or type what the agent should do.", value: 200 }, - { text: busyText, value: 200 }, - { text: screenError, value: 230 }, - ], - busyText ? "" : `${GESTURE_CLICK} speak ${GESTURE_DOUBLE_CLICK} cancel`, - ); + const image = new GrayImage(win.viewportWidth, win.viewportHeight, 0); + const f = font(); + const step = lineStep(f); + const title = "New thread"; + image.drawText(f, TITLE_X, TITLE_Y, title, 220); + const status = busyText || notice?.text || ""; + if (status) { + const statusX = TITLE_X + f.measureText(title) + 16; + image.drawText(f, statusX, TITLE_Y, truncateText(f, status, win.viewportWidth - statusX - 14), 150); + } + // Below the list: an error, or why nothing can start here. + const resolved = composeResolved(target); + const problem = screenError || (resolved.model ? "" : (resolved.blocker ?? "")); + const problemLines = problem ? wrapText(f, problem, win.viewportWidth - TITLE_X * 2).slice(0, 3) : []; + const listTop = TITLE_Y + step + 8; + const listWidth = win.viewportWidth - LIST_LEFT - LIST_RIGHT; + const listHeight = win.viewportHeight - 6 - listTop - (problemLines.length ? problemLines.length * step + 6 : 0); + const items = composeItems(); + const widest = Math.max(0, ...items.filter((item) => item.kind === "field").map((item) => f.measureText(item.label))); + composeValueColumn = Math.min(widest + 16, Math.floor(listWidth * 0.4)); + const menu = list(); + menu.setItems(items, selectionIndex(items)); + composeSelectionKey = menu.selectedItem?.key ?? composeSelectionKey; + menu.paint(image, { x: LIST_LEFT, y: listTop - 2, width: listWidth, height: listHeight }, win.focused); + menu.drawScrollbar(image, win.viewportWidth - 10, listTop, listHeight - 4); + problemLines.forEach((line, index) => image.drawText(f, TITLE_X, listTop + listHeight + 4 + index * step, line, 230)); + return image; } // --------------------------------------------------------------------------- @@ -1735,10 +2218,16 @@ async function handleTool(name: string, args: any): Promise { if (!choice) { return { ok: false, error: `No project matches "${args?.project}". Projects: ${projectChoices().map(({ project }) => project.title).join(", ")}` }; } - const selection = defaultModelSelection(choice.client.settings, choice.client.providers, choice.project); - if (!selection) return { ok: false, error: "No usable agent provider on that computer." }; - const threadId = await choice.client.launchThread(launchThreadPayload(choice.project, text, selection, choice.client.settings)); - return { ok: true, content: `Started a thread in ${choice.project.title} (id ${threadId}).` }; + const { client, project } = choice; + const resolved = await defaultLaunch(client, project); + if (resolved.blocker) return { ok: false, error: resolved.blocker }; + const threadId = await client.launchThread(launchThreadPayload(project, text, resolved, client.providers)); + const how = [ + modelDisplayName(client.providers, resolved.model), + runtimeModeLabel(resolved.runtimeMode).toLowerCase(), + resolved.workspaceMode === "worktree" ? `new worktree from ${resolved.branch}` : resolved.branch ? `on ${resolved.branch}` : "", + ].filter(Boolean); + return { ok: true, content: `Started a thread in ${project.title} (${how.join(", ")}; id ${threadId}).` }; } default: return { ok: false, error: `Unknown T3 Code tool: ${name}` }; diff --git a/app/apps/teleprompter/index.ts b/app/apps/teleprompter/index.ts index f2abbcf1..e382ea7b 100644 --- a/app/apps/teleprompter/index.ts +++ b/app/apps/teleprompter/index.ts @@ -9,8 +9,8 @@ const teleprompterApp: AppDefinition = { ctx.launchInProcessApp(TELEPROMPTER_WINDOW_ID, TELEPROMPTER_SURFACE_ID, (options) => createTeleprompterAppWindow({ ...options, - startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture("teleprompter"), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("teleprompter"), appendLog: (message) => ctx.appendLog(message), }), ), diff --git a/app/apps/terminal/terminal-app.worker.ts b/app/apps/terminal/terminal-app.worker.ts index 6cf44429..87bf9fdc 100644 --- a/app/apps/terminal/terminal-app.worker.ts +++ b/app/apps/terminal/terminal-app.worker.ts @@ -55,10 +55,9 @@ import { clamp } from "../../util/numeric-util"; import { enumSettingMenuItem, terminalAutoReconnectSetting, - terminalDisplayModeSetting, + terminalDisplayLayoutSetting, terminalLaunchPresetsSetting, terminalNewConnectionSetting, - terminalVerticalPositionSetting, terminalWakeOnBellSetting, textSettingMenuItem, toggleSettingMenuItem, @@ -1118,8 +1117,7 @@ function windowMenuItems(window: TerminalWindow): MenuItem[] { */ function settingsMenuItems(): MenuItem[] { return [ - enumSettingMenuItem(terminalDisplayModeSetting), - enumSettingMenuItem(terminalVerticalPositionSetting), + enumSettingMenuItem(terminalDisplayLayoutSetting), terminalFontPickerMenuItem(), // Opens the phone text editor; the settings listener repaints its page. textSettingMenuItem(terminalLaunchPresetsSetting), diff --git a/app/apps/timer/index.ts b/app/apps/timer/index.ts index a2073eb6..165fd48d 100644 --- a/app/apps/timer/index.ts +++ b/app/apps/timer/index.ts @@ -1,8 +1,8 @@ -import { type AppDefinition } from "../app-definition"; +import { inProcessTrayWidget, type AppDefinition } from "../app-definition"; import { shell } from "../../ui/shell/shell"; import { registerTimerTools } from "../../assistant/timer-tools"; import { timerEngine, timersWakeSetting } from "./timer-engine"; -import { createTimerAppWindow, startTimersTrayIcon, TIMER_SURFACE_ID, TIMER_WINDOW_ID } from "./timer-app"; +import { createTimerAppWindow, showTrayTimer, startTimersTrayIcon, TIMER_SURFACE_ID, TIMER_WINDOW_ID } from "./timer-app"; /** * Timers, stopwatch and alarms. The clock engine boots with the shell so @@ -33,7 +33,8 @@ const timerApp: AppDefinition = { .catch((error) => console.error(`timers ring launch failed: ${error}`)); }, }); - startTimersTrayIcon(); + // Selected in the switcher, the tray readout shows its timer in the app's window. + startTimersTrayIcon(inProcessTrayWidget(ctx, TIMER_WINDOW_ID, TIMER_SURFACE_ID, createTimerAppWindow, showTrayTimer)); registerTimerTools((appId) => ctx.launchApp(appId)); }, launch: (ctx) => ctx.launchInProcessApp(TIMER_WINDOW_ID, TIMER_SURFACE_ID, createTimerAppWindow), diff --git a/app/apps/timer/timer-app.ts b/app/apps/timer/timer-app.ts index a08fec9a..bd9f0293 100644 --- a/app/apps/timer/timer-app.ts +++ b/app/apps/timer/timer-app.ts @@ -39,7 +39,7 @@ import { import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; import { createInProcessWindow, type InProcessAppOptions, type InProcessWindow } from "../../ui/shell/in-process-window"; -import { shell } from "../../ui/shell/shell"; +import { shell, type TrayWidget } from "../../ui/shell/shell"; import { WindowMenuLayer } from "../../ui/window-menu"; import { clamp } from "../../util/numeric-util"; import { @@ -68,6 +68,7 @@ import { formatTimeOfDay, nearestLadderIndex, sortAlarms, + soonestRunningTimer, sortTimers, stopwatchElapsedMs, stopwatchIsRunning, @@ -271,6 +272,19 @@ export class TimersLayer implements Layer { if (index !== null) this.selectionKey = rows[index]!.key; } + /** + * Put a timer on the stage, out of any picker: the tray widget's view of + * the timer it counts down. Lands the selection first, if this is before + * the first paint, so that doesn't move it again. + */ + showTimer(timer: CountdownTimer): void { + this.reconcileSelection(this.buildRows(Date.now())); + this.selectionKey = timerKey(timer); + this.mode = { kind: "list" }; + this.ringButton = 0; + this.notice = ""; + } + /** Called on every engine change: jump to a newly ringing item. */ onEngineChanged(): void { const ringing = timerEngine.ringingItems(); @@ -1291,8 +1305,12 @@ function pad2(value: number): string { // --------------------------------------------------------------------------- // Window +/** The open Timers window's layer, for the tray widget (see showTrayTimer). */ +let windowLayer: TimersLayer | null = null; + export function createTimerAppWindow(options: InProcessAppOptions): InProcessWindow { const layer = new TimersLayer(); + windowLayer = layer; let tick: ReturnType | null = null; let unsubscribe: (() => void) | null = null; const stopTick = () => { @@ -1324,6 +1342,7 @@ export function createTimerAppWindow(options: InProcessAppOptions): InProcessWin stopTick(); unsubscribe?.(); unsubscribe = null; + if (windowLayer === layer) windowLayer = null; options.onClosed(); }, }); @@ -1348,9 +1367,10 @@ let trayTick: ReturnType | null = null; /** * Keep a small "soonest timer" readout in the top bar while a timer runs, * so any app can glance at it. Minute granularity until the last minute, - * so the shell isn't repainted every second for an hour. + * so the shell isn't repainted every second for an hour. `widget` makes it + * selectable in the switcher (see showTrayTimer). */ -export function startTimersTrayIcon(): void { +export function startTimersTrayIcon(widget?: TrayWidget): void { const sync = () => { const remaining = soonestRunningTimerRemainingMs(timerEngine); let text: string | null = null; @@ -1359,7 +1379,7 @@ export function startTimersTrayIcon(): void { } if (text === trayText) return; trayText = text; - shell.setTrayIcon("timer", text === null ? null : renderTrayIcon(text)); + shell.setTrayIcon("timer", text === null ? null : renderTrayIcon(text), widget); if (text !== null && trayTick === null) { trayTick = setInterval(sync, 1000); } else if (text === null && trayTick !== null) { @@ -1371,6 +1391,16 @@ export function startTimersTrayIcon(): void { sync(); } +/** + * The tray widget's view: the open Timers window with the timer the tray + * icon counts down on its stage. Called each time the switcher's selection + * moves onto the widget, once the window is open. + */ +export function showTrayTimer(): void { + const timer = soonestRunningTimer(timerEngine.state.timers, Date.now()); + if (timer) windowLayer?.showTimer(timer); +} + function renderTrayIcon(text: string): GrayImage { const font = getFont("terminus12"); const dial = 12; diff --git a/app/apps/timer/timer-engine.ts b/app/apps/timer/timer-engine.ts index 34dbf3d7..296ec103 100644 --- a/app/apps/timer/timer-engine.ts +++ b/app/apps/timer/timer-engine.ts @@ -44,6 +44,7 @@ import { parseTimersState, pushRecentDuration, serializeTimersState, + soonestRunningTimer, stopwatchElapsedMs, timerRemainingMs, type Alarm, @@ -685,13 +686,8 @@ function alarmNotificationText(alarm: Alarm): string { /** Remaining time of the timer that ends soonest, for the tray icon; null when none runs. */ export function soonestRunningTimerRemainingMs(engine: TimerEngine, now = Date.now()): number | null { - let soonest: number | null = null; - for (const timer of engine.state.timers) { - if (timer.endsAtMs === null || timer.rungAtMs !== null) continue; - const remaining = timerRemainingMs(timer, now); - if (soonest === null || remaining < soonest) soonest = remaining; - } - return soonest; + const timer = soonestRunningTimer(engine.state.timers, now); + return timer ? timerRemainingMs(timer, now) : null; } export const timerEngine = new TimerEngine(); diff --git a/app/apps/timer/timer-model.ts b/app/apps/timer/timer-model.ts index 3afc6860..bf627731 100644 --- a/app/apps/timer/timer-model.ts +++ b/app/apps/timer/timer-model.ts @@ -185,6 +185,16 @@ export function sortTimers(timers: readonly CountdownTimer[], nowMs: number): Co }); } +/** The running timer that ends soonest (the one the tray icon counts down); null when none runs. */ +export function soonestRunningTimer(timers: readonly CountdownTimer[], nowMs: number): CountdownTimer | null { + let soonest: CountdownTimer | null = null; + for (const timer of timers) { + if (timer.endsAtMs === null || timer.rungAtMs !== null) continue; + if (!soonest || timerRemainingMs(timer, nowMs) < timerRemainingMs(soonest, nowMs)) soonest = timer; + } + return soonest; +} + export function pushRecentDuration(recent: readonly number[], durationMs: number): number[] { return [durationMs, ...recent.filter((value) => value !== durationMs)].slice(0, RECENT_DURATION_COUNT); } diff --git a/app/apps/transcribe/index.ts b/app/apps/transcribe/index.ts index ebd3c0f3..d5e504c6 100644 --- a/app/apps/transcribe/index.ts +++ b/app/apps/transcribe/index.ts @@ -9,8 +9,8 @@ const transcribeApp: AppDefinition = { ctx.launchInProcessApp(TRANSCRIBE_WINDOW_ID, TRANSCRIBE_SURFACE_ID, (options) => createTranscribeAppWindow({ ...options, - startContinuousVoiceCapture: () => void options.actions.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture("transcribe", capture), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("transcribe"), }), ), }; diff --git a/app/apps/transcribe/transcribe-app.ts b/app/apps/transcribe/transcribe-app.ts index 21ac40d4..71c1b4c6 100644 --- a/app/apps/transcribe/transcribe-app.ts +++ b/app/apps/transcribe/transcribe-app.ts @@ -1,5 +1,5 @@ import { imageFromAsciiArt } from "../../graphics/image"; -import { TranscribeLayer } from "./transcribe"; +import { TranscribeLayer, transcriptTimestamp } from "./transcribe"; import { createInProcessWindow, YieldAtRootLayer, @@ -13,8 +13,8 @@ export const TRANSCRIBE_SURFACE_ID = "window:transcribe"; const TRAY_ICON_ID = "transcribe"; -// 16x18 microphone glyph for the top-bar tray. -const MIC_ICON = imageFromAsciiArt( +// 16x18 microphone glyph for the top-bar tray (also Translate's). +export const MIC_ICON = imageFromAsciiArt( [ " #### ", " ###### ", @@ -37,7 +37,7 @@ const MIC_ICON = imageFromAsciiArt( export type TranscribeAppOptions = InProcessAppOptions & { /** Begin continuous mic capture (kept on while the window is open). */ - startContinuousVoiceCapture: () => void; + startContinuousVoiceCapture: (options?: { recordAs?: string }) => void; stopContinuousVoiceCapture: () => void; }; @@ -47,7 +47,9 @@ export type TranscribeAppOptions = InProcessAppOptions & { * shows a microphone tray icon in the top bar. */ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProcessWindow { - const layer = new TranscribeLayer(); + // One timestamp per session names both the audio recording and a saved transcript. + const stamp = transcriptTimestamp(); + const layer = new TranscribeLayer(stamp); const app = createInProcessWindow({ appId: "transcribe", windowId: TRANSCRIBE_WINDOW_ID, @@ -77,7 +79,7 @@ export function createTranscribeAppWindow(options: TranscribeAppOptions): InProc // Wire async state changes to this window's renderer, rather than the // controller action from which the window-specific actions were derived. layer.start(app.requestRender); - options.startContinuousVoiceCapture(); + options.startContinuousVoiceCapture({ recordAs: `transcribe-${stamp}` }); shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); return app; } diff --git a/app/apps/transcribe/transcribe.ts b/app/apps/transcribe/transcribe.ts index 3da56357..251b1807 100644 --- a/app/apps/transcribe/transcribe.ts +++ b/app/apps/transcribe/transcribe.ts @@ -23,6 +23,9 @@ export class TranscribeLayer implements Layer { private unsubscribePause: (() => void) | null = null; private unsubscribeStatus: (() => void) | null = null; + /** `stamp` names the saved transcript, matching the session's audio recording. */ + constructor(private readonly stamp: string = transcriptTimestamp()) {} + start(requestRender: () => void): void { this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { this.transcript.accept(event); @@ -93,13 +96,13 @@ export class TranscribeLayer implements Layer { this.saveNotice = "Nothing to save yet."; return; } - const filename = `transcript-${transcriptTimestamp()}.txt`; + const filename = `transcript-${this.stamp}.txt`; const path = writeTextToDownloads(filename, `${text}\n`); this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; } } -function transcriptTimestamp(): string { +export function transcriptTimestamp(): string { const now = new Date(); const pad = (value: number) => String(value).padStart(2, "0"); return ( diff --git a/app/apps/translate/index.ts b/app/apps/translate/index.ts new file mode 100644 index 00000000..c4ddb944 --- /dev/null +++ b/app/apps/translate/index.ts @@ -0,0 +1,18 @@ +import { type AppDefinition } from "../app-definition"; +import { createTranslateAppWindow, TRANSLATE_SURFACE_ID, TRANSLATE_WINDOW_ID } from "./translate-app"; + +const translateApp: AppDefinition = { + appId: "translate", + title: "Translate", + icon: "languages", + launch: (ctx) => + ctx.launchInProcessApp(TRANSLATE_WINDOW_ID, TRANSLATE_SURFACE_ID, (options) => + createTranslateAppWindow({ + ...options, + startContinuousVoiceCapture: (capture) => void options.actions.startContinuousVoiceCapture("translate", capture), + stopContinuousVoiceCapture: () => void options.actions.stopContinuousVoiceCapture("translate"), + }), + ), +}; + +export default translateApp; diff --git a/app/apps/translate/translate-app.ts b/app/apps/translate/translate-app.ts new file mode 100644 index 00000000..7cbd57e3 --- /dev/null +++ b/app/apps/translate/translate-app.ts @@ -0,0 +1,99 @@ +import { TranslateLayer } from "./translate"; +import { resolveTranslationModel } from "./translate-llm"; +import { resolveTranslateProvider, translateProviderSetting, translateShowOriginalSetting } from "./translate-settings"; +import { MIC_ICON } from "../transcribe/transcribe-app"; +import { transcriptTimestamp } from "../transcribe/transcribe"; +import { type ContinuousCaptureOptions } from "../../ui/layers"; +import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel"; +import { enumSettingMenuItem, toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "../../ui/shell/in-process-window"; +import { shell } from "../../ui/shell/shell"; + +export const TRANSLATE_WINDOW_ID = "translate"; +export const TRANSLATE_SURFACE_ID = "window:translate"; + +const TRAY_ICON_ID = "translate"; + +const NO_PROVIDER_PROBLEM = + "Translate needs a transcription provider that understands any language: add an ElevenLabs, OpenAI, or Soniox API key (Settings > API Keys), or a Whisper server URL (Settings > Voice)."; +const NO_MODEL_PROBLEM = global.isIOS + ? "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys)." + : "Translate needs a language model: add an Anthropic or OpenAI API key (Settings > API Keys), or download the on-phone model (Settings > Assistant)."; + +export type TranslateAppOptions = InProcessAppOptions & { + /** Begin (or retarget) continuous mic capture, kept on while the window is open. */ + startContinuousVoiceCapture: (options: ContinuousCaptureOptions) => void; + stopContinuousVoiceCapture: () => void; +}; + +/** + * The Translate app: live speech in any language, translated into English by + * the assistant's model, in its own window. Like Transcribe it holds + * continuous mic capture while open, but always with a multilingual cloud + * provider (Translate settings), which a shared capture switches to. + */ +export function createTranslateAppWindow(options: TranslateAppOptions): InProcessWindow { + // Checks the setup and (re)starts capture with the chosen provider: at + // launch, when the provider setting changes, and on a tap after a problem. + const startCapture = () => { + const provider = resolveTranslateProvider(); + const problem = !provider ? NO_PROVIDER_PROBLEM : !resolveTranslationModel() ? NO_MODEL_PROBLEM : null; + layer.setProblem(problem); + if (!provider || problem) { + options.stopContinuousVoiceCapture(); + shell.setTrayIcon(TRAY_ICON_ID, null); + return; + } + options.startContinuousVoiceCapture({ multilingualProvider: provider }); + shell.setTrayIcon(TRAY_ICON_ID, MIC_ICON); + }; + const layer = new TranslateLayer(transcriptTimestamp(), startCapture); + const app = createInProcessWindow({ + appId: "translate", + windowId: TRANSLATE_WINDOW_ID, + title: "Translate", + iconLetter: "Tl", + icon: "languages", + closeable: true, + actions: options.actions, + baseLayer: new YieldAtRootLayer(layer), + menuItems: () => [ + { + label: "Save", + onSelect: (ctx) => { + ctx.stack.pop(); + layer.saveTranscript(); + }, + }, + { + label: "Translate settings", + onSelect: (ctx) => { + ctx.stack.pop(); + openSettingsSubMenu(ctx, "Translate settings", [ + enumSettingMenuItem(translateProviderSetting, { onChange: startCapture }), + toggleSettingMenuItem(translateShowOriginalSetting), + ]); + }, + }, + ], + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + onClosed: () => { + layer.onRemoved(); + options.stopContinuousVoiceCapture(); + shell.setTrayIcon(TRAY_ICON_ID, null); + options.onClosed(); + }, + }); + // Wire async state changes to this window's renderer, rather than the + // controller action from which the window-specific actions were derived. + layer.start(app.requestRender); + startCapture(); + return app; +} diff --git a/app/apps/translate/translate-llm.ts b/app/apps/translate/translate-llm.ts new file mode 100644 index 00000000..26021899 --- /dev/null +++ b/app/apps/translate/translate-llm.ts @@ -0,0 +1,65 @@ +/** + * Streamed translation against the assistant's configured model (Settings > + * Assistant). Each call translates one stretch of speech into English; see + * TranslationModel for how stretches are batched and given context. + */ +import { streamAnthropicMessage } from "../../native/anthropic"; +import { streamLocalQwen } from "../../native/llama"; +import { streamOpenAiResponse } from "../../native/openai"; +import { resolveAssistantModel, type ResolvedAssistantModel } from "../../assistant/models"; +import { type LlmStreamHandle, type LlmStreamOptions } from "../../assistant/llm-protocol"; +import { anthropicApiKeySetting, assistantModelSetting, openAiApiKeySetting } from "../../ui/dashboard-settings"; +import { TRANSLATE_SYSTEM_PROMPT, buildTranslateUserMessage } from "../../prompts"; +import { type TranslationTurn } from "./translation-model"; + +/** + * The model Translate uses, or null when none is configured. Hermes is an + * agent with its own tools and memory rather than a translator (every + * utterance would land in its session), so it falls back to Auto: the model + * for whichever API key is set, else the on-phone model. + */ +export function resolveTranslationModel(): ResolvedAssistantModel | null { + const selection = assistantModelSetting.get(); + return resolveAssistantModel(selection === "hermes" ? "auto" : selection, { + anthropic: anthropicApiKeySetting.get(), + openai: openAiApiKeySetting.get(), + }); +} + +export type TranslationRequest = { + speech: string; + context: TranslationTurn[]; + onProgress: (textSoFar: string) => void; + onDone: (translation: string) => void; + onError: (message: string) => void; +}; + +export function streamTranslation(model: ResolvedAssistantModel, request: TranslationRequest): LlmStreamHandle { + const options: LlmStreamOptions = { + apiKey: model.apiKey, + model: model.model, + system: TRANSLATE_SYSTEM_PROMPT, + effort: model.effort, + maxTokens: 1024, + messages: [{ role: "user", content: buildTranslateUserMessage(request.speech, request.context) }], + onTextDelta: (_delta, textSoFar) => request.onProgress(cleanTranslation(textSoFar)), + onDone: (result) => { + if (result.stopReason === "refusal") request.onError("The model declined to translate this."); + else request.onDone(cleanTranslation(result.text)); + }, + onError: request.onError, + }; + switch (model.provider) { + case "openai": + return streamOpenAiResponse(options); + case "local": + return streamLocalQwen(options); + default: + return streamAnthropicMessage(options); + } +} + +/** The context turns are tagged ; a model may echo the tag around its reply. */ +function cleanTranslation(text: string): string { + return text.replace(/^\s*/i, "").replace(/<\/english>\s*$/i, "").trim(); +} diff --git a/app/apps/translate/translate-settings.ts b/app/apps/translate/translate-settings.ts new file mode 100644 index 00000000..c75d1ef3 --- /dev/null +++ b/app/apps/translate/translate-settings.ts @@ -0,0 +1,56 @@ +import { ConfigSettingBoolean, ConfigSettingEnum, voiceProviderSetting, type VoiceProvider } from "../../ui/dashboard-settings"; + +/** + * The transcription providers that recognize any spoken language, in picker + * order. The on-device engines can't: Moonshine and Whisper base.en are + * English-only, and Apple's recognizer hears only the iPhone's speech language. + */ +const MULTILINGUAL_PROVIDERS = ["elevenlabs", "whisper", "soniox", "whisper-server"] as const satisfies readonly VoiceProvider[]; + +export type MultilingualProvider = (typeof MULTILINGUAL_PROVIDERS)[number]; +export type TranslateProvider = "auto" | MultilingualProvider; + +function isMultilingual(provider: VoiceProvider): provider is MultilingualProvider { + return (MULTILINGUAL_PROVIDERS as readonly VoiceProvider[]).includes(provider); +} + +/** Whether the provider has its API key (or, for Whisper server, its URL). */ +function isSetUp(provider: MultilingualProvider): boolean { + return !voiceProviderSetting.isDisabled(provider); +} + +/** + * The provider Translate transcribes with, or null when none is set up. Auto + * uses the main Transcription Provider when it is multilingual, else the + * first multilingual provider that is set up. + */ +export function resolveTranslateProvider(choice: TranslateProvider = translateProviderSetting.get()): MultilingualProvider | null { + if (choice !== "auto") return isSetUp(choice) ? choice : null; + const main = voiceProviderSetting.get(); + if (isMultilingual(main) && isSetUp(main)) return main; + return MULTILINGUAL_PROVIDERS.find(isSetUp) ?? null; +} + +export const translateProviderSetting = new ConfigSettingEnum({ + id: "translate-provider", + label: "Transcription provider", + storageKey: "translate.provider", + defaultValue: "auto", + values: ["auto", ...MULTILINGUAL_PROVIDERS], + formatValue: (value) => { + if (value !== "auto") return voiceProviderSetting.displayValue(value); + const resolved = resolveTranslateProvider("auto"); + return `Auto (${resolved ? voiceProviderSetting.displayValue(resolved) : "none set up"})`; + }, + isDisabled: (value) => value !== "auto" && !isSetUp(value), + description: + "Speech-to-text for Translate. Only cloud providers recognize any language (the on-device engines don't), so each needs its API key in Settings > API Keys, or a Whisper server URL in Settings > Voice. Auto uses the main Transcription Provider when it is one of these.", +}); + +export const translateShowOriginalSetting = new ConfigSettingBoolean({ + id: "translate-show-original", + label: "Show original", + storageKey: "translate.showOriginal", + defaultValue: false, + description: "Show what was said, dimmed, above each translation.", +}); diff --git a/app/apps/translate/translate.ts b/app/apps/translate/translate.ts new file mode 100644 index 00000000..252da0d4 --- /dev/null +++ b/app/apps/translate/translate.ts @@ -0,0 +1,185 @@ +import { GrayImage } from "../../graphics/image"; +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { lineStep } from "../../ui/metrics"; +import { writeTextToDownloads } from "../../native/file-access"; +import { voiceControlBridge } from "../../native/voice-control"; +import { Layer, type LayerContext } from "../../ui/layers"; +import { type InputEvent } from "../../ui/gestures"; +import { type LlmStreamHandle } from "../../assistant/llm-protocol"; +import { TranscriptScroll, wrapTranscribeText } from "../transcribe/transcript-model"; +import { TranslationModel, type TranslationJob, type TranslationLine } from "./translation-model"; +import { resolveTranslationModel, streamTranslation } from "./translate-llm"; +import { translateShowOriginalSetting } from "./translate-settings"; + +const NO_MODEL_NOTICE = "No language model is set up."; + +/** + * Live translation view. Continuous mic capture is owned by the app wrapper + * (createTranslateAppWindow); this layer feeds finalized speech to the + * assistant's model one stretch at a time, shows the English as it streams + * in, and saves the session to Downloads from the app menu. + */ +export class TranslateLayer implements Layer { + private status = "Starting microphone..."; + private listening = false; + private detail = ""; + private saveNotice = ""; + private translationError = ""; + // Why the app can't run (no multilingual provider or no model); shown in + // place of the transcript, and a tap asks the app wrapper to check again. + private problem: string | null = null; + private readonly model = new TranslationModel(); + private readonly scroll = new TranscriptScroll(); + private activeJob: TranslationJob | null = null; + private activeHandle: LlmStreamHandle | null = null; + private requestRender: () => void = () => {}; + private removed = false; + private unsubscribeTranscript: (() => void) | null = null; + private unsubscribeStatus: (() => void) | null = null; + + /** `stamp` names the saved transcript; `retry` re-runs the app's setup check. */ + constructor(private readonly stamp: string, private readonly retry: () => void) {} + + start(requestRender: () => void): void { + this.requestRender = requestRender; + this.unsubscribeTranscript = voiceControlBridge.onTranscript((event) => { + this.model.accept(event); + this.pump(); + requestRender(); + }); + this.unsubscribeStatus = voiceControlBridge.onStatus((state) => { + this.status = state.status; + this.listening = state.listening; + this.detail = state.detail; + this.saveNotice = ""; + requestRender(); + }); + } + + setProblem(problem: string | null): void { + this.problem = problem; + this.requestRender(); + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + const measure = (line: string) => font.measureText(line); + const maxWidth = width - 64; + + image.drawText(font, 24, 20, "Translate", 200); + const status = this.problem ? "Setup needed" : this.translationError || this.status; + image.drawText(font, 24, 40, this.saveNotice || status, 110); + + let lines: TranslationLine[]; + if (this.problem) { + lines = wrapTranscribeText(measure, `${this.problem}\n\nTap to check again.`, maxWidth).map((text) => ({ text, gray: 230 })); + } else { + lines = this.model.lines(measure, maxWidth, translateShowOriginalSetting.get()); + if (!lines.length) { + const placeholder = this.listening ? "Listening..." : this.detail; + lines = wrapTranscribeText(measure, placeholder, maxWidth).map((text) => ({ text, gray: 230 })); + } + } + + const step = lineStep(font) + 2; + const bodyTop = 62; + const bodyLines = Math.max(1, Math.floor((height - 6 - bodyTop - font.lineHeight) / step) + 1); + const firstLine = this.scroll.layout(lines.length, bodyLines); + for (let index = firstLine; index < Math.min(lines.length, firstLine + bodyLines); index++) { + const y = bodyTop + (index - firstLine) * step; + image.drawText(font, 32, y, lines[index]!.text, lines[index]!.gray); + } + + if (lines.length > bodyLines) { + const trackHeight = height - bodyTop - 6; + const thumbHeight = Math.max(8, Math.round(trackHeight * bodyLines / lines.length)); + const thumbY = bodyTop + Math.round((trackHeight - thumbHeight) * firstLine / (lines.length - bodyLines)); + image.fillRect(width - 14, bodyTop, 2, trackHeight, 45); + image.fillRect(width - 14, thumbY, 2, thumbHeight, 140); + } + return image; + } + + handleInput(event: InputEvent, ctx: LayerContext): void { + if (event.type === "scroll-up" || event.type === "scroll-down") { + this.scroll.scroll(event.type === "scroll-up" ? -3 : 3); + ctx.actions.requestRender(); + return; + } + if (event.type === "click" && this.problem) { + this.retry(); + return; + } + if (event.type === "double-click") { + ctx.stack.pop(); + } + } + + onRemoved(): void { + this.removed = true; + this.activeJob = null; + this.activeHandle?.cancel(); + this.activeHandle = null; + this.unsubscribeTranscript?.(); + this.unsubscribeTranscript = null; + this.unsubscribeStatus?.(); + this.unsubscribeStatus = null; + } + + saveTranscript(): void { + const text = this.model.savedText().trim(); + if (!text) { + this.saveNotice = "Nothing to save yet."; + return; + } + const filename = `translation-${this.stamp}.txt`; + const path = writeTextToDownloads(filename, `${text}\n`); + this.saveNotice = path ? `Saved ${filename}` : "Save failed (check file access)."; + } + + /** Start translating the next waiting stretch of speech, unless one is in flight. */ + private pump(): void { + if (this.removed) return; + const job = this.model.nextJob(); + if (!job) return; + const llm = resolveTranslationModel(); + if (!llm) { + // The model went away mid-session (a key was cleared): keep the original. + this.model.fail(job.segment); + this.translationError = NO_MODEL_NOTICE; + this.pump(); + return; + } + this.activeJob = job; + const handle = streamTranslation(llm, { + speech: job.segment.source, + context: job.context, + onProgress: (textSoFar) => { + if (this.activeJob !== job) return; + this.model.progress(job.segment, textSoFar); + this.requestRender(); + }, + onDone: (translation) => this.settle(job, () => { + this.model.complete(job.segment, translation); + this.translationError = ""; + }), + onError: (message) => this.settle(job, () => { + this.model.fail(job.segment); + this.translationError = `Translation failed: ${message}`; + }), + }); + // A provider that fails synchronously has already settled (and moved on). + if (this.activeJob === job) this.activeHandle = handle; + } + + private settle(job: TranslationJob, apply: () => void): void { + if (this.activeJob !== job) return; + this.activeJob = null; + this.activeHandle = null; + apply(); + this.pump(); + this.requestRender(); + } +} diff --git a/app/apps/translate/translation-model.ts b/app/apps/translate/translation-model.ts new file mode 100644 index 00000000..ea0e9204 --- /dev/null +++ b/app/apps/translate/translation-model.ts @@ -0,0 +1,129 @@ +import { wrapTranscribeText } from "../transcribe/transcript-model"; + +type TranscriptEvent = { text: string; isFinal: boolean }; + +export type TranslationState = "pending" | "translating" | "done" | "failed"; + +/** One stretch of finalized speech and its English translation. */ +export type TranslationSegment = { + source: string; + /** The translation so far (streamed while translating). */ + translation: string; + state: TranslationState; +}; + +export type TranslationTurn = { source: string; translation: string }; + +/** A segment handed to the model, with the finished turns before it. */ +export type TranslationJob = { + segment: TranslationSegment; + /** Earlier translated turns, oldest first. */ + context: TranslationTurn[]; +}; + +export type TranslationLine = { text: string; gray: number }; + +// Earlier turns sent with each request, so pronouns, names, and topic carry +// over between requests; bounded so a long session's requests stay small. +const CONTEXT_TURNS = 4; +const CONTEXT_CHARS = 800; + +const TRANSLATION_GRAY = 230; +const ORIGINAL_GRAY = 120; +const LIVE_GRAY = 90; + +/** + * The Translate app's conversation: finalized speech segments with their + * translations, plus the live partial of the utterance in progress. + * Translation is serial (one request at a time, in order, so each can carry + * the turns before it as context); finals that arrive while a request is in + * flight merge into one pending segment, so a slow model batches the backlog + * instead of falling further behind one request per pause. + */ +export class TranslationModel { + readonly segments: TranslationSegment[] = []; + /** The utterance in progress (replace semantics), not yet final. */ + live = ""; + + accept(event: TranscriptEvent): void { + const text = event.text.trim(); + if (!event.isFinal) { + this.live = text; + return; + } + const finalText = text || this.live; + this.live = ""; + if (!finalText) return; + const last = this.segments[this.segments.length - 1]; + if (last?.state === "pending") last.source = `${last.source} ${finalText}`; + else this.segments.push({ source: finalText, translation: "", state: "pending" }); + } + + /** Take the next segment to translate, or null while one is in flight or none waits. */ + nextJob(): TranslationJob | null { + if (this.segments.some((segment) => segment.state === "translating")) return null; + const segment = this.segments.find((candidate) => candidate.state === "pending"); + if (!segment) return null; + segment.state = "translating"; + return { segment, context: this.contextBefore(segment) }; + } + + progress(segment: TranslationSegment, textSoFar: string): void { + if (segment.state === "translating") segment.translation = textSoFar; + } + + complete(segment: TranslationSegment, translation: string): void { + segment.translation = translation; + segment.state = "done"; + } + + fail(segment: TranslationSegment): void { + segment.translation = ""; + segment.state = "failed"; + } + + private contextBefore(segment: TranslationSegment): TranslationTurn[] { + const turns: TranslationTurn[] = []; + let chars = 0; + for (let index = this.segments.indexOf(segment) - 1; index >= 0 && turns.length < CONTEXT_TURNS; index--) { + const earlier = this.segments[index]!; + if (earlier.state !== "done" || !earlier.translation) continue; + chars += earlier.source.length + earlier.translation.length; + if (turns.length && chars > CONTEXT_CHARS) break; + turns.unshift({ source: earlier.source, translation: earlier.translation }); + } + return turns; + } + + /** + * Display lines: each segment's translation, bright; its original, dim, + * above it when `showOriginal` is on and in its place until a translation + * arrives (or when it failed); then the live partial, dimmer. + */ + lines(measure: (text: string) => number, maxWidth: number, showOriginal: boolean): TranslationLine[] { + const lines: TranslationLine[] = []; + const push = (text: string, gray: number) => { + for (const line of wrapTranscribeText(measure, text, maxWidth)) lines.push({ text: line, gray }); + }; + for (const segment of this.segments) { + const translation = segment.translation.trim(); + // A finished, empty translation is noise the model chose to drop. + if (showOriginal || (!translation && segment.state !== "done")) push(segment.source, ORIGINAL_GRAY); + if (translation) push(translation, TRANSLATION_GRAY); + } + if (this.live) push(this.live, LIVE_GRAY); + return lines; + } + + /** Plain-text transcript for saving: each original, then its translation. */ + savedText(): string { + return this.segments + .map((segment) => { + const translation = segment.translation.trim(); + if (segment.state === "failed") return `${segment.source}\n→ (not translated)`; + if (!translation || translation === segment.source.trim()) return segment.source; + return `${segment.source}\n→ ${translation}`; + }) + .join("\n\n"); + } +} diff --git a/app/apps/weather/index.ts b/app/apps/weather/index.ts index 73f12e7d..50410ce6 100644 --- a/app/apps/weather/index.ts +++ b/app/apps/weather/index.ts @@ -1,4 +1,4 @@ -import { type AppDefinition } from "../app-definition"; +import { inProcessTrayWidget, type AppDefinition } from "../app-definition"; import { createWeatherAppWindow, WEATHER_SURFACE_ID, WEATHER_WINDOW_ID } from "./weather-app"; import { startWeatherIndicators } from "./weather-indicators"; @@ -6,7 +6,10 @@ const weatherApp: AppDefinition = { appId: "weather", title: "Weather", icon: "cloud-sun", - boot: () => startWeatherIndicators(), + // Selected in the switcher, the status-bar indicator shows the app's window. + boot: (ctx) => startWeatherIndicators( + inProcessTrayWidget(ctx, WEATHER_WINDOW_ID, WEATHER_SURFACE_ID, createWeatherAppWindow), + ), launch: (ctx) => ctx.launchInProcessApp(WEATHER_WINDOW_ID, WEATHER_SURFACE_ID, createWeatherAppWindow), }; diff --git a/app/apps/weather/temperature-format.ts b/app/apps/weather/temperature-format.ts new file mode 100644 index 00000000..9fdadcc8 --- /dev/null +++ b/app/apps/weather/temperature-format.ts @@ -0,0 +1,17 @@ +import { temperatureUnitSetting } from "../../ui/dashboard-settings"; + +/** + * A temperature (weather data is kept in Fahrenheit) in the unit chosen in + * Settings > Customization > Temperature unit, rounded: "72°F" / "22°C", or + * "72°" without the unit letter. + */ +export function formatTemperature(fahrenheit: number, withUnit = true): string { + const celsius = temperatureUnitSetting.get() === "celsius"; + const value = Math.round(celsius ? (fahrenheit - 32) * 5 / 9 : fahrenheit); + return withUnit ? `${value}°${celsius ? "C" : "F"}` : `${value}°`; +} + +/** The widest temperature the forecast's column is sized for. */ +export function widestTemperatureSample(): string { + return temperatureUnitSetting.get() === "celsius" ? "-20°C" : "100°F"; +} diff --git a/app/apps/weather/weather-app.ts b/app/apps/weather/weather-app.ts index 2e4bd3e5..96d2ec83 100644 --- a/app/apps/weather/weather-app.ts +++ b/app/apps/weather/weather-app.ts @@ -1,6 +1,6 @@ import { ensureLocationPermission, hasLocationPermission } from "../../native/location-permissions"; import { weatherBridge } from "../../native/weather"; -import { toggleSettingMenuItem } from "../../ui/dashboard-settings"; +import { onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings"; import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel"; import { WeatherLayer } from "./weather"; import { weatherShowInStatusBarSetting, weatherShowOnSystemCardSetting } from "./weather-settings"; @@ -14,11 +14,12 @@ import { export const WEATHER_WINDOW_ID = "weather"; export const WEATHER_SURFACE_ID = "window:weather"; -/** Local current conditions and forecast from the National Weather Service. */ +/** Local current conditions and forecast from the National Weather Service, or Open-Meteo outside the US. */ export function createWeatherAppWindow(options: InProcessAppOptions): InProcessWindow { let closed = false; let requestingPermission = false; let unsubscribe: (() => void) | null = null; + let unsubscribeSettings: (() => void) | null = null; const requestUpdate = () => { if (closed || requestingPermission) return; @@ -81,11 +82,15 @@ export function createWeatherAppWindow(options: InProcessAppOptions): InProcessW closed = true; unsubscribe?.(); unsubscribe = null; + unsubscribeSettings?.(); + unsubscribeSettings = null; weatherBridge.stop(); options.onClosed(); }, }); unsubscribe = weatherBridge.onStateChange(() => app.requestRender()); + // Repaint for a temperature-unit change made in the Settings app. + unsubscribeSettings = onAnySettingChanged(() => app.requestRender()); requestUpdate(); return app; } diff --git a/app/apps/weather/weather-icons.ts b/app/apps/weather/weather-icons.ts index 0c2a3008..0a21173f 100644 --- a/app/apps/weather/weather-icons.ts +++ b/app/apps/weather/weather-icons.ts @@ -1,6 +1,7 @@ import { renderSvgIcon } from "../../graphics/icons"; import { type GrayImage, type UiFont } from "../../graphics/image"; import { type WeatherCondition, type WeatherState } from "../../native/weather"; +import { formatTemperature } from "./temperature-format"; // Lucide weather icons (ISC licensed), kept verbatim so they can be diffed // against upstream. @@ -90,7 +91,7 @@ export function weatherReading(state: WeatherState, nowMs: number): WeatherReadi return { condition: current.condition, isDaytime: current.isDaytime, - temperature: current.temperatureF === null ? "--°" : `${Math.round(current.temperatureF)}°`, + temperature: current.temperatureF === null ? "--°" : formatTemperature(current.temperatureF, false), }; } diff --git a/app/apps/weather/weather-indicators.ts b/app/apps/weather/weather-indicators.ts index 4ff28fef..0e2be397 100644 --- a/app/apps/weather/weather-indicators.ts +++ b/app/apps/weather/weather-indicators.ts @@ -2,7 +2,8 @@ import { GrayImage, type UiFont } from "../../graphics/image"; import { getDefaultSmallFont } from "../../graphics/ui-fonts"; import { weatherBridge } from "../../native/weather"; import { onAnySettingChanged } from "../../ui/dashboard-settings"; -import { shell } from "../../ui/shell/shell"; +import { shell, type TrayWidget } from "../../ui/shell/shell"; +import { isSystemCardWidgetInUse } from "../glanceboard/configuration"; import { drawWeatherReading, measureWeatherReading, weatherReading, type WeatherReading } from "./weather-icons"; import { weatherShowInStatusBarSetting, weatherShowOnSystemCardSetting } from "./weather-settings"; @@ -11,31 +12,35 @@ const TRAY_HEIGHT = 24; const TRAY_ICON_SIZE = 20; let started = false; +let trayWidget: TrayWidget | undefined; /** What the tray currently shows, to skip rebuilding an unchanged icon. */ let trayShown: { key: string; font: UiFont } | null = null; /** * Shell-lifetime subscription: keeps the hourly background refresh running - * while the status-bar or system-card indicator is enabled, and the tray - * icon in sync. The system card reads the bridge itself while it shows. + * while the status-bar indicator is enabled, or the system-card one is and + * the system card is on an enabled Glanceboard, and the tray icon in sync. + * The system card reads the bridge itself while it shows. `widget` makes + * the tray icon selectable in the switcher. */ -export function startWeatherIndicators(): void { +export function startWeatherIndicators(widget?: TrayWidget): void { if (started) return; started = true; + trayWidget = widget; weatherBridge.onStateChange(syncWeatherIndicators); onAnySettingChanged(syncWeatherIndicators); } function syncWeatherIndicators(): void { const inStatusBar = weatherShowInStatusBarSetting.get(); - weatherBridge.setBackgroundRefresh(inStatusBar || weatherShowOnSystemCardSetting.get()); + weatherBridge.setBackgroundRefresh(inStatusBar || (weatherShowOnSystemCardSetting.get() && isSystemCardWidgetInUse())); const reading = inStatusBar ? weatherReading(weatherBridge.snapshot(), Date.now()) : null; const font = getDefaultSmallFont(); const key = reading ? `${reading.condition}|${reading.isDaytime}|${reading.temperature}` : null; if (key === (trayShown?.key ?? null) && (!trayShown || trayShown.font === font)) return; trayShown = key === null ? null : { key, font }; - shell.setTrayIcon(TRAY_ICON_ID, reading ? buildWeatherTrayIcon(reading, font) : null); + shell.setTrayIcon(TRAY_ICON_ID, reading ? buildWeatherTrayIcon(reading, font) : null, trayWidget); } /** Condition icon and temperature, baked to a raster so the tray keeps one stable image. */ diff --git a/app/apps/weather/weather-settings.ts b/app/apps/weather/weather-settings.ts index df86efa4..92bd1f6f 100644 --- a/app/apps/weather/weather-settings.ts +++ b/app/apps/weather/weather-settings.ts @@ -14,11 +14,15 @@ export const weatherShowInStatusBarSetting = new ConfigSettingBoolean({ "Show the current conditions and temperature in the top bar. Weather for your current location is fetched once an hour while the glasses are connected.", }); +/** + * Also the Weather toggle in the system card's Glanceboard settings. On by + * default: it only fetches while the system card is on an enabled board. + */ export const weatherShowOnSystemCardSetting = new ConfigSettingBoolean({ id: "weather-show-on-system-card", label: "Show on system card", storageKey: "weather.showOnSystemCard", - defaultValue: false, + defaultValue: true, description: - "Show the current conditions and temperature on the Glanceboard's system card. Weather for your current location is fetched once an hour while the glasses are connected.", + "Show the current conditions and temperature on the Glanceboard's system card. Weather for your current location is fetched once an hour while the glasses are connected and the system card is on the Glanceboard.", }); diff --git a/app/apps/weather/weather.ts b/app/apps/weather/weather.ts index db97c810..d6c1e2af 100644 --- a/app/apps/weather/weather.ts +++ b/app/apps/weather/weather.ts @@ -1,12 +1,13 @@ import { getDefaultLargeFont, getDefaultMediumFont, getDefaultSmallFont } from "../../graphics/ui-fonts"; import { GrayImage, type UiFont } from "../../graphics/image"; import { wrapText, truncateText } from "../../graphics/textwrap"; -import { type ForecastPeriod, type WeatherState } from "../../native/weather"; +import { WEATHER_SOURCE_NAMES, type ForecastPeriod, type WeatherState } from "../../native/weather"; import { GESTURE_CLICK, type InputEvent } from "../../ui/gestures"; import { type Layer, type LayerContext } from "../../ui/layers"; import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics"; import { renderWeatherIcon } from "./weather-icons"; +import { formatTemperature, widestTemperatureSample } from "./temperature-format"; const PAGE_X = 18; const HEADER_Y = 8; @@ -115,7 +116,7 @@ export class WeatherLayer implements Layer { const large = getDefaultLargeFont(); const medium = getDefaultMediumFont(); const small = getDefaultSmallFont(); - const temperature = current.temperatureF === null ? "--°" : `${Math.round(current.temperatureF)}°F`; + const temperature = current.temperatureF === null ? "--°" : formatTemperature(current.temperatureF); const temperatureY = CURRENT_TOP + 8; let temperatureX = PAGE_X; if (current.condition) { @@ -145,7 +146,9 @@ export class WeatherLayer implements Layer { } image.drawText(small, textX, cy, details.join(" ") || "Current forecast", 170); cy += lineStep(small); - const source = current.observed ? "Observed" : "Forecast"; + // Naming the source also credits Open-Meteo, as its CC BY licence asks. + const sourceName = weather.source ? `${WEATHER_SOURCE_NAMES[weather.source]} ` : ""; + const source = `${sourceName}${current.observed ? "observed" : "forecast"}`; const age = current.timestampMs ? formatAge(Date.now() - current.timestampMs) : ""; const updating = weather.phase === "locating" || weather.phase === "loading" ? " · updating" : ""; image.drawText(small, textX, cy, `${source}${age ? ` ${age}` : ""}${updating}`, 105); @@ -180,7 +183,7 @@ export class WeatherLayer implements Layer { ...forecast.map((period) => font.measureText(period.name)), ); const tempX = PAGE_X + nameWidth + FORECAST_COL_GAP; - const precipX = tempX + Math.max(font.measureText("Temp"), font.measureText("100°F")) + FORECAST_COL_GAP; + const precipX = tempX + Math.max(font.measureText("Temp"), font.measureText(widestTemperatureSample())) + FORECAST_COL_GAP; const summaryX = precipX + Math.max(font.measureText("Rain"), font.measureText("100%")) + FORECAST_COL_GAP; image.drawText(font, PAGE_X, FORECAST_HEADER_Y, "Upcoming", 150); @@ -207,7 +210,7 @@ export class WeatherLayer implements Layer { const shade = selected ? 245 : 190; const textY = centeredTextY(font, y, height); image.drawText(font, x + columns.name, textY, period.name, shade); - image.drawText(font, x + columns.temp, textY, period.temperatureF === null ? "--" : `${Math.round(period.temperatureF)}°F`, shade); + image.drawText(font, x + columns.temp, textY, period.temperatureF === null ? "--" : formatTemperature(period.temperatureF), shade); image.drawText(font, x + columns.precip, textY, period.precipitationPercent === null ? "--" : `${Math.round(period.precipitationPercent)}%`, selected ? 220 : 155); // The summary stops PAGE_X short of the viewport's right edge, as the name starts PAGE_X in from the left. const summaryWidth = width - columns.summary - columns.name; diff --git a/app/assistant/direct-backend.ts b/app/assistant/direct-backend.ts index 753a89cf..20c6b2e6 100644 --- a/app/assistant/direct-backend.ts +++ b/app/assistant/direct-backend.ts @@ -1,4 +1,5 @@ import { streamAnthropicMessage } from "../native/anthropic"; +import { streamHermesChat } from "../native/hermes"; import { streamLocalQwen } from "../native/llama"; import { streamOpenAiResponse } from "../native/openai"; import type { @@ -24,12 +25,22 @@ const MAX_ITERATIONS = 8; /** The on-phone model is slower per step and unreliable on long tool chains. */ const MAX_ITERATIONS_LOCAL = 4; const MAX_TURN_MS = 2 * 60 * 1000; +/** + * Hermes runs whole multi-tool agent tasks in one request, and dropping the + * stream interrupts the run on the server, so its cap is only a backstop. A + * dead server is caught sooner: Hermes sends keepalives every 10s, and both + * transports fail a stream that goes quiet for 180s. + */ +const MAX_TURN_MS_HERMES = 20 * 60 * 1000; export type DirectTurnOptions = { provider: AssistantProvider; apiKey: string; model: string; effort?: "low" | "medium" | "high"; + /** Hermes: the API root, and the server-side session this conversation maps to. */ + baseUrl?: string; + sessionId?: string; system: string; /** Owned by the session; this loop appends assistant + tool_result messages. */ messages: LlmMessage[]; @@ -62,13 +73,14 @@ export class DirectAssistantBackend { }; const maxIterations = options.provider === "local" ? MAX_ITERATIONS_LOCAL : MAX_ITERATIONS; + const maxTurnMs = options.provider === "hermes" ? MAX_TURN_MS_HERMES : MAX_TURN_MS; const runIteration = (iteration: number): void => { if (cancelled) return; if (iteration >= maxIterations) { finishError("Assistant stopped: too many tool steps"); return; } - if (Date.now() - startMs > MAX_TURN_MS) { + if (Date.now() - startMs > maxTurnMs) { finishError("Assistant stopped: turn took too long"); return; } @@ -79,7 +91,8 @@ export class DirectAssistantBackend { system: options.system, effort: options.effort, messages: options.messages, - tools: options.buildTools(), + // Hermes runs its own tools; Faceclaw's are not offered to it. + tools: options.provider === "hermes" ? [] : options.buildTools(), onTextDelta: (delta, textSoFar) => { if (cancelled) return; options.callbacks.onTextDelta(delta, priorText + textSoFar); @@ -128,14 +141,16 @@ export class DirectAssistantBackend { }, }; streamHandle = - options.provider === "openai" + options.provider === "hermes" + ? streamHermesChat({ ...streamOptions, baseUrl: options.baseUrl ?? "", sessionId: options.sessionId ?? "" }) + : options.provider === "openai" ? streamOpenAiResponse(streamOptions) : options.provider === "local" ? streamLocalQwen(streamOptions) : streamAnthropicMessage(streamOptions); }; - deadline = setTimeout(() => finishError("Assistant stopped: turn took too long"), MAX_TURN_MS); + deadline = setTimeout(() => finishError("Assistant stopped: turn took too long"), maxTurnMs); runIteration(0); return { diff --git a/app/assistant/models.ts b/app/assistant/models.ts index 66d67670..567de0ad 100644 --- a/app/assistant/models.ts +++ b/app/assistant/models.ts @@ -10,6 +10,7 @@ export const ASSISTANT_MODEL_VALUES = [ "terra", "sol", "qwen", + "hermes", ] as const; export type AssistantModel = (typeof ASSISTANT_MODEL_VALUES)[number]; @@ -20,11 +21,14 @@ export function supportedAssistantModel(value: AssistantModel): AssistantModel { return global.isIOS && value === "qwen" ? "auto" : value; } -export type AssistantProvider = "anthropic" | "openai" | "local"; +export type AssistantProvider = "anthropic" | "openai" | "local" | "hermes"; export type AssistantApiKeys = { anthropic: string; openai: string; + /** A Hermes Agent API server; only the voice assistant passes these. */ + hermesUrl?: string; + hermesKey?: string; }; export type ResolvedAssistantModel = { @@ -33,6 +37,8 @@ export type ResolvedAssistantModel = { model: string; apiKey: string; effort?: "low" | "medium" | "high"; + /** API root for providers that are not fixed services (Hermes). */ + baseUrl?: string; }; type ModelDefinition = { @@ -55,6 +61,8 @@ const MODEL_DEFINITIONS: Record, ModelDefinition // Runs on the phone itself (llama.cpp); needs the downloaded model file, // not an API key. The default for users who haven't set any key. qwen: { label: `${LOCAL_MODEL.label} (on-phone)`, provider: "local", model: LOCAL_MODEL.id }, + // Your own Hermes Agent (its API server), with its own tools and memory. + hermes: { label: "Hermes agent", provider: "hermes", model: "hermes-agent" }, }; export function assistantModelLabel(value: AssistantModel): string { @@ -93,6 +101,11 @@ export function resolveAssistantModel( } const definition = MODEL_DEFINITIONS[resolvedSelection as Exclude]; + if (definition.provider === "hermes") { + const baseUrl = keys.hermesUrl?.trim() ?? ""; + if (!baseUrl) return null; + return { selection, provider: "hermes", model: definition.model, apiKey: keys.hermesKey?.trim() ?? "", baseUrl }; + } if (definition.provider === "local") { if (!isLocalModelReady()) return null; return { diff --git a/app/assistant/session.ts b/app/assistant/session.ts index 69a915fc..148d5e79 100644 --- a/app/assistant/session.ts +++ b/app/assistant/session.ts @@ -22,6 +22,8 @@ export type AssistantSessionHistory = { transcript: AssistantTranscriptEntry[]; /** Model identity for safe replay after Auto resolves differently on restart. */ engine?: string; + /** The Hermes server-side session for this conversation (Hermes keeps the history). */ + hermesSessionId?: string; }; /** Which engine answers utterances, plus what it needs to do so. */ @@ -32,6 +34,16 @@ export type AssistantBackendConfig = /** Trim history from the head once it grows past this many messages. */ const MAX_HISTORY_MESSAGES = 40; +/** The longest tool name every provider accepts (OpenAI allows 64; Anthropic, 128). */ +const MAX_API_TOOL_NAME_LENGTH = 64; +/** Characters of the sanitized name kept at the start when shortening it. */ +const SHORTENED_TOOL_NAME_HEAD = 16; + + +/** Hermes brings its own persona and tools; it only needs to know where the reply is shown. */ +const HERMES_SYSTEM_PROMPT = + "The user is talking to you through smart glasses with a small text display. Reply briefly in plain text, without Markdown."; + export class AssistantSession { private readonly directBackend = new DirectAssistantBackend(); private readonly messages: LlmMessage[] = []; @@ -43,12 +55,18 @@ export class AssistantSession { // API-safe tool name -> canonical registry name. Provider APIs restrict tool // names, so dotted registry names are sanitized and mapped back on calls. private readonly toolNameMap = new Map(); + // Saved with the history: Faceclaw only sends Hermes the latest message, so a + // restored conversation must keep addressing the same server-side session. + private readonly hermesSessionId: string; constructor( private config: AssistantBackendConfig, private readonly registry: ToolRegistry = toolRegistry, history?: AssistantSessionHistory, ) { + this.hermesSessionId = typeof history?.hermesSessionId === "string" && history.hermesSessionId + ? history.hermesSessionId + : `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 10)}`; if (history) { this.messages.push(...history.messages); this.transcript.push(...history.transcript); @@ -68,7 +86,7 @@ export class AssistantSession { } history(): AssistantSessionHistory { - return { messages: this.messages, transcript: this.transcript, engine: this.engine() }; + return { messages: this.messages, transcript: this.transcript, engine: this.engine(), hermesSessionId: this.hermesSessionId }; } configure(config: AssistantBackendConfig): void { @@ -173,7 +191,10 @@ export class AssistantSession { apiKey: llm.apiKey, model: llm.model, effort: llm.effort, - system: isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), + baseUrl: llm.baseUrl, + sessionId: this.hermesSessionId, + system: llm.provider === "hermes" ? HERMES_SYSTEM_PROMPT + : isLocal ? ASSISTANT_SYSTEM_PROMPT_BASE : buildAssistantSystemPrompt(ctx), messages: turnMessages, buildTools: () => this.buildToolDefinitions(), registry: this.registry, @@ -221,13 +242,20 @@ export class AssistantSession { /** Convert a canonical registry name to a provider-legal tool name. */ private toApiToolName(name: string): string { let base = name.replace(/[^a-zA-Z0-9_-]/g, "_"); - // Disambiguate the rare case where two canonical names sanitize alike. - if (this.toolNameMap.has(base)) { - let i = 2; - while (this.toolNameMap.has(`${base}_${i}`)) i++; - base = `${base}_${i}`; + if (base.length > MAX_API_TOOL_NAME_LENGTH) { + // Keep the start (the namespace) and the end (the tool's own name). The + // hash of the whole name keeps shortened names distinct and stable. + const hash = fnv32(name).toString(16).padStart(8, "0"); + const tail = MAX_API_TOOL_NAME_LENGTH - SHORTENED_TOOL_NAME_HEAD - hash.length - 2; + base = `${base.slice(0, SHORTENED_TOOL_NAME_HEAD)}_${hash}_${base.slice(-tail)}`; } - return base; + // Disambiguate the rare case where two canonical names sanitize alike. + let apiName = base; + for (let i = 2; this.toolNameMap.has(apiName); i++) { + const suffix = `_${i}`; + apiName = base.slice(0, MAX_API_TOOL_NAME_LENGTH - suffix.length) + suffix; + } + return apiName; } private trimHistory(): void { @@ -250,3 +278,13 @@ function startsWithToolResult(message: LlmMessage): boolean { message.content[0]!.type === "tool_result" ); } + +/** FNV-1a over UTF-16 code units: a small stable hash for shortened tool names. */ +function fnv32(text: string): number { + let hash = 2166136261; + for (let i = 0; i < text.length; i++) { + hash ^= text.charCodeAt(i); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} diff --git a/app/assistant/system-tools.ts b/app/assistant/system-tools.ts index 75309775..9baacf56 100644 --- a/app/assistant/system-tools.ts +++ b/app/assistant/system-tools.ts @@ -21,7 +21,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void registered = true; // Keep a media listener warm so "what's playing" works without opening Music. - if (!global.isIOS) void mediaControllerBridge.start(); + void mediaControllerBridge.start(); registry.registerSystemTool( { @@ -86,9 +86,6 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void }, ); - // Phone media and Android notification tools are still platform-specific. - if (global.isIOS) return; - registry.registerSystemTool( { name: "media.now_playing", @@ -127,6 +124,9 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void }, ); + // Android notification tools are still platform-specific. + if (global.isIOS) return; + registry.registerSystemTool( { name: "notifications.list", @@ -181,7 +181,7 @@ function describeState(): string { function describeNowPlaying(): string { const state = mediaControllerBridge.snapshot(); - if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable."; + if (!state.accessEnabled) return `Media state is unavailable (${mediaControllerBridge.accessLabel.toLowerCase()}).`; if (state.playbackState === "playing" || state.playbackState === "paused") { const verb = state.playbackState === "playing" ? "Playing" : "Paused"; const title = state.title || "(unknown title)"; diff --git a/app/bootstrap.android.ts b/app/bootstrap.android.ts index c46446a0..82c23dd5 100644 --- a/app/bootstrap.android.ts +++ b/app/bootstrap.android.ts @@ -7,6 +7,11 @@ import { startLiveHealthSync, startAlignedRingPull } from './health/health-live' installNativeUserAgent() registerShareIntentHandler() registerPhoneRotation() +// Opt-in analytics (app/analytics/), started once the app is up rather than +// loading its imports on the boot path. +Application.on(Application.launchEvent, () => { + setTimeout(() => require('./analytics/analytics-sources').startAppAnalytics(), 0) +}) // Ring health records live only in the communicator's memory until something // stores them. Both health surfaces store on open, but a pull that lands while diff --git a/app/bootstrap.ios.ts b/app/bootstrap.ios.ts index 7ef6831e..14195c12 100644 --- a/app/bootstrap.ios.ts +++ b/app/bootstrap.ios.ts @@ -4,6 +4,10 @@ declare const FaceclawConfigPort: any // Apply transferred preferences before settings getters or app workers load. FaceclawConfigPort.processPendingRequest() require('./phone-ui/onboarding-register.ios') +// Opt-in analytics (app/analytics/), started once the app is up. +Application.on(Application.launchEvent, () => { + setTimeout(() => require('./analytics/analytics-sources').startAppAnalytics(), 0) +}) const { hasCompletedOnboarding } = require('./phone-ui/onboarding-state') as typeof import('./phone-ui/onboarding-state') Application.run({ create: () => { const frame = new Frame() diff --git a/app/g2/dashboard-controller.ts b/app/g2/dashboard-controller.ts index 30966522..0d24ecf4 100644 --- a/app/g2/dashboard-controller.ts +++ b/app/g2/dashboard-controller.ts @@ -15,8 +15,9 @@ import { startForegroundNotification, stopForegroundNotification, updateForegrou import { mediaControllerBridge } from "../native/media-controller"; import { nightscoutBridge } from "../native/nightscout-bridge"; import { weatherBridge } from "../native/weather"; -import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, readActiveNotifications } from "../native/notification-icons"; +import { ALL_NOTIFICATIONS, onAndroidNotificationPosted, onAndroidNotificationRemoved, readActiveNotifications } from "../native/notification-icons"; import { shouldShowNotificationOnGlasses } from "../native/notification-sources"; +import { watchNotificationListenerBinding } from "../native/notification-access"; import { openEvenAppSettings, readEvenAppNotificationState } from "../native/even-app-conflict"; import { defaultRingName, setGlassesRingLink } from "../native/ring-status"; import { getBooleanSetting, setBooleanSetting } from "../native/settings-store"; @@ -47,17 +48,20 @@ const MIRROR_TOUCH_GESTURES: Record, WearRemoteI "swipe-left": "swipe-left", "swipe-right": "swipe-right", }; -import { findSoundEffect, playSoundEffect } from "../ui/sound-effects"; import { GlanceHost } from "./glance-host"; +import { mirrorToPanelFraction, phonePreviewCrop } from "./phone-preview-crop"; import { type GlanceEvent } from "./glance-state"; -import { isPreviewOnlyMode, isWelcomeSoundPending, setWelcomeSoundPending } from "../phone-ui/onboarding-state"; +import { isPreviewOnlyMode } from "../phone-ui/onboarding-state"; +import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from "../apps/onboarding/onboarding-progress"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { voiceControlBridge } from "../native/voice-control"; +import { voiceActivity } from "../ui/shell/voice-activity"; import { G2_LENS_HEIGHT, G2_LENS_WIDTH, GrayImage } from "../graphics/image"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import { prepareFrameDraws } from "../graphics/glyph-wire"; import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from "./lock-screen"; import { rawInputEventToInputEvent, shell, type ShellInputOutcome } from "../ui/shell/shell"; +import { NotificationPopupDelay, notificationPopupDelayMs } from "../ui/shell/notification-popup-delay"; import { type InputEvent } from "../ui/gestures"; import { registerSystemTools } from "../assistant/system-tools"; import { updateGlassesPresence } from "./glasses-presence"; @@ -71,10 +75,10 @@ import { ALL_APPS } from "../apps/all-apps"; import { type AppContext, type AppDefinition, type AppLaunchParams, type TextEditorHost } from "../apps/app-definition"; import { type InProcessAppOptions, type InProcessWindow } from "../ui/shell/in-process-window"; import { loadPersistedOpenApps, savePersistedOpenApps } from "../ui/shell/open-apps-persistence"; -import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type WindowHeightMode } from "../ui/shell/geometry"; -import { type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; +import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect, type WindowHeightMode } from "../ui/shell/geometry"; +import { type ContinuousCaptureOptions, type LayerActions, type TextSettingsEditToggle } from "../ui/layers"; import { type KeyboardInputSession } from "../ui/shell/keyboard-input"; -import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayModeSetting, navigateDisplayModeSetting, navigateVerticalPositionSetting, terminalDisplayModeSetting, terminalVerticalPositionSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, verticalPositionSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString } from "../ui/dashboard-settings"; +import { appSwitcherPositionSetting, statusBarPositionSetting, statusBarVisibilitySetting, windowBorderSetting, assistantAllowProactiveSetting, assistantBackendSetting, assistantBridgeHostSetting, assistantBridgePortSetting, assistantBridgeTokenSetting, getBrightnessPreferences, displayLayoutSetting, navigateDisplayLayoutSetting, terminalDisplayLayoutSetting, elevenLabsApiKeySetting, getStringSettingById, openAiApiKeySetting, nightscoutApiTokenSetting, firmwareDebugFlagsSetting, whisperServerApiKeySetting, whisperServerLanguageSetting, whisperServerModelSetting, whisperServerUrlSetting, lockScreenEnabledSetting, nightscoutSiteUrlSetting, onAnySettingChanged, previewColorSetting, ringConnectionModeSetting, type RingConnectionMode, saveVoiceRecordingsSetting, sonioxApiKeySetting, screenTimeoutSetting, screenTimeoutSettingToMs, suspendEvenHubWhenScreenOffSetting, voiceProviderSetting, wakeWordActionSetting, type ConfigSettingString, type VoiceProvider } from "../ui/dashboard-settings"; import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization"; import { getInstalledEvenHubAppById, @@ -94,7 +98,7 @@ import { createNotificationsAppWindow, NOTIFICATIONS_SURFACE_ID, NOTIFICATIONS_W import { isInstalledPackagePresent } from "../apps/evenhub/updates"; import { wearerVerificationOptions } from "../apps/microphones/speakers"; import { micSession } from "../apps/microphones/mic-session"; -import { glassesDisplayLabel } from "./glasses-display-state"; +import { glassesDisplayAsleep, glassesDisplayLabel, type GlassesDisplayState } from "./glasses-display-state"; import { PreviewDisplayTarget, type DisplayTarget } from "../native/preview-display"; type ConnectionPhase = "disconnected" | "connecting" | "connected" | "charging" | "disconnecting"; @@ -164,6 +168,9 @@ const SHELL_REFRESH_INTERVAL_MS = 60_000; // schedules a refresh), so this just covers anything that slips through. const PREVIEW_INTERVAL_MS = 1_000; const SCREEN_TIMEOUT_CHECK_MS = 1_000; +// voiceActivity holder for continuous capture (Transcribe, Calculator's +// continuous mode, the Teleprompter reader). +const CONTINUOUS_CAPTURE_HOLDER = "continuous-capture"; // The lock screen is a short notice, not something to read at leisure; it // ignores the screen-timeout setting (including "never"). const LOCK_SCREEN_TIMEOUT_MS = 15_000; @@ -186,24 +193,26 @@ const EVEN_APP_DETECTED_MESSAGE = const LAUNCHABLE_APPS = ALL_APPS.filter((app) => app.showInLauncher !== false); /** - * The global settings window viewport sizes depend on: the display mode, - * whether the app switcher takes width beside windows, height below them - * or neither (a popup), and the chrome rows each window loses to it: below - * them, whether the status bar joins the row (dropping the top bar); with a - * popup, whether the status bar keeps rows of its own and the window frame - * (where the status bar goes only moves windows). + * The global settings window viewport sizes depend on: the display mode's + * size (a band's position only moves windows), whether the app switcher + * takes width beside windows, height below them or neither (a popup), and + * the chrome rows each window loses to it: below them, whether the status + * bar joins the row (dropping the top bar); with a popup, whether the + * status bar keeps rows of its own and the window frame (where the status + * bar goes only moves windows). */ function viewportSizesKey(): string { const position = appSwitcherPositionSetting.get(); const switcher = position === "bottom" ? `row|${statusBarPositionSetting.get()}` : position === "popup" ? `popup|${statusBarVisibilitySetting.get()}|${windowBorderSetting.get()}` : "strip"; - return `${displayModeSetting.get()}|${switcher}`; + const layout = displayLayoutSetting.get(); + return `${layout.startsWith("band-") ? "band" : layout}|${switcher}`; } /** The global settings that move window surfaces without resizing them. */ function windowPlacementKey(): string { - return `${verticalPositionSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`; + return `${displayLayoutSetting.get()}|${appSwitcherPositionSetting.get()}|${statusBarPositionSetting.get()}`; } function createInitialDisplayPreview(): ImageSource | null { @@ -276,6 +285,8 @@ class DashboardController { private extractedFontsConfirmed = false; private screenRecordingActive = false; private displayPreview: ImageSource | null = createInitialDisplayPreview(); + /** The screen area displayPreview shows (Preview scaling), or null for the whole panel. */ + private displayPreviewCrop: ScreenRect | null = null; private silentMode = false; // Brightness value last pushed to the glasses; see pushBrightness. private lastPushedBrightness: string | null = null; @@ -283,9 +294,6 @@ class DashboardController { // drop-off right after a low reading can be explained as a flat battery. private lastHeadsetBattery: number | null = null; private readonly listeners = new Set(); - // Set at connect time from the persisted flag; the one-time post-onboarding - // welcome sound plays on the first rendered frame (proof the session is warm). - private welcomeSoundArmed = false; private communicator: FaceclawCommunicatorBridge | null = null; // Headless stand-in for the compositor when no glasses are paired @@ -343,6 +351,9 @@ class DashboardController { private offFirmwareInfo: (() => void) | null = null; private offVoiceStatus: (() => void) | null = null; private offAndroidNotification: (() => void) | null = null; + private offAndroidNotificationRemoved: (() => void) | null = null; + /** New notifications wait here briefly before popping up (see NotificationPopupDelay). */ + private readonly notificationPopups = new NotificationPopupDelay((notificationKey) => this.popUpAndroidNotification(notificationKey)); private lastInput = "waiting..."; private lastSys = "none yet"; private shellRenderInProgress = false; @@ -389,8 +400,9 @@ class DashboardController { endTextSettingEdit: () => this.endTextSettingEdit(), startVoiceCapture: (endpointing = false) => this.startVoiceCapture(endpointing), stopVoiceCapture: () => this.stopVoiceCapture(), - startContinuousVoiceCapture: () => this.startContinuousVoiceCapture(), - stopContinuousVoiceCapture: () => this.stopContinuousVoiceCapture(), + startContinuousVoiceCapture: (client: string, options?: ContinuousCaptureOptions) => + this.startContinuousVoiceCapture(client, options), + stopContinuousVoiceCapture: (client: string) => this.stopContinuousVoiceCapture(client), playBuzzerSequence: (payload: Uint8Array) => this.playBuzzerSequence(payload), }; this.sharedActions = sharedActions; @@ -431,6 +443,7 @@ class DashboardController { }, openNotificationsWindow: () => this.openInProcessAppInBackground(NOTIFICATIONS_WINDOW_ID, NOTIFICATIONS_SURFACE_ID, createNotificationsAppWindow), + onSwitcherBarHeightChanged: () => this.applySwitcherBarHeight(), onScreenStateChanged: (on) => { // Any wake of the regular UI replaces a showing Glanceboard. if (on) this.glance.dismiss(); @@ -445,11 +458,9 @@ class DashboardController { app.boot?.(this.buildAppContext(app)); } timerEngine.onChange(() => this.refreshAlarmReliabilityWarning()); - this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => { - void this.handleAndroidNotificationPosted(notificationKey).catch((error) => { - this.appendLog(`notification wake failed: ${this.formatError(error)}`); - }); - }); + this.offAndroidNotification = onAndroidNotificationPosted((notificationKey) => this.handleAndroidNotificationPosted(notificationKey)); + this.offAndroidNotificationRemoved = onAndroidNotificationRemoved((notificationKey) => this.handleAndroidNotificationRemoved(notificationKey)); + watchNotificationListenerBinding(); readActiveNotifications(ALL_NOTIFICATIONS); // Settings toggled from the glasses can change what the phone UI shows // (e.g. the text-setting editor), so re-emit the snapshot on any change. @@ -459,15 +470,16 @@ class DashboardController { this.pushFirmwareDebugFlags(); this.pushBrightness(); this.syncEvenHubScreenOffSetting(); - // Display mode first: a reflow repositions every surface too, so the - // vertical-position check after it then has nothing left to do. + // Display mode size first: a reflow repositions every surface too, so + // the band-position check after it then has nothing left to do. this.applyDisplayModeIfChanged(); this.applyVerticalPositionIfChanged(); this.applyAppLayoutsIfChanged(); this.syncAssistantBridgeIfChanged(); this.syncLockScreenSettingIfChanged(); - // A Preview color change should show on the mirror at once, not at the - // next composited frame (cheap: floor-limited, dedup'd on the Java side). + // A Preview color or scaling change should show on the mirror at once, + // not at the next composited frame (cheap: floor-limited, dedup'd on + // the Java side). this.schedulePreviewUpdate(); }); // Connect to the external agent bridge at boot if configured; the @@ -540,7 +552,7 @@ class DashboardController { }); } - // Vertical-position changes (and moving the app switcher between the left + // Band position changes (and moving the app switcher between the left // and right edges, or a popup switcher's status bar between the top and // bottom) move every window surface and the chrome that aligns with them; // the values are tracked so unrelated setting changes don't trigger a full @@ -550,7 +562,7 @@ class DashboardController { private lastViewportSizes = viewportSizesKey(); /** - * Display mode changed (Settings > Display, or the phone page's picker), + * Display mode size changed (Settings > Display, or the phone page's picker), * or the app switcher or its chrome changed what windows lose to them * (see viewportSizesKey): every window's viewport size changes. In-process windows re-measure in place; workers * that support resizing receive the new viewport. Other worker windows are @@ -592,12 +604,35 @@ class DashboardController { }); } + /** + * The bottom switcher's layout changed its height (shell.ts + * onSwitcherBarHeightChanged): window bands move where the slack under + * them runs out, and full-height windows resize; the rest stay as they + * are. Windows that can't resize keep the surface they have, the switcher + * covering any of it that it now overlaps, rather than being relaunched + * for it. + */ + private applySwitcherBarHeight(): void { + void (async () => { + for (const window of Array.from(shell.getWindows())) { + const rect = JSON.stringify(appViewportRect(window.heightMode, window.appId)); + if (!window.relayout || this.windowSurfaceRects.get(window.surfaceId) === rect) continue; + window.relayout(); + await this.configureWindowSurface(window.surfaceId, this.isForegroundWindow(window.windowId), window.heightMode); + } + shell.foregroundWindow()?.requestRender(); + this.requestShellRender(); + })().catch((error) => { + this.appendLog(`switcher height change failed: ${this.formatError(error)}`); + }); + } + private lastAppLayouts = this.appLayouts(); private appLayouts(): Record { return { - navigate: `${navigateDisplayModeSetting.get()}:${navigateVerticalPositionSetting.get()}`, - terminal: `${terminalDisplayModeSetting.get()}:${terminalVerticalPositionSetting.get()}`, + navigate: navigateDisplayLayoutSetting.get(), + terminal: terminalDisplayLayoutSetting.get(), }; } @@ -1170,14 +1205,23 @@ class DashboardController { /** What occupies the foreground-title line on the watch-style controls. */ glassesDisplayLabel(): string { - return glassesDisplayLabel({ + return glassesDisplayLabel(this.glassesDisplayState()); + } + + /** Whether that line reads "Display off" (the watch face adds a wake hint under it). */ + glassesDisplayAsleep(): boolean { + return glassesDisplayAsleep(this.glassesDisplayState()); + } + + private glassesDisplayState(): GlassesDisplayState { + return { phase: this.phase, silentMode: this.silentMode, screenOn: shell.isScreenOn(), battery: this.lastHeadsetBattery, foregroundTitle: shell.getForegroundApp()?.title ?? null, previewMode: this.isPreviewDisplayActive(), - }); + }; } /** @@ -1439,7 +1483,6 @@ class DashboardController { this.teardownPreviewDisplay(); this.lastInput = "waiting..."; this.lastSys = "none yet"; - this.welcomeSoundArmed = isWelcomeSoundPending(); this.firmwareWarningMessage = ""; this.glassesWorn = null; this.wakeAfterChargingReconnect = false; @@ -1614,11 +1657,12 @@ class DashboardController { // parked by the previous session (or by an EvenHub suspend) resumes. this.resumeVoiceCapture(); // A rendered frame means the session is warmed up (fixedLayoutCreated), - // so the buzzer won't be dropped. Play the one-time welcome sound now. - if (this.welcomeSoundArmed) { - this.welcomeSoundArmed = false; - setWelcomeSoundPending(false); - void this.playWelcomeSound(); + // so the onboarding's welcome jingle won't be dropped. Open the + // on-glasses onboarding if this version hasn't been through it. + if (claimGlassesOnboardingAutoLaunch()) { + this.launchApp(ONBOARDING_APP_ID).catch((error) => { + this.appendLog(`onboarding launch failed: ${this.formatError(error)}`); + }); } } }); @@ -1731,6 +1775,10 @@ class DashboardController { await nightscoutBridge.stop().catch(() => {}); weatherBridge.setSessionActive(false); voiceControlBridge.stop(); + // The parked continuous capture went with it; the next session starts + // it afresh for any window still asking. + this.continuousAttemptedOn = undefined; + this.syncContinuousClaim(); if (communicator) { await communicator.setFaceclawWakeLeaseEnabled(false).catch(() => false); await communicator.close().catch(() => {}); @@ -1966,13 +2014,14 @@ class DashboardController { await this.injectSyntheticRingInput(MIRROR_TOUCH_GESTURES[kind], "watch"); return; } - const x = Math.round(Math.min(1, Math.max(0, nx)) * G2_LENS_WIDTH); - const y = Math.round(Math.min(1, Math.max(0, ny)) * G2_LENS_HEIGHT); + const at = mirrorToPanelFraction(this.displayPreviewCrop, nx, ny); + const x = Math.round(Math.min(1, Math.max(0, at.nx)) * G2_LENS_WIDTH); + const y = Math.round(Math.min(1, Math.max(0, at.ny)) * G2_LENS_HEIGHT); const foreground = shell.foregroundWindow(); const stripShown = sidebarStripVisible(shell.getFocus(), foreground?.appId); this.appendLog(`mirror tap at ${x},${y}`); - if (!shell.hasOverlay() && shell.focusNotificationAt(x, y)) { - this.appendLog("mirror tap: notification icon"); + if (!shell.hasOverlay() && shell.focusStatusItemAt(x, y)) { + this.appendLog("mirror tap: status bar item"); this.requestShellRender(); return; } @@ -2054,12 +2103,73 @@ class DashboardController { return voiceControlBridge.stopPushToTalk(); } - private startContinuousVoiceCapture(): void { - void this.beginVoiceCapture("continuous"); + // Windows that want continuous capture, and what each needs of it. They + // share one stream, which runs while any of them wants it: one window + // letting go must not cut off another. + private readonly continuousClients = new Map(); + // The connection a continuous-capture start was last tried on (undefined: + // none since the last teardown), so a window waiting on the glasses is + // retried once per new session rather than on every frame. + private continuousAttemptedOn: FaceclawCommunicatorBridge | null | undefined = undefined; + + private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { + this.continuousClients.set(client, options); + this.retargetContinuousCapture(); + this.ensureContinuousCapture(true); } - private stopContinuousVoiceCapture(): void { - voiceControlBridge.stopContinuousCapture(); + private stopContinuousVoiceCapture(client: string): void { + if (!this.continuousClients.delete(client)) return; + if (this.continuousClients.size > 0) { + // The window that needed its own provider may be the one that left. + this.retargetContinuousCapture(); + return; + } + try { + voiceControlBridge.stopContinuousCapture(); + } finally { + this.syncContinuousClaim(); + } + } + + /** + * Start continuous capture for the windows asking for it, unless it is + * running or parked across a session outage. A window restored at launch + * asks before the glasses connect, so this is retried on each new session + * (from resumeVoiceCapture); `force` retries on the same one. + */ + private ensureContinuousCapture(force = false): void { + if (this.continuousClients.size === 0 || voiceControlBridge.holdsCapture("continuous")) return; + if (!force && this.continuousAttemptedOn === this.communicator) return; + this.continuousAttemptedOn = this.communicator; + void this.beginVoiceCapture("continuous").finally(() => this.syncContinuousClaim()); + } + + /** Move a running continuous capture onto the provider its windows now call for. */ + private retargetContinuousCapture(): void { + voiceControlBridge.retargetContinuousCapture(this.voiceCaptureOptions(this.communicator)); + } + + /** The provider a continuous-capture window needs (Translate's multilingual one), if any. */ + private multilingualProvider(): VoiceProvider | undefined { + for (const options of this.continuousClients.values()) { + if (options.multilingualProvider) return options.multilingualProvider; + } + return undefined; + } + + /** + * Continuous capture's claim on the mic (the Microphones app and EvenHub + * mic apps stand down while it is held) follows whether the capture really + * holds the mic, not merely whether a window asked: a start that never + * happened (no glasses yet, permission refused) must not idle every other + * mic user. + */ + private syncContinuousClaim(): void { + voiceActivity.setActive( + this.continuousClients.size > 0 && voiceControlBridge.holdsCapture("continuous"), + CONTINUOUS_CAPTURE_HOLDER, + ); } /** @@ -2068,13 +2178,21 @@ class DashboardController { * stands in for the G2 mic. */ private voiceCaptureOptions(communicator: FaceclawCommunicatorBridge | null) { + // A window that needs a multilingual provider sets it for every capture + // while it is open, so push-to-talk still shares its stream. + const multilingual = this.multilingualProvider(); return { communicator: communicator?.getNativeCommunicator() ?? null, usePhoneMic: communicator === null, - provider: voiceProviderSetting.get(), + provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + whisperServerUrl: whisperServerUrlSetting.get(), + whisperServerModel: whisperServerModelSetting.get(), + // Whisper detects the language itself when the speech may be in any. + whisperServerLanguage: multilingual ? "" : whisperServerLanguageSetting.get(), + whisperServerApiKey: whisperServerApiKeySetting.get(), saveRecording: saveVoiceRecordingsSetting.get(), // "My voice only" (Microphones app): verify command utterances // against the enrolled wearer voice-print; non-matching speakers' @@ -2087,15 +2205,19 @@ class DashboardController { } /** - * Restart the mic for a capture that a previous session left holding. Cheap - * to call per frame: it does nothing unless one is actually parked. + * Restart the mic for a capture that a previous session left holding, and + * start a continuous capture asked for before this session. Cheap to call + * per frame: it does nothing unless one is actually waiting. */ private resumeVoiceCapture(): void { - if (!voiceControlBridge.hasSuspendedCapture()) return; const communicator = this.communicator; if (!communicator || this.phase !== "connected") return; - this.appendLog("resuming voice capture on the new glasses session"); - voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); + if (voiceControlBridge.hasSuspendedCapture()) { + this.appendLog("resuming voice capture on the new glasses session"); + voiceControlBridge.resumeCapture(this.voiceCaptureOptions(communicator)); + } + // A continuous capture asked for before this session existed. + this.ensureContinuousCapture(); } /** @@ -2140,6 +2262,11 @@ class DashboardController { if (kind === "ptt") { voiceControlBridge.startPushToTalk(options); } else { + // Every window may have let go while permission was pending. + if (this.continuousClients.size === 0) return; + // Claimed before the capture starts so the Microphones app and + // EvenHub mic apps stand down first, as they do for the voice dialog. + voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER); voiceControlBridge.startContinuousCapture(options); } }) @@ -2294,7 +2421,7 @@ class DashboardController { let shellPreWoke = false; if (wakewordShouldWake || displayShouldWake) { if (!shell.isScreenOn()) { - shellPreWoke = shell.wake("sidebar"); + shellPreWoke = shell.wake(displayShouldWake ? shell.userWakeFocus() : "sidebar"); } if (shellPreWoke || this.evenHubSessionSuspended || this.evenHubResumePromise) { const ready = await frameTimings.spanAsync(frameId, "wake-barrier", () => @@ -2394,7 +2521,7 @@ class DashboardController { * An in-process singleton app's window, opened in the background (neither * foregrounded nor focused) unless it is open already: for the shell to * bring forward itself (the Notifications window under a notification - * selected in the switcher). The bridge runs calls in order, so the + * selected in the switcher, or an app widget's window under the widget). The bridge runs calls in order, so the * shell may foreground it at once: its surface's visibility is settled * when the configure call is queued, ahead of any frame. */ @@ -2483,6 +2610,8 @@ class DashboardController { launchApp: (appId, params) => this.launchApp(appId, params), uninstallApp: (appId) => this.uninstallApp(appId), launchInProcessApp: (windowId, surfaceId, create) => this.launchInProcessApp(windowId, surfaceId, create), + openInProcessAppInBackground: (windowId, surfaceId, create) => + this.openInProcessAppInBackground(windowId, surfaceId, create), ensureWorkerHost: (createWorker) => this.ensureWorkerHost(app.appId, createWorker), submitWindowFrame: (surfaceId, planes, paintMs, frameId) => this.submitWindowFrame(surfaceId, planes, paintMs, frameId), @@ -2600,20 +2729,22 @@ class DashboardController { target: DisplayTarget | null = this.display, ): Promise { if (!target) return; - await target.configureSurface(surfaceId, { - ...appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId), - zOrder: 0, - transparency: "opaque", - }); + const rect = appViewportRect(heightMode, shell.getWindows().find((window) => window.surfaceId === surfaceId)?.appId); + this.windowSurfaceRects.set(surfaceId, JSON.stringify(rect)); + await target.configureSurface(surfaceId, { ...rect, zOrder: 0, transparency: "opaque" }); await target.setSurfaceVisible(surfaceId, typeof visible === "function" ? visible() : visible); } + /** Each window surface's rect as last configured (applySwitcherBarHeight skips those that stay). */ + private readonly windowSurfaceRects = new Map(); + /** A late answer to "is this window in front?", for configureWindowSurface. */ private isForegroundWindow(windowId: string): () => boolean { return () => shell.foregroundWindow()?.windowId === windowId; } private removeWindowSurface(surfaceId: string): void { + this.windowSurfaceRects.delete(surfaceId); const display = this.display; if (!display) return; void display.removeSurface(surfaceId).catch((error) => { @@ -2751,23 +2882,47 @@ class DashboardController { this.schedulePreviewUpdate(); } - private async handleAndroidNotificationPosted(notificationKey: string): Promise { + private handleAndroidNotificationPosted(notificationKey: string): void { + // The tray shows the notification at once; its popup waits out a short + // delay, so one withdrawn right away (see NotificationPopupDelay) never shows. const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey); - if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) { - this.requestShellRender(); - return; + if (notification && shouldShowNotificationOnGlasses(notification.packageName)) { + this.notificationPopups.post(notificationKey, notificationPopupDelayMs(notification)); } - // New notifications open a shell modal over the app viewport; if the - // screen was off, wake for it and go back to sleep when it is closed. - // Waking while already on would steal focus, so only wake from sleep. - const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar"); - if (wokeScreen) { - this.appendLog("android notification woke the screen"); - } - shell.openNotificationModal(notificationKey, wokeScreen); this.requestShellRender(); } + private handleAndroidNotificationRemoved(notificationKey: string): void { + if (this.notificationPopups.withdraw(notificationKey)) { + this.appendLog(`android notification ${notificationKey} withdrawn before its popup`); + } + try { + shell.notificationRemoved(notificationKey); + } catch (error) { + this.appendLog(`closing a removed notification's popup failed: ${this.formatError(error)}`); + } + this.requestShellRender(); + } + + /** A posted notification waited out its popup delay: show it, if it is still there. */ + private popUpAndroidNotification(notificationKey: string): void { + try { + const notification = readActiveNotifications(ALL_NOTIFICATIONS).find((item) => item.key === notificationKey); + if (!notification || !shouldShowNotificationOnGlasses(notification.packageName)) return; + // New notifications open a shell modal over the app viewport; if the + // screen was off, wake for it and go back to sleep when it is closed. + // Waking while already on would steal focus, so only wake from sleep. + const wokeScreen = shell.isScreenOn() ? false : shell.wake("sidebar"); + if (wokeScreen) { + this.appendLog("android notification woke the screen"); + } + shell.openNotificationModal(notificationKey, wokeScreen); + this.requestShellRender(); + } catch (error) { + this.appendLog(`notification popup failed: ${this.formatError(error)}`); + } + } + private async playBuzzerSequence(payload: Uint8Array): Promise { if (this.phase !== "connected" || !this.communicator) { return; @@ -2775,21 +2930,6 @@ class DashboardController { await this.communicator.playBuzzerSequence(payload); } - /** One-time celebratory jingle on the first connection after onboarding. */ - private async playWelcomeSound(): Promise { - const effect = findSoundEffect("questcomplete"); - if (!effect) return; - try { - await playSoundEffect( - effect, - (payload) => this.playBuzzerSequence(payload), - (ms) => new Promise((resolve) => setTimeout(resolve, ms)), - ); - } catch (error) { - this.appendLog(`welcome sound failed: ${this.formatError(error)}`); - } - } - private clearDashboardTimer(): void { this.cancelEvenHubSuspendTimer(); if (this.shellRefreshTimer) { @@ -2933,8 +3073,10 @@ class DashboardController { const power = activity.getSystemService(android.content.Context.POWER_SERVICE) as android.os.PowerManager; if (!power?.isInteractive()) return; } - const preview = display.getCompositePreview(previewColorSetting.get() === "green"); + const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect()); + const preview = display.getCompositePreview(previewColorSetting.get() === "green", crop); if (preview) { + this.displayPreviewCrop = crop; this.setDisplayPreview(preview); } } diff --git a/app/g2/glance-host.ts b/app/g2/glance-host.ts index e92613ad..cf5d05ff 100644 --- a/app/g2/glance-host.ts +++ b/app/g2/glance-host.ts @@ -12,7 +12,7 @@ import { prepareFrameDraws } from "../graphics/glyph-wire"; import { flattenPlanesWithDraws, planesFingerprint, type Plane } from "../graphics/plane"; import * as frameTimings from "../native/frame-timings"; import { type DisplayTarget } from "../native/preview-display"; -import { minWindowTop } from "../ui/shell/geometry"; +import { minWindowTop, type ScreenRect } from "../ui/shell/geometry"; import { beginRenderPass, endRenderPass } from "../util/render-freshness"; import { type GlanceBoardInstance, type GlanceboardProvider } from "../apps/app-definition"; import { GLANCE_HIDDEN, glanceEventForGesture, reduceGlance, type GlanceEvent, type GlanceState } from "./glance-state"; @@ -59,6 +59,8 @@ export class GlanceHost { private rendering = false; private renderPending = false; private nextRenderWantsFreshData = false; + /** Where the last render put the board on screen. */ + private boardRect: ScreenRect | null = null; constructor(private readonly options: GlanceHostOptions) {} @@ -66,6 +68,11 @@ export class GlanceHost { return this.state.visible; } + /** The board's area of the screen while it shows (for the phone mirror's Scaled preview), else null. */ + screenRect(): ScreenRect | null { + return this.state.visible ? this.boardRect : null; + } + /** Whether sleep-time gestures should reach the board at all. */ isEnabled(): boolean { return this.options.getProvider()?.isEnabled() ?? false; @@ -249,6 +256,7 @@ export class GlanceHost { const x = Math.max(0, ((G2_LENS_WIDTH - boardImage.width) / 2) | 0); const y = Math.max(0, Math.min(G2_LENS_HEIGHT - boardImage.height, minWindowTop())); boardImage.composeInto(full, x, y); + this.boardRect = { x, y, width: boardImage.width, height: boardImage.height }; return full; }), ); diff --git a/app/g2/glasses-display-state.ts b/app/g2/glasses-display-state.ts index ebda5fa2..c9e51e4b 100644 --- a/app/g2/glasses-display-state.ts +++ b/app/g2/glasses-display-state.ts @@ -23,12 +23,25 @@ export function glassesDisplayLabel(state: GlassesDisplayState): string { if (state.silentMode) { return "Silent mode"; } - if (!state.screenOn) { + if (glassesDisplayAsleep(state)) { return "Display off"; } return state.foregroundTitle || "Launcher"; } +/** + * True when the label reads "Display off": the screen is dark but the glasses + * (or the preview standing in for them) would light it on a wake gesture. + */ +export function glassesDisplayAsleep(state: GlassesDisplayState): boolean { + return ( + state.phase !== "charging" && + (state.phase === "connected" || state.previewMode) && + !state.silentMode && + !state.screenOn + ); +} + export function chargingDisplayLabel(battery: number | null): string { return validBatteryLevel(battery) ? `Charging · G2 ${battery}%` : "Charging"; } diff --git a/app/g2/ios-preview-controller.ts b/app/g2/ios-preview-controller.ts index 12c1574e..982f0904 100644 --- a/app/g2/ios-preview-controller.ts +++ b/app/g2/ios-preview-controller.ts @@ -7,7 +7,7 @@ import { launcherEntries } from '../apps/launcher' import { getInstalledEvenHubAppById, installedEvenHubPackageId, uninstallEvenHubPackage } from '../apps/evenhub/installed-apps' import { isInstalledPackagePresent } from '../apps/evenhub/updates' import { openEvenHubStoreForPackage } from '../apps/evenhub' -import { launchInstalledPackage, closeRunningPackage } from '../apps/evenhub/manager' +import { launchInstalledPackage, closeRunningPackage, onPhoneShownChanged, phoneUiButton, showOnPhone, type PhoneUiButton } from '../apps/evenhub/manager' import { registerSystemTools } from '../assistant/system-tools' import { registerWindowTools } from '../assistant/window-tools' import { registerNavigateTools } from '../assistant/navigate-tools' @@ -16,6 +16,7 @@ import { IosNavigationSensors } from '../native/ios-navigation-sensors' import { bindCompassSession, receiveCompassEvent } from '../native/compass.ios' import { Dialogs, File, knownFolders, path, Utils, type ImageSource } from '@nativescript/core' import { iosVoiceInput } from '../native/ios-voice-input' +import { voiceActivity } from '../ui/shell/voice-activity' import { nightscoutBridge } from '../native/nightscout-bridge' import { weatherBridge } from '../native/weather' import { FaceclawCommunicatorBridge, resolveIosPeripherals, type CommunicatorState, type RawInputEvent } from '../native/faceclaw-communicator.ios' @@ -25,6 +26,7 @@ import { GlanceHost, type GlanceDisplay } from './glance-host' import { createLockScreenImage, LOCK_SCREEN_SURFACE_ID } from './lock-screen' import { OsEventTypeList } from './events' import { loadDeviceAddresses } from './device-addresses' +import { updateGlassesPresence } from './glasses-presence' import { firmwareIncompatibilityMessage, hasCompatibleFirmware } from './firmware-compat' import { isHaltedSessionPhase, suppressAutoReconnect } from './reconnect-policy' import { deviceAddressError } from './ios-peripheral-identity' @@ -39,18 +41,23 @@ import { flattenPlanesWithDraws, planesFingerprint, type Plane } from '../graphi import { prepareFrameDraws } from '../graphics/glyph-wire' import { G2_LENS_WIDTH, G2_LENS_HEIGHT } from '../graphics/image' import { makeInputEvent, type InputEvent, type InputEventPayload } from '../ui/gestures' -import { noopLayerActions, type LayerActions } from '../ui/layers' +import { noopLayerActions, type ContinuousCaptureOptions, type LayerActions } from '../ui/layers' import { TextViewerLayer } from '../apps/files/text-viewer' import { shell, rawInputEventToInputEvent, type ShellWindow } from '../ui/shell/shell' -import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible } from '../ui/shell/geometry' -import { DISPLAY_MODE_VALUES, displayModeLabel, displayModeSetting, onAnySettingChanged, +import { appViewportRect, isOnSwitcherEdge, sidebarStripVisible, type ScreenRect } from '../ui/shell/geometry' +import { mirrorToPanelFraction, phonePreviewCrop } from './phone-preview-crop' +import { onAnySettingChanged, previewColorSetting, lockScreenEnabledSetting, getBrightnessPreferences, getStringSettingById, - nightscoutSiteUrlSetting, nightscoutApiTokenSetting } from '../ui/dashboard-settings' + nightscoutSiteUrlSetting, nightscoutApiTokenSetting, voiceProviderSetting, elevenLabsApiKeySetting, + openAiApiKeySetting, sonioxApiKeySetting, whisperServerApiKeySetting, whisperServerLanguageSetting, + whisperServerModelSetting, whisperServerUrlSetting, ringConnectionModeSetting, saveTranscribeAudioSetting } from '../ui/dashboard-settings' +import { transcriptionKey, type VoiceProviderSelection } from '../native/cloud-stt-provider' import type { PhoneGesture } from '../phone-ui/phone-gestures' -import { isWelcomeSoundPending, setWelcomeSoundPending } from '../phone-ui/onboarding-state' -import { findSoundEffect, playSoundEffect } from '../ui/sound-effects' +import { claimGlassesOnboardingAutoLaunch, ONBOARDING_APP_ID } from '../apps/onboarding/onboarding-progress' const SHELL_SURFACE_ID = 'shell' +// voiceActivity holder for continuous capture (Transcribe), as on Android. +const CONTINUOUS_CAPTURE_HOLDER = 'continuous-capture' const SHELL_SURFACE_Z_ORDER = 1 const LOCK_SURFACE_Z_ORDER = 1000 /** The shell render loop waits this long for the glasses to take a frame before moving on. */ @@ -114,6 +121,8 @@ export class IosPreviewController { appendLog: message => this.logBluetooth(message), }) private previewTimer: ReturnType | null = null + /** The screen area the mirror's image shows (Preview scaling), or null for the whole panel. */ + private previewCrop: ScreenRect | null = null private clockTimer: ReturnType | null = null private offSettings: (() => void) | null = null private lastBrightness: string | null = null @@ -126,6 +135,8 @@ export class IosPreviewController { private nextWorkerFrame = 0 private inputQueue: Promise = Promise.resolve() private lastLayout = '' + /** The phone-UI button last pushed to the page (onEvenHubPhoneUiChanged). */ + private evenHubPhoneUi: PhoneUiButton | null = null private readonly appHosts = new Map() private readonly inProcessApps = new Map() private readonly batteryObservers: any[] = [] @@ -137,6 +148,20 @@ export class IosPreviewController { private acknowledgedFrames = 0 private logLines: string[] = [] private logTimer: ReturnType | null = null + // Windows that want continuous capture, and what each needs of it (a + // recording, a multilingual provider). They share one capture, which runs + // while any of them wants it. + private readonly continuousClients = new Map() + // transcriptionKey of the running continuous capture's provider; one that no + // longer matches what its windows need is drained and restarted. + private continuousTranscription = '' + // Continuous capture should (re)start once audio is available: a window + // asked, the glasses connected or disconnected, or the glasses lock or the + // app lifecycle cut it short. A capture that fails doesn't set it, so a + // broken start is retried on the next such change rather than in a loop. + private continuousPending = false + private continuousCaptureToken = 0 + private sessionConnected = false private readonly actions: LayerActions = { ...noopLayerActions, playBuzzerSequence: payload => this.communicator?.playBuzzerSequence(payload), @@ -144,7 +169,8 @@ export class IosPreviewController { disconnect: () => this.disconnect(), startVoiceCapture: endpointing => this.startVoiceCapture(endpointing), stopVoiceCapture: () => iosVoiceInput.stopPushToTalk(), - startContinuousVoiceCapture: () => this.onError("Voice capture is not available on iOS yet."), + startContinuousVoiceCapture: (client, options) => this.startContinuousVoiceCapture(client, options), + stopContinuousVoiceCapture: client => this.stopContinuousVoiceCapture(client), startTextSettingEdit: async setting => { await this.editSetting(setting) }, startTextSettingsEdit: async (settings, title, finished, toggle, cancelled) => { for (const setting of settings) if (!await this.editSetting(setting)) { cancelled?.(); return } @@ -160,7 +186,8 @@ export class IosPreviewController { constructor(private readonly onFrame: (image: ImageSource, focus: string) => void, private readonly onError: (message: string) => void, private readonly onConnectionState: (state: SessionState) => void = () => {}, - private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}) { + private readonly onKeyboardInputChanged: (session: KeyboardInputSession | null) => void = () => {}, + private readonly onEvenHubPhoneUiChanged: (button: PhoneUiButton | null) => void = () => {}) { this.ensurePreviewDisplay() shell.configure({ actions: this.actions, @@ -175,7 +202,12 @@ export class IosPreviewController { prepareVoiceCapture: () => this.prepareVoiceCapture(), getScreenTimeoutMs: () => null, requestShellRender: () => this.requestShellRender(), - onWindowsChanged: () => this.requestShellRender(), + onWindowsChanged: () => { this.requestShellRender(); this.syncEvenHubPhoneUi() }, + onSwitcherBarHeightChanged: () => { + this.relayout() + shell.foregroundWindow()?.requestRender() + this.requestShellRender() + }, onScreenStateChanged: on => { if (on) this.glance.dismiss() void this.display?.setScreenBlanked(!on).catch(error => this.fail(error)) @@ -218,11 +250,15 @@ export class IosPreviewController { assistantAvailable: () => shell.isAssistantAvailable(), assistant: text => shell.sendToAssistant(text), }) + // The header's EvenHub phone-UI button hides while that UI is showing. + onPhoneShownChanged(() => this.syncEvenHubPhoneUi()) registerSystemTools() registerWindowTools({ apps: ALL_APPS.filter(app => !iosAppUnavailableReason(app.appId)), launchApp: id => this.launchApp(id), requestShellRender: () => this.requestShellRender() }) registerNavigateTools(id => this.launchApp(id)) registerRoamTools(id => this.launchApp(id)) + // Every capture end (drained, failed, replaced) may leave continuous capture to restart. + iosVoiceInput.onSpeechEnd(() => this.syncContinuousCapture()) for (const app of ALL_APPS) if (app.appId !== 'launcher' && !iosAppUnavailableReason(app.appId)) app.boot?.(this.buildAppContext(app)) } @@ -277,16 +313,35 @@ export class IosPreviewController { this.relayout() shell.foregroundWindow()?.requestRender() this.requestShellRender() + this.syncEvenHubPhoneUi() + this.syncContinuousCapture() } pause(): void { if (!this.active) return this.active = false + if (iosVoiceInput.continuousSource === 'phone') this.continuousPending = true iosVoiceInput.stopPhoneCapture() // The phone preview is hidden; the glasses still own their screen and input. this.syncRuntime() this.logBluetooth(`Phone background; glasses ${this.state.phase}`) this.flushBluetoothLog() } + /** Overlay an EvenHub app's phone UI (the header's app-icon button). */ + showEvenHubPhoneUi(windowId: string): void { + showOnPhone(windowId) + } + /** + * Tell the page when the foreground window's EvenHub phone-UI button + * changes. A paused controller skips this and catches up on resume. + */ + private syncEvenHubPhoneUi(): void { + if (!this.active) return + const button = phoneUiButton() + const current = this.evenHubPhoneUi + if (current?.windowId === button?.windowId && current?.icon === button?.icon) return + this.evenHubPhoneUi = button + this.onEvenHubPhoneUiChanged(button) + } private get runtimeNeeded(): boolean { return this.active || ['connected', 'connecting', 'retrying', 'disconnecting'].includes(this.state.phase) } @@ -373,6 +428,7 @@ export class IosPreviewController { // a session's first snapshot) wakes the screen, to the lock notice if locked. const putOn = wearing && this.glassesWorn === false this.glassesWorn = wearing + this.syncPresence() this.logBluetooth(`Glasses wear state: ${wearing ? 'ON_HEAD' : 'OFF_HEAD'}`) if (!wearing && this.phoneLocked && this.lockEnabled) this.setGlassesLocked(true) if (putOn && this.runtimeRunning && !shell.isScreenOn()) shell.wake('sidebar') @@ -382,8 +438,10 @@ export class IosPreviewController { this.glassesLocked = locked if (locked) { this.glance.dismiss() - iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.') - } + if (iosVoiceInput.continuousSource === 'glasses') this.continuousPending = true + iosVoiceInput.handleSessionEnded('Glasses locked. Unlock your phone to start voice input again.', + 'Glasses locked. Transcription resumes when you unlock your phone.') + } else this.syncContinuousCapture() void (async () => { await this.displayReady const display = this.display @@ -418,10 +476,11 @@ export class IosPreviewController { } } /** Submit a painted frame for an in-process window (e.g. the launcher). */ - private async submit(id: string, planes: Plane[]): Promise { + private async submit(id: string, planes: Plane[], live: () => boolean = () => true): Promise { await this.displayReady const display = this.display - if (!display) return + // The window can close (removing its surface) while this frame waits. + if (!display || !live()) return const fingerprint = planesFingerprint(planes) const { image, draws } = flattenPlanesWithDraws(planes) await display.submitSurfaceFrame(id, image.pixels, { x: 0, y: 0, width: image.width, height: image.height }, @@ -481,8 +540,10 @@ export class IosPreviewController { this.previewTimer = null if (!this.active) return try { - const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green') + const crop = phonePreviewCrop(this.glassesLocked, this.glance.screenRect()) + const image = this.display?.getCompositePreview(previewColorSetting.get() === 'green', crop) if (!image) return + this.previewCrop = crop this.onFrame(image, this.glassesLocked ? 'Glasses locked' : this.glance.isVisible() ? 'Glanceboard' : `${shell.foregroundWindow()?.title ?? 'Apps'} · ${shell.getFocus() === 'sidebar' ? 'App switcher' : 'App'}`) } catch (error) { this.fail(error) } @@ -536,7 +597,8 @@ export class IosPreviewController { await shell.receiveInput(event) this.requestShellRender() } - private async mirrorTap(nx: number, ny: number): Promise { + private async mirrorTap(mirrorNx: number, mirrorNy: number): Promise { + const { nx, ny } = mirrorToPanelFraction(this.previewCrop, mirrorNx, mirrorNy) const x = Math.max(0, Math.min(639, Math.floor(nx * 640))) const y = Math.max(0, Math.min(479, Math.floor(ny * 480))) const window = shell.foregroundWindow() @@ -610,7 +672,7 @@ export class IosPreviewController { // The explicit render below supplies a fresh frame after configuration. let surfaceReady = false const app = create({ - actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes) : Promise.resolve(), + actions: this.actions, submitFrame: planes => surfaceReady ? this.submit(surfaceId, planes, () => surfaceReady) : Promise.resolve(), setSurfaceVisible: visible => this.setWindowSurfaceVisible(surfaceId, visible), removeSurface: () => { surfaceReady = false; this.removeWindowSurface(surfaceId) }, reconfigureSurface: () => { @@ -657,11 +719,6 @@ export class IosPreviewController { this.appHosts.set(appId, host) return host } - cycleDisplayMode(): void { - const values = DISPLAY_MODE_VALUES - displayModeSetting.set(values[(values.indexOf(displayModeSetting.get()) + 1) % values.length]) - } - get displayModeLabel(): string { return displayModeLabel(displayModeSetting.get()) } get connectionState(): SessionState | null { return this.sessionCreated ? this.state : null } get connectionDetails(): string { const state = this.connectionState @@ -672,13 +729,19 @@ export class IosPreviewController { this.logBluetooth(status) this.emitState() } + /** Mirror the session into glasses-presence (read by EvenHub's device status and Timers). */ + private syncPresence(): void { + updateGlassesPresence({ connected: this.state.phase === 'connected', worn: this.glassesWorn, charging: this.state.charging ?? false }) + } private emitState(): void { if (this.state.phase !== 'connected') resetRingInputFilter() this.pushBrightness() - this.maybePlayWelcomeSound(this.state) + this.maybeLaunchOnboarding(this.state) if (this.state.phase !== 'connected' && this.state.phase !== 'connecting') this.glassesWorn = null if (this.state.phase !== 'connected') iosVoiceInput.handleSessionEnded() + this.syncContinuousSession() shell.setBatteryLevels({ headset: this.state.battery, headsetCharging: this.state.charging }) + this.syncPresence() this.syncRuntime() if (this.active) this.onConnectionState({ ...this.state }) this.requestShellRender() @@ -695,7 +758,10 @@ export class IosPreviewController { this.firmwareCompatible = false this.update('connecting', 'Finding configured devices…') let identifiers - try { identifiers = await resolveIosPeripherals(addresses) } + // As on Android, connect to the ring only in "Direct" mode; otherwise a phone-side connect + // fails and retries every few seconds, starving the glasses link. + const ring = ringConnectionModeSetting.get() === 'direct' ? addresses.ring : '' + try { identifiers = await resolveIosPeripherals({ ...addresses, ring }) } catch (failure) { this.update('error', failure instanceof Error ? failure.message : String(failure)) return @@ -763,7 +829,7 @@ export class IosPreviewController { if (this.communicator !== communicator) return this.state = { ...this.state, frames: this.state.frames + 1 } this.onActivity() - this.maybePlayWelcomeSound(this.state) + this.maybeLaunchOnboarding(this.state) if (this.active) this.onConnectionState({ ...this.state }) this.schedulePreviewUpdate() }), @@ -871,23 +937,20 @@ export class IosPreviewController { this.glance.reset() await communicator.close().catch(error => this.logBluetooth(`Close: ${String(error)}`)) } - private maybePlayWelcomeSound(state: SessionState): void { + private maybeLaunchOnboarding(state: SessionState): void { const firstNewFrame = state.frames > this.acknowledgedFrames this.acknowledgedFrames = state.frames - // Match Android: wait for an acknowledged frame before consuming the jingle. - if (!firstNewFrame || state.phase !== 'connected' || !isWelcomeSoundPending()) return - setWelcomeSoundPending(false) - const effect = findSoundEffect('questcomplete') - if (effect) void playSoundEffect(effect, payload => this.actions.playBuzzerSequence(payload), - ms => new Promise(resolve => setTimeout(resolve, ms))) - .catch(error => this.logBluetooth(`Welcome sound failed: ${error}`)) + // Match Android: wait for an acknowledged frame (a warm session, so the + // onboarding's welcome jingle isn't dropped) before opening it. + if (!firstNewFrame || state.phase !== 'connected' || !claimGlassesOnboardingAutoLaunch()) return + void this.launchApp(ONBOARDING_APP_ID) } startVoiceInput(): void { if (!this.glassesLocked) shell.startVoiceInput() } private async prepareVoiceCapture(): Promise { if (this.glassesLocked) return false const usePhoneMic = this.state.phase !== 'connected' if (usePhoneMic && !this.active) return false - const ready = await iosVoiceInput.prepare(this.active, usePhoneMic) + const ready = await iosVoiceInput.prepare(this.active, usePhoneMic, this.voiceProvider()) if (!ready && this.active) this.onError(iosVoiceInput.statusText) return ready && !this.glassesLocked && (this.state.phase === 'connected' || this.active) } @@ -895,14 +958,116 @@ export class IosPreviewController { if (this.glassesLocked) return const log = (message: string) => this.logBluetooth(message) const communicator = this.communicator + const voice = this.voiceProvider() if (communicator && this.state.phase === 'connected') { - await iosVoiceInput.startGlassesCapture(communicator, log, endpointing) + await iosVoiceInput.startGlassesCapture(communicator, voice, log, endpointing) } else if (this.active) { // Recheck microphone permission if the glasses disconnected after prepare. - if (!await iosVoiceInput.prepare(this.active, true) || !this.active || this.glassesLocked) return - await iosVoiceInput.startPhoneCapture(log, endpointing) + if (!await iosVoiceInput.prepare(this.active, true, voice) || !this.active || this.glassesLocked) return + await iosVoiceInput.startPhoneCapture(voice, log, endpointing) } } + /** + * Continuous capture on behalf of `client` (one id per window: Transcribe, + * Translate, Teleprompter, Calculator); it runs while any client wants it, as + * on Android. `recordAs` names a WAV of the audio in Downloads when Save + * Transcribe audio is on. + */ + private startContinuousVoiceCapture(client: string, options: ContinuousCaptureOptions = {}): void { + this.continuousClients.set(client, options) + if (!iosVoiceInput.supportsContinuous(this.voiceProvider())) { + this.continuousClients.delete(client) + this.onError('Transcribe on iOS needs a cloud or Whisper server transcription provider (Settings > Voice > Transcription Provider).') + return + } + this.syncContinuousRecording() + // A listening capture is shared as it is; otherwise start one (after any drain). + if (!iosVoiceInput.continuousSource) this.continuousPending = true + this.syncContinuousCapture() + } + private stopContinuousVoiceCapture(client: string): void { + if (!this.continuousClients.delete(client)) return + this.syncContinuousRecording() + // The window that needed its own provider may be the one that left. + if (this.continuousClients.size > 0) { this.syncContinuousCapture(); return } + // A start still awaiting permission or the mic backs off. + ++this.continuousCaptureToken + this.continuousPending = false + this.syncContinuousClaim() + void iosVoiceInput.stopContinuousCapture().catch(error => this.logBluetooth(`Stop continuous capture: ${String(error)}`)) + } + /** Connecting or losing the glasses moves continuous capture to the new audio source. */ + private syncContinuousSession(): void { + const connected = this.state.phase === 'connected' + if (connected !== this.sessionConnected && this.continuousClients.size > 0) this.continuousPending = true + this.sessionConnected = connected + this.syncContinuousCapture() + } + /** + * Keep continuous capture running while a window wants it: start a pending + * one once audio is available, and move a phone capture to the glasses once + * they connect. A glasses session supplies the audio, even while it + * reconnects (as on Android); the phone mic stands in only without one, while + * Faceclaw is in the foreground. Cheap enough to call on every state change. + */ + private syncContinuousCapture(): void { + this.syncContinuousClaim() + if (this.continuousClients.size === 0 || this.glassesLocked) return + const glassesReady = !!this.communicator && this.state.phase === 'connected' + if (iosVoiceInput.continuousActive) { + // Draining or listening already. A phone capture from before the glasses + // connected is drained here, and its end restarts it on the glasses; so + // is one transcribing with another provider than its windows now need. + const retarget = this.continuousTranscription !== transcriptionKey(this.voiceProvider()) + if (iosVoiceInput.continuousSource && (retarget || glassesReady && iosVoiceInput.continuousSource === 'phone')) { + this.continuousPending = true + void iosVoiceInput.stopContinuousCapture() + } + return + } + if (!this.continuousPending || (this.communicator ? !glassesReady : !this.active)) return + this.continuousPending = false + void this.beginContinuousCapture() + } + private async beginContinuousCapture(): Promise { + const voice = this.voiceProvider() + if (!iosVoiceInput.supportsContinuous(voice)) return + this.continuousTranscription = transcriptionKey(voice) + // A stop during an await below bumps the token; this start then backs off. + const token = ++this.continuousCaptureToken + const current = () => token === this.continuousCaptureToken + const log = (message: string) => this.logBluetooth(message) + // Claimed before the capture starts so other mic users stand down first, as on Android. + voiceActivity.setActive(true, CONTINUOUS_CAPTURE_HOLDER) + const communicator = this.communicator + if (communicator) { + await iosVoiceInput.startGlassesCapture(communicator, voice, log, false, true) + } else if (await iosVoiceInput.prepare(this.active, true, voice) && current() && this.active && !this.glassesLocked && !this.communicator) { + await iosVoiceInput.startPhoneCapture(voice, log, false, true) + } + if (current()) this.syncContinuousClaim() + } + /** Other mic users stand down while continuous capture really holds the mic, not merely while a window wants it. */ + private syncContinuousClaim(): void { + voiceActivity.setActive(this.continuousClients.size > 0 && iosVoiceInput.continuousActive, CONTINUOUS_CAPTURE_HOLDER) + } + /** Record to the first client's requested name while Save Transcribe audio is on. */ + private syncContinuousRecording(): void { + const name = saveTranscribeAudioSetting.get() ? [...this.continuousClients.values()].map(options => options.recordAs).find(Boolean) ?? '' : '' + iosVoiceInput.recordTo(name) + } + /** + * The transcription provider setting and the keys the cloud ones need. A + * window that needs a multilingual provider (Translate) sets it for every + * capture while it is open, with Whisper left to detect the language. + */ + private voiceProvider(): VoiceProviderSelection { + const multilingual = [...this.continuousClients.values()].find(options => options.multilingualProvider)?.multilingualProvider + return { provider: multilingual ?? voiceProviderSetting.get(), elevenLabsApiKey: elevenLabsApiKeySetting.get(), + openAiApiKey: openAiApiKeySetting.get(), sonioxApiKey: sonioxApiKeySetting.get(), + whisperServerUrl: whisperServerUrlSetting.get(), whisperServerModel: whisperServerModelSetting.get(), + whisperServerLanguage: multilingual ? '' : whisperServerLanguageSetting.get(), whisperServerApiKey: whisperServerApiKeySetting.get() } + } private logBluetooth(message: string): void { const line = `${new Date().toISOString()} [${this.active ? 'foreground' : 'background'}${UIApplication.sharedApplication.protectedDataAvailable ? '' : ',protected-data-unavailable'}] ${message}` console.log(`[ios-ble] ${line}`); this.logLines.push(line) diff --git a/app/g2/phone-preview-crop.ts b/app/g2/phone-preview-crop.ts new file mode 100644 index 00000000..2eafaa51 --- /dev/null +++ b/app/g2/phone-preview-crop.ts @@ -0,0 +1,33 @@ +import { G2_LENS_HEIGHT, G2_LENS_WIDTH } from "../graphics/image"; +import { previewScalingSetting } from "../ui/dashboard-settings"; +import { phonePreviewRect, usedScreenRect, type ScreenRect } from "../ui/shell/geometry"; +import { shell } from "../ui/shell/shell"; + +/** + * The screen area the phone's mirror shows (Settings > Phone display > + * Preview scaling), or null for the whole panel. Scaled zooms to the + * Glanceboard while it shows, else to the area the shell uses around the + * foreground window, centred at the panel's aspect. A locked display gets + * the whole panel: its message box is centred in the panel, not the band. + */ +export function phonePreviewCrop(locked: boolean, glanceRect: ScreenRect | null): ScreenRect | null { + if (locked || previewScalingSetting.get() === "raw") return null; + const window = shell.foregroundWindow(); + const crop = phonePreviewRect(glanceRect ?? usedScreenRect(window?.heightMode ?? "min", window?.appId)); + const wholePanel = crop.x === 0 && crop.y === 0 && crop.width === G2_LENS_WIDTH && crop.height === G2_LENS_HEIGHT; + return wholePanel ? null : crop; +} + +/** + * A point on the mirror, as fractions of the mirror's width and height, + * as fractions of the panel's, given the crop the mirror is showing. + * Points in the black margins past the panel's edges come out below 0 or + * above 1. + */ +export function mirrorToPanelFraction(crop: ScreenRect | null, nx: number, ny: number): { nx: number; ny: number } { + if (!crop) return { nx, ny }; + return { + nx: (crop.x + nx * crop.width) / G2_LENS_WIDTH, + ny: (crop.y + ny * crop.height) / G2_LENS_HEIGHT, + }; +} diff --git a/app/g2/wear-remote.ts b/app/g2/wear-remote.ts index 84e04dee..9ce66457 100644 --- a/app/g2/wear-remote.ts +++ b/app/g2/wear-remote.ts @@ -23,12 +23,11 @@ * Transport is app/native/wear-bridge.ts (Java FaceclawWearBridge); the * message shapes are documented in wear/PROTOCOL.md. */ +import { flagAnalyticsEvent } from "../analytics/analytics"; import { type AppDefinition } from "../apps/app-definition"; import { getInstalledEvenHubApps, installedEvenHubAppId } from "../apps/evenhub/installed-apps"; import { WEAR_PATHS, wearBridge, type WearMessage } from "../native/wear-bridge"; import { - DISPLAY_MODE_VALUES, - displayModeSetting, onAnySettingChanged, watchCanUnlockSetting, watchCrownClockwiseNextSetting, @@ -87,7 +86,6 @@ const COMMANDS = [ "connect", "disconnect", "close-assistant", - "display-mode", ] as const; type WearCommand = (typeof COMMANDS)[number]; @@ -167,6 +165,7 @@ export class WearRemote { // with it (no placeholder). if (changed) this.watchBatteryPollsUnanswered = 0; if (connection.reachable) { + flagAnalyticsEvent("minimal", "device.wear.reachable"); this.publishNow(true); this.pollWatchBattery(true); } else if (changed) { @@ -275,7 +274,6 @@ export class WearRemote { acceptsText: typeof window.receiveTextInput === "function", })), apps: this.listApps(), - displayMode: displayModeSetting.get(), remoteEnabled: watchRemoteEnabledSetting.get(), crownClockwiseNext: watchCrownClockwiseNextSetting.get(), canUnlock: watchCanUnlockSetting.get(), @@ -326,6 +324,7 @@ export class WearRemote { return; } + flagAnalyticsEvent("minimal", "device.wear.used"); switch (message.path) { case WEAR_PATHS.input: await this.handleInput(message.payload, ack); @@ -519,17 +518,6 @@ export class WearRemote { // but tell the watch which of the two happened. ack(true, shell.closeAssistant() ? "" : "The assistant was not open."); return; - case "display-mode": { - const value = String(payload.value ?? ""); - if (!(DISPLAY_MODE_VALUES as readonly string[]).includes(value)) { - ack(false, `Unknown display mode: ${value}`); - return; - } - displayModeSetting.set(value as (typeof DISPLAY_MODE_VALUES)[number]); - this.schedulePublish(); - ack(true, `Display mode: ${value}`); - return; - } default: ack(false, `Unhandled command: ${command}`); } diff --git a/app/graphics/battery.ts b/app/graphics/battery.ts index 3c8a66fe..d8d1ec6f 100644 --- a/app/graphics/battery.ts +++ b/app/graphics/battery.ts @@ -182,6 +182,16 @@ export function drawDenseBatteries( return x + DENSE_COLUMN_GAP; } +/** The width drawDenseBatteries would take for `items` (0 when empty), to place the block before drawing it. */ +export function denseBatteriesWidth(items: readonly DenseBatteryItem[]): number { + let width = 0; + for (let end = items.length; end > 0; end -= 2) { + const column = items.slice(Math.max(0, end - 2), end); + width += Math.max(...column.map((item) => DEVICE_ICONS[item.device].width)) + DENSE_ICON_GAP + BATTERY_ICON_WIDTH + DENSE_COLUMN_GAP; + } + return Math.max(0, width - DENSE_COLUMN_GAP); +} + /** The empty outline, built once from the geometry constants. */ const EMPTY_BATTERY_ICON = buildEmptyBattery(); diff --git a/app/graphics/bdffont.ts b/app/graphics/bdffont.ts index 8bf9dfec..1d43ad05 100644 --- a/app/graphics/bdffont.ts +++ b/app/graphics/bdffont.ts @@ -248,6 +248,29 @@ export function getFont(font: EmbeddedFontName): BdfFont { return loadEmbeddedFont(font); } +/** Every Terminus size bundled under fonts/terminus, each in normal and bold. */ +export const TERMINUS_SIZES = [12, 14, 16, 18, 20, 22, 24, 28, 32] as const; + +const terminusFonts = new Map(); + +/** + * Terminus at any bundled size, normal or bold (EvenHub apps pick fonts by + * size and weight). Null for a size that isn't bundled. The sizes the UI + * already uses come back as the same objects getFont returns. + */ +export function getTerminusFont(size: number, bold: boolean): BdfFont | null { + if (!(TERMINUS_SIZES as readonly number[]).includes(size)) return null; + const name = `terminus${size}${bold ? "b" : ""}`; + if (name in embeddedFonts) return loadEmbeddedFont(name as EmbeddedFontName); + let font = terminusFonts.get(name); + if (!font) { + const text = knownFolders.currentApp().getFile(`fonts/terminus/ter-u${size}${bold ? "b" : "n"}.bdf`).readTextSync(); + font = BdfFont.parse(text, { atlasKey: name }); + terminusFonts.set(name, font); + } + return font; +} + // NOTE: the default UI font getters (getDefaultSmallFont & co.) live in // ./ui-fonts, which resolves the user's font settings to a BdfFont or // TtfFont. This module stays a pure BDF loader. diff --git a/app/graphics/icons.ts b/app/graphics/icons.ts index c18b69ae..c7310294 100644 --- a/app/graphics/icons.ts +++ b/app/graphics/icons.ts @@ -70,16 +70,39 @@ export const ICON_SVGS = { '', calendar: '', + // iOS notification categories (with bell, message-circle and calendar). + "phone-incoming": + '', + "phone-missed": + '', + voicemail: + '', + mail: + '', + newspaper: + '', + "heart-pulse": + '', + briefcase: + '', + "map-pin": + '', + tv: + '', "cloud-sun": '', "flask-conical": '', wrench: '', + sparkles: + '', settings: '', mic: '', + languages: + '', map: '', compass: diff --git a/app/graphics/imagefile.ts b/app/graphics/imagefile.ts index 2acf9ef0..d2479e36 100644 --- a/app/graphics/imagefile.ts +++ b/app/graphics/imagefile.ts @@ -6,7 +6,12 @@ declare const interop: any; const UPNG = require("upng-js"); -export function loadPngAsGrayImage(path: string): GrayImage { +/** + * Decode a PNG to grayscale, premultiplied by alpha so transparency reads as + * black. With `invert`, the PNG is dark art on a light or transparent ground + * (e.g. the phone-side logo) and its ink is drawn white instead. + */ +export function loadPngAsGrayImage(path: string, options: { invert?: boolean } = {}): GrayImage { const file = resolveFile(path); const raw = file.readSync(); const pngBuffer = toArrayBuffer(raw); @@ -25,7 +30,8 @@ export function loadPngAsGrayImage(path: string): GrayImage { const g = rgba[offset + 1] ?? 0; const b = rgba[offset + 2] ?? 0; const a = (rgba[offset + 3] ?? 0) / 255; - const gray = Math.round((0.2126 * r + 0.7152 * g + 0.0722 * b) * a); + const luminance = 0.2126 * r + 0.7152 * g + 0.0722 * b; + const gray = Math.round((options.invert ? 255 - luminance : luminance) * a); image.pixels[y * decoded.width + x] = gray; } } diff --git a/app/graphics/installed-fonts.ts b/app/graphics/installed-fonts.ts index 8eec0c04..afba6f49 100644 --- a/app/graphics/installed-fonts.ts +++ b/app/graphics/installed-fonts.ts @@ -44,6 +44,11 @@ export type InstalledFont = { monospace: boolean; }; +/** Whether `fileName` is one of the fonts the app ships, rather than one the user installed. */ +export function isPreinstalledFont(fileName: string): boolean { + return (PREINSTALLED_FONT_FILES as readonly string[]).includes(fileName); +} + /** The app-internal fonts directory (created on first use). */ export function installedFontsDir(): string { return knownFolders.documents().getFolder("fonts").path; diff --git a/app/graphics/logo.ts b/app/graphics/logo.ts index 8515515f..c6932348 100644 --- a/app/graphics/logo.ts +++ b/app/graphics/logo.ts @@ -14,3 +14,57 @@ export function getDashboardLogo(): GrayImage | null { } return cachedDashboardLogo; } + +const FULL_LOGO_PATH = "images/faceclaw-logo.png"; +let fullLogoSource: GrayImage | null | undefined; +const scaledFullLogos = new Map(); + +/** + * The full Faceclaw logo (the lobster over the smiley face), white on black, + * scaled to `height` pixels tall for splash screens. The source is the phone + * UI's black-on-transparent PNG, inverted, and area-averaged down so the + * edges stay antialiased. + */ +export function getFullLogo(height: number): GrayImage | null { + const targetHeight = Math.max(1, Math.round(height)); + const cached = scaledFullLogos.get(targetHeight); + if (cached) return cached; + if (fullLogoSource === undefined) { + try { + fullLogoSource = loadPngAsGrayImage(FULL_LOGO_PATH, { invert: true }); + } catch (error) { + console.warn(`Could not load ${FULL_LOGO_PATH}: ${error}`); + fullLogoSource = null; + } + } + if (!fullLogoSource) return null; + const scale = Math.min(1, targetHeight / fullLogoSource.height); + const scaled = downscaleArea( + fullLogoSource, + Math.max(1, Math.round(fullLogoSource.width * scale)), + Math.max(1, Math.round(fullLogoSource.height * scale)), + ); + scaledFullLogos.set(targetHeight, scaled); + return scaled; +} + +/** Box-filter downscale: each output pixel averages the source pixels it covers. */ +function downscaleArea(source: GrayImage, width: number, height: number): GrayImage { + if (width === source.width && height === source.height) return source; + const output = new GrayImage(width, height, 0); + for (let y = 0; y < height; y++) { + const y0 = Math.floor((y * source.height) / height); + const y1 = Math.max(y0 + 1, Math.floor(((y + 1) * source.height) / height)); + for (let x = 0; x < width; x++) { + const x0 = Math.floor((x * source.width) / width); + const x1 = Math.max(x0 + 1, Math.floor(((x + 1) * source.width) / width)); + let sum = 0; + for (let sy = y0; sy < y1; sy++) { + const row = sy * source.width; + for (let sx = x0; sx < x1; sx++) sum += source.pixels[row + sx]!; + } + output.pixels[y * width + x] = Math.round(sum / ((y1 - y0) * (x1 - x0))); + } + } + return output; +} diff --git a/app/graphics/surface-compositor.ios.ts b/app/graphics/surface-compositor.ios.ts index 17adfa02..500f54da 100644 --- a/app/graphics/surface-compositor.ios.ts +++ b/app/graphics/surface-compositor.ios.ts @@ -20,17 +20,25 @@ export class SurfaceCompositor { this.surfaces.add(id) } removeSurface(id: string): void { this.native.removeId(id); this.surfaces.delete(id) } - setSurfaceVisible(id: string, visible: boolean): void { this.native.visibleIdVisible(id, visible) } - setSurfaceDepth(id: string, depth: number): void { this.native.depthIdDepth(id, depth) } + hasSurface(id: string): boolean { return this.surfaces.has(id) } + // Keep lifecycle errors catchable in JS; an undeclared Kotlin exception + // crossing the Objective-C boundary terminates the process. + private requireSurface(id: string): void { + if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`) + } + setSurfaceVisible(id: string, visible: boolean): void { this.requireSurface(id); this.native.visibleIdVisible(id, visible) } + setSurfaceDepth(id: string, depth: number): void { + this.requireSurface(id) + if (!Number.isInteger(depth) || depth < -128 || depth > 127) throw new Error(`Invalid surface depth ${depth} for ${id}`) + this.native.depthIdDepth(id, depth) + } setUnderlayDim(belowZOrder: number, factor: number): void { if (!Number.isFinite(factor)) throw new Error('Invalid dim factor') this.native.dimBelowFactor(belowZOrder, factor) } setScreenBlanked(blanked: boolean): void { this.native.blankBlanked(blanked) } submitSurfaceFrame(id: string, pixels: Uint8Array, rect: SurfaceRect, draws: ArrayBuffer | null = null): void { - // Keep lifecycle errors catchable in JS; an undeclared Kotlin exception - // crossing the Objective-C boundary terminates the process. - if (!this.surfaces.has(id)) throw new Error(`Unknown surface: ${id}`) + this.requireSurface(id) if (![rect.x, rect.y, rect.width, rect.height].every(Number.isInteger) || rect.width <= 0 || rect.height <= 0 || pixels.length !== rect.width * rect.height) throw new Error('Invalid frame buffer or rectangle') this.native.submitDrawsIdDataXYWidthHeightDraws(id, toData(pixels), rect.x, rect.y, rect.width, rect.height, diff --git a/app/native/binary-http.ios.ts b/app/native/binary-http.ios.ts index d96d9aad..e65172ab 100644 --- a/app/native/binary-http.ios.ts +++ b/app/native/binary-http.ios.ts @@ -1,6 +1,6 @@ import { Http } from '@nativescript/core'; import { withUserAgent } from '../util/http'; -import type { BinaryResponse } from './binary-http'; +import type { BinaryPostResponse, BinaryResponse } from './binary-http'; /** Bulk NSData -> ArrayBuffer; bypass fetch's Blob/FileReader copies. */ export async function downloadBinary(url: string, timeoutMs: number): Promise { @@ -8,3 +8,9 @@ export async function downloadBinary(url: string, timeoutMs: number): Promise name.toLowerCase() === 'content-length')?.[1]; return { status: response.statusCode, bytes: new Uint8Array(response.content.toArrayBuffer()), contentLength: Number(length ?? 0) }; } + +/** POST a binary body and read the response as text; NativeScript hands an ArrayBuffer to NSURLRequest as one NSData. */ +export async function postBinary(url: string, body: ArrayBuffer, headers: Record, timeoutMs: number): Promise { + const response = await Http.request({ url, method: 'POST', headers: withUserAgent(headers), content: body, timeout: timeoutMs }); + return { status: response.statusCode, text: response.content ? String(response.content.toString()) : '' }; +} diff --git a/app/native/binary-http.ts b/app/native/binary-http.ts index 12a79d32..27833ea6 100644 --- a/app/native/binary-http.ts +++ b/app/native/binary-http.ts @@ -2,6 +2,7 @@ declare const com: any; declare const java: any; export type BinaryResponse = { status: number; bytes: Uint8Array; contentLength: number }; +export type BinaryPostResponse = { status: number; text: string }; let nextDownloadSerial = 0; @@ -35,3 +36,25 @@ export function downloadBinary(url: string, timeoutMs: number): Promise, timeoutMs: number): Promise { + // Required lazily so Node tests can load this module's importers. + const { copyToJavaByteBuffer } = require("./java-direct-buffer") as typeof import("./java-direct-buffer"); + const entries = Object.entries(headers); + const nativeHeaders = Array.create("java.lang.String", entries.length * 2); + entries.forEach(([name, value], index) => { nativeHeaders[index * 2] = name; nativeHeaders[index * 2 + 1] = value; }); + return new Promise((resolve, reject) => { + const listener = new com.faceclaw.app.FaceclawBinaryPostListener({ + onResponse: (status: number, text: string | null) => resolve({ status, text: String(text ?? "") }), + onFailure: (message: string | null) => reject(new Error(String(message ?? "Request failed."))), + }); + // The User-Agent comes from the okhttp client's interceptor. + new com.faceclaw.app.FaceclawBinaryPost(url, copyToJavaByteBuffer(new Uint8Array(body)), nativeHeaders, timeoutMs, listener).start(); + }); +} diff --git a/app/native/build-info.ios.ts b/app/native/build-info.ios.ts new file mode 100644 index 00000000..d8d7ddef --- /dev/null +++ b/app/native/build-info.ios.ts @@ -0,0 +1,19 @@ +import type { BuildKind } from './build-info' + +declare const NSBundle: any +declare const NSProcessInfo: any + +let cached: BuildKind | null = null + +/** + * App Store (and TestFlight) builds carry no provisioning profile; anything + * installed from Xcode or as an ad hoc build does. Simulator builds have none + * either, so they're recognized by the simulator's environment. + */ +export function buildKind(): BuildKind { + if (cached) return cached + if (NSProcessInfo.processInfo.environment.objectForKey('SIMULATOR_DEVICE_NAME')) cached = 'debug' + else if (NSBundle.mainBundle.pathForResourceOfType('embedded', 'mobileprovision')) cached = 'self-built' + else cached = 'official' + return cached +} diff --git a/app/native/build-info.ts b/app/native/build-info.ts new file mode 100644 index 00000000..cd35d5cc --- /dev/null +++ b/app/native/build-info.ts @@ -0,0 +1,54 @@ +/** + * Which kind of build this is, for analytics: an official release, or one + * compiled by somebody else. Official releases are signed with the key + * scripts/release.sh uses; anything else was built by whoever installed it. + */ +import { Utils } from "@nativescript/core"; + +declare const android: any; +declare const java: any; + +/** + * `debug` is a debuggable build (build.sh / build_and_run.sh); `self-built` + * is a release build signed with someone else's key. + */ +export type BuildKind = "official" | "self-built" | "debug" | "unknown"; + +/** SHA-256 of the release signing certificate (`apksigner verify --print-certs dist/Faceclaw-*.apk`). */ +const OFFICIAL_SIGNING_CERT_SHA256 = "c1efe42d74f3fc4e9a13bcfc8c99e018344a55525e5b300fa960c89431e382ad"; + +let cached: BuildKind | null = null; + +export function buildKind(): BuildKind { + cached ??= detectBuildKind(); + return cached; +} + +function detectBuildKind(): BuildKind { + try { + const context = Utils.android.getApplicationContext(); + const packageManager = context.getPackageManager(); + const PackageManager = android.content.pm.PackageManager; + const signatures = + android.os.Build.VERSION.SDK_INT >= 28 + ? packageManager + .getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNING_CERTIFICATES) + .signingInfo.getApkContentsSigners() + : packageManager.getPackageInfo(context.getPackageName(), PackageManager.GET_SIGNATURES).signatures; + for (let index = 0; index < signatures.length; index++) { + const digest = java.security.MessageDigest.getInstance("SHA-256").digest(signatures[index].toByteArray()); + if (hex(digest) === OFFICIAL_SIGNING_CERT_SHA256) return "official"; + } + const debuggable = (context.getApplicationInfo().flags & android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) !== 0; + return debuggable ? "debug" : "self-built"; + } catch (error) { + console.warn(`build-info: could not read the signing certificate: ${error}`); + return "unknown"; + } +} + +function hex(bytes: any): string { + let text = ""; + for (let index = 0; index < bytes.length; index++) text += ((bytes[index] & 0xff) + 0x100).toString(16).slice(1); + return text; +} diff --git a/app/native/cloud-stt-provider.ts b/app/native/cloud-stt-provider.ts new file mode 100644 index 00000000..2ab6ea08 --- /dev/null +++ b/app/native/cloud-stt-provider.ts @@ -0,0 +1,112 @@ +import type { CloudSttClient, CloudSttOptions } from "./cloud-stt"; +import { ElevenLabsSttClient } from "./elevenlabs-stt"; +import { OpenAiRealtimeSttClient } from "./openai-stt"; +import { ReconnectingSttClient } from "./reconnecting-stt"; +import { SonioxSttClient } from "./soniox-stt"; +import type { VoiceProviderKind } from "./voice-control"; +import { normalizeWhisperServerUrl, WHISPER_SERVER_FINAL_TIMEOUT_MS, WhisperServerSttClient } from "./whisper-server-stt"; + +/** + * The transcription-provider setting plus the keys the cloud providers need. + * Shared by the Android and iOS voice bridges. + */ +export type VoiceProviderSelection = { + provider: VoiceProviderKind; + elevenLabsApiKey: string; + openAiApiKey: string; + sonioxApiKey: string; + /** Self-hosted Whisper ("whisper-server"); the key is optional there. */ + whisperServerUrl?: string; + whisperServerModel?: string; + whisperServerLanguage?: string; + whisperServerApiKey?: string; +}; + +type CloudProvider = { + name: string; + apiKey: (selection: VoiceProviderSelection) => string; + /** Whether the provider can run; defaults to having a key. */ + configured?: (selection: VoiceProviderSelection) => boolean; + /** What to tell the user when it can't. */ + missing?: string; + create: (options: CloudSttOptions, selection: VoiceProviderSelection) => CloudSttClient; + finalTimeoutMs?: number; +}; + +// Providers absent here ("onboard", "onboard-whisper") transcribe on-device. +const CLOUD_PROVIDERS: Partial> = { + elevenlabs: { + name: "ElevenLabs", + apiKey: (selection) => selection.elevenLabsApiKey, + create: (options) => new ElevenLabsSttClient(options), + }, + soniox: { + name: "Soniox", + apiKey: (selection) => selection.sonioxApiKey, + create: (options) => new SonioxSttClient(options), + }, + whisper: { + name: "OpenAI", + apiKey: (selection) => selection.openAiApiKey, + create: (options) => new OpenAiRealtimeSttClient(options), + }, + "whisper-server": { + name: "Whisper server", + apiKey: (selection) => selection.whisperServerApiKey ?? "", + configured: (selection) => Boolean(normalizeWhisperServerUrl(selection.whisperServerUrl ?? "")), + missing: "No Whisper server URL set; using on-device voice.", + create: (options, selection) => new WhisperServerSttClient(options, { + baseUrl: selection.whisperServerUrl ?? "", + model: selection.whisperServerModel ?? "", + language: selection.whisperServerLanguage ?? "", + }), + finalTimeoutMs: WHISPER_SERVER_FINAL_TIMEOUT_MS, + }, +}; + +function isConfigured(provider: CloudProvider, selection: VoiceProviderSelection): boolean { + return provider.configured ? provider.configured(selection) : Boolean(provider.apiKey(selection).trim()); +} + +/** + * What a capture transcribes with: the provider, and for Whisper server the + * language it is told to expect. A running capture is shared only by holders + * asking for the same. + */ +export function transcriptionKey(selection: Pick): string { + return selection.provider === "whisper-server" + ? `whisper-server:${(selection.whisperServerLanguage ?? "").trim().toLowerCase()}` + : selection.provider; +} + +/** Whether createCloudSttClient would return a cloud client for this selection. */ +export function usesCloudStt(selection: VoiceProviderSelection): boolean { + const provider = CLOUD_PROVIDERS[selection.provider]; + return Boolean(provider && isConfigured(provider, selection)); +} + +/** + * The cloud provider for a capture, or null to transcribe on-device. A cloud + * provider whose API key is missing falls back to on-device (reported via + * onStatus) rather than failing the capture outright. `continuous` says + * whether the capture is open-ended (Transcribe), where pauses commit segments. + */ +export function createCloudSttClient( + selection: VoiceProviderSelection, + options: Omit, + continuous: () => boolean, +): ReconnectingSttClient | null { + const provider = CLOUD_PROVIDERS[selection.provider]; + if (!provider) return null; + if (!isConfigured(provider, selection)) { + options.onStatus(provider.missing ?? `No ${provider.name} key set; using on-device voice.`); + return null; + } + const apiKey = provider.apiKey(selection).trim(); + return new ReconnectingSttClient( + (clientOptions) => provider.create(clientOptions, selection), + { ...options, apiKey }, + continuous, + provider.finalTimeoutMs, + ); +} diff --git a/app/native/cloud-stt.ts b/app/native/cloud-stt.ts index f9a77167..2600f802 100644 --- a/app/native/cloud-stt.ts +++ b/app/native/cloud-stt.ts @@ -2,8 +2,9 @@ * Shared shape for the cloud speech-to-text providers (ElevenLabs, Whisper, * Soniox). * The voice bridge holds one of these while a cloud provider owns the - * transcript; the Java controller then only decodes LC3 to PCM and hands it - * over via acceptPcm. + * transcript; the native capture (FaceclawVoiceController on Android, + * FaceclawSpeech on iOS) then only decodes audio to PCM and hands it over + * via acceptPcm. */ export type CloudSttTranscriptEvent = { @@ -51,8 +52,13 @@ export function toJavaBytes(bytes: Uint8Array): any { return javaBytes; } -/** Base64 for audio payloads, via the Android SDK (no JS base64 in NS core). */ +/** Base64 for audio payloads, via the platform SDK (no JS base64 in NS core). */ export function encodeBase64(bytes: Uint8Array): string { + if (global.isIOS) { + const copy = bytes.slice(); + return NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) + .base64EncodedStringWithOptions(0 as NSDataBase64EncodingOptions); + } if (!global.isAndroid) return ""; return String(android.util.Base64.encodeToString(toJavaBytes(bytes), android.util.Base64.NO_WRAP)); } diff --git a/app/native/connection-counters.ios.ts b/app/native/connection-counters.ios.ts new file mode 100644 index 00000000..786ae3ab --- /dev/null +++ b/app/native/connection-counters.ios.ts @@ -0,0 +1,10 @@ +declare const FaceclawKitConnectionCounters: any + +export function drainConnectionCounters(): Record { + const counts: Record = {} + const parsed = JSON.parse(String(FaceclawKitConnectionCounters.shared.drain())) as Record + for (const [name, count] of Object.entries(parsed)) { + if (typeof count === 'number' && count > 0) counts[name] = count + } + return counts +} diff --git a/app/native/connection-counters.ts b/app/native/connection-counters.ts new file mode 100644 index 00000000..3863496d --- /dev/null +++ b/app/native/connection-counters.ts @@ -0,0 +1,14 @@ +declare const com: any; + +/** + * Connection-reliability event counts since the last call, from the shared + * Kotlin session core (ConnectionCounters.kt), which keeps them process-wide. + */ +export function drainConnectionCounters(): Record { + const counts: Record = {}; + const parsed = JSON.parse(String(com.faceclaw.app.ConnectionCounters.drain())) as Record; + for (const [name, count] of Object.entries(parsed)) { + if (typeof count === "number" && count > 0) counts[name] = count; + } + return counts; +} diff --git a/app/native/elevenlabs-stt.ts b/app/native/elevenlabs-stt.ts index 6fa555fc..bc493fb9 100644 --- a/app/native/elevenlabs-stt.ts +++ b/app/native/elevenlabs-stt.ts @@ -1,6 +1,5 @@ import { type CloudSttOptions, type CloudSttTranscriptEvent, encodeBase64 } from "./cloud-stt"; - -declare const com: any; +import { openSocket, type SocketConnection } from "./socket"; /** * ElevenLabs realtime speech-to-text over WebSocket. @@ -22,8 +21,7 @@ export type ElevenLabsTranscriptEvent = CloudSttTranscriptEvent; export type ElevenLabsSttOptions = CloudSttOptions; export class ElevenLabsSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // PCM that arrived before the socket finished opening; flushed on open. @@ -36,7 +34,7 @@ export class ElevenLabsSttClient { start(): void { if (this.closed || this.ws) return; const url = `${WS_URL}?model_id=${MODEL_ID}&audio_format=pcm_${SAMPLE_RATE}&commit_strategy=manual`; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; this.open = true; @@ -56,12 +54,12 @@ export class ElevenLabsSttClient { if (this.closed) return; this.options.onDisconnected?.(`ElevenLabs connection failed: ${String(message)}`); }, - }); + }; try { - // The API key rides an xi-api-key header, added by FaceclawWebSocket. + // The API key rides an xi-api-key header on the handshake. const key = this.options.apiKey; console.log(`[elevenlabs] connecting; apiKey length=${key.length} prefix=${key.slice(0, 4)}`); - this.ws = new com.faceclaw.app.FaceclawWebSocket(url, this.listenerProxy, "xi-api-key", key); + this.ws = openSocket(url, listener, { name: "xi-api-key", value: key }); this.options.onStatus("Connecting to ElevenLabs..."); } catch (error) { this.options.onDisconnected?.(`ElevenLabs connection failed: ${String((error as Error)?.message ?? error)}`); @@ -106,7 +104,6 @@ export class ElevenLabsSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingChunks.length = 0; } diff --git a/app/native/faceclaw-communicator.ios.ts b/app/native/faceclaw-communicator.ios.ts index aa5a9408..6a865fe8 100644 --- a/app/native/faceclaw-communicator.ios.ts +++ b/app/native/faceclaw-communicator.ios.ts @@ -141,7 +141,7 @@ class AncsListener extends NSObject { class AudioPacketListener extends NSObject { static ObjCProtocols = [FaceclawKitIosAudioPacketListener]; handler!: (packet: Uint8Array, arm: string, arrivalMs: number) => void; - onAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void { + onNsDataAudioPacketDataArmArrivalMs(data: NSData, arm: string, arrivalMs: number): void { this.handler(fromData(data), String(arm ?? ""), Number(arrivalMs)); } } @@ -178,6 +178,8 @@ export class FaceclawCommunicatorBridge { private ancsProxy: AncsListener | null = null; private nativeCallQueue: Promise = Promise.resolve(); private queuedNativeCalls = 0; + // Surfaces Kotlin holds, as of the queued call now running (see hasNativeSurface). + private readonly nativeSurfaces = new Set(); private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>(); private readonly finishedFrameOutcomes = new Map(); private readonly frameFinishedWaiters = new Map void>>(); @@ -493,13 +495,13 @@ export class FaceclawCommunicatorBridge { }); } - /** Phone-UI preview of the current composited screen, or null if none yet. */ - getCompositePreview(green = false): ImageSource | null { + /** Phone-UI preview of the current composited screen (or of `crop`'s area of it), or null if none yet. */ + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { const frame = this.communicator.previewComposite(); if (!frame) return null; const width = Number(this.communicator.compositeWidth()), height = Number(this.communicator.compositeHeight()); if (width <= 0 || height <= 0) return null; - return previewPixels(fromData(frame.pixels), width, height, green); + return previewPixels(fromData(frame.pixels), width, height, green, crop); } /** Save the current composite as a 4-bit grayscale PNG under Documents; returns the path (empty if none). Cropping is not supported on iOS. */ @@ -522,15 +524,32 @@ export class FaceclawCommunicatorBridge { async configureSurface(id: string, options: SurfaceOptions): Promise { const transparency = options.transparency === "color-key" ? 1 : 0; await this.enqueueNativeCall(() => { - this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency( + const configured = this.communicator.configureSurfaceIdXYWidthHeightZOrderTransparency( id, Math.round(options.x), Math.round(options.y), Math.round(options.width), Math.round(options.height), Math.round(options.zOrder), transparency, ); + // Kotlin logged why; reject so callers don't treat the surface as live (as on Android, + // where the compositor's exception reaches JS). + if (configured === false) throw new Error(`Could not configure surface ${id}`); + this.nativeSurfaces.add(id); }); } async removeSurface(id: string): Promise { - await this.enqueueNativeCall(() => { this.communicator.removeSurfaceId(id); }); + await this.enqueueNativeCall(() => { + this.nativeSurfaces.delete(id); + this.communicator.removeSurfaceId(id); + }); + } + + /** + * Checked inside each queued call: a render in flight when its window closed lands + * behind the removal, and Kotlin aborts on a frame for an unknown surface. + */ + private hasNativeSurface(id: string, what: string): boolean { + if (this.nativeSurfaces.has(id)) return true; + console.log(`[ios-ble] ${what} for removed surface ${id} dropped`); + return false; } async submitShellScene(bytes: Uint8Array, paintMs = 0, frameId = 0): Promise { @@ -545,11 +564,15 @@ export class FaceclawCommunicatorBridge { } async setSurfaceVisible(id: string, visible: boolean): Promise { - await this.enqueueNativeCall(() => { this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); }); + await this.enqueueNativeCall(() => { + if (this.hasNativeSurface(id, "setSurfaceVisible")) this.communicator.setSurfaceVisibleIdVisible(id, Boolean(visible)); + }); } async setSurfaceDepth(id: string, depth: number): Promise { - await this.enqueueNativeCall(() => { this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); }); + await this.enqueueNativeCall(() => { + if (this.hasNativeSurface(id, "setSurfaceDepth")) this.communicator.setSurfaceDepthIdDepth(id, Math.round(depth)); + }); } async setScreenBlanked(blanked: boolean): Promise { @@ -576,6 +599,7 @@ export class FaceclawCommunicatorBridge { const snapshot = new Uint8Array(pixels8bpp); const draws = glyphs ? toData(new Uint8Array(glyphs)) : null; await this.enqueueNativeCall(() => { + if (!this.hasNativeSurface(surfaceId, "frame")) return; this.communicator.submitSurfaceFramePixels8bppSurfaceIdRectXRectYRectWidthRectHeightContentFingerprintPaintMsFrameIdGlyphs( toData(snapshot), surfaceId, Math.round(rect.x), Math.round(rect.y), Math.round(rect.width), Math.round(rect.height), fingerprint, Math.round(nonNegativeNumber(paintMs)), Math.round(nonNegativeNumber(frameId)), draws, diff --git a/app/native/faceclaw-communicator.ts b/app/native/faceclaw-communicator.ts index 1e1ffeec..1f7f3965 100644 --- a/app/native/faceclaw-communicator.ts +++ b/app/native/faceclaw-communicator.ts @@ -509,11 +509,21 @@ export class FaceclawCommunicatorBridge { /** * Phone-UI preview of the current composited screen, or null if none yet. * `green` tints it green-on-black (the Preview color setting) instead of - * grayscale. + * grayscale; `crop` limits it to a screen area (the Preview scaling + * setting), black where that runs past the screen's edges. */ - getCompositePreview(green = false): ImageSource | null { + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { if (!global.isAndroid) return null; - const bitmap = this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); + const bitmap = crop + ? this.communicator.getCompositePreviewBitmap( + PREVIEW_BRIGHTEN_GAMMA, + green, + Math.round(crop.x), + Math.round(crop.y), + Math.round(crop.width), + Math.round(crop.height), + ) + : this.communicator.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); return bitmap ? new ImageSource(bitmap) : null; } diff --git a/app/native/gray-image-preview.ios.ts b/app/native/gray-image-preview.ios.ts new file mode 100644 index 00000000..da6c89d6 --- /dev/null +++ b/app/native/gray-image-preview.ios.ts @@ -0,0 +1,19 @@ +import { type ImageSource } from '@nativescript/core' + +import { type GrayImage } from '../graphics/image' +import { previewPixels, whiteIconPixels } from './ios-graphics' + +/** Build the phone-UI preview for a frame (brightened like Android's). */ +export function grayImageToPreviewSource(image: GrayImage): ImageSource | null { + const baked = image.withDrawsBaked() + return previewPixels(baked.pixels, baked.width, baked.height) +} + +/** + * A glasses icon as a white-on-transparent phone image (gray becomes alpha), + * for the dark header. + */ +export function grayImageToWhiteIconSource(image: GrayImage): ImageSource | null { + const baked = image.withDrawsBaked() + return whiteIconPixels(baked.pixels, baked.width, baked.height) +} diff --git a/app/native/hermes.ts b/app/native/hermes.ts new file mode 100644 index 00000000..2e6eed76 --- /dev/null +++ b/app/native/hermes.ts @@ -0,0 +1,136 @@ +import type { LlmStreamHandle, LlmStreamOptions } from "../assistant/llm-protocol"; +import { openSseRequest, type SseListener } from "./sse"; + +/** + * Streaming client for a Hermes Agent gateway's OpenAI-compatible API server + * (POST {baseUrl}/chat/completions). Hermes is an agent with its own tools and + * memory, so only the latest user message is sent: the X-Hermes-Session-Id header + * keeps the conversation on the server. Faceclaw's own tools are not offered. + */ + +export type HermesStreamOptions = LlmStreamOptions & { + baseUrl: string; + sessionId: string; +}; + +/** The API root from what a user typed: adds /v1 to a bare host. Empty when unusable. */ +export function normalizeHermesUrl(input: string): string { + let url = input.trim().replace(/\/+$/, ""); + if (!/^https?:\/\/[^/\s]+/i.test(url)) return ""; + url = url.replace(/\/chat\/completions$/i, ""); + if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1"; + return url; +} + +export function streamHermesChat(options: HermesStreamOptions): LlmStreamHandle { + let request: { cancel(): void } | null = null; + let settled = false; + let text = ""; + let finishReason: string | null = null; + + const fail = (message: string) => { + if (settled) return; + settled = true; + request?.cancel(); + options.onError(message); + }; + const done = () => { + if (settled) return; + // A failed or interrupted run still ends with a finish chunk and [DONE]; the + // chunk carries an error object only when Hermes has a message to show. + if (finishReason === "error") { + fail("Hermes: the agent run failed"); + return; + } + if (finishReason === "cancelled") { + fail("Hermes: the agent run was interrupted"); + return; + } + if (!text) { + fail("Hermes ended the turn without a reply"); + return; + } + settled = true; + request?.cancel(); + options.onDone({ text, content: [{ type: "text", text }], stopReason: finishReason === "length" ? "max_tokens" : "end_turn" }); + }; + + const baseUrl = normalizeHermesUrl(options.baseUrl); + const last = [...options.messages].reverse().find((message) => message.role === "user"); + const prompt = typeof last?.content === "string" ? last.content : ""; + if (!baseUrl || !prompt) { + setTimeout(() => fail(baseUrl ? "Nothing to send to Hermes" : "No Hermes URL set"), 0); + return { cancel: () => { settled = true; } }; + } + + const messages = [ + ...(options.system ? [{ role: "system", content: options.system }] : []), + { role: "user", content: prompt }, + ]; + const body = JSON.stringify({ model: options.model, stream: true, messages }); + const headers: Record = { "Content-Type": "application/json", "X-Hermes-Session-Id": options.sessionId }; + if (options.apiKey.trim()) headers.Authorization = `Bearer ${options.apiKey.trim()}`; + + const listener: SseListener = { + onLine: (line: string) => { + if (settled || !line.startsWith("data:")) return; + const payload = line.slice(5).trim(); + if (payload === "[DONE]") { + done(); + return; + } + let event: any; + try { + event = JSON.parse(payload); + } catch { + return; + } + if (event?.error) { + fail(`Hermes: ${String(event.error.message ?? event.error)}`); + return; + } + const choice = event?.choices?.[0]; + // reasoning_content is the agent's thinking; only the answer goes to the glasses. + // Hermes opens its answer with blank lines after the reasoning; drop them. + const delta = text ? choice?.delta?.content : String(choice?.delta?.content ?? "").trimStart(); + if (typeof delta === "string" && delta) { + text += delta; + options.onTextDelta?.(delta, text); + } + if (choice?.finish_reason) finishReason = String(choice.finish_reason); + }, + onHttpError: (code: number, errorBody: string) => fail(describeHttpError(Number(code), String(errorBody))), + // Some servers close without [DONE] after the final chunk. Hermes always sends + // a finish_reason, so a stream closed without one was cut off. + onComplete: () => (finishReason ? done() : fail(text ? "Hermes: the reply was cut off" : "Hermes stream ended without a reply")), + onFailure: (message: string) => fail(`Hermes connection failed: ${String(message)}`), + }; + + try { + request = openSseRequest(`${baseUrl}/chat/completions`, body, headers, listener); + } catch (error) { + setTimeout(() => fail(`Hermes request failed: ${String((error as Error)?.message ?? error)}`), 0); + return { cancel: () => { settled = true; } }; + } + return { + cancel: () => { + settled = true; + try { + request?.cancel(); + } catch { + // ignore + } + }, + }; +} + +function describeHttpError(code: number, body: string): string { + if (code === 401 || code === 403) return "Hermes: the API server rejected the key"; + let detail = ""; + try { + detail = String(JSON.parse(body)?.error?.message ?? ""); + } catch { + // non-JSON error body + } + return `Hermes: HTTP ${code}${detail ? ` (${detail})` : ""}`; +} diff --git a/app/native/ios-graphics.ts b/app/native/ios-graphics.ts index 6052d625..1a57d828 100644 --- a/app/native/ios-graphics.ts +++ b/app/native/ios-graphics.ts @@ -3,12 +3,35 @@ import { GrayImage } from '../graphics/image' declare const FaceclawGraphics: any -export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false): ImageSource { +type CropRect = { x: number; y: number; width: number; height: number } + +/** The gray pixels of a screen area; black where it runs past the frame's edges. */ +function cropGray(pixels: Uint8Array, width: number, height: number, crop: CropRect): Uint8Array { + const cropped = new Uint8Array(crop.width * crop.height) + const left = Math.max(crop.x, 0), right = Math.min(crop.x + crop.width, width) + if (right <= left) return cropped + for (let y = Math.max(crop.y, 0); y < Math.min(crop.y + crop.height, height); y++) { + cropped.set(pixels.subarray(y * width + left, y * width + right), (y - crop.y) * crop.width + (left - crop.x)) + } + return cropped +} + +/** Gray pixels as the phone mirror's image, optionally cropped to a screen area (Preview scaling). */ +export function previewPixels(pixels: Uint8Array, width: number, height: number, green = false, crop: CropRect | null = null): ImageSource { // NSData copies the buffer; the UIImage owns its pixels after this call. + const copy = crop ? cropGray(pixels, width, height, crop) : new Uint8Array(pixels) + const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) + const native = FaceclawGraphics.previewWidthHeightGreen(data, crop?.width ?? width, crop?.height ?? height, green) + if (!native) throw new Error('Could not create preview image') + return new ImageSource(native) +} + +/** Gray pixels as a white-on-transparent image (gray becomes alpha). */ +export function whiteIconPixels(pixels: Uint8Array, width: number, height: number): ImageSource { const copy = new Uint8Array(pixels) const data = NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength) - const native = FaceclawGraphics.previewWidthHeightGreen(data, width, height, green) - if (!native) throw new Error('Could not create preview image') + const native = FaceclawGraphics.whiteIconWidthHeight(data, width, height) + if (!native) throw new Error('Could not create icon image') return new ImageSource(native) } diff --git a/app/native/location-types.ts b/app/native/location-types.ts index bd29a813..e5b576b9 100644 --- a/app/native/location-types.ts +++ b/app/native/location-types.ts @@ -5,6 +5,15 @@ export type CurrentLocation = { timestampMs: number; }; +/** A reverse-geocoded place; parts the geocoder doesn't know are empty. */ +export type Place = { + /** City or town. */ + locality: string; + /** State, province or similar. */ + region: string; + country: string; +}; + export type TrackedLocation = { latitude: number; longitude: number; diff --git a/app/native/media-browser.ios.ts b/app/native/media-browser.ios.ts new file mode 100644 index 00000000..ac1ded94 --- /dev/null +++ b/app/native/media-browser.ios.ts @@ -0,0 +1,70 @@ +import { + APPLE_MUSIC_APP_NAME, + APPLE_MUSIC_PACKAGE, + parseMediaBrowseItems, + type MediaBrowseItem, + type MediaBrowserApp, + type MediaBrowserBridge, +} from "./media-types"; + +export type { MediaBrowseItem, MediaBrowserApp } from "./media-types"; + +declare const FaceclawAppleMusic: any; + +const APPLE_MUSIC_APP: MediaBrowserApp = { packageName: APPLE_MUSIC_PACKAGE, serviceClass: "", appName: APPLE_MUSIC_APP_NAME }; + +/** + * The Apple Music library (playlists, artists, albums, songs in the + * on-device / iCloud music library) as a browse tree, through + * FaceclawAppleMusic's MediaPlayer queries. Playing an item queues its + * collection on the system Music player. + */ +export class AppleMusicBrowserBridge implements MediaBrowserBridge { + private connected = false; + /** Bumped on every connect/disconnect; answers to older requests are dropped. */ + private generation = 0; + + listBrowsableApps(): MediaBrowserApp[] { + return FaceclawAppleMusic.shared().hasPermission() ? [APPLE_MUSIC_APP] : []; + } + + connect(app: MediaBrowserApp): Promise { + this.disconnect(); + if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS")); + if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed")); + this.connected = true; + return Promise.resolve("root"); + } + + browse(parentId: string): Promise { + if (!this.connected) return Promise.reject(new Error("Not connected")); + const generation = this.generation; + return new Promise((resolve, reject) => { + FaceclawAppleMusic.shared().browseCompletion(parentId, (json: string | null, error: string | null) => { + if (generation !== this.generation) reject(new Error("Disconnected")); + else if (error || json == null) reject(new Error(String(error || "Browse failed"))); + else { + try { + resolve(parseMediaBrowseItems(String(json))); + } catch (parseError) { + reject(new Error(`Browse result parse failed: ${parseError}`)); + } + } + }); + }); + } + + playFromMediaId(mediaId: string): void { + if (!this.connected) return; + FaceclawAppleMusic.shared().playCompletion(mediaId, (error: string | null) => { + if (error) console.warn(`[apple-music] play ${mediaId} failed: ${error}`); + }); + } + + disconnect(): void { + this.connected = false; + this.generation++; + } +} + +export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge(); diff --git a/app/native/media-browser.ts b/app/native/media-browser.ts index 7da4d72a..0061f9fe 100644 --- a/app/native/media-browser.ts +++ b/app/native/media-browser.ts @@ -1,22 +1,11 @@ import { Utils } from "@nativescript/core"; import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps"; import { onSettingsStoreChanged } from "./settings-store"; +import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types"; declare const com: any; -export type MediaBrowserApp = { - packageName: string; - serviceClass: string; - appName: string; -}; - -export type MediaBrowseItem = { - mediaId: string; - title: string; - subtitle: string; - browsable: boolean; - playable: boolean; -}; +export type { MediaBrowseItem, MediaBrowserApp } from "./media-types"; type PendingRequest = { resolve: (value: T) => void; @@ -29,7 +18,7 @@ type PendingRequest = { * start playback, including starting a player that is not currently running. * Holds at most one connection at a time. */ -export class FaceclawMediaBrowserBridge { +export class FaceclawMediaBrowserBridge implements MediaBrowserBridge { private browser: any = null; private listenerProxy: any = null; private nextRequestId = 1; @@ -155,22 +144,7 @@ export class FaceclawMediaBrowserBridge { return; } try { - const raw = JSON.parse(json) as Array<{ - mediaId?: string; - title?: string; - subtitle?: string; - browsable?: boolean; - playable?: boolean; - }>; - pending.resolve( - raw.map((item) => ({ - mediaId: String(item.mediaId ?? ""), - title: String(item.title ?? ""), - subtitle: String(item.subtitle ?? ""), - browsable: Boolean(item.browsable), - playable: Boolean(item.playable), - })), - ); + pending.resolve(parseMediaBrowseItems(json)); } catch (parseError) { pending.reject(new Error(`Browse result parse failed: ${parseError}`)); } @@ -183,4 +157,4 @@ export class FaceclawMediaBrowserBridge { } } -export const mediaBrowserBridge = new FaceclawMediaBrowserBridge(); +export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge(); diff --git a/app/native/media-controller.ios.ts b/app/native/media-controller.ios.ts new file mode 100644 index 00000000..1200c7ce --- /dev/null +++ b/app/native/media-controller.ios.ts @@ -0,0 +1,231 @@ +import { Dialogs, Utils } from "@nativescript/core"; +import type { GrayImage } from "../graphics/image"; +import { grayImageFromPacket, PHOTO_TONE } from "./image-files"; +import { fromData } from "./kotlin-data"; +import { + APPLE_MUSIC_APP_NAME, + APPLE_MUSIC_PACKAGE, + DEFAULT_MEDIA_STATE, + type MediaControllerBridge, + type MediaControllerState, + type MediaPlaybackState, + type MediaQueueItem, +} from "./media-types"; + +export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types"; + +declare const FaceclawAppleMusic: any; +declare const FaceclawKitIosProtocol: any; + +/** + * Re-read the system player this often while something listens: its change + * notifications are not delivered while Faceclaw is suspended, so a track + * change in the background would otherwise wait for the next one. + */ +const POLL_MS = 2_000; + +/** Normalize FaceclawAppleMusic.stateJson()'s object. */ +export function appleMusicState(raw: any): MediaControllerState { + if (!raw || raw.access !== true) { + return { ...DEFAULT_MEDIA_STATE, status: "Apple Music access is required." }; + } + const hasItem = Boolean(raw.hasItem); + const playbackState: MediaPlaybackState = !hasItem + ? "idle" + : raw.state === "playing" || raw.state === "paused" || raw.state === "stopped" ? raw.state : "unknown"; + const rate = Number(raw.rate); + return { + available: hasItem, + accessEnabled: true, + packageName: APPLE_MUSIC_PACKAGE, + appName: APPLE_MUSIC_APP_NAME, + playbackState, + title: String(raw.title ?? ""), + artist: String(raw.artist ?? ""), + album: String(raw.album ?? ""), + positionMs: finiteOr(raw.positionMs, -1), + durationMs: finiteOr(raw.durationMs, -1), + // The system player reports a zero rate while it is still starting up. + playbackSpeed: playbackState === "playing" ? (rate > 0 ? rate : 1) : 0, + status: hasItem ? "" : "Nothing is playing in Apple Music.", + canPlayPause: hasItem, + canSkipNext: hasItem, + canSkipPrevious: hasItem, + artVersion: finiteOr(raw.artVersion, 0), + }; +} + +function finiteOr(value: unknown, fallback: number): number { + const number = Number(value); + return typeof value === "number" && Number.isFinite(number) ? number : fallback; +} + +/** Everything a listener would redraw for; the position advances on its own. */ +function stateSignature(state: MediaControllerState): string { + return JSON.stringify({ ...state, positionMs: 0 }); +} + +/** + * Apple Music through FaceclawAppleMusic (MediaPlayer's system player): iOS + * gives other apps' players no shared session API, so the system Music app is + * the only player Faceclaw can follow and control. Snapshots read the player + * directly (cheap property reads), so they are current even when a change + * notification was missed. + */ +export class AppleMusicControllerBridge implements MediaControllerBridge { + readonly accessPrompt = "Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone."; + readonly accessLabel = "Apple Music access needed"; + readonly accessAction = "allow access"; + readonly skipRebuildsQueue = false; + private readonly stateListeners = new Set<(state: MediaControllerState) => void>(); + private bound = false; + private started = false; + private pollTimer: ReturnType | null = null; + private lastSignature = ""; + private pendingAccess: Promise | null = null; + private protocol: any = null; + + onStateChange(listener: (state: MediaControllerState) => void): () => void { + void this.start(); + this.stateListeners.add(listener); + this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS); + listener(this.snapshot()); + return () => { + this.stateListeners.delete(listener); + if (!this.stateListeners.size && this.pollTimer !== null) { + clearInterval(this.pollTimer); + this.pollTimer = null; + } + }; + } + + snapshot(): MediaControllerState { + let raw: unknown = null; + try { + raw = JSON.parse(String(this.native().stateJson())); + } catch (error) { + console.warn(`[apple-music] state read failed: ${error}`); + } + return appleMusicState(raw); + } + + async start(): Promise { + if (this.started) return; + this.started = true; + this.native().start(); + } + + async stop(): Promise { + if (!this.started) return; + this.started = false; + this.native().stop(); + } + + async playPause(): Promise { + this.native().playPause(); + } + + async skipNext(): Promise { + this.native().skipNext(); + } + + async skipPrevious(): Promise { + this.native().skipPrevious(); + } + + hasAccess(): boolean { + return Boolean(this.native().hasPermission()); + } + + /** Concurrent requests (glasses tap, phone Permissions card) share one prompt. */ + requestAccess(): Promise { + if (this.hasAccess()) return Promise.resolve(true); + this.pendingAccess ??= this.askForAccess().finally(() => { + this.pendingAccess = null; + this.refresh(); + }); + return this.pendingAccess; + } + + async skipToQueueItem(id: number): Promise { + this.native().skipToQueueIndex(Math.round(id)); + } + + getMediaVolumePercent(): number { + const volume = Number(this.native().volumePercent()); + return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1; + } + + setMediaVolumePercent(volumePercent: number): void { + this.native().setVolumePercent(Math.max(0, Math.min(100, Math.round(volumePercent)))); + } + + getAlbumArt(maxSize: number): GrayImage | null { + const packet = this.native().albumArt(Math.round(maxSize)); + if (!packet) return null; + // The shared Kotlin tone curve and dither, as on Android. + this.protocol ??= FaceclawKitIosProtocol.new(); + return grayImageFromPacket(fromData(this.protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither))); + } + + getQueue(): MediaQueueItem[] { + try { + const raw = JSON.parse(String(this.native().queueJson())); + if (!Array.isArray(raw)) return []; + return raw.map((item: { id?: number; title?: string; active?: boolean }) => ({ + id: Number(item?.id ?? -1), + title: String(item?.title ?? ""), + active: Boolean(item?.active), + })); + } catch (error) { + console.warn(`[apple-music] queue parse failed: ${error}`); + return []; + } + } + + private native(): any { + const music = FaceclawAppleMusic.shared(); + if (!this.bound) { + this.bound = true; + music.changeHandler = () => this.refresh(); + } + return music; + } + + private refresh(): void { + if (!this.stateListeners.size) return; + const state = this.snapshot(); + const signature = stateSignature(state); + if (signature === this.lastSignature) return; + this.lastSignature = signature; + for (const listener of this.stateListeners) listener(state); + } + + private async askForAccess(): Promise { + const music = this.native(); + const status = String(music.permissionStatus()); + if (status === "denied") { + const open = await Dialogs.confirm({ + title: "Apple Music Access", + message: "Allow Media & Apple Music access in Settings to show and control Apple Music on your glasses.", + okButtonText: "Open Settings", + cancelButtonText: "Cancel", + }); + if (open) Utils.openUrl("app-settings:"); + return false; + } + if (status === "restricted") { + await Dialogs.alert({ + title: "Apple Music Access Restricted", + message: "Media library access is restricted on this iPhone. Check Screen Time or device management settings.", + okButtonText: "OK", + }); + return false; + } + return new Promise((resolve) => { + music.requestPermission((granted: boolean) => resolve(Boolean(granted))); + }); + } +} + +export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge(); diff --git a/app/native/media-controller.ts b/app/native/media-controller.ts index e81a98e9..c6baeb2e 100644 --- a/app/native/media-controller.ts +++ b/app/native/media-controller.ts @@ -3,61 +3,25 @@ import { GrayImage } from "../graphics/image"; import { grayImageFromPacket, PHOTO_TONE } from "./image-files"; import { ignoredMediaPackages, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps"; import { onSettingsStoreChanged } from "./settings-store"; +import { isNotificationListenerEnabled } from "./notification-access"; +import { + DEFAULT_MEDIA_STATE, + interpolateMediaState, + type MediaControllerBridge, + type MediaControllerState, + type MediaPlaybackState, + type MediaQueueItem, +} from "./media-types"; declare const com: any; -export type MediaQueueItem = { - id: number; - title: string; - active: boolean; -}; +export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types"; -export type MediaPlaybackState = - | "notification-access-required" - | "idle" - | "playing" - | "paused" - | "buffering" - | "stopped" - | "unknown"; - -export type MediaControllerState = { - available: boolean; - accessEnabled: boolean; - packageName: string; - appName: string; - playbackState: MediaPlaybackState; - title: string; - artist: string; - album: string; - positionMs: number; - durationMs: number; - playbackSpeed: number; - status: string; - canPlayPause: boolean; - canSkipNext: boolean; - canSkipPrevious: boolean; -}; - -const DEFAULT_MEDIA_STATE: MediaControllerState = { - available: false, - accessEnabled: false, - packageName: "", - appName: "", - playbackState: "idle", - title: "", - artist: "", - album: "", - positionMs: -1, - durationMs: -1, - playbackSpeed: 0, - status: "Media controller unavailable.", - canPlayPause: false, - canSkipNext: false, - canSkipPrevious: false, -}; - -export class FaceclawMediaControllerBridge { +export class FaceclawMediaControllerBridge implements MediaControllerBridge { + readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings."; + readonly accessLabel = "Notification access needed"; + readonly accessAction = "open settings"; + readonly skipRebuildsQueue = true; private readonly stateListeners = new Set<(state: MediaControllerState) => void>(); private controller: any = null; private listenerProxy: any = null; @@ -72,12 +36,7 @@ export class FaceclawMediaControllerBridge { } snapshot(): MediaControllerState { - const state = { ...this.state }; - if (state.playbackState === "playing" && state.positionMs >= 0 && state.playbackSpeed !== 0) { - state.positionMs += (Date.now() - this.stateUpdatedAtMs) * state.playbackSpeed; - state.positionMs = Math.max(0, state.durationMs > 0 ? Math.min(state.positionMs, state.durationMs) : state.positionMs); - } - return state; + return interpolateMediaState(this.state, this.stateUpdatedAtMs, Date.now()); } async start(): Promise { @@ -108,9 +67,15 @@ export class FaceclawMediaControllerBridge { this.controller?.skipPrevious(); } - openNotificationAccessSettings(): void { + hasAccess(): boolean { + return isNotificationListenerEnabled(); + } + + /** Opens the notification-access settings screen; the grant is only visible once the app resumes. */ + async requestAccess(): Promise { this.ensureController(); this.controller?.openNotificationAccessSettings(); + return false; } /** Jump playback to a queue item (see getQueue). */ @@ -206,6 +171,7 @@ export class FaceclawMediaControllerBridge { canSkipPrevious: boolean, accessEnabled: boolean, status: string, + artVersion: number, ) => { this.state = { available: Boolean(title || artist || album || packageName) || String(playbackState) !== "idle", @@ -223,6 +189,7 @@ export class FaceclawMediaControllerBridge { canPlayPause: Boolean(canPlayPause), canSkipNext: Boolean(canSkipNext), canSkipPrevious: Boolean(canSkipPrevious), + artVersion: Number(artVersion), }; this.stateUpdatedAtMs = Date.now(); for (const listener of this.stateListeners) { @@ -248,4 +215,4 @@ function normalizePlaybackState(value: string): MediaPlaybackState { } } -export const mediaControllerBridge = new FaceclawMediaControllerBridge(); +export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge(); diff --git a/app/native/media-types.ts b/app/native/media-types.ts new file mode 100644 index 00000000..84449b30 --- /dev/null +++ b/app/native/media-types.ts @@ -0,0 +1,161 @@ +import type { GrayImage } from "../graphics/image"; + +/** + * Shapes shared by the media bridges: media-controller.ts / media-browser.ts + * drive Android media sessions and MediaBrowserServices; their .ios.ts + * counterparts drive Apple Music through the MediaPlayer framework. + */ + +/** The player behind the iOS bridges: the system Music app. */ +export const APPLE_MUSIC_APP_NAME = "Apple Music"; +export const APPLE_MUSIC_PACKAGE = "com.apple.Music"; + +export type MediaQueueItem = { + id: number; + title: string; + active: boolean; +}; + +export type MediaPlaybackState = + | "notification-access-required" + | "idle" + | "playing" + | "paused" + | "buffering" + | "stopped" + | "unknown"; + +export type MediaControllerState = { + available: boolean; + accessEnabled: boolean; + packageName: string; + appName: string; + playbackState: MediaPlaybackState; + title: string; + artist: string; + album: string; + positionMs: number; + durationMs: number; + playbackSpeed: number; + status: string; + canPlayPause: boolean; + canSkipNext: boolean; + canSkipPrevious: boolean; + /** + * Changes whenever getAlbumArt's result may have changed (any metadata + * update or session switch). Key cached art on this, not the title: + * players re-publish the same strings with a different bitmap. + */ + artVersion: number; +}; + +export const DEFAULT_MEDIA_STATE: MediaControllerState = { + available: false, + accessEnabled: false, + packageName: "", + appName: "", + playbackState: "idle", + title: "", + artist: "", + album: "", + positionMs: -1, + durationMs: -1, + playbackSpeed: 0, + status: "Media controller unavailable.", + canPlayPause: false, + canSkipNext: false, + canSkipPrevious: false, + artVersion: 0, +}; + +/** Interpolate the position of a playing item from the last state update. */ +export function interpolateMediaState(state: MediaControllerState, updatedAtMs: number, nowMs: number): MediaControllerState { + const result = { ...state }; + if (result.playbackState === "playing" && result.positionMs >= 0 && result.playbackSpeed !== 0) { + result.positionMs += (nowMs - updatedAtMs) * result.playbackSpeed; + result.positionMs = Math.max(0, result.durationMs > 0 ? Math.min(result.positionMs, result.durationMs) : result.positionMs); + } + return result; +} + +export interface MediaControllerBridge { + /** Explains, on the glasses, what access is missing and that a tap asks for it. */ + readonly accessPrompt: string; + /** The missing access as a short card label. */ + readonly accessLabel: string; + /** What clicking the access prompt does, for its gesture hint. */ + readonly accessAction: string; + /** + * Whether jumping to a queue item rebuilds the queue with that item first + * (Android players do); otherwise the queue keeps its order. + */ + readonly skipRebuildsQueue: boolean; + onStateChange(listener: (state: MediaControllerState) => void): () => void; + snapshot(): MediaControllerState; + start(): Promise; + stop(): Promise; + playPause(): Promise; + skipNext(): Promise; + skipPrevious(): Promise; + hasAccess(): boolean; + /** Show the grant UI: a system settings screen or a permission prompt on the phone. */ + requestAccess(): Promise; + /** Jump playback to a queue item (see getQueue). */ + skipToQueueItem(id: number): Promise; + /** Media volume normalized to 0..100; -1 when unavailable. */ + getMediaVolumePercent(): number; + setMediaVolumePercent(volumePercent: number): void; + /** + * Album art for the current item, grayscale with the shared photo tone + * (gamma-corrected and dithered onto the display's 16 levels), scaled to + * fit maxSize; null when the player provides none. + */ + getAlbumArt(maxSize: number): GrayImage | null; + /** The player's queue (playlist), empty when the player exposes none. */ + getQueue(): MediaQueueItem[]; +} + +export type MediaBrowserApp = { + packageName: string; + serviceClass: string; + appName: string; +}; + +export type MediaBrowseItem = { + mediaId: string; + title: string; + subtitle: string; + browsable: boolean; + playable: boolean; +}; + +export interface MediaBrowserBridge { + /** Apps whose library can be browsed; cached after the first call. */ + listBrowsableApps(refresh?: boolean): MediaBrowserApp[]; + /** Connect to an app's library, replacing any existing connection. Resolves to the browse-tree root id. */ + connect(app: MediaBrowserApp): Promise; + /** Fetch one level of the connected library's tree. */ + browse(parentId: string): Promise; + /** Start playback of a browsed item. */ + playFromMediaId(mediaId: string): void; + /** Drop the connection; in-flight requests are rejected. */ + disconnect(): void; +} + +/** Parse a browse listing's JSON rows. */ +export function parseMediaBrowseItems(json: string): MediaBrowseItem[] { + const raw = JSON.parse(json) as Array<{ + mediaId?: string; + title?: string; + subtitle?: string; + browsable?: boolean; + playable?: boolean; + }>; + return raw.map((item) => ({ + mediaId: String(item.mediaId ?? ""), + title: String(item.title ?? ""), + subtitle: String(item.subtitle ?? ""), + browsable: Boolean(item.browsable), + playable: Boolean(item.playable), + })); +} diff --git a/app/native/nightscout-bridge.ts b/app/native/nightscout-bridge.ts index 9271d7c2..51be4f57 100644 --- a/app/native/nightscout-bridge.ts +++ b/app/native/nightscout-bridge.ts @@ -1,5 +1,6 @@ import { nightscoutApiTokenSetting, nightscoutSiteUrlSetting } from "../ui/dashboard-settings"; import { fetchTextWithUserAgent } from "../util/http"; +import { bolusKindFromTreatment, type BolusKind } from "../apps/nightscout/missed-bolus-detector"; export type NightscoutPoint = { timestampMs: number; @@ -14,6 +15,8 @@ export type NightscoutCarbEvent = { export type NightscoutBolusEvent = { timestampMs: number; insulin: number; + /** Meal, manual, loop-automatic, or unmarked (see BolusKind). */ + kind: BolusKind; }; export type NightscoutBasalEvent = { @@ -102,10 +105,17 @@ type NightscoutTreatmentResponse = { absolute?: unknown; rate?: unknown; duration?: unknown; + automatic?: unknown; + isSMB?: unknown; + type?: unknown; + wizard?: unknown; }; const NIGHTSCOUT_REFRESH_MS = 60_000; -const NIGHTSCOUT_HISTORY_COUNT = 30; +// Glucose history by time rather than count, so a 1-minute CGM still covers +// the graphs' 2 hours and the missed-bolus detector's 90-minute lookback. +const NIGHTSCOUT_HISTORY_WINDOW_MS = 3 * 60 * 60 * 1000; +const NIGHTSCOUT_HISTORY_MAX_COUNT = 200; const NIGHTSCOUT_TREATMENT_LOOKBACK_MS = 3 * 60 * 60 * 1000; // The v1 treatments API silently adds `created_at >= now - 4 days` to any // query without an explicit date filter (lib/server/query.js enforceDateFilter), @@ -208,13 +218,17 @@ export class NightscoutBridge { this.emit(); return; } + const historyStartMs = nowMs - NIGHTSCOUT_HISTORY_WINDOW_MS; const treatmentStartIso = new Date(nowMs - NIGHTSCOUT_TREATMENT_LOOKBACK_MS).toISOString(); const siteChangeStartIso = new Date(nowMs - NIGHTSCOUT_SITE_CHANGE_LOOKBACK_MS).toISOString(); const tokenQuery = `token=${encodeURIComponent(apiToken)}`; const [entries, deviceStatus, status, treatments, siteChanges] = await Promise.all([ fetchJson( - `${siteUrl}/api/v1/entries.json?count=${NIGHTSCOUT_HISTORY_COUNT}&${tokenQuery}`, - ), + `${siteUrl}/api/v1/entries.json?find%5Bdate%5D%5B%24gte%5D=${historyStartMs}&count=${NIGHTSCOUT_HISTORY_MAX_COUNT}&${tokenQuery}`, + ).then(async (recent) => recent?.length + ? recent + // Nothing in the window: still show the last reading, as stale. + : await fetchJson(`${siteUrl}/api/v1/entries.json?count=1&${tokenQuery}`)), fetchJson( `${siteUrl}/api/v1/devicestatus.json?count=1&${tokenQuery}`, ), @@ -334,7 +348,7 @@ function parseTreatments(treatments: NightscoutTreatmentResponse[], nowMs: numbe const insulinValue = firstFiniteNumber(treatment.insulin); if (insulinValue !== null && insulinValue > 0) { - boluses.push({ timestampMs, insulin: insulinValue }); + boluses.push({ timestampMs, insulin: insulinValue, kind: bolusKindFromTreatment(treatment) }); } const eventType = typeof treatment.eventType === "string" ? treatment.eventType : ""; diff --git a/app/native/notification-access.ios.ts b/app/native/notification-access.ios.ts index de94e22c..7842b604 100644 --- a/app/native/notification-access.ios.ts +++ b/app/native/notification-access.ios.ts @@ -2,3 +2,6 @@ import { iosNotificationState, iosNotificationMessage } from './notification-ico export function isNotificationListenerEnabled(): boolean { return iosNotificationState() === 'ready' } export function requestNotificationListenerAccess(): void { /* iOS exposes no deep link to this Bluetooth setting. */ } export function notificationEmptyMessage(): string { return iosNotificationMessage() } +export type NotificationListenerState = "connected" | "no-access" | "connecting" | "stuck" +export function notificationListenerState(): NotificationListenerState { return isNotificationListenerEnabled() ? "connected" : "no-access" } +export function watchNotificationListenerBinding(): void { /* ANCS has no listener binding to watch. */ } diff --git a/app/native/notification-access.ts b/app/native/notification-access.ts index 97823a72..39c6b7aa 100644 --- a/app/native/notification-access.ts +++ b/app/native/notification-access.ts @@ -32,7 +32,44 @@ export function requestNotificationListenerAccess(): void { com.faceclaw.app.FaceclawEvenAppDetector.openNotificationAccessSettings(context); } +/** + * Whether Android has the listener connected, and if not, why not: + * "connecting" is access granted but not (yet, or again) bound; "stuck" is + * still unbound after the native watchdog's forced rebinds, which only the + * user toggling notification access is sure to fix. + */ +export type NotificationListenerState = "connected" | "no-access" | "connecting" | "stuck"; + +export function notificationListenerState(): NotificationListenerState { + if (!global.isAndroid) return "connected"; + const context = Utils.android.getApplicationContext(); + if (!context) return "no-access"; + return String( + com.faceclaw.app.FaceclawMediaNotificationListenerService.getListenerState(context), + ) as NotificationListenerState; +} + +/** + * Start the native watchdog that rebinds the listener when Android drops it + * while access is granted (see FaceclawMediaNotificationListenerService + * .watchListenerBinding). Called once at boot. + */ +export function watchNotificationListenerBinding(): void { + if (!global.isAndroid) return; + const context = Utils.android.getApplicationContext(); + if (!context) return; + com.faceclaw.app.FaceclawMediaNotificationListenerService.watchListenerBinding(context); +} + export function notificationEmptyMessage(): string { - return isNotificationListenerEnabled() ? "No current Android notifications." - : "Grant permission on your phone to view notifications on the glasses."; + switch (notificationListenerState()) { + case "no-access": + return "Grant permission on your phone to view notifications on the glasses."; + case "connecting": + return "Connecting to Android notifications..."; + case "stuck": + return "Android stopped sending notifications to Faceclaw. Turn Faceclaw's notification access off and on again in phone settings."; + case "connected": + return "No current Android notifications."; + } } diff --git a/app/native/notification-icons.ios.ts b/app/native/notification-icons.ios.ts index 95a387ed..c908c838 100644 --- a/app/native/notification-icons.ios.ts +++ b/app/native/notification-icons.ios.ts @@ -1,8 +1,10 @@ import { GrayImage } from '../graphics/image' +import { renderIcon, type IconName } from '../graphics/icons' import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-client' import { rememberNotificationSources } from './notification-sources' +import type { AndroidNotification } from './notification-types' export type { AndroidNotification, AndroidNotificationAction } from './notification-types' -export type { NotificationIconsResult, NotificationIconResult } from './notification-icons' +export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons' export const ALL_NOTIFICATIONS = 0x7fffffff let active: AncsClient | null = null @@ -26,8 +28,21 @@ export function dismissNotification(key: string): boolean { return active?.dismi export function onAndroidNotificationPosted(listener: (key: string) => void): () => void { listeners.add(listener); return () => { listeners.delete(listener) } } -// ANCS does not provide app icons. Use a local bell; never fetch notification -// sources or content from a third-party icon service. +// ANCS removals carry no key and already reach the posted listeners above +// (they hear every change), and an open popup closes on the repaint that follows. +export function onAndroidNotificationRemoved(_listener: (key: string) => void): () => void { return () => {} } +export function onAndroidNotificationsChanged(listener: () => void): () => void { return onAndroidNotificationPosted(listener) } +// ANCS does not provide app icons, so show one for the notification's ANCS +// category; never fetch notification sources or content from a third-party +// icon service. Indexed by CategoryID: Other, IncomingCall, MissedCall, +// Voicemail, Social, Schedule, Email, News, HealthAndFitness, +// BusinessAndFinance, Location, Entertainment. +const CATEGORY_ICONS: readonly IconName[] = ['bell','phone-incoming','phone-missed','voicemail','message-circle', + 'calendar','mail','newspaper','heart-pulse','briefcase','map-pin','tv'] +function categoryIcon(category: string): GrayImage { + return renderIcon(CATEGORY_ICONS[Number(category)] ?? 'bell',24)?.clone() ?? bell() +} +// Drawn fallback for when the SVG rasterizer fails. function bell(): GrayImage { const image = new GrayImage(24,24,0) image.fillRoundedRect(6,5,12,13,220,5) @@ -37,11 +52,16 @@ function bell(): GrayImage { } export function readActiveNotificationIcons(maxIcons: number, _allowStale: boolean) { // One icon per source, standing for its first notification. - const sources = new Map() - for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n.key) - const keys = Array.from(sources.values()).slice(0,Math.max(0,maxIcons)) - return {icons: keys.map(() => bell()),keys,stale:false} + const sources = new Map() + for (const n of readActiveNotifications(ALL_NOTIFICATIONS)) if (!sources.has(n.packageName)) sources.set(n.packageName, n) + const shown = Array.from(sources.values()).slice(0,Math.max(0,maxIcons)) + return {icons: shown.map(n => categoryIcon(n.category)),keys: shown.map(n => n.key),stale:false} } export function readNotificationIconByKey(key: string, _allowStale: boolean) { - return {icon: readActiveNotifications(ALL_NOTIFICATIONS).some(n => n.key === key) ? bell() : null,stale:false} + const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key) + return {icon: notification ? categoryIcon(notification.category) : null,stale:false} +} +// ANCS carries no attachments. +export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) { + return {picture: null,stale:false} } diff --git a/app/native/notification-icons.ts b/app/native/notification-icons.ts index 7620ab24..aa652096 100644 --- a/app/native/notification-icons.ts +++ b/app/native/notification-icons.ts @@ -2,6 +2,7 @@ import { GrayImage } from "../graphics/image"; import { logCurrent, spanCurrent } from "./frame-timings"; import { toUint8Array } from "../util/array-util"; import { rememberNotificationSources } from "./notification-sources"; +import { PHOTO_TONE, grayImageFromPacket } from "./image-files"; declare const com: any; declare const java: any; @@ -10,7 +11,7 @@ const ICON_SIZE = 24; /** Ask the native listener for every active source, including those below the list's display limit. */ export const ALL_NOTIFICATIONS = 0x7fffffff; // Backstop TTL only: the icon caches are invalidated eagerly whenever a -// notification is posted or dismissed (see invalidateIconCaches callers), so +// notification is posted or removed (see invalidateIconCaches callers), so // the tray is kept fresh by invalidation, not by expiry. A short TTL just // forced a blocking main-thread re-fetch (~40-100ms of Java icon // rasterization) every few seconds — usually producing a no-change frame that @@ -22,8 +23,16 @@ let cachedKeys: string[] = []; let cachedAtMs = 0; const keyedIconCache = new Map(); const KEYED_ICON_CACHE_MAX = 128; +/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */ +const pictureCache = new Map(); +const PICTURE_CACHE_MAX = 8; let notificationListenerProxy: any = null; const notificationPostedListeners = new Set<(notificationKey: string) => void>(); +const notificationRemovedListeners = new Set<(notificationKey: string) => void>(); +let listenerStateObserver: any = null; +const listenerStateListeners = new Set<() => void>(); function invalidateIconCaches(): void { cachedAtMs = 0; @@ -58,19 +67,21 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea const now = Date.now(); if (cachedAtMs > 0 && now - cachedAtMs < ICON_CACHE_MS) { logCurrent("notification icons served from cache"); - return { icons: cachedIcons.map(icon => icon.clone()), keys: cachedKeys.slice(), stale: false }; + return cachedIconsResult(maxIcons, false); } if (allowStale) { logCurrent("notification icons served stale"); - return { icons: cachedIcons.map(icon => icon.clone()), keys: cachedKeys.slice(), stale: true }; + return cachedIconsResult(maxIcons, true); } + // Every icon, whatever this caller wants: the cache serves callers asking + // for different counts (the switcher row needs them all to lay itself out). const keyList = new java.util.ArrayList(); const bytes = spanCurrent("fetch-notification-icons", () => toUint8Array( com.faceclaw.app.FaceclawMediaNotificationListenerService.getActiveNotificationIconGrays( ICON_SIZE, - maxIcons, + ALL_NOTIFICATIONS, keyList, ), ), @@ -91,7 +102,12 @@ export function readActiveNotificationIcons(maxIcons: number, allowStale: boolea cachedIcons = icons; cachedKeys = keys; cachedAtMs = now; - return { icons: icons.map(icon => icon.clone()), keys: keys.slice(), stale: false }; + return cachedIconsResult(maxIcons, false); +} + +/** Copies of the first maxIcons cached icons, with their keys. */ +function cachedIconsResult(maxIcons: number, stale: boolean): NotificationIconsResult { + return { icons: cachedIcons.slice(0, maxIcons).map(icon => icon.clone()), keys: cachedKeys.slice(0, maxIcons), stale }; } export type NotificationIconResult = { @@ -143,6 +159,71 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not return { icon: icon?.clone() ?? null, stale: false }; } +export type NotificationPictureResult = { + /** Shared with the cache: blit it, don't draw on it. */ + picture: GrayImage | null; + /** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */ + stale: boolean; +}; + +/** + * The picture attached to a notification (hasPicture), scaled to fit + * maxWidth x maxHeight with the photo tone and inverted if it is drawn on a + * light background (GrayPacket.hasLightBackground), under the allow-stale contract of + * readNotificationIconByKey. Cached per post of the notification: an ongoing + * notification updated in place (Dexcom redraws its graph every few minutes) + * fires no posted event, but its postTime moves. ICON_CACHE_MS is the + * backstop here too. Under allowStale an earlier fetch stands in, if it fits. + */ +export function readNotificationPicture( + notification: AndroidNotification, + maxWidth: number, + maxHeight: number, + allowStale: boolean, +): NotificationPictureResult { + const width = Math.round(maxWidth); + const height = Math.round(maxHeight); + if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) { + return { picture: null, stale: false }; + } + + const now = Date.now(); + const cached = pictureCache.get(notification.key); + if ( + cached && cached.postTime === notification.postTime && cached.maxWidth === width + && cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS + ) { + return { picture: cached.picture, stale: false }; + } + if (allowStale) { + const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height; + return { picture: fits ? cached!.picture : null, stale: true }; + } + + const picture = spanCurrent("fetch-notification-picture", () => + grayImageFromPacket( + com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray( + notification.key, + width, + height, + PHOTO_TONE.gamma, + PHOTO_TONE.dither, + // A chart or screenshot made for a light theme would be a glaring + // lit panel on the glasses: show it light-on-dark. + true, + ), + ), + ); + pictureCache.delete(notification.key); + pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now }); + while (pictureCache.size > PICTURE_CACHE_MAX) { + const oldestKey = pictureCache.keys().next().value; + if (oldestKey === undefined) break; + pictureCache.delete(oldestKey); + } + return { picture, stale: false }; +} + export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] { if (!global.isAndroid || maxNotifications <= 0) return []; try { @@ -183,12 +264,68 @@ export function dismissNotification(notificationKey: string): boolean { } export function onAndroidNotificationPosted(listener: (notificationKey: string) => void): () => void { - notificationPostedListeners.add(listener); - ensureNotificationPostedListener(); + return addNotificationListener(notificationPostedListeners, listener); +} + +/** + * A listed notification went away: cancelled by its source (including one + * withdrawn moments after it was posted), dismissed on the phone or from the + * glasses, or updated into one the glasses don't list. + */ +export function onAndroidNotificationRemoved(listener: (notificationKey: string) => void): () => void { + return addNotificationListener(notificationRemovedListeners, listener); +} + +/** + * Any change to the listed notifications (a post, a removal, or the listener + * connecting or going missing), for views that just repaint. + */ +export function onAndroidNotificationsChanged(listener: () => void): () => void { + const offPosted = onAndroidNotificationPosted(listener); + const offRemoved = onAndroidNotificationRemoved(listener); + const offState = onListenerStateChanged(listener); return () => { - notificationPostedListeners.delete(listener); - if (notificationPostedListeners.size === 0) { - removeNotificationPostedListener(); + offPosted(); + offRemoved(); + offState(); + }; +} + +function onListenerStateChanged(listener: () => void): () => void { + listenerStateListeners.add(listener); + if (global.isAndroid && !listenerStateObserver) { + listenerStateObserver = new java.lang.Runnable({ + run: () => { + invalidateIconCaches(); + const snapshot = Array.from(listenerStateListeners); + setTimeout(() => { + for (const each of snapshot) { + each(); + } + }, 0); + }, + }); + com.faceclaw.app.FaceclawMediaNotificationListenerService.addListenerStateObserver(listenerStateObserver); + } + return () => { + listenerStateListeners.delete(listener); + if (listenerStateListeners.size === 0 && listenerStateObserver) { + com.faceclaw.app.FaceclawMediaNotificationListenerService.removeListenerStateObserver(listenerStateObserver); + listenerStateObserver = null; + } + }; +} + +function addNotificationListener( + listeners: Set<(notificationKey: string) => void>, + listener: (notificationKey: string) => void, +): () => void { + listeners.add(listener); + ensureNativeNotificationListener(); + return () => { + listeners.delete(listener); + if (notificationPostedListeners.size === 0 && notificationRemovedListeners.size === 0) { + removeNativeNotificationListener(); } }; } @@ -215,21 +352,19 @@ function normalizeNotification(value: any): AndroidNotification | null { postTime: Number(value.postTime) || 0, when: Number(value.when) || 0, actions, + foregroundService: Boolean(value.foregroundService), + hasPicture: Boolean(value.hasPicture), }; } -function ensureNotificationPostedListener(): void { +function ensureNativeNotificationListener(): void { if (!global.isAndroid || notificationListenerProxy) return; notificationListenerProxy = new com.faceclaw.app.FaceclawNotificationListener({ onNotificationPosted: (notificationKey: string) => { - invalidateIconCaches(); - const key = String(notificationKey); - const listeners = Array.from(notificationPostedListeners); - setTimeout(() => { - for (const listener of listeners) { - listener(key); - } - }, 0); + dispatchNotificationEvent(notificationPostedListeners, notificationKey); + }, + onNotificationRemoved: (notificationKey: string) => { + dispatchNotificationEvent(notificationRemovedListeners, notificationKey); }, }); com.faceclaw.app.FaceclawMediaNotificationListenerService.addNotificationListener( @@ -237,7 +372,18 @@ function ensureNotificationPostedListener(): void { ); } -function removeNotificationPostedListener(): void { +function dispatchNotificationEvent(listeners: Set<(notificationKey: string) => void>, notificationKey: string): void { + invalidateIconCaches(); + const key = String(notificationKey); + const snapshot = Array.from(listeners); + setTimeout(() => { + for (const listener of snapshot) { + listener(key); + } + }, 0); +} + +function removeNativeNotificationListener(): void { if (!global.isAndroid || !notificationListenerProxy) return; com.faceclaw.app.FaceclawMediaNotificationListenerService.removeNotificationListener( notificationListenerProxy, diff --git a/app/native/notification-types.ts b/app/native/notification-types.ts index b7a6581f..78d35ac0 100644 --- a/app/native/notification-types.ts +++ b/app/native/notification-types.ts @@ -5,6 +5,10 @@ export type AndroidNotification = { bigText: string; subText: string; infoText: string; summaryText: string; category: string; lines: string[]; postTime: number; when: number; actions: AndroidNotificationAction[]; dismissLabel?: string; + /** Posted by a foreground service (Android): a "this app is working" status notification. */ + foregroundService?: boolean; + /** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */ + hasPicture?: boolean; /** Full message size reported by iOS, even when the fetched text is bounded. */ messageSize?: number; }; diff --git a/app/native/openai-stt.ts b/app/native/openai-stt.ts index 2949be00..063e0a39 100644 --- a/app/native/openai-stt.ts +++ b/app/native/openai-stt.ts @@ -1,6 +1,5 @@ import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt"; - -declare const com: any; +import { openSocket, type SocketConnection } from "./socket"; /** * OpenAI realtime speech-to-text over WebSocket (a transcription-only session @@ -23,8 +22,7 @@ const TARGET_SAMPLE_RATE = 24000; export type OpenAiSttOptions = CloudSttOptions; export class OpenAiRealtimeSttClient implements CloudSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // Base64 audio (or the commit sentinel) queued until the socket opens. @@ -38,7 +36,7 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { start(): void { if (this.closed || this.ws) return; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; this.open = true; @@ -59,14 +57,9 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { if (this.closed) return; this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`); }, - }); + }; try { - this.ws = new com.faceclaw.app.FaceclawWebSocket( - WS_URL, - this.listenerProxy, - "Authorization", - `Bearer ${this.options.apiKey}`, - ); + this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` }); this.options.onStatus("Connecting to OpenAI..."); } catch (error) { this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`); @@ -117,7 +110,6 @@ export class OpenAiRealtimeSttClient implements CloudSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingChunks.length = 0; } diff --git a/app/native/preview-display.ios.ts b/app/native/preview-display.ios.ts index c25628a6..d01d3f5f 100644 --- a/app/native/preview-display.ios.ts +++ b/app/native/preview-display.ios.ts @@ -85,12 +85,25 @@ export class PreviewDisplayTarget implements DisplayTarget { this.composited(); } + /** + * As on the live bridge (FaceclawCommunicatorBridge.hasNativeSurface): a call that lands + * after its window closed, or on a target that replaced the one it was configured on, is + * dropped rather than failing the caller. + */ + private hasSurface(id: string, what: string): boolean { + if (this.compositor.hasSurface(id)) return true; + console.log(`[ios-preview] ${what} for unknown surface ${id} dropped`); + return false; + } + async setSurfaceVisible(id: string, visible: boolean): Promise { + if (!this.hasSurface(id, "setSurfaceVisible")) return; this.compositor.setSurfaceVisible(id, Boolean(visible)); this.composited(); } async setSurfaceDepth(id: string, depth: number): Promise { + if (!this.hasSurface(id, "setSurfaceDepth")) return; this.compositor.setSurfaceDepth(id, Math.round(depth)); } @@ -113,6 +126,7 @@ export class PreviewDisplayTarget implements DisplayTarget { _frameId = 0, glyphs: ArrayBuffer | null = null, ): Promise { + if (!this.hasSurface(surfaceId, "frame")) return; this.compositor.submitSurfaceFrame(surfaceId, pixels8bpp, { x: Math.round(rect.x), y: Math.round(rect.y), width: Math.round(rect.width), height: Math.round(rect.height) }, glyphs); this.composited(); @@ -138,8 +152,8 @@ export class PreviewDisplayTarget implements DisplayTarget { return this.compositor.composite(); } - getCompositePreview(green = false): ImageSource | null { - return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green); + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { + return previewPixels(this.compositor.composite(), this.compositor.width, this.compositor.height, green, crop); } /** Save the current composite as a 4-bit grayscale PNG under Documents; cropping is not supported on iOS. */ diff --git a/app/native/preview-display.ts b/app/native/preview-display.ts index 4f6d7531..8df3f5af 100644 --- a/app/native/preview-display.ts +++ b/app/native/preview-display.ts @@ -160,9 +160,18 @@ export class PreviewDisplayTarget implements DisplayTarget { return Promise.resolve("composited"); } - getCompositePreview(green = false): ImageSource | null { + getCompositePreview(green = false, crop: { x: number; y: number; width: number; height: number } | null = null): ImageSource | null { if (!global.isAndroid) return null; - const bitmap = this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); + const bitmap = crop + ? this.compositor.getCompositePreviewBitmap( + PREVIEW_BRIGHTEN_GAMMA, + green, + Math.round(crop.x), + Math.round(crop.y), + Math.round(crop.width), + Math.round(crop.height), + ) + : this.compositor.getCompositePreviewBitmap(PREVIEW_BRIGHTEN_GAMMA, green); return bitmap ? new ImageSource(bitmap) : null; } diff --git a/app/native/reconnecting-stt.ts b/app/native/reconnecting-stt.ts index 1221bb8d..2b860e24 100644 --- a/app/native/reconnecting-stt.ts +++ b/app/native/reconnecting-stt.ts @@ -24,6 +24,8 @@ export class ReconnectingSttClient implements CloudSttClient { private readonly create: (options: CloudSttOptions) => CloudSttClient, private readonly options: CloudSttOptions, private readonly continuous: () => boolean, + /** How long finish() waits for the provider's final transcript. */ + readonly finalTimeoutMs = 5000, ) {} start(): void { @@ -90,7 +92,7 @@ export class ReconnectingSttClient implements CloudSttClient { this.stop(); return; } - this.finishTimeout = setTimeout(() => this.stop(), 5000); + this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs); if (this.ready) this.client?.finish(); } diff --git a/app/native/reverse-geocode.ios.ts b/app/native/reverse-geocode.ios.ts new file mode 100644 index 00000000..d1ff4b78 --- /dev/null +++ b/app/native/reverse-geocode.ios.ts @@ -0,0 +1,21 @@ +import { type Place } from './location-types' +declare const CLGeocoder: any, CLLocation: any + +/** Geocoders with a lookup in flight, held until their callback so collection can't drop it. */ +const pending = new Set() + +/** The place at a coordinate from Apple's geocoder, or null when it can't name it. */ +export function reverseGeocode(latitude: number, longitude: number): Promise { + return new Promise(resolve => { + const geocoder = CLGeocoder.new() + pending.add(geocoder) + const location = CLLocation.alloc().initWithLatitudeLongitude(latitude, longitude) + geocoder.reverseGeocodeLocationCompletionHandler(location, (placemarks: any, error: any) => { + pending.delete(geocoder) + const placemark = !error && placemarks && placemarks.count ? placemarks.objectAtIndex(0) : null + resolve(placemark + ? { locality: placemark.locality ?? '', region: placemark.administrativeArea ?? '', country: placemark.country ?? '' } + : null) + }) + }) +} diff --git a/app/native/reverse-geocode.ts b/app/native/reverse-geocode.ts new file mode 100644 index 00000000..d09a5b9f --- /dev/null +++ b/app/native/reverse-geocode.ts @@ -0,0 +1,20 @@ +import { Utils } from "@nativescript/core"; + +import { type Place } from "./location-types"; + +declare const com: any; + +/** The place at a coordinate from Android's Geocoder, or null when the phone can't name it. */ +export function reverseGeocode(latitude: number, longitude: number): Promise { + const context = Utils.android.getApplicationContext(); + if (!context) return Promise.resolve(null); + return new Promise((resolve) => { + const geocoder = new com.faceclaw.app.FaceclawReverseGeocoder(context); + // The lookup thread holds the listener proxy until its one callback. + geocoder.lookup(latitude, longitude, new com.faceclaw.app.FaceclawReverseGeocoder.Listener({ + onResult: (locality: string | null, region: string | null, country: string | null) => { + resolve({ locality: locality ? String(locality) : "", region: region ? String(region) : "", country: country ? String(country) : "" }); + }, + })); + }); +} diff --git a/app/native/socket.ios.ts b/app/native/socket.ios.ts index e0bb5897..dd9ba7a5 100644 --- a/app/native/socket.ios.ts +++ b/app/native/socket.ios.ts @@ -1,7 +1,7 @@ -export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void } +import type { SocketConnection, SocketHeader, SocketListener } from './socket' declare const FaceclawSocket: any -export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } { - const native = FaceclawSocket.alloc().initWithURL(url) +export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection { + const native = FaceclawSocket.alloc().initWithURLHeaderNameHeaderValue(url, header?.name ?? null, header?.value ?? null) let ended = false const close = () => { if (ended) return; ended = true; clearInterval(timer); native.close() } const timer = setInterval(() => { @@ -12,5 +12,15 @@ export function openSocket(url: string, listener: SocketListener): { sendText(te else { close(); if (event.kind === 'closed') listener.onClosed(event.code, event.text); else listener.onFailure(event.text) } } }, 25) - return { sendText: text => { if (!ended) native.sendText(text) }, close } + // NSURLSession queues every send; a failed one surfaces as an error event. + return { + sendText: text => { if (!ended) native.sendText(text); return !ended }, + sendBinary: bytes => { + if (ended) return false + const copy = bytes.slice() + native.sendData(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) + return true + }, + close, + } } diff --git a/app/native/socket.ts b/app/native/socket.ts index 7b952acc..8677b878 100644 --- a/app/native/socket.ts +++ b/app/native/socket.ts @@ -1,9 +1,20 @@ +import { toJavaBytes } from './cloud-stt' + declare const com: any export type SocketListener = { onOpen(): void; onTextMessage(message: string): void; onClosed(code: number, reason: string): void; onFailure(message: string): void } -export function openSocket(url: string, listener: SocketListener): { sendText(text: string): void; close(code: number, reason: string): void } { +/** One extra handshake header, e.g. an API key. */ +export type SocketHeader = { name: string; value: string } +export type SocketConnection = { + /** False when the frame was refused outright; later failures arrive via onFailure. */ + sendText(text: string): boolean + sendBinary(bytes: Uint8Array): boolean + close(code: number, reason: string): void +} +export function openSocket(url: string, listener: SocketListener, header?: SocketHeader): SocketConnection { const proxy = new com.faceclaw.app.FaceclawWebSocketListener(listener) - const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, null, null) - const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text), + const native = new com.faceclaw.app.FaceclawWebSocket(url, proxy, header?.name ?? null, header?.value ?? null) + const connection = { listenerProxy: proxy, sendText: (text: string) => native.sendText(text) !== false, + sendBinary: (bytes: Uint8Array) => native.sendBinary(toJavaBytes(bytes)) !== false, close: (code: number, reason: string) => native.close(code, reason) } return connection } diff --git a/app/native/soniox-stt.ts b/app/native/soniox-stt.ts index e52454e7..aa0d210c 100644 --- a/app/native/soniox-stt.ts +++ b/app/native/soniox-stt.ts @@ -1,8 +1,7 @@ -import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, toJavaBytes } from "./cloud-stt"; +import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions } from "./cloud-stt"; +import { openSocket, type SocketConnection } from "./socket"; import { TimedTranscript } from "./transcript-format"; -declare const com: any; - /** * Soniox realtime speech-to-text over WebSocket. * https://soniox.com/docs/stt/rt/real-time-transcription @@ -24,8 +23,7 @@ const MODEL_ID = "stt-rt-v5"; export type SonioxSttOptions = CloudSttOptions; export class SonioxSttClient implements CloudSttClient { - private ws: any = null; - private listenerProxy: any = null; + private ws: SocketConnection | null = null; private open = false; private closed = false; // PCM queued until the socket opens and the config message is sent. @@ -40,7 +38,7 @@ export class SonioxSttClient implements CloudSttClient { start(): void { if (this.closed || this.ws) return; - this.listenerProxy = new com.faceclaw.app.FaceclawWebSocketListener({ + const listener = { onOpen: () => { if (this.closed) return; // The API key rides in the config message; there is no auth header. @@ -77,9 +75,9 @@ export class SonioxSttClient implements CloudSttClient { if (this.closed) return; this.options.onDisconnected?.(`Soniox connection failed: ${String(message)}`); }, - }); + }; try { - this.ws = new com.faceclaw.app.FaceclawWebSocket(WS_URL, this.listenerProxy, null, null); + this.ws = openSocket(WS_URL, listener); this.options.onStatus("Connecting to Soniox..."); } catch (error) { this.options.onDisconnected?.(`Soniox connection failed: ${String((error as Error)?.message ?? error)}`); @@ -122,13 +120,12 @@ export class SonioxSttClient implements CloudSttClient { } this.ws = null; } - this.listenerProxy = null; this.pendingPcm.length = 0; } private sendPcm(pcm: Uint8Array): void { try { - if (this.ws?.sendBinary(toJavaBytes(pcm)) === false) { + if (this.ws?.sendBinary(pcm) === false) { this.options.onDisconnected?.("Soniox audio send failed."); } } catch (error) { diff --git a/app/native/voice-control.ios.ts b/app/native/voice-control.ios.ts index 2896e21d..38dd996f 100644 --- a/app/native/voice-control.ios.ts +++ b/app/native/voice-control.ios.ts @@ -1,12 +1,23 @@ import type { VoiceControlState, VoiceTranscriptEvent } from './voice-control' +import type { CloudSttTranscriptEvent } from './cloud-stt' +import type { ReconnectingSttClient } from './reconnecting-stt' +import { createCloudSttClient, usesCloudStt, type VoiceProviderSelection } from './cloud-stt-provider' +import { WavRecorder } from './wav-recorder.ios' declare const FaceclawSpeech: any export type IosMicrophoneSession = { setMicrophone(enabled: boolean, listener?: (packet: Uint8Array) => void): Promise } +/** How long a cloud provider gets, after the last audio, to send its final transcript. */ +const CLOUD_FINAL_TIMEOUT_MS = 5000 /** iOS implementation of the shared voice dialog's event bridge. Audio comes - * from the glasses when connected, or the default phone input in preview mode. + * from the glasses when connected, or the default phone input in preview mode, + * and is transcribed by Apple's on-device recognizer or a cloud provider. + * Continuous capture (Transcribe, Teleprompter) needs a cloud provider: only + * the PCM capture runs open-ended, and pauses commit its segments. When the + * session or the app lifecycle cuts one short it is drained, not discarded; + * restarting it is the controller's call. */ export class IosVoiceControlBridge { private native: any = null @@ -24,6 +35,22 @@ export class IosVoiceControlBridge { private audioTimer: ReturnType | null = null private packets = 0 private log: (message: string) => void = () => {} + // Non-null while a cloud provider owns the transcript; FaceclawSpeech then + // only supplies PCM. cloudEndStatus is set once the audio has ended and the + // provider was asked to finalize; it is the status shown when it does. + private cloud: ReconnectingSttClient | null = null + private cloudEndStatus: string | null = null + private cloudTimer: ReturnType | null = null + // Open-ended capture: runs until stopContinuousCapture, and push-to-talk + // joins it instead of restarting the mic (transcripts reach every listener). + private continuous = false + // Shown instead of the native "ended" status once a drain the bridge started completes. + private drainStatus: string | null = null + // Continuous captures' audio goes to Downloads/.wav while it is + // set. The file outlives a capture, so a restarted one keeps appending to it, + // and it is created by the first audio, so a failed start leaves no file. + private recordingName = '' + private recorder: WavRecorder | null = null onStatus(listener: (state: VoiceControlState) => void): () => void { this.statuses.add(listener); listener({ status: this.status, listening: this.capturing, detail: '' }); return () => { this.statuses.delete(listener) } @@ -38,16 +65,24 @@ export class IosVoiceControlBridge { if (!this.native) { this.native = FaceclawSpeech.new() this.native.eventHandler = (json: string) => this.receive(JSON.parse(json)) + this.native.pcmHandler = (pcm: NSData) => { + const bytes = new Uint8Array(interop.bufferFromData(pcm)) + this.cloud?.acceptPcm(bytes) + if (this.continuous) this.record(bytes) + } } return this.native } - async prepare(foreground: boolean, usePhoneMic = false): Promise { - let status = FaceclawSpeech.authorizationStatus() - if (status === 0) { - if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false } - status = await new Promise(resolve => FaceclawSpeech.requestAuthorization(resolve)) + async prepare(foreground: boolean, usePhoneMic: boolean, voice: VoiceProviderSelection): Promise { + // Only Apple's recognizer needs Speech Recognition; a cloud provider just needs audio. + if (!usesCloudStt(voice)) { + let status = FaceclawSpeech.authorizationStatus() + if (status === 0) { + if (!foreground) { this.setStatus('Open Faceclaw on the phone once to allow Speech Recognition.'); return false } + status = await new Promise(resolve => FaceclawSpeech.requestAuthorization(resolve)) + } + if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false } } - if (status !== 3) { this.setStatus('Allow Speech Recognition for Faceclaw in iPhone Settings.'); return false } if (usePhoneMic) { let microphoneStatus = FaceclawSpeech.microphoneAuthorizationStatus() if (microphoneStatus === 0) { @@ -59,12 +94,14 @@ export class IosVoiceControlBridge { return true } get statusText(): string { return this.status } - async startGlassesCapture(session: IosMicrophoneSession, log: (message: string) => void, endpointing = false): Promise { + async startGlassesCapture(session: IosMicrophoneSession, voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { + if (this.joinsContinuous(continuous)) return this.stop() const generation = ++this.generation + this.continuous = continuous this.log = log; this.source = 'glasses'; this.session = session; this.packets = 0; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) - const startError = String(this.ensureNative().startWithEndpointing(endpointing) ?? '') + const startError = this.startNative(voice, endpointing, false) if (startError) { this.stop(); this.setStatus(startError); this.emitEnd(); return } try { await session.setMicrophone(true, packet => { @@ -74,7 +111,7 @@ export class IosVoiceControlBridge { this.native.acceptPacket(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength)) }) if (generation !== this.generation || !this.capturing) return - log('Voice: glasses microphone enabled; using on-device recognition') + log(`Voice: glasses microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) this.audioTimer = setTimeout(() => { this.audioTimer = null if (generation === this.generation && this.capturing && !this.packets) { @@ -86,35 +123,103 @@ export class IosVoiceControlBridge { this.stop(); this.setStatus(error instanceof Error ? error.message : String(error)); this.emitEnd() } } - async startPhoneCapture(log: (message: string) => void, endpointing = false): Promise { + async startPhoneCapture(voice: VoiceProviderSelection, log: (message: string) => void, endpointing = false, continuous = false): Promise { + if (this.joinsContinuous(continuous)) return this.stop() + this.continuous = continuous this.log = log; this.source = 'phone'; this.capturing = true this.completion = new Promise(resolve => { this.resolveCompletion = resolve }) - const error = String(this.ensureNative().startPhoneWithEndpointing(endpointing) ?? '') + const error = this.startNative(voice, endpointing, true) if (error) { this.stop(); this.setStatus(error); this.emitEnd(); return } - log('Voice: phone default microphone enabled; using on-device recognition') + log(`Voice: phone default microphone enabled; using ${this.cloud ? 'cloud' : 'on-device'} recognition`) + } + /** Starts the native capture for the chosen provider; returns an actionable error, or ''. */ + private startNative(voice: VoiceProviderSelection, endpointing: boolean, phone: boolean): string { + const native = this.ensureNative() + const generation = this.generation + const current = () => generation === this.generation + const cloud = createCloudSttClient(voice, { + onTranscript: event => { if (current()) this.receiveCloudTranscript(event) }, + onStatus: status => { if (current()) this.setStatus(status) }, + onError: message => { if (current()) { this.stop(); this.setStatus(message); this.emitEnd() } }, + }, () => this.continuous) + if (!cloud) return String((phone ? native.startPhoneWithEndpointing(endpointing) : native.startWithEndpointing(endpointing)) ?? '') + const error = String(native.startPcmCaptureWithEndpointingPhone(endpointing, phone) ?? '') + if (error) return error + this.cloud = cloud + cloud.start() + return '' } stopPhoneCapture(): void { if (this.source !== 'phone') return + if (this.continuous) { if (this.capturing) void this.finalizeCapture('Paused while Faceclaw is in the background.'); return } this.stop(); this.setStatus('Phone voice input stopped.'); this.emitEnd() } + /** Whether continuous capture can run with this provider (Apple's recognizer can't). */ + supportsContinuous(voice: VoiceProviderSelection): boolean { return usesCloudStt(voice) } + get continuousActive(): boolean { return this.continuous && (this.capturing || this.finalizing) } + /** Where a listening continuous capture gets its audio; null when none is listening. */ + get continuousSource(): 'glasses' | 'phone' | null { return this.continuous && this.capturing ? this.source : null } + /** + * Record continuous captures' audio to Downloads/.wav, across restarts, + * until called with another name or ''. Only cloud captures supply PCM. + */ + recordTo(name: string): void { + if (name === this.recordingName) return + this.recorder?.close(); this.recorder = null + this.recordingName = name + } + private record(pcm: Uint8Array): void { + if (!this.recorder && this.recordingName) { + this.recorder = WavRecorder.open(this.recordingName) + this.log(this.recorder ? `Voice: recording audio to ${this.recorder.path}` : 'Voice: could not create the audio recording') + // One attempt per recording, not one per packet. + if (!this.recorder) this.recordingName = '' + } + this.recorder?.append(pcm) + } + /** End continuous capture, transcribing the audio since the last pause. */ + stopContinuousCapture(): Promise { + if (!this.continuous) return Promise.resolve() + return this.finalizeCapture() + } + /** Push-to-talk shares a listening continuous capture (not one that is only finalizing). */ + private joinsContinuous(continuous: boolean): boolean { + if (continuous || !this.continuous || !this.capturing) return false + this.log('Voice: push-to-talk joins the running continuous capture') + return true + } stopPushToTalk(): Promise { + // The continuous capture owns the mic and keeps it. + if (this.continuous) return Promise.resolve() + return this.finalizeCapture() + } + private finalizeCapture(status: string | null = null): Promise { const completion = this.completion if (!this.capturing) return completion this.capturing = false; this.finalizing = true + this.drainStatus = status this.clearAudioTimer(); this.releaseMicrophone() this.setStatus('Recognizing…'); this.native?.finish() return completion } stop(): void { ++this.generation - this.capturing = this.finalizing = false - this.source = null + this.capturing = this.finalizing = this.continuous = false + this.source = null; this.drainStatus = null this.clearAudioTimer(); this.releaseMicrophone(); this.native?.cancel() + this.clearCloud() this.finishCompletion() } - handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.'): void { + /** + * The glasses session went away (or the glasses locked). Push-to-talk ends + * with `message`; continuous capture transcribes what it already heard and + * shows `continuousMessage`. + */ + handleSessionEnded(message = 'Glasses disconnected. Start voice input again after reconnecting.', + continuousMessage = 'Glasses disconnected. Transcription resumes when they reconnect.'): void { if (this.source !== 'glasses' || !this.capturing && !this.finalizing) return + if (this.continuous) { if (this.capturing) void this.finalizeCapture(continuousMessage); return } this.stop(); this.setStatus(message); this.emitEnd() } private clearAudioTimer(): void { if (this.audioTimer !== null) clearTimeout(this.audioTimer); this.audioTimer = null } @@ -122,6 +227,34 @@ export class IosVoiceControlBridge { const session = this.session; this.session = null if (session) void session.setMicrophone(false).catch(error => this.log(`Voice microphone cleanup: ${error}`)) } + private clearCloud(): void { + if (this.cloudTimer !== null) clearTimeout(this.cloudTimer) + this.cloudTimer = null + this.cloud?.stop(); this.cloud = null; this.cloudEndStatus = null + } + /** The native audio has ended; have the provider finalize what it heard. */ + private finishCloud(status: string): void { + this.capturing = false; this.finalizing = true + this.cloudEndStatus = status + this.setStatus('Recognizing…') + // Armed first: finish() can deliver the final synchronously. + this.cloudTimer = setTimeout(() => this.completeCloud(), this.cloud?.finalTimeoutMs ?? CLOUD_FINAL_TIMEOUT_MS) + this.cloud?.finish() + } + private completeCloud(): void { + const status = this.cloudEndStatus ?? 'Ready to send' + this.clearCloud() + this.capturing = this.finalizing = this.continuous = false + this.finishCompletion() + this.setStatus(status); this.emitEnd() + } + private receiveCloudTranscript(event: CloudSttTranscriptEvent): void { + if (!this.capturing && !this.finalizing) return + for (const fn of [...this.transcripts]) fn(event) + // Push-to-talk commits once, so the first final after it is the last. + // In continuous capture, earlier pause segments (paragraphBreakAfter) may still arrive. + if (event.isFinal && this.cloudEndStatus !== null && !(this.continuous && event.paragraphBreakAfter)) this.completeCloud() + } private emitEnd(): void { for (const fn of [...this.ends]) fn() } private finishCompletion(): void { const resolve = this.resolveCompletion; this.resolveCompletion = null; resolve?.() @@ -134,15 +267,18 @@ export class IosVoiceControlBridge { } else if (event.kind === 'status') this.setStatus(event.message ?? '') else if (event.kind === 'audio') { this.log(`Voice audio: ${event.seconds?.toFixed(1)}s, packets=${event.packets}, RMS=${event.rms?.toFixed(4)}, missing=${event.missing}, errors=${event.errors}`) - if (this.capturing) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`) + // A cloud provider's own status (connecting, listening, reconnecting) stays up. + if (this.capturing && !this.cloud) this.setStatus(`Listening on ${this.source === 'phone' ? 'phone' : 'glasses'}… ${event.seconds?.toFixed(0)}s`) } else if (event.kind === 'finishing') { void this.stopPushToTalk(); this.emitEnd() } else if (event.kind === 'ended') { - this.source = null - this.capturing = this.finalizing = false + const status = this.drainStatus ?? event.message ?? 'Ready to send' + this.source = null; this.drainStatus = null this.clearAudioTimer(); this.releaseMicrophone() + if (this.cloud) { this.finishCloud(status); return } + this.capturing = this.finalizing = false this.finishCompletion() - this.setStatus(event.message ?? 'Ready to send'); this.emitEnd() + this.setStatus(status); this.emitEnd() } } } diff --git a/app/native/voice-control.ts b/app/native/voice-control.ts index cdaedf74..c80d98ca 100644 --- a/app/native/voice-control.ts +++ b/app/native/voice-control.ts @@ -1,11 +1,8 @@ import { Utils } from "@nativescript/core"; -import { type CloudSttTranscriptEvent, type CloudSttOptions, CloudSttClient } from "./cloud-stt"; -import { ReconnectingSttClient } from "./reconnecting-stt"; +import { type CloudSttTranscriptEvent, CloudSttClient } from "./cloud-stt"; +import { createCloudSttClient, transcriptionKey } from "./cloud-stt-provider"; import { SpeechPauseDetector } from "./speech-pause"; -import { ElevenLabsSttClient } from "./elevenlabs-stt"; -import { OpenAiRealtimeSttClient } from "./openai-stt"; -import { SonioxSttClient } from "./soniox-stt"; import { toUint8Array } from "../util/array-util"; declare const com: any; @@ -31,7 +28,9 @@ export type VoiceControlState = { // family, not with "onboard-whisper" below, which is the on-device sherpa-onnx // Whisper backend added alongside "onboard" (Moonshine). Kept as-is rather than // renamed, since it's a persisted settings value on real installs already. -export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox"; +// "whisper-server" is a self-hosted OpenAI-compatible Whisper over HTTP +// (whisper-server-stt.ts). +export type VoiceProviderKind = "onboard" | "onboard-whisper" | "elevenlabs" | "whisper" | "soniox" | "whisper-server"; export type VoiceTranscriptEvent = CloudSttTranscriptEvent & { /** @@ -55,6 +54,10 @@ export type PushToTalkOptions = { elevenLabsApiKey: string; openAiApiKey: string; sonioxApiKey: string; + whisperServerUrl?: string; + whisperServerModel?: string; + whisperServerLanguage?: string; + whisperServerApiKey?: string; saveRecording: boolean; /** * Watch the mic and fire onSpeechEnd when the speaker stops. For hands-free @@ -125,13 +128,20 @@ export class FaceclawVoiceControlBridge { private captureEndpointing = false; // Non-null while a cloud provider owns the transcript; Java only decodes PCM. private cloudClient: CloudSttClient | null = null; - // Raw-PCM tap (EvenHub mic apps): when active, the controller runs in the - // decode-only "cloud" mode and every decoded PCM frame is broadcast to these - // listeners. STT capture preempts it — a live assistant/transcribe session - // owns the single G2 mic and the raw tap is stopped for its duration. + // What the running STT stream transcribes with (transcriptionKey); a holder + // asking for something else restarts the stream rather than sharing it. + private streamProvider: string | null = null; + // Raw-PCM tap (EvenHub mic apps, the Microphones app): when active, the + // controller runs in the decode-only "cloud" mode and every decoded PCM + // frame is broadcast to these listeners. STT capture preempts it — a live + // assistant/transcribe session owns the single G2 mic and the raw tap is + // stopped for its duration. private readonly rawPcmListeners = new Set(); private readonly frameMetaListeners = new Set(); private rawActive = false; + // Who wants the raw tap. It runs while any owner does, so one owner letting + // go (an EvenHub app backgrounding) doesn't silence another. + private readonly rawOwners = new Set(); // Set when the current capture session runs "my voice only" verification; // a non-wearer verdict arrives just before the final transcript and makes // the bridge swallow it. @@ -210,6 +220,17 @@ export class FaceclawVoiceControlBridge { this.releaseCapture("continuous", false); } + /** + * Restart the running continuous capture when it transcribes with another + * provider than `options` asks for (a window that needs a multilingual one + * opened or closed), keeping every holder on the new stream. A parked + * capture is left alone: it takes the options when it resumes. + */ + retargetContinuousCapture(options: PushToTalkOptions): void { + if (!this.captureHolders.has("continuous") || !this.micIsLive()) return; + this.acquireCapture("continuous", options); + } + /** Subscribe to decoded raw mic PCM (16 kHz mono S16LE). */ onRawPcm(listener: RawPcmListener): () => void { this.rawPcmListeners.add(listener); @@ -223,11 +244,33 @@ export class FaceclawVoiceControlBridge { } /** - * Start the decode-only raw-PCM tap for an EvenHub mic app. Fails (returns - * false) if an STT capture already owns the mic; the caller re-tries when its - * eligibility is re-evaluated. Idempotent while already running. + * Start the decode-only raw-PCM tap on behalf of `owner`. Fails (returns + * false) if an STT capture already owns the mic; the owner stays registered + * and re-tries later (the EvenHub router when eligibility is re-evaluated, + * the Microphones app from its capture check). Idempotent while already + * running. */ - startRawCapture(options: { communicator: any }): boolean { + startRawCapture(options: { communicator: any; owner: string }): boolean { + if (!global.isAndroid) return false; + this.rawOwners.add(options.owner); + return this.startRawTap(options.communicator); + } + + /** `owner` no longer wants the raw tap; it stops once no owner does. */ + stopRawCapture(owner: string): void { + this.rawOwners.delete(owner); + if (this.rawOwners.size === 0) this.endRawTap(); + } + + /** + * Whether the raw tap is running and its glasses mic enable still holds + * (or is still coming up). False after an STT preemption or a session drop. + */ + isRawCaptureLive(): boolean { + return this.rawActive && this.micIsLive(); + } + + private startRawTap(communicator: any): boolean { if (!global.isAndroid) return false; if (this.rawActive && this.micIsLive()) return true; // An STT holder owns the mic even while its stream waits on a new glasses @@ -237,7 +280,7 @@ export class FaceclawVoiceControlBridge { if (this.started || this.rawActive) this.teardownCapture(); this.suspendedRaw = false; this.ensureController(); - this.controller?.setCommunicator(options.communicator); + this.controller?.setCommunicator(communicator); // The raw tap is strictly the G2 stream; clear any phone-mic flag a // preview-mode capture may have left on the shared controller. this.controller?.setUsePhoneMic(false); @@ -257,7 +300,7 @@ export class FaceclawVoiceControlBridge { return true; } - stopRawCapture(): void { + private endRawTap(): void { this.suspendedRaw = false; if (!this.rawActive) return; this.rawActive = false; @@ -269,7 +312,7 @@ export class FaceclawVoiceControlBridge { if (!global.isAndroid) return; // STT preempts the raw-PCM tap: a live transcription session owns the mic, // and an EvenHub app gets no audio for its duration. - if (this.rawActive) this.stopRawCapture(); + if (this.rawActive) this.endRawTap(); // Whether the mic is already running is a question about the stream, not // about this set: a holder whose capture died with the glasses session // (or whose mic enable was refused) is still in it. Trusting the set here @@ -278,13 +321,15 @@ export class FaceclawVoiceControlBridge { const micLive = this.micIsLive(); this.suspendedHolders.delete(holder); this.captureHolders.add(holder); - if (micLive) { + if (micLive && this.streamProvider === transcriptionKey(options)) { // Mic already running; the new holder just shares the existing stream // (transcripts are already broadcast to its listeners). return; } - // Any leftover capture is dead; drop it before starting the new one. + // Any leftover capture is dead, or transcribes with another provider than + // this holder needs; drop it before starting the new one. if (this.started) this.teardownCapture(); + this.streamProvider = transcriptionKey(options); this.ensureController(); this.controller?.setCommunicator(options.communicator); this.controller?.setUsePhoneMic(Boolean(options.usePhoneMic)); @@ -341,38 +386,12 @@ export class FaceclawVoiceControlBridge { * than failing the capture outright. */ private createCloudClient(options: PushToTalkOptions): CloudSttClient | null { - if (options.provider === "onboard" || options.provider === "onboard-whisper") return null; - const sttOptions = { - apiKey: "", + return createCloudSttClient(options, { onTranscript: (event: CloudSttTranscriptEvent) => this.emitTranscript(event.text, event.isFinal, event), onStatus: (status: string) => this.setStatus(status), onError: (message: string) => this.setStatus(message), - }; - const reconnecting = (create: (options: CloudSttOptions) => CloudSttClient, apiKey: string) => - new ReconnectingSttClient(create, { ...sttOptions, apiKey }, () => this.captureHolders.has("continuous")); - if (options.provider === "elevenlabs") { - const apiKey = options.elevenLabsApiKey.trim(); - if (!apiKey) { - this.setStatus("No ElevenLabs key set; using on-device voice."); - return null; - } - return reconnecting((config) => new ElevenLabsSttClient(config), apiKey); - } - if (options.provider === "soniox") { - const apiKey = options.sonioxApiKey.trim(); - if (!apiKey) { - this.setStatus("No Soniox key set; using on-device voice."); - return null; - } - return reconnecting((config) => new SonioxSttClient(config), apiKey); - } - const apiKey = options.openAiApiKey.trim(); - if (!apiKey) { - this.setStatus("No OpenAI key set; using on-device voice."); - return null; - } - return reconnecting((config) => new OpenAiRealtimeSttClient(config), apiKey); + }, () => this.captureHolders.has("continuous")); } private releaseCapture(holder: CaptureHolder, commit: boolean): void { @@ -437,6 +456,11 @@ export class FaceclawVoiceControlBridge { return this.suspendedHolders.size > 0 || this.suspendedRaw; } + /** Whether `holder` holds the mic, including across a session outage. */ + holdsCapture(holder: CaptureHolder): boolean { + return this.captureHolders.has(holder) || this.suspendedHolders.has(holder); + } + /** Whether anyone holds the mic, including across a session outage. */ isCaptureHeld(): boolean { return this.captureHolders.size > 0 || this.suspendedHolders.size > 0; @@ -457,8 +481,8 @@ export class FaceclawVoiceControlBridge { for (const holder of holders) { this.acquireCapture(holder, { ...options, endpointing: this.captureEndpointing }); } - if (resumeRaw && !holders.length) { - this.startRawCapture({ communicator: options.communicator }); + if (resumeRaw && !holders.length && this.rawOwners.size > 0) { + this.startRawTap(options.communicator); } } @@ -476,6 +500,7 @@ export class FaceclawVoiceControlBridge { /** Stop the native capture and any cloud client; leaves holders alone. */ private teardownCapture(): void { this.rawActive = false; + this.streamProvider = null; if (global.isAndroid) { this.controller?.stop(); } diff --git a/app/native/wav-header.ts b/app/native/wav-header.ts new file mode 100644 index 00000000..a85ba2e0 --- /dev/null +++ b/app/native/wav-header.ts @@ -0,0 +1,22 @@ +/** The 44-byte header of a 16 kHz mono PCM16 WAV file holding `dataBytes` of samples. */ +export function wavHeader(dataBytes: number): Uint8Array { + const header = new Uint8Array(44); + const view = new DataView(header.buffer); + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) header[offset + i] = text.charCodeAt(i); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + dataBytes, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, 16000, true); + view.setUint32(28, 32000, true); // byte rate + view.setUint16(32, 2, true); // block align + view.setUint16(34, 16, true); // bits per sample + ascii(36, "data"); + view.setUint32(40, dataBytes, true); + return header; +} diff --git a/app/native/wav-recorder.ios.ts b/app/native/wav-recorder.ios.ts new file mode 100644 index 00000000..ff5e815a --- /dev/null +++ b/app/native/wav-recorder.ios.ts @@ -0,0 +1,75 @@ +import { knownFolders, path as paths } from '@nativescript/core' +import { wavHeader } from './wav-header' + +// About a second of 16 kHz mono PCM16: written in batches, not per 50 ms packet. +const FLUSH_BYTES = 32000 + +/** + * Appends 16 kHz mono PCM16 to a WAV file in Documents/Downloads while it is + * captured. The header's sizes are rewritten on every flush, so a crash leaves a + * playable file of everything up to the last second. + */ +export class WavRecorder { + private pending: Uint8Array[] = [] + private pendingBytes = 0 + private dataBytes = 0 + + private constructor(private handle: NSFileHandle | null, readonly path: string) {} + + /** A recorder for Downloads/.wav (a -2, -3... suffix if taken), or null. */ + static open(name: string): WavRecorder | null { + const fm = NSFileManager.defaultManager + const dir = paths.join(knownFolders.documents().path, 'Downloads') + try { + fm.createDirectoryAtPathWithIntermediateDirectoriesAttributesError(dir, true, null) + let path = paths.join(dir, `${name}.wav`) + for (let n = 2; fm.fileExistsAtPath(path); n++) path = paths.join(dir, `${name}-${n}.wav`) + if (!fm.createFileAtPathContentsAttributes(path, toData(wavHeader(0)), null)) return null + const handle = NSFileHandle.fileHandleForWritingAtPath(path) + if (!handle || !handle.seekToEndReturningOffsetError(null)) return null + return new WavRecorder(handle, path) + } catch { + return null + } + } + + append(pcm: Uint8Array): void { + if (!this.handle || !pcm.length) return + this.pending.push(pcm.slice()) + this.pendingBytes += pcm.length + if (this.pendingBytes >= FLUSH_BYTES) this.flush() + } + + close(): void { + this.flush() + try { this.handle?.closeAndReturnError() } catch { /* already closed */ } + this.handle = null + } + + private flush(): void { + const handle = this.handle + if (!handle || !this.pendingBytes) return + const chunk = new Uint8Array(this.pendingBytes) + let offset = 0 + for (const part of this.pending) { chunk.set(part, offset); offset += part.length } + this.pending = [] + this.pendingBytes = 0 + try { + // NativeScript throws a failed call's NSError (storage full, file removed). + handle.writeDataError(toData(chunk)) + handle.seekToOffsetError(0) + handle.writeDataError(toData(wavHeader(this.dataBytes + chunk.length))) + handle.seekToEndReturningOffsetError(null) + this.dataBytes += chunk.length + return + } catch { + // Stop recording rather than retry. + } + try { handle.closeAndReturnError() } catch { /* ignore */ } + this.handle = null + } +} + +function toData(bytes: Uint8Array): NSData { + return NSData.dataWithBytesLength(interop.handleof(bytes.buffer), bytes.byteLength) +} diff --git a/app/native/weather.ts b/app/native/weather.ts index e8e55dcb..f72e1176 100644 --- a/app/native/weather.ts +++ b/app/native/weather.ts @@ -1,11 +1,22 @@ import { hasLocationPermission } from "./location-permissions"; import { getCurrentLocation, type CurrentLocation } from "./location"; +import { type Place } from "./location-types"; +import { reverseGeocode } from "./reverse-geocode"; import { getStringSetting, setStringSetting } from "./settings-store"; import { fetchWithUserAgent } from "../util/http"; import { USER_AGENT } from "../version"; export type WeatherPhase = "permission-required" | "locating" | "loading" | "ready" | "error"; +/** National Weather Service in the US, Open-Meteo everywhere else. */ +export type WeatherSource = "nws" | "open-meteo"; + +/** Short source names for the Weather app's credit line. */ +export const WEATHER_SOURCE_NAMES: Record = { + nws: "NWS", + "open-meteo": "Open-Meteo", +}; + /** Sky and precipitation state, as coarse as the icons that show it. */ export type WeatherCondition = | "clear" @@ -48,6 +59,8 @@ export type ForecastPeriod = { export type WeatherState = { phase: WeatherPhase; status: string; + /** Where current and forecast came from; null until the first refresh. */ + source: WeatherSource | null; locationName: string; current: CurrentWeather | null; forecast: ForecastPeriod[]; @@ -106,12 +119,56 @@ type NwsMeasure = { unitCode?: unknown; }; +type OpenMeteoResponse = { + utc_offset_seconds?: unknown; + current?: { + time?: unknown; + temperature_2m?: unknown; + relative_humidity_2m?: unknown; + is_day?: unknown; + weather_code?: unknown; + wind_speed_10m?: unknown; + wind_direction_10m?: unknown; + }; + hourly?: { + time?: unknown; + temperature_2m?: unknown; + precipitation_probability?: unknown; + weather_code?: unknown; + wind_speed_10m?: unknown; + wind_direction_10m?: unknown; + }; +}; + +type LoadedWeather = { + source: WeatherSource; + locationName: string; + current: CurrentWeather; + forecast: ForecastPeriod[]; +}; + const NWS_API_ROOT = "https://api.weather.gov"; const NWS_HEADERS = { Accept: "application/geo+json", // NWS asks callers to identify themselves with a contact address. "User-Agent": `${USER_AGENT} (https://github.com/jimrandomh/faceclaw)`, }; +// Free for non-commercial use without a key; data is CC BY 4.0, credited +// on the Weather app's source line and in ACKNOWLEDGEMENTS.md. +const OPEN_METEO_FORECAST_URL = "https://api.open-meteo.com/v1/forecast"; +/** + * Days of hourly forecast to ask Open-Meteo for, from local midnight today: + * enough that MAX_FORECAST_PERIODS periods from any hour end before it does. + */ +const OPEN_METEO_FORECAST_DAYS = 8; +/** Local hours Open-Meteo forecast days and nights start at, as NWS periods do. */ +const DAY_START_HOUR = 6; +const NIGHT_START_HOUR = 18; +/** How long a place-name lookup may hold up an Open-Meteo refresh. */ +const GEOCODE_TIMEOUT_MS = 10_000; +const HOUR_MS = 60 * 60 * 1000; +const DAY_MS = 24 * HOUR_MS; +const WEEKDAYS = ["Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday"]; /** Refresh period while the Weather app is open. */ const APP_REFRESH_MS = 30 * 60 * 1000; /** Refresh period for the status-bar and system-card indicators. */ @@ -127,6 +184,7 @@ const CACHE_VERSION = 1; const DEFAULT_STATE: WeatherState = { phase: "permission-required", status: "Location permission is required for local weather.", + source: null, locationName: "", current: null, forecast: [], @@ -135,6 +193,8 @@ const DEFAULT_STATE: WeatherState = { type WeatherCache = { version: number; + /** Absent from caches written before Open-Meteo, which all came from NWS. */ + source?: WeatherSource; locationName: string; current: CurrentWeather | null; forecast: ForecastPeriod[]; @@ -262,12 +322,13 @@ export class WeatherBridge { this.emit(); const location = await getCurrentLocation(); - this.state = { ...this.state, phase: "loading", status: "Loading National Weather Service data..." }; + this.state = { ...this.state, phase: "loading", status: "Loading weather..." }; this.emit(); - const weather = await loadNwsWeather(location); + const weather = await loadWeather(location); this.state = { phase: "ready", status: "Weather updated.", + source: weather.source, locationName: weather.locationName, current: weather.current, forecast: weather.forecast, @@ -311,6 +372,7 @@ function loadCachedState(): WeatherState { return cloneState({ phase: "ready", status: "Weather updated.", + source: cache.source === "open-meteo" ? "open-meteo" : "nws", locationName: typeof cache.locationName === "string" ? cache.locationName : "", current: cache.current && typeof cache.current === "object" ? cache.current : null, forecast: cache.forecast, @@ -326,6 +388,7 @@ function saveCachedState(state: WeatherState): void { if (state.lastUpdatedMs === null) return; const cache: WeatherCache = { version: CACHE_VERSION, + source: state.source ?? undefined, locationName: state.locationName, current: state.current, forecast: state.forecast, @@ -338,15 +401,23 @@ function saveCachedState(state: WeatherState): void { } } -async function loadNwsWeather(location: CurrentLocation): Promise<{ - locationName: string; - current: CurrentWeather; - forecast: ForecastPeriod[]; -}> { +/** NWS where it has coverage (the US and its territories), Open-Meteo everywhere else. */ +async function loadWeather(location: CurrentLocation): Promise { + return (await loadNwsWeather(location)) ?? loadOpenMeteoWeather(location); +} + +/** NWS weather, or null when the location is outside its coverage. */ +async function loadNwsWeather(location: CurrentLocation): Promise { // NWS recommends no more than four decimals for its point lookup. const latitude = location.latitude.toFixed(4); const longitude = location.longitude.toFixed(4); - const point = await fetchNwsJson(`${NWS_API_ROOT}/points/${latitude},${longitude}`); + let point: NwsPointResponse; + try { + point = await fetchNwsJson(`${NWS_API_ROOT}/points/${latitude},${longitude}`); + } catch (error) { + if (error instanceof WeatherHttpError && error.status === 404) return null; + throw error; + } const properties = point.properties; const forecastUrl = stringValue(properties?.forecast); const hourlyUrl = stringValue(properties?.forecastHourly); @@ -369,6 +440,7 @@ async function loadNwsWeather(location: CurrentLocation): Promise<{ .find((period): period is ForecastPeriod => period !== null); return { + source: "nws", locationName: pointLocationName(point, latitude, longitude), current: observation ? normalizeObservation(observation, hourly) : currentFromHourly(hourly, forecast[0]!), forecast, @@ -382,20 +454,26 @@ async function loadLatestObservation(stationsUrl: string): Promise(`${stationUrl}/observations/latest`); } -async function fetchNwsJson(url: string): Promise { - const request = fetchWithUserAgent(url, { headers: NWS_HEADERS }); +function fetchNwsJson(url: string): Promise { + return fetchWeatherJson(url, "National Weather Service", NWS_HEADERS); +} + +/** A weather request the server answered with an error status. */ +class WeatherHttpError extends Error { + constructor(sourceName: string, readonly status: number) { + super(`${sourceName} request failed (HTTP ${status}).`); + } +} + +async function fetchWeatherJson(url: string, sourceName: string, headers?: Record): Promise { + const request = fetchWithUserAgent(url, headers ? { headers } : undefined); let timeoutHandle: ReturnType | null = null; const timeout = new Promise((_resolve, reject) => { timeoutHandle = setTimeout(() => reject(new Error("Weather request timed out.")), FETCH_TIMEOUT_MS); }); try { const response = await Promise.race([request, timeout]); - if (!response.ok) { - if (response.status === 404 && url.includes("/points/")) { - throw new Error("This location is outside National Weather Service coverage."); - } - throw new Error(`National Weather Service request failed (HTTP ${response.status}).`); - } + if (!response.ok) throw new WeatherHttpError(sourceName, response.status); return (await response.json()) as T; } finally { if (timeoutHandle) clearTimeout(timeoutHandle); @@ -526,6 +604,193 @@ function conditionFromText(text: string): WeatherCondition | null { return null; } +async function loadOpenMeteoWeather(location: CurrentLocation): Promise { + const latitude = location.latitude.toFixed(4); + const longitude = location.longitude.toFixed(4); + const query = [ + `latitude=${latitude}`, + `longitude=${longitude}`, + "current=temperature_2m,relative_humidity_2m,is_day,weather_code,wind_speed_10m,wind_direction_10m", + "hourly=temperature_2m,precipitation_probability,weather_code,wind_speed_10m,wind_direction_10m", + `forecast_days=${OPEN_METEO_FORECAST_DAYS}`, + // Local-time periods need the location's UTC offset, which timezone=auto adds. + "timezone=auto", + "timeformat=unixtime", + "temperature_unit=fahrenheit", + "wind_speed_unit=mph", + ].join("&"); + const [response, placeName] = await Promise.all([ + fetchWeatherJson(`${OPEN_METEO_FORECAST_URL}?${query}`, "Open-Meteo"), + lookUpPlaceName(location), + ]); + const forecast = openMeteoForecastPeriods(response, Date.now()).slice(0, MAX_FORECAST_PERIODS); + if (!forecast.length) throw new Error("Open-Meteo returned an empty forecast."); + return { + source: "open-meteo", + locationName: placeName || `${latitude}, ${longitude}`, + current: normalizeOpenMeteoCurrent(response) ?? currentFromHourly(undefined, forecast[0]!), + forecast, + }; +} + +/** Open-Meteo's current conditions, which come from its forecast models rather than a station. */ +function normalizeOpenMeteoCurrent(response: OpenMeteoResponse): CurrentWeather | null { + const current = response.current; + if (!current || typeof current !== "object") return null; + const wmo = wmoCode(finiteNumber(current.weather_code)); + const windDegrees = finiteNumber(current.wind_direction_10m); + const time = finiteNumber(current.time); + const isDay = finiteNumber(current.is_day); + return { + temperatureF: finiteNumber(current.temperature_2m), + description: wmo?.description ?? "Current conditions", + humidityPercent: finiteNumber(current.relative_humidity_2m), + windSpeedMph: finiteNumber(current.wind_speed_10m), + windDirection: windDegrees === null ? "" : degreesToCompass(windDegrees), + timestampMs: time === null ? null : time * 1000, + observed: false, + condition: wmo?.condition ?? null, + isDaytime: isDay === null ? isLocalDaytime() : isDay === 1, + }; +} + +/** + * NWS-style day (6am-6pm) and night periods from Open-Meteo's hourly + * forecast, in the location's local time: each has its warmest daytime or + * coolest nighttime temperature, its highest chance of rain, and the weather + * from periodWeatherCode. The current period counts only the hours left in + * it. Local time uses today's UTC offset, so periods after a DST change are + * an hour off. + */ +function openMeteoForecastPeriods(response: OpenMeteoResponse, nowMs: number): ForecastPeriod[] { + const hourly = response.hourly; + const times = numberArray(hourly?.time); + const temperatures = numberArray(hourly?.temperature_2m); + const precipitation = numberArray(hourly?.precipitation_probability); + const codes = numberArray(hourly?.weather_code); + const windSpeeds = numberArray(hourly?.wind_speed_10m); + const windDirections = numberArray(hourly?.wind_direction_10m); + const offsetMs = (finiteNumber(response.utc_offset_seconds) ?? 0) * 1000; + const todayIndex = Math.floor((nowMs + offsetMs) / DAY_MS); + + // The hourly indexes of each period still to come, in order. + const groups: Array<{ dayIndex: number; isDaytime: boolean; hours: number[] }> = []; + times.forEach((time, index) => { + if (time === null || time * 1000 + HOUR_MS <= nowMs) return; + const localMs = time * 1000 + offsetMs; + const hour = new Date(localMs).getUTCHours(); + const isDaytime = hour >= DAY_START_HOUR && hour < NIGHT_START_HOUR; + // A night belongs to the date it starts on. + const dayIndex = Math.floor(localMs / DAY_MS) - (hour < DAY_START_HOUR ? 1 : 0); + const last = groups[groups.length - 1]; + if (last && last.dayIndex === dayIndex && last.isDaytime === isDaytime) last.hours.push(index); + else groups.push({ dayIndex, isDaytime, hours: [index] }); + }); + + return groups.map(({ dayIndex, isDaytime, hours }) => { + const during = (values: Array) => hours.map((index) => values[index] ?? null); + const wmo = wmoCode(periodWeatherCode(during(codes))); + const speeds = during(windSpeeds); + const windiest = maxValue(speeds); + const windDegrees = windiest === null ? null : windDirections[hours[speeds.indexOf(windiest)]!] ?? null; + return { + name: openMeteoPeriodName(dayIndex, isDaytime, todayIndex), + startTimeMs: times[hours[0]!]! * 1000, + temperatureF: isDaytime ? maxValue(during(temperatures)) : minValue(during(temperatures)), + shortForecast: wmo?.description ?? "", + detailedForecast: "", + precipitationPercent: maxValue(during(precipitation)), + windSpeed: windiest === null ? "" : `${Math.round(windiest)} mph`, + windDirection: windDegrees === null ? "" : degreesToCompass(windDegrees), + isDaytime, + condition: wmo?.condition ?? null, + }; + }); +} + +/** + * The most severe weather code that lasts at least two hours of a period (or + * its only hour), so one stray hour of drizzle doesn't name it. WMO codes + * rise with severity, as Open-Meteo's own daily summaries assume. + */ +function periodWeatherCode(codes: Array): number | null { + const known = codes.filter((code): code is number => code !== null).sort((a, b) => b - a); + return known.length ? known[Math.min(1, known.length - 1)]! : null; +} + +/** "Today", "Tonight", "Friday", "Friday Night", or "Overnight" for last night's remainder. */ +function openMeteoPeriodName(dayIndex: number, isDaytime: boolean, todayIndex: number): string { + if (dayIndex < todayIndex) return "Overnight"; + if (dayIndex === todayIndex) return isDaytime ? "Today" : "Tonight"; + const weekday = WEEKDAYS[new Date(dayIndex * DAY_MS).getUTCDay()]!; + return isDaytime ? weekday : `${weekday} Night`; +} + +// WMO weather interpretation codes, as Open-Meteo reports them +// (https://open-meteo.com/en/docs#weather_variable_documentation). +const WMO_CODES: Record = { + 0: { description: "Clear", condition: "clear" }, + 1: { description: "Mostly Clear", condition: "partly-cloudy" }, + 2: { description: "Partly Cloudy", condition: "partly-cloudy" }, + 3: { description: "Overcast", condition: "cloudy" }, + 45: { description: "Fog", condition: "fog" }, + 48: { description: "Freezing Fog", condition: "fog" }, + 51: { description: "Light Drizzle", condition: "drizzle" }, + 53: { description: "Drizzle", condition: "drizzle" }, + 55: { description: "Heavy Drizzle", condition: "drizzle" }, + 56: { description: "Light Freezing Drizzle", condition: "sleet" }, + 57: { description: "Freezing Drizzle", condition: "sleet" }, + 61: { description: "Light Rain", condition: "rain" }, + 63: { description: "Rain", condition: "rain" }, + 65: { description: "Heavy Rain", condition: "rain" }, + 66: { description: "Light Freezing Rain", condition: "sleet" }, + 67: { description: "Freezing Rain", condition: "sleet" }, + 71: { description: "Light Snow", condition: "snow" }, + 73: { description: "Snow", condition: "snow" }, + 75: { description: "Heavy Snow", condition: "snow" }, + 77: { description: "Snow Grains", condition: "snow" }, + 80: { description: "Light Rain Showers", condition: "showers" }, + 81: { description: "Rain Showers", condition: "showers" }, + 82: { description: "Heavy Rain Showers", condition: "rain" }, + 85: { description: "Light Snow Showers", condition: "snow" }, + 86: { description: "Heavy Snow Showers", condition: "snow" }, + 95: { description: "Thunderstorms", condition: "thunderstorm" }, + 96: { description: "Thunderstorms and Hail", condition: "thunderstorm" }, + 99: { description: "Thunderstorms and Heavy Hail", condition: "thunderstorm" }, +}; + +function wmoCode(code: number | null): { description: string; condition: WeatherCondition } | undefined { + return code === null ? undefined : WMO_CODES[code]; +} + +/** "Sydney, Australia" from the phone's geocoder, or "" when it can't name the place in time. */ +async function lookUpPlaceName(location: CurrentLocation): Promise { + let timeoutHandle: ReturnType | null = null; + const timeout = new Promise((resolve) => { + timeoutHandle = setTimeout(() => resolve(null), GEOCODE_TIMEOUT_MS); + }); + try { + const place = await Promise.race([ + reverseGeocode(location.latitude, location.longitude).catch((error) => { + console.warn(`weather place lookup failed: ${error}`); + return null; + }), + timeout, + ]); + return placeLabel(place); + } finally { + if (timeoutHandle) clearTimeout(timeoutHandle); + } +} + +/** The city (or failing that, the region) and country. */ +function placeLabel(place: Place | null): string { + if (!place) return ""; + const local = place.locality || place.region; + if (!local || local === place.country) return place.country; + return place.country ? `${local}, ${place.country}` : local; +} + /** Rough day/night by the phone's clock, for when NWS gives no hint. */ function isLocalDaytime(): boolean { const hour = new Date().getHours(); @@ -573,6 +838,20 @@ function finiteNumber(value: unknown): number | null { return Number.isFinite(number) ? number : null; } +function numberArray(value: unknown): Array { + return Array.isArray(value) ? value.map((item) => finiteNumber(item)) : []; +} + +function maxValue(values: Array): number | null { + const numbers = values.filter((value): value is number => value !== null); + return numbers.length ? Math.max(...numbers) : null; +} + +function minValue(values: Array): number | null { + const numbers = values.filter((value): value is number => value !== null); + return numbers.length ? Math.min(...numbers) : null; +} + function timestampValue(value: unknown): number | null { if (typeof value !== "string") return null; const timestamp = Date.parse(value); @@ -586,7 +865,7 @@ function stringValue(value: unknown): string { function friendlyWeatherError(error: unknown): string { const message = (error as Error)?.message || String(error); if (/network request failed|failed to fetch|unable to resolve host/i.test(message)) { - return "Couldn't reach the National Weather Service. Check the phone's connection and retry."; + return "Couldn't reach the weather service. Check the phone's connection and retry."; } return message; } diff --git a/app/native/whisper-server-stt.ts b/app/native/whisper-server-stt.ts new file mode 100644 index 00000000..fe115df0 --- /dev/null +++ b/app/native/whisper-server-stt.ts @@ -0,0 +1,456 @@ +import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt"; + +/** + * Speech-to-text against a self-hosted OpenAI-compatible Whisper server (vLLM, + * speaches, LocalAI): batch POST {baseUrl}/audio/transcriptions with a WAV file. + * Each segment (a push-to-talk utterance, or a Transcribe span ended by a pause) + * is one request, finalized in audio order; an open segment is re-sent every + * INTERIM_STEP_BYTES for provisional text. A segment that runs SEGMENT_MAX_BYTES + * without a pause is cut into requests that still leave as one final. start() + * lists {baseUrl}/models to check reachability and pick a model when none is + * configured. + */ + +export type WhisperServerConfig = { + /** OpenAI-style API root, e.g. http://host:8004/v1. */ + baseUrl: string; + /** Model id to request; empty uses the first one the server lists. */ + model: string; + /** ISO-639-1 language code; empty lets Whisper detect it. */ + language: string; +}; + +export type WhisperHttpRequest = { + url: string; + method: "GET" | "POST"; + headers: Record; + body?: ArrayBuffer; + timeoutMs: number; +}; +export type WhisperHttpResponse = { status: number; text: string }; +export type WhisperHttp = (request: WhisperHttpRequest) => Promise; + +const BYTES_PER_SECOND = CLOUD_STT_SAMPLE_RATE * 2; +// Provisional transcripts: first after one second of audio, then every 1.5s. +const INTERIM_MIN_BYTES = BYTES_PER_SECOND; +const INTERIM_STEP_BYTES = BYTES_PER_SECOND * 1.5; +// Steady speech or background noise can keep the pause detector from ever +// ending a span. Such a segment is cut at this length, inside Whisper's 30s +// window and well within FINAL_TIMEOUT_MS, after its quietest chunk among the +// last SPLIT_SEARCH_BYTES so a word is less likely to straddle the cut. +const SEGMENT_MAX_BYTES = BYTES_PER_SECOND * 25; +const SPLIT_SEARCH_BYTES = BYTES_PER_SECOND * 5; +const MODELS_TIMEOUT_MS = 5000; +const INTERIM_TIMEOUT_MS = 10000; +const FINAL_TIMEOUT_MS = 12000; +const FINAL_RETRY_DELAY_MS = 750; +// Whisper invents text ("Thank you.") for silence, so a segment whose loudest +// 50ms frame stays below this RMS is not sent at all. Normal speech on the G2 +// mic measures in the thousands; the pause detector's noise floor starts at 220. +const SILENCE_RMS = 300; + +/** Time the voice bridges should allow a final transcript before giving up. */ +export const WHISPER_SERVER_FINAL_TIMEOUT_MS = FINAL_TIMEOUT_MS * 2 + FINAL_RETRY_DELAY_MS + 2000; + +type Segment = { + chunks: Uint8Array[]; + /** RMS of each chunk. */ + levels: number[]; + bytes: number; + peakRms: number; + text: string; + final: boolean; + closed: boolean; + paragraphBreakAfter: boolean; + /** Emit a final even when empty: the end of push-to-talk needs one. */ + required: boolean; + /** Cut for length: its final leaves joined with the next segment's. */ + continues: boolean; + interimInFlight: boolean; + interimBytes: number; +}; + +/** + * The API root from what a user typed: tolerates a trailing slash or the full + * transcription endpoint, and adds /v1 to a bare host. Empty when unusable. + */ +export function normalizeWhisperServerUrl(input: string): string { + let url = input.trim().replace(/\/+$/, ""); + if (!/^https?:\/\/[^/\s]+/i.test(url)) return ""; + url = url.replace(/\/audio\/transcriptions$/i, ""); + if (/^https?:\/\/[^/]+$/i.test(url)) url += "/v1"; + return url; +} + +export class WhisperServerSttClient implements CloudSttClient { + private readonly baseUrl: string; + private model: string; + private ready = false; + private closed = false; + private readonly segments: Segment[] = []; + + constructor( + private readonly options: CloudSttOptions, + private readonly config: WhisperServerConfig, + private readonly http: WhisperHttp = nativeHttp, + ) { + this.baseUrl = normalizeWhisperServerUrl(config.baseUrl); + this.model = config.model.trim(); + } + + start(): void { + if (this.closed || this.ready) return; + if (!this.baseUrl) { + this.options.onError("Whisper server URL is not a valid http(s) address."); + return; + } + this.options.onStatus("Connecting to Whisper server..."); + this.http({ url: `${this.baseUrl}/models`, method: "GET", headers: this.authHeaders(), timeoutMs: MODELS_TIMEOUT_MS }) + .then((response) => { + if (this.closed) return; + if (response.status < 200 || response.status >= 300) { + this.options.onDisconnected?.(`Whisper server returned ${response.status} for /models.`); + return; + } + if (!this.model) this.model = firstModelId(response.text); + if (!this.model) { + this.options.onError("Whisper server lists no models; set the model in Settings > Voice."); + return; + } + this.ready = true; + this.options.onStatus("Listening (Whisper server)..."); + this.options.onReady?.(); + }) + .catch((error) => { + if (!this.closed) this.options.onDisconnected?.(`Whisper server unreachable: ${describeError(error)}`); + }); + } + + /** PCM16LE at 16 kHz, appended to the open segment. */ + acceptPcm(pcm: Uint8Array): void { + if (this.closed || !pcm.length) return; + const segment = this.openSegment(); + const level = rms(pcm); + segment.chunks.push(pcm.slice()); + segment.levels.push(level); + segment.bytes += pcm.length; + segment.peakRms = Math.max(segment.peakRms, level); + if (segment.bytes >= SEGMENT_MAX_BYTES) this.splitSegment(segment); + this.maybeRequestInterim(this.segments[this.segments.length - 1]!); + } + + /** A pause in continuous capture: transcribe what came before it. */ + commitSegment(): void { + this.closeSegment(true, false); + } + + /** End of the utterance: transcribe the rest and deliver a final. */ + finish(): void { + this.closeSegment(false, true); + } + + stop(): void { + this.closed = true; + this.segments.length = 0; + } + + private openSegment(): Segment { + const last = this.segments[this.segments.length - 1]; + if (last && !last.closed) return last; + const segment: Segment = { + chunks: [], levels: [], bytes: 0, peakRms: 0, text: "", final: false, closed: false, + paragraphBreakAfter: false, required: false, continues: false, interimInFlight: false, interimBytes: 0, + }; + this.segments.push(segment); + return segment; + } + + /** + * Close the open segment at SEGMENT_MAX_BYTES and carry the audio after its + * cut into a new one. The cut is after the quietest chunk among the last + * SPLIT_SEARCH_BYTES; the first chunk always stays. + */ + private splitSegment(segment: Segment): void { + let cut = segment.chunks.length - 1; + let quietest = Infinity; + let carried = 0; + for (let i = segment.chunks.length - 1; i > 0 && carried <= SPLIT_SEARCH_BYTES; i--) { + if (segment.levels[i]! < quietest) { + quietest = segment.levels[i]!; + cut = i; + } + carried += segment.chunks[i]!.length; + } + const chunks = segment.chunks.splice(cut + 1); + const levels = segment.levels.splice(cut + 1); + for (const chunk of chunks) segment.bytes -= chunk.length; + segment.peakRms = peak(segment.levels); + segment.continues = true; + this.closeSegment(false, false); + if (!chunks.length) return; + const next = this.openSegment(); + next.chunks = chunks; + next.levels = levels; + for (const chunk of chunks) next.bytes += chunk.length; + next.peakRms = peak(levels); + } + + private closeSegment(paragraphBreakAfter: boolean, required: boolean): void { + if (this.closed) return; + const last = this.segments[this.segments.length - 1]; + if (last?.closed && last.continues) { + // Cut for length with nothing after it yet: this boundary ends its run. + last.continues = false; + last.paragraphBreakAfter = paragraphBreakAfter; + last.required = required; + if (last.final) this.emit(); + return; + } + if (!last || last.closed) { + // Nothing new since the last boundary. Push-to-talk still needs a final. + if (required) this.finalize(this.openSegment(), "", paragraphBreakAfter, required); + return; + } + if (last.peakRms < SILENCE_RMS) { + this.finalize(last, "", paragraphBreakAfter, required); + return; + } + last.closed = true; + last.paragraphBreakAfter = paragraphBreakAfter; + last.required = required; + void this.requestFinal(last, 0); + } + + private async requestFinal(segment: Segment, attempt: number): Promise { + try { + const text = await this.transcribe(segment, segment.bytes, FINAL_TIMEOUT_MS); + if (!this.closed) this.finalize(segment, text, segment.paragraphBreakAfter, segment.required); + } catch (error) { + if (this.closed) return; + if (error instanceof WhisperAuthError) { + this.options.onError(error.message); + return; + } + if (attempt === 0) { + setTimeout(() => { void this.requestFinal(segment, 1); }, FINAL_RETRY_DELAY_MS); + return; + } + // Keep the capture alive and keep whatever the interim pass heard. + this.options.onStatus(`Whisper server request failed: ${describeError(error)}`); + this.finalize(segment, segment.text, segment.paragraphBreakAfter, segment.required); + } + } + + private maybeRequestInterim(segment: Segment): void { + if (!this.ready || segment.interimInFlight || segment.closed) return; + if (segment.bytes < INTERIM_MIN_BYTES) return; + if (segment.bytes - segment.interimBytes < INTERIM_STEP_BYTES) return; + if (segment.peakRms < SILENCE_RMS) return; + segment.interimInFlight = true; + segment.interimBytes = segment.bytes; + this.transcribe(segment, segment.bytes, INTERIM_TIMEOUT_MS) + .then((text) => { + if (this.closed || segment.final) return; + segment.text = text; + this.emit(); + }) + .catch(() => { + // Provisional only; the final request reports real failures. + }) + .finally(() => { + segment.interimInFlight = false; + }); + } + + private finalize(segment: Segment, text: string, paragraphBreakAfter: boolean, required: boolean): void { + segment.closed = segment.final = true; + segment.text = text; + segment.paragraphBreakAfter = paragraphBreakAfter; + segment.required = required; + segment.chunks = []; + this.emit(); + } + + /** + * Finals leave in audio order, even when a later request answers first, and + * segments cut for length leave as one final with the segment that ends + * their run; whatever is not final yet is shown as one provisional transcript. + */ + private emit(): void { + for (;;) { + let end = 0; + while (this.segments[end]?.final && this.segments[end]!.continues) end++; + if (!this.segments[end]?.final) break; + const run = this.segments.splice(0, end + 1); + const last = run[run.length - 1]!; + const text = run.map((segment) => segment.text.trim()).filter(Boolean).join(" "); + if (!text && !last.required) continue; + this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: last.paragraphBreakAfter }); + } + const pending = this.segments.filter((segment) => segment.text.trim()); + if (!pending.length) return; + const text = pending.map((segment) => segment.text.trim()).join(" "); + const transcribeText = pending.map((segment, index) => + (index && pending[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + segment.text.trim(), + ).join(""); + this.options.onTranscript({ text, transcribeText, isFinal: false }); + } + + private async transcribe(segment: Segment, bytes: number, timeoutMs: number): Promise { + const fields: Record = { model: this.model, response_format: "json", temperature: "0" }; + const language = this.config.language.trim().toLowerCase(); + if (language) fields.language = language; + const boundary = `faceclaw-${Date.now().toString(36)}-${Math.random().toString(36).slice(2)}`; + const body = multipartBody(boundary, fields, wavFile(segment.chunks, bytes)); + const response = await this.http({ + url: `${this.baseUrl}/audio/transcriptions`, + method: "POST", + headers: { ...this.authHeaders(), "Content-Type": `multipart/form-data; boundary=${boundary}` }, + body, + timeoutMs, + }); + if (response.status === 401 || response.status === 403) { + throw new WhisperAuthError(`Whisper server rejected the key (${response.status}).`); + } + if (response.status < 200 || response.status >= 300) { + throw new Error(`HTTP ${response.status}: ${response.text.slice(0, 200)}`); + } + return parseTranscript(response.text); + } + + private authHeaders(): Record { + const key = this.options.apiKey.trim(); + return key ? { Authorization: `Bearer ${key}` } : {}; + } +} + +class WhisperAuthError extends Error {} + +/** The transcript from a JSON response; tolerates servers that send plain text. */ +export function parseTranscript(body: string): string { + const raw = body.trim(); + try { + const parsed = JSON.parse(raw); + if (parsed && typeof parsed.text === "string") return parsed.text.trim(); + } catch { + // Not JSON: a server that answered response_format=text anyway. + } + return raw; +} + +function firstModelId(body: string): string { + try { + const parsed = JSON.parse(body); + const id = parsed?.data?.[0]?.id; + return typeof id === "string" ? id : ""; + } catch { + return ""; + } +} + +function peak(levels: number[]): number { + let max = 0; + for (const level of levels) max = Math.max(max, level); + return max; +} + +/** RMS of a PCM16LE chunk. */ +function rms(pcm: Uint8Array): number { + const count = pcm.length >> 1; + if (!count) return 0; + let squares = 0; + for (let i = 0; i < count; i++) { + const unsigned = pcm[i * 2]! | (pcm[i * 2 + 1]! << 8); + const sample = unsigned >= 32768 ? unsigned - 65536 : unsigned; + squares += sample * sample; + } + return Math.sqrt(squares / count); +} + +/** The first `bytes` of the chunks as a 16 kHz mono 16-bit WAV file. */ +export function wavFile(chunks: Uint8Array[], bytes: number): Uint8Array { + const length = bytes & ~1; + const out = new Uint8Array(44 + length); + const view = new DataView(out.buffer); + const ascii = (offset: number, text: string) => { + for (let i = 0; i < text.length; i++) out[offset + i] = text.charCodeAt(i); + }; + ascii(0, "RIFF"); + view.setUint32(4, 36 + length, true); + ascii(8, "WAVE"); + ascii(12, "fmt "); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // mono + view.setUint32(24, CLOUD_STT_SAMPLE_RATE, true); + view.setUint32(28, BYTES_PER_SECOND, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + ascii(36, "data"); + view.setUint32(40, length, true); + let offset = 44; + for (const chunk of chunks) { + const take = Math.min(chunk.length, out.length - offset); + if (take <= 0) break; + out.set(chunk.subarray(0, take), offset); + offset += take; + } + return out; +} + +/** multipart/form-data with text fields and one WAV part named "file". */ +export function multipartBody(boundary: string, fields: Record, wav: Uint8Array): ArrayBuffer { + let head = ""; + for (const [name, value] of Object.entries(fields)) { + head += `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\n\r\n${value}\r\n`; + } + head += `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="audio.wav"\r\nContent-Type: audio/wav\r\n\r\n`; + const headBytes = utf8(head); + const tailBytes = utf8(`\r\n--${boundary}--\r\n`); + const out = new Uint8Array(headBytes.length + wav.length + tailBytes.length); + out.set(headBytes, 0); + out.set(wav, headBytes.length); + out.set(tailBytes, headBytes.length + wav.length); + return out.buffer; +} + +function utf8(text: string): Uint8Array { + const out: number[] = []; + for (const char of text) { + const code = char.codePointAt(0)!; + if (code < 0x80) out.push(code); + else if (code < 0x800) out.push(0xc0 | (code >> 6), 0x80 | (code & 63)); + else if (code < 0x10000) out.push(0xe0 | (code >> 12), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63)); + else out.push(0xf0 | (code >> 18), 0x80 | ((code >> 12) & 63), 0x80 | ((code >> 6) & 63), 0x80 | (code & 63)); + } + return Uint8Array.from(out); +} + +function describeError(error: unknown): string { + return String((error as Error)?.message ?? error); +} + +/** + * Uploads go through postBinary (on Android NativeScript's HTTP module would + * copy each body into Java a byte at a time); the model list through + * NativeScript's HTTP module. + */ +const nativeHttp: WhisperHttp = async (request) => { + // Required lazily so Node tests can load this module without NativeScript. + if (request.body) { + const { postBinary } = require("./binary-http") as typeof import("./binary-http"); + return postBinary(request.url, request.body, request.headers, request.timeoutMs); + } + const { withUserAgent } = require("../util/http") as typeof import("../util/http"); + const { request: send } = require("@nativescript/core/http") as typeof import("@nativescript/core/http"); + const response = await send({ + url: request.url, + method: request.method, + headers: withUserAgent(request.headers), + timeout: request.timeoutMs, + }); + const content = response.content as any; + let text = ""; + if (content) text = global.isAndroid ? String(content.raw.toString("UTF-8")) : String(content.toString()); + return { status: response.statusCode, text }; +}; diff --git a/app/phone-ui/main-page.ios.ts b/app/phone-ui/main-page.ios.ts index 00c15893..5ce75e5d 100644 --- a/app/phone-ui/main-page.ios.ts +++ b/app/phone-ui/main-page.ios.ts @@ -20,26 +20,37 @@ export function createMainPage(): Page { root.rows = 'auto,*' page.content = root const header = new GridLayout() - header.columns = '*,auto,auto,auto'; header.className = 'main-header' + header.columns = '*,auto,auto,auto,auto'; header.className = 'main-header' const title = new Label() title.text = 'Faceclaw'; title.className = 'action-bar-title' header.addChild(title) + // The foreground EvenHub app's icon, which opens its phone UI. UIImageView + // ignores padding, so a wrapper supplies Android's wider tap target. + const appIconButton = new GridLayout() + appIconButton.className = 'action-bar-app-icon'; appIconButton.verticalAlignment = 'middle' + appIconButton.visibility = 'collapse'; appIconButton.accessible = true + const appIcon = new Image() + appIcon.width = 21; appIcon.height = 21 + appIconButton.addChild(appIcon) + let phoneUiWindowId: string | null = null + appIconButton.on('tap', () => { if (phoneUiWindowId) controller.showEvenHubPhoneUi(phoneUiWindowId) }) + GridLayout.setColumn(appIconButton, 1); header.addChild(appIconButton) const state = new Label() state.text = 'Preview'; state.className = 'action-bar-status' state.verticalAlignment = 'middle' state.on('tap', () => { void openDevices() }) - GridLayout.setColumn(state, 1); header.addChild(state) + GridLayout.setColumn(state, 2); header.addChild(state) // Android's warning triangle: shown while the glasses report firmware Faceclaw can't run. const warning = new Label() warning.text = '\u26A0\uFE0F'; warning.className = 'action-bar-warning' warning.accessibilityLabel = 'Attention needed'; warning.verticalAlignment = 'middle' warning.visibility = 'collapse' warning.on('tap', () => { void showFirmwareWarning() }) - GridLayout.setColumn(warning, 2); header.addChild(warning) + GridLayout.setColumn(warning, 3); header.addChild(warning) const overflow = new Button() overflow.text = '⋮'; overflow.accessibilityLabel = 'More options'; overflow.className = 'overflow-button' overflow.on('tap', () => openMenu()) - GridLayout.setColumn(overflow, 3); header.addChild(overflow); root.addChild(header) + GridLayout.setColumn(overflow, 4); header.addChild(overflow); root.addChild(header) const body = new GridLayout() GridLayout.setRow(body, 1); root.addChild(body) const mirror = new Image() @@ -76,6 +87,11 @@ export function createMainPage(): Page { if (session) { focusTimer = setTimeout(() => { focusTimer = null; keyboardField()?.focus() }, 0) } else keyboardField()?.dismissSoftInput() + }, button => { + phoneUiWindowId = button?.icon ? button.windowId : null + appIcon.imageSource = button?.icon ?? null + appIconButton.visibility = phoneUiWindowId ? 'visible' : 'collapse' + appIconButton.accessibilityLabel = button ? `Open ${button.title} on phone` : '' }) async function navigateDevicePage(moduleName: string, context?: object): Promise { suppressAutoReconnect() diff --git a/app/phone-ui/main-view-model.ts b/app/phone-ui/main-view-model.ts index a413f432..8f4a3a53 100644 --- a/app/phone-ui/main-view-model.ts +++ b/app/phone-ui/main-view-model.ts @@ -1127,8 +1127,14 @@ export class MainViewModel extends RemoteControlsViewModel { return dashboardController.glassesDisplayLabel(); } + protected override get padDisplayAsleep(): boolean { + return dashboardController.glassesDisplayAsleep(); + } + private refreshPadFocusLine(): void { this.notifyPropertyChange("padFocusLine", this.padFocusLine); + this.notifyPropertyChange("padFocusClass", this.padFocusClass); + this.notifyPropertyChange("padWakeHintVisibility", this.padWakeHintVisibility); } async onPadTap(): Promise { diff --git a/app/phone-ui/onboarding-state.ts b/app/phone-ui/onboarding-state.ts index 451a5ef4..9f478373 100644 --- a/app/phone-ui/onboarding-state.ts +++ b/app/phone-ui/onboarding-state.ts @@ -2,32 +2,17 @@ import { ApplicationSettings } from "@nativescript/core"; const ONBOARDING_COMPLETE_KEY = "onboarding.complete"; const PREVIEW_ONLY_KEY = "onboarding.previewOnly"; -const WELCOME_SOUND_PENDING_KEY = "onboarding.welcomeSoundPending"; export function hasCompletedOnboarding(): boolean { return ApplicationSettings.getBoolean(ONBOARDING_COMPLETE_KEY, false); } -export function setOnboardingCompleted(completed: boolean): void { - const wasCompleted = hasCompletedOnboarding(); - ApplicationSettings.setBoolean(ONBOARDING_COMPLETE_KEY, completed); - // Arm the one-time welcome sound on the first-ever completion (not on the - // idempotent re-completions that happen when installing from the main menu). - if (completed && !wasCompleted) { - setWelcomeSoundPending(true); - } -} - /** - * True when onboarding just completed and the celebratory sound hasn't played - * yet. Consumed (and cleared) on the first successful glasses connection. + * Phone-side onboarding. The on-glasses onboarding that follows it on the + * first connection is tracked separately (app/apps/onboarding/). */ -export function isWelcomeSoundPending(): boolean { - return ApplicationSettings.getBoolean(WELCOME_SOUND_PENDING_KEY, false); -} - -export function setWelcomeSoundPending(pending: boolean): void { - ApplicationSettings.setBoolean(WELCOME_SOUND_PENDING_KEY, pending); +export function setOnboardingCompleted(completed: boolean): void { + ApplicationSettings.setBoolean(ONBOARDING_COMPLETE_KEY, completed); } /** diff --git a/app/phone-ui/onboarding-view-model.ts b/app/phone-ui/onboarding-view-model.ts index 8774b35c..897c2ffc 100644 --- a/app/phone-ui/onboarding-view-model.ts +++ b/app/phone-ui/onboarding-view-model.ts @@ -1,5 +1,6 @@ import { finishOnboardingNavigation } from "./onboarding-navigation"; import { Frame, Observable, Screen } from "@nativescript/core"; +import { IOS_PRIVACY_NOTICE } from "../analytics/privacy-disclosure"; import { setOnboardingCompleted, setPreviewOnlyMode } from "./onboarding-state"; @@ -29,7 +30,8 @@ const STEP_CONTENT: Record = { 2: { headline: "Before You Continue", tagline: "", - body: "This software provides a custom user interface and functionality for the Even Realities G2 smart glasses. It is not created or supported by Even Realities. If this software somehow breaks your headset, this is not Even's fault and is not covered by the hardware's warranty. If this software doesn't break you headset, using this software may void the hardware's warranty anyways, at the sole discretion of Even Realities. This software is beta; don't rely on it for anything critically important.", + body: "This software provides a custom user interface and functionality for the Even Realities G2 smart glasses. It is not created or supported by Even Realities. If this software somehow breaks your headset, this is not Even's fault and is not covered by the hardware's warranty. If this software doesn't break you headset, using this software may void the hardware's warranty anyways, at the sole discretion of Even Realities. This software is beta; don't rely on it for anything critically important." + + (global.isIOS ? "\n\niOS / TestFlight privacy\n\n" + IOS_PRIVACY_NOTICE : ""), primaryLabel: "Agree", secondaryLabel: "Back", splash: false, diff --git a/app/phone-ui/pairing-page.xml b/app/phone-ui/pairing-page.xml index b1704d29..60af9a44 100644 --- a/app/phone-ui/pairing-page.xml +++ b/app/phone-ui/pairing-page.xml @@ -82,8 +82,8 @@