Merge branch 'main' into ring-health-fixes

This commit is contained in:
James Babcock committed 2026-10-25 01:06:38 -07:00
commit 4f8c082b52
368 files changed
+26238 -3401

No files matched your search

+23 -11
View File
@@ -46,6 +46,17 @@ def faceclawNdkHostTag = {
return "linux-x86_64"
}
// Host tools carry .exe on Windows. Call the binaries themselves: the NDK's
// target-prefixed clang wrappers are bash scripts (or .cmd files) that only add
// --target.
def faceclawHostExecutable = { File dir, String name ->
return file("$dir/$name${Os.isFamily(Os.FAMILY_WINDOWS) ? ".exe" : ""}")
}
def faceclawNdkTool = { File ndkRoot, String name ->
return faceclawHostExecutable(file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin"), name)
}
def faceclawFindNdkRoot = {
def explicitNdk = System.getenv("ANDROID_NDK_HOME")
if (explicitNdk != null && file(explicitNdk).exists()) {
@@ -111,7 +122,7 @@ tasks.register("prepareFaceclawNativeLibs") {
if (ndkRoot == null || !ndkRoot.exists()) {
throw new GradleException("Android NDK is required to build libfaceclaw_lc3.so")
}
def clang = file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/aarch64-linux-android24-clang")
def clang = faceclawNdkTool(ndkRoot, "clang")
if (!clang.exists()) {
throw new GradleException("Could not find Android clang at $clang")
}
@@ -125,6 +136,7 @@ tasks.register("prepareFaceclawNativeLibs") {
exec {
commandLine([
clang.absolutePath,
"--target=aarch64-linux-android24",
"-shared",
"-fPIC",
"-O3",
@@ -192,8 +204,8 @@ tasks.register("prepareFaceclawLlama") {
if (cmakeDir == null) {
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
}
def cmake = file("$cmakeDir/bin/cmake")
def ninja = file("$cmakeDir/bin/ninja")
def cmake = faceclawHostExecutable(file("$cmakeDir/bin"), "cmake")
def ninja = faceclawHostExecutable(file("$cmakeDir/bin"), "ninja")
def march = "-march=armv8.2-a+dotprod+fp16"
// Configure once; afterwards the ninja no-op build keeps this cheap while
@@ -227,14 +239,13 @@ tasks.register("prepareFaceclawLlama") {
}
// The static-linked lib carries llama.cpp's full debug info (~50MB);
// strip it so the APK only pays for the code (~8MB).
def ndkRootForStrip = faceclawFindNdkRoot()
if (ndkRootForStrip != null) {
def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip")
if (llvmStrip.exists()) {
exec {
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
}
def llvmStrip = faceclawNdkTool(ndkRoot, "llvm-strip")
if (llvmStrip.exists()) {
exec {
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
}
} else {
logger.warn("Could not find llvm-strip at $llvmStrip; packaging an unstripped libfaceclaw_llama.so (~50MB)")
}
}
}
@@ -251,7 +262,8 @@ if (faceclawVersionFile.exists()) {
}
}
def faceclawVersionCode = {
def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger()
// Prerelease labels belong in versionName, not the numeric versionCode.
def parts = (faceclawVersionName.split('-')[0].tokenize('.') + ["0", "0"]).take(3)*.toInteger()
return parts[0] * 10000 + parts[1] * 100 + parts[2]
}()
@@ -13,7 +13,8 @@ import java.io.IOException
/**
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
* container, cutting speech recordings to roughly a tenth of their WAV size.
* Runs on its own thread; the source WAV is deleted on success when requested.
* Runs on its own thread; when requested, the source WAV is deleted once the
* listener's onDone has returned normally.
*/
class FaceclawAudioTranscoder(
private val wavPath: String?,
@@ -66,19 +67,38 @@ class FaceclawAudioTranscoder(
private fun run() {
try {
transcode()
val size = File(outputPath).length()
if (deleteSourceOnSuccess) {
File(wavPath).delete()
}
if (listener != null) {
listener.onDone(outputPath, size)
}
} catch (t: Throwable) {
Log.w(TAG, "transcode failed for $wavPath", t)
File(outputPath).delete()
if (listener != null) {
listener.onError(if (t.message == null) t.toString() else t.message)
try {
listener?.onError(if (t.message == null) t.toString() else t.message)
} catch (listenerError: Throwable) {
Log.w(TAG, "transcode error listener failed", listenerError)
}
return
}
val size = File(outputPath).length()
if (size <= 0) {
Log.w(TAG, "transcode produced no output for $wavPath")
File(outputPath).delete()
try {
listener?.onError("transcode produced no output")
} catch (listenerError: Throwable) {
Log.w(TAG, "transcode error listener failed", listenerError)
}
return
}
try {
listener?.onDone(outputPath, size)
} catch (t: Throwable) {
// The caller never took up the new file, so whatever it recorded still names the
// source: keep that and drop the output, rather than lose both.
Log.w(TAG, "transcode listener failed for $wavPath", t)
File(outputPath).delete()
return
}
if (deleteSourceOnSuccess) {
File(wavPath).delete()
}
}
@@ -93,6 +113,7 @@ class FaceclawAudioTranscoder(
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
var muxer: MediaMuxer? = null
var muxerStopped = false
var input: FileInputStream? = null
try {
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
@@ -159,6 +180,11 @@ class FaceclawAudioTranscoder(
}
}
}
// stop() writes the index (moov); without it the file won't play, so its failure
// (storage full, I/O error, a muxer that never started) fails the transcode rather
// than reporting a broken output as done.
muxerStopped = true
muxer.stop()
} finally {
if (input != null) {
try {
@@ -172,9 +198,12 @@ class FaceclawAudioTranscoder(
}
codec.release()
if (muxer != null) {
try {
muxer.stop()
} catch (ignored: Throwable) {
if (!muxerStopped) {
// Already failing; the output is deleted either way.
try {
muxer.stop()
} catch (ignored: Throwable) {
}
}
muxer.release()
}
@@ -0,0 +1,103 @@
package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import java.io.IOException
import java.nio.ByteBuffer
import java.util.concurrent.TimeUnit
import okhttp3.Call
import okhttp3.Callback
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
/**
* One POST of a binary body (a Whisper server WAV upload), sent by start().
* NativeScript's http module hands an ArrayBuffer body to Java one bridge call
* per byte, which for a ~1 MB upload stalls the JS thread; here the body
* arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk.
* headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and
* may carry the Content-Type. The response text and status reach the listener
* on the constructing thread's Looper.
*/
class FaceclawBinaryPost(
url: String?,
body: ByteBuffer?,
headers: Array<String?>?,
timeoutMs: Long,
listener: FaceclawBinaryPostListener?
) {
private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
private val listener: FaceclawBinaryPostListener =
listener ?: throw IllegalArgumentException("listener is required")
private val call: Call
@Volatile
private var cancelled = false
init {
if (url == null || url.trim().isEmpty()) {
throw IllegalArgumentException("url is required")
}
val bytes = ByteArray(body?.remaining() ?: 0)
body?.duplicate()?.get(bytes)
// No media type on the body, so okhttp keeps the caller's Content-Type header.
val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody())
for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value)
call = getClient().newCall(builder.build())
call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS)
}
fun start() {
call.enqueue(object : Callback {
override fun onFailure(call: Call, e: IOException) {
post { listener.onFailure(e.message ?: e.toString()) }
}
override fun onResponse(call: Call, response: Response) {
val text = try {
response.use { it.body?.string() ?: "" }
} catch (e: IOException) {
post { listener.onFailure(e.message ?: e.toString()) }
return
}
post { listener.onResponse(response.code, text) }
}
})
}
/** Aborts the request; no callback follows. */
fun cancel() {
cancelled = true
call.cancel()
}
private fun post(runnable: () -> Unit) {
callbackHandler.post {
if (!cancelled) runnable()
}
}
companion object {
@Volatile
private var sharedClient: OkHttpClient? = null
private fun getClient(): OkHttpClient {
if (sharedClient == null) {
synchronized(FaceclawBinaryPost::class.java) {
if (sharedClient == null) {
sharedClient = OkHttpClient.Builder()
.connectTimeout(30, TimeUnit.SECONDS)
.readTimeout(60, TimeUnit.SECONDS)
.addInterceptor(FaceclawHttp.userAgentInterceptor())
.build()
}
}
}
return sharedClient!!
}
}
}
@@ -294,7 +294,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener)
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) =
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
@@ -329,6 +329,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
}
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
val composite = core.previewComposite() ?: return null
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
cropX, cropY, cropWidth, cropHeight)
}
/**
* Posted to the main looper after each step of an animation the preview is replaying (menu
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
@@ -550,6 +550,7 @@ class FaceclawBleManager(context: Context) {
override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) {
val address = gatt.device.address
Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState")
if (status != BluetoothGatt.GATT_SUCCESS) ConnectionCounters.increment("ble.gatt-status.$status")
val link = links[address]
val current = link?.gatt
if (link == null || (current != null && current !== gatt)) {
@@ -68,6 +68,11 @@ class FaceclawCaptionEngine {
session.stop()
}
/** Stop without blocking; the listener's onStopped follows the trailing utterance. */
fun requestStop() {
session.requestStop()
}
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
fun acceptPcm(pcm16le: ByteArray?) {
session.acceptPcm(pcm16le)
@@ -718,6 +718,94 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
}
}
// ---- crash recovery ----
/**
* Close out sessions a dead process left open (ended_at NULL): the end becomes the last
* stored segment's end (or the start), and the tone average comes from the stored segments.
* excludeIdsJson is a JSON array of session ids that are legitimately still open. Returns
* the sessions it closed as a JSON array of {id, audioPath, audioCodec}.
*/
fun closeInterruptedSessions(excludeIdsJson: String?): String {
val exclude = HashSet<Long>()
if (excludeIdsJson != null && !excludeIdsJson.isEmpty()) {
val ids = JSONArray(excludeIdsJson)
for (i in 0 until ids.length()) {
exclude.add(ids.getLong(i))
}
}
val db = writableDatabase
val closed = JSONArray()
val closedIds = ArrayList<Long>()
db.rawQuery("SELECT id, audio_path, audio_codec FROM sessions WHERE ended_at IS NULL", null).use { c ->
while (c.moveToNext()) {
val id = c.getLong(0)
if (id in exclude) continue
closedIds.add(id)
val row = JSONObject()
row.put("id", id)
row.put("audioPath", if (c.isNull(1)) JSONObject.NULL else c.getString(1))
row.put("audioCodec", if (c.isNull(2)) JSONObject.NULL else c.getString(2))
closed.put(row)
}
}
for (id in closedIds) {
db.execSQL(
"UPDATE sessions SET"
+ " ended_at = COALESCE((SELECT MAX(ended_at) FROM segments WHERE session_id = ?), started_at),"
+ " avg_sentiment = COALESCE((SELECT AVG(sentiment) FROM segments WHERE session_id = ?), 0)"
+ " WHERE id = ?",
arrayOf<Any?>(id, id, id))
}
return closed.toString()
}
/**
* Closed sessions whose audio is still the WAV: a transcode that never finished (the process
* died mid-transcode, or recovery failed partway on an earlier boot). Returns a JSON array of
* {id, audioPath}.
*/
fun listUntranscodedSessions(): String {
val sessions = JSONArray()
readableDatabase.rawQuery(
"SELECT id, audio_path FROM sessions"
+ " WHERE ended_at IS NOT NULL AND audio_codec = 'wav' AND audio_path IS NOT NULL",
null,
).use { c ->
while (c.moveToNext()) {
val row = JSONObject()
row.put("id", c.getLong(0))
row.put("audioPath", c.getString(1))
sessions.put(row)
}
}
return sessions.toString()
}
/**
* Delete recordings in dirPath that no session's audio_path names: files a session row never
* learned about, or partial transcoder output. Matched by file name, so a storage move that
* changed the directory's absolute path can't make live recordings look orphaned. Only call
* while no recording or transcode is in flight. Returns how many files were deleted.
*/
fun deleteUnreferencedRecordings(dirPath: String?): Int {
if (dirPath == null) return 0
val files = File(dirPath).listFiles() ?: return 0
val referenced = HashSet<String>()
readableDatabase.rawQuery("SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL", null).use { c ->
while (c.moveToNext()) {
referenced.add(File(c.getString(0)).name)
}
}
var deleted = 0
for (file in files) {
if (file.isFile && file.name !in referenced && file.delete()) {
deleted++
}
}
return deleted
}
fun deleteSession(sessionId: Long) {
val db = writableDatabase
val path = getSessionAudioPath(sessionId)
@@ -1,32 +1,100 @@
package com.faceclaw.app
import android.net.Uri
import android.os.Handler
import android.os.Looper
import android.webkit.JavascriptInterface
import android.util.Log
import android.webkit.WebView
import androidx.webkit.JavaScriptReplyProxy
import androidx.webkit.WebMessageCompat
import androidx.webkit.WebViewCompat
import androidx.webkit.WebViewFeature
import org.json.JSONArray
import org.json.JSONException
/**
* The object injected into a hosted EvenHub app's WebView as
* window.__faceclawEvenHub. The document-start shim (served inline by
* FaceclawEvenHubWebViewClient) wraps it in a promise-returning
* window.flutter_inappwebview.callHandler, which is what the EvenHub SDK
* actually talks to.
* The host end of a hosted EvenHub app's bridge: window.__faceclawEvenHubPort,
* a WebMessageListener object that WebView injects only into frames from the
* app's own origin. Sites the app navigates to, and other-origin iframes it
* embeds, never see it, and messages from the app's own subframes are dropped,
* so only the app's main document reaches the host (its SDK storage, sensors,
* glasses display and the Faceclaw extensions, API keys included). This
* replaced an addJavascriptInterface object, which WebView exposes to every
* frame of every page.
*
* JavascriptInterface methods run on a WebView-internal thread; everything
* is bounced to the main thread before touching the NativeScript listener.
* The transport script (webview.ts) wraps the port as window.__faceclawEvenHub,
* whose postMessage(name, argsJson, id) and wakeTimers() the stock bridge shim
* and the extensions call. Messages are JSON arrays: ["call", name, argsJson,
* id] or ["wake"].
*/
class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) {
class FaceclawEvenHubJsBridge private constructor(
private val listener: FaceclawEvenHubListener
) : WebViewCompat.WebMessageListener {
private val mainHandler = Handler(Looper.getMainLooper())
@JavascriptInterface
fun postMessage(handlerName: String?, argsJson: String?, callId: Int) {
mainHandler.post {
listener.onEvenAppMessage(handlerName, argsJson, callId)
override fun onPostMessage(
view: WebView,
message: WebMessageCompat,
sourceOrigin: Uri,
isMainFrame: Boolean,
replyProxy: JavaScriptReplyProxy
) {
if (!isMainFrame) return
val parts = try {
JSONArray(message.data ?: return)
} catch (e: JSONException) {
return
}
when (parts.optString(0)) {
"call" -> {
val name = parts.optString(1)
val argsJson = parts.optString(2)
val callId = parts.optInt(3)
mainHandler.post { listener.onEvenAppMessage(name, argsJson, callId) }
}
// The page's timer shim has work due before the host's next tick.
"wake" -> FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
}
}
/** The page's timer shim has work due before the host's next tick. */
@JavascriptInterface
fun wakeTimers() {
FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
companion object {
private const val TAG = "FaceclawEvenHub"
private const val OBJECT_NAME = "__faceclawEvenHubPort"
/**
* The origin rule matching exactly [url]'s origin (default port elided),
* or null when it isn't an http(s) URL with a host.
*/
@JvmStatic
fun originRule(url: String): String? {
val uri = Uri.parse(url)
val scheme = uri.scheme?.lowercase()
val host = uri.host?.lowercase()
if ((scheme != "http" && scheme != "https") || host.isNullOrEmpty()) return null
val defaultPort = if (scheme == "https") 443 else 80
return if (uri.port == -1 || uri.port == defaultPort) "$scheme://$host" else "$scheme://$host:${uri.port}"
}
/**
* Expose the bridge to [webView]'s frames from [originRule]. Must run
* before the WebView loads anything.
*
* @return null on success, else why the bridge could not be installed.
*/
@JvmStatic
fun install(webView: WebView, originRule: String, listener: FaceclawEvenHubListener): String? {
if (!WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
Log.w(TAG, "WebMessageListener unsupported; not exposing the EvenHub bridge")
return "This Android System WebView is too old to run EvenHub apps safely. Update it, then reopen the app."
}
return try {
WebViewCompat.addWebMessageListener(webView, OBJECT_NAME, setOf(originRule), FaceclawEvenHubJsBridge(listener))
null
} catch (e: IllegalArgumentException) {
Log.w(TAG, "bridge origin rule rejected: $originRule", e)
"The app's address ($originRule) can't be used as a web origin."
}
}
}
}
@@ -0,0 +1,54 @@
package com.faceclaw.app
import android.util.Log
import android.webkit.WebView
import androidx.webkit.ProfileStore
import androidx.webkit.WebViewCompat
import androidx.webkit.WebViewFeature
import java.security.MessageDigest
/**
* Gives each EvenHub package its own WebView profile, so one app's browser
* storage (cookies, localStorage, IndexedDB, caches, service workers) is out of
* reach of every other app. That includes the storage of third-party sites an
* app navigates to, such as a sign-in provider's session cookies, which in the
* shared default profile any other app's WebView could reuse. The per-app fake
* origins alone don't cover that, and `*.evenhub.invalid` hosts share a site,
* so domain cookies would cross between packaged apps too.
*
* The profile persists across launches, and is shared by every instance of the
* same package, like the iOS per-package WKWebsiteDataStore.
*/
class FaceclawEvenHubProfile private constructor() {
companion object {
private const val TAG = "FaceclawEvenHub"
/** Stable per package; the hash input matches the iOS data-store identifier's. */
@JvmStatic
fun nameFor(packageId: String): String {
val digest = MessageDigest.getInstance("SHA-256")
.digest("com.faceclaw.ehpk:$packageId".toByteArray(Charsets.UTF_8))
return "evenhub-" + digest.take(16).joinToString("") { "%02x".format(it) }
}
/**
* Put a new WebView in [packageId]'s profile. Must run on the main thread
* before anything else is done with the WebView.
*
* @return false when the WebView provider lacks multi-profile support,
* leaving the WebView in the shared default profile.
*/
@JvmStatic
fun apply(webView: WebView, packageId: String): Boolean {
if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) {
Log.w(TAG, "multi-profile WebView unsupported; $packageId shares the default profile")
return false
}
val name = nameFor(packageId)
ProfileStore.getInstance().getOrCreateProfile(name)
WebViewCompat.setProfile(webView, name)
return true
}
}
}
@@ -2,6 +2,7 @@ package com.faceclaw.app
import android.os.Handler
import android.os.Looper
import android.webkit.RenderProcessGoneDetail
import android.webkit.WebResourceRequest
import android.webkit.WebResourceResponse
import android.webkit.WebView
@@ -33,6 +34,11 @@ import java.nio.charset.StandardCharsets
* The shim is registered as a document-start script instead
* (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for
* WebView versions that lack that API.
*
* An app can crash its renderer, or the system can kill it for memory. Unless
* every WebView that renderer served handles that, WebView aborts the whole
* process, taking Faceclaw down with it; so onRenderProcessGone retires the
* affected WebView and reports it instead.
*/
class FaceclawEvenHubWebViewClient(
private val rootDir: String?,
@@ -93,6 +99,15 @@ class FaceclawEvenHubWebViewClient(
}
}
override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean {
// Called on the main thread for each WebView the renderer served. This
// one can't be used again: drop it, and the app shows as failed.
Log.w(TAG, "renderer gone (crashed=${detail.didCrash()}) for $host")
FaceclawEvenHubWebViewHost.getInstance().detach(view)
listener.onRenderProcessGone(detail.didCrash())
return true
}
companion object {
private const val TAG = "FaceclawEvenHub"
@@ -59,8 +59,8 @@ import java.util.HashMap
* requestAnimationFrame) with JS queues fired by window.__fcTick(), and this
* host ticks those on the main thread regardless of screen state. The tick
* returns when the page next has work, so an idle page isn't evaluated at
* 60Hz; work scheduled between ticks wakes the host through
* FaceclawEvenHubJsBridge.wakeTimers. The main Looper keeps running under the
* 60Hz; work scheduled between ticks wakes the host through the bridge's
* "wake" message (FaceclawEvenHubJsBridge). The main Looper keeps running under the
* foreground service, so the tick survives the screen turning off. (Matches the
* official Even app's approach.)
*
@@ -105,8 +105,12 @@ class FaceclawEvenHubWebViewHost {
* the page wakes it. It also re-ticks after IDLE_POLL_MS regardless, so a
* lost evaluateJavascript callback (e.g. mid-navigation) or a page whose
* wake didn't reach us stalls its timers for at most that long.
*
* The tick only runs in the app's own documents ([appCheck], a JS
* expression): after a navigation the WebView may show another site.
*/
private inner class Ticker(private val web: WebView) : Runnable {
private inner class Ticker(private val web: WebView, appCheck: String) : Runnable {
private val tickJs = "if ($appCheck) { $TICK_JS }"
private var inFlight = false
private var wakePending = false
private var stopped = false
@@ -114,7 +118,7 @@ class FaceclawEvenHubWebViewHost {
override fun run() {
if (stopped) return
inFlight = true
web.evaluateJavascript(TICK_JS) { result -> onTicked(result) }
web.evaluateJavascript(tickJs) { result -> onTicked(result) }
schedule(IDLE_POLL_MS)
}
@@ -278,8 +282,11 @@ class FaceclawEvenHubWebViewHost {
(web.context as? MutableContextWrapper)?.baseContext = activity
}
/** Add a WebView to the host, full-size and rendering, hidden behind the UI. */
fun attach(activity: Activity, web: WebView) {
/**
* Add a WebView to the host, full-size and rendering, hidden behind the UI.
* [appCheck] is a JS expression that is true only in the app's own documents.
*/
fun attach(activity: Activity, web: WebView, appCheck: String) {
ensureOverlay(activity)
rebaseContext(web, activity)
web.visibility = View.VISIBLE
@@ -301,7 +308,7 @@ class FaceclawEvenHubWebViewHost {
}
if (!webViews.contains(web)) webViews.add(web)
if (!tickers.containsKey(web)) {
val ticker = Ticker(web)
val ticker = Ticker(web, appCheck)
tickers[web] = ticker
mainHandler.postDelayed(ticker, TICK_MS)
}
@@ -348,9 +355,9 @@ class FaceclawEvenHubWebViewHost {
return shown
}
/** Remove and destroy a WebView (its app is closing). */
/** Remove and destroy a WebView (its app is closing, or its renderer is gone). Idempotent. */
fun detach(web: WebView) {
webViews.remove(web)
if (!webViews.remove(web)) return
tickers.remove(web)?.stop()
stack?.removeView(web)
web.destroy()
@@ -46,6 +46,7 @@ class FaceclawMediaController(context: Context) {
override fun onMetadataChanged(metadata: MediaMetadata?) {
synchronized(lock) {
artVersion++
emitStateLocked()
}
}
@@ -92,6 +93,15 @@ class FaceclawMediaController(context: Context) {
private var sessionsListenerRegistered = false
private var ignoredPackages: Set<String> = HashSet()
/**
* Bumped whenever the art getAlbumArtGray would return may have changed:
* every metadata update and every switch of active session. Players
* re-publish the same title/album with a different bitmap (art attached
* after download, or a stale bitmap from the previous track replaced
* after a quick double skip), so the strings can't key an art cache.
*/
private var artVersion = 0
init {
this.listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java)
this.sessionManager = appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as MediaSessionManager?
@@ -404,6 +414,7 @@ class FaceclawMediaController(context: Context) {
previous.unregisterCallback(controllerCallback)
}
activeController = controller
artVersion++
if (controller != null) {
controller.registerCallback(controllerCallback, mainHandler)
}
@@ -503,6 +514,7 @@ class FaceclawMediaController(context: Context) {
val finalCanSkipPrevious = canSkipPrevious
val finalAccessEnabled = accessEnabled
val finalStatus = status
val finalArtVersion = artVersion
mainHandler.post {
currentListener.onStateChange(
@@ -519,7 +531,8 @@ class FaceclawMediaController(context: Context) {
finalCanSkipNext,
finalCanSkipPrevious,
finalAccessEnabled,
finalStatus
finalStatus,
finalArtVersion
)
}
}
@@ -3,15 +3,20 @@ package com.faceclaw.app
import android.app.Notification
import android.app.NotificationManager
import android.app.PendingIntent
import android.content.ComponentName
import android.content.Context
import android.database.ContentObserver
import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.drawable.BitmapDrawable
import android.graphics.drawable.Drawable
import android.graphics.drawable.Icon
import android.os.Build
import android.os.Bundle
import android.os.Handler
import android.os.Looper
import android.os.SystemClock
import android.provider.Settings
import android.service.notification.NotificationListenerService
import android.service.notification.StatusBarNotification
import android.util.Log
@@ -29,15 +34,177 @@ import org.json.JSONObject
class FaceclawMediaNotificationListenerService : NotificationListenerService() {
companion object {
private const val TAG = "FaceclawNotify"
private const val NOTIFICATION_ICON_GAMMA = 1.6
private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName"
/**
* How long notification access can be granted with no listener
* connected before the watchdog forces a rebind: longer than a
* normal bind at process start, and than the system's own single
* retry 10 s after a binding dies. Later attempts double from here.
*/
private const val REBIND_GRACE_MS = 15_000L
private const val REBIND_MAX_DELAY_MS = 15 * 60_000L
/** Missing this long, the listener needs the user to toggle notification access. */
private const val LISTENER_STUCK_MS = 30_000L
/** Notifications posted while disconnected alert on reconnect only if this recent. */
private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L
/** The connected listener; null until onListenerConnected and after onListenerDisconnected. */
@Volatile
private var activeService: FaceclawMediaNotificationListenerService? = null
private val mainHandler = Handler(Looper.getMainLooper())
private val notificationListeners: MutableSet<FaceclawNotificationListener> = CopyOnWriteArraySet()
private val listenerStateObservers: MutableSet<Runnable> = CopyOnWriteArraySet()
private val activeNotificationWakeKeys: MutableSet<String> = HashSet()
// Binding watchdog (see watchListenerBinding); written on the main thread only.
private var watchdogContext: Context? = null
private var rebindScheduled = false
private var rebindDelayMs = REBIND_GRACE_MS
/** elapsedRealtime when the watchdog first saw access granted with no listener; 0 when not missing. */
@Volatile
private var listenerMissingSinceMs = 0L
/** Wall-clock time of the last disconnect in this process (compared with postTime); 0 if none. */
private var lastListenerDisconnectAtMs = 0L
private val forceRebindTask = Runnable { forceRebindIfStillMissing() }
private val accessObserver = object : ContentObserver(mainHandler) {
override fun onChange(selfChange: Boolean) {
checkListenerBinding()
notifyListenerStateChanged()
}
}
/**
* Start watching for the listener going missing while notification
* access is granted. Android can drop a listener's binding, or give
* up rebinding it after the process dies, and not bring it back;
* every read then comes back empty for the life of the process.
* Called at app boot; disconnects and access changes re-check.
*/
@JvmStatic
fun watchListenerBinding(context: Context?) {
if (context == null) {
return
}
val appContext = context.applicationContext
mainHandler.post {
if (watchdogContext == null) {
watchdogContext = appContext
try {
appContext.contentResolver.registerContentObserver(
Settings.Secure.getUriFor("enabled_notification_listeners"), false, accessObserver)
} catch (t: Throwable) {
Log.w(TAG, "notification access observer registration failed", t)
}
}
checkListenerBinding()
}
}
/**
* "connected"; "no-access" (notification access not granted);
* "connecting" (granted, listener not connected yet); or "stuck"
* (granted, but still missing past LISTENER_STUCK_MS, which forced
* rebinds can't always fix: the user has to toggle access).
*/
@JvmStatic
fun getListenerState(context: Context?): String {
if (activeService != null) {
return "connected"
}
if (!FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
return "no-access"
}
val missingSince = listenerMissingSinceMs
if (missingSince > 0 && SystemClock.elapsedRealtime() - missingSince >= LISTENER_STUCK_MS) {
return "stuck"
}
return "connecting"
}
/** Run on the main thread whenever getListenerState may have changed. */
@JvmStatic
fun addListenerStateObserver(observer: Runnable?) {
if (observer != null) {
listenerStateObservers.add(observer)
}
}
@JvmStatic
fun removeListenerStateObserver(observer: Runnable?) {
if (observer != null) {
listenerStateObservers.remove(observer)
}
}
private fun notifyListenerStateChanged() {
for (observer in listenerStateObservers) {
mainHandler.post {
try {
observer.run()
} catch (t: Throwable) {
Log.w(TAG, "listener state observer failed", t)
}
}
}
}
/** Main thread. Starts the forced-rebind countdown when access is granted and no listener is connected. */
private fun checkListenerBinding() {
val context = watchdogContext ?: return
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
resetRebindWatchdog()
return
}
if (listenerMissingSinceMs == 0L) {
listenerMissingSinceMs = SystemClock.elapsedRealtime()
}
scheduleForcedRebind()
}
private fun scheduleForcedRebind() {
if (!rebindScheduled) {
rebindScheduled = true
mainHandler.postDelayed(forceRebindTask, rebindDelayMs)
}
}
private fun resetRebindWatchdog() {
mainHandler.removeCallbacks(forceRebindTask)
rebindScheduled = false
rebindDelayMs = REBIND_GRACE_MS
listenerMissingSinceMs = 0L
}
private fun forceRebindIfStillMissing() {
rebindScheduled = false
val context = watchdogContext ?: return
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
resetRebindWatchdog()
return
}
val missingSeconds = (SystemClock.elapsedRealtime() - listenerMissingSinceMs) / 1000
Log.w(TAG, "notification access granted but no listener connected for ${missingSeconds}s; forcing a rebind")
val component = ComponentName(context, FaceclawMediaNotificationListenerService::class.java)
try {
// requestRebind only undoes a requestUnbind, so alone it can't
// bring back a binding the system dropped. Unbinding by
// component first marks the listener snoozed and drops any
// record of it; the rebind then un-snoozes it and binds afresh.
// Below API 34 there is no public equivalent (and flipping the
// component's enabled state can make the system revoke
// access), so there the state goes "stuck" for the user.
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
NotificationListenerService.requestUnbind(component)
}
NotificationListenerService.requestRebind(component)
} catch (t: Throwable) {
Log.w(TAG, "forced notification listener rebind failed", t)
}
rebindDelayMs = Math.min(rebindDelayMs * 2, REBIND_MAX_DELAY_MS)
scheduleForcedRebind()
notifyListenerStateChanged()
}
@JvmStatic
fun addNotificationListener(listener: FaceclawNotificationListener?) {
if (listener != null) {
@@ -127,15 +294,16 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) {
continue
}
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
if (drawable == null) {
Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.packageName)
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
getNotificationAppName(service, statusBarNotification)
}
if (icon == null) {
Log.i(TAG, "icon skipped (nothing to draw): " + statusBarNotification.packageName)
continue
}
Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.packageName
+ " drawable=" + drawable.javaClass.simpleName
+ " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight)
appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName)
+ " source=" + icon.source + " kind=" + icon.art.kind)
out.write(icon.art.gray)
keysOut?.add(statusBarNotification.key ?: "")
if (dedupeGroupKey != null) {
emittedGroupKeys.add(dedupeGroupKey)
@@ -165,13 +333,37 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
if (statusBarNotification == null || statusBarNotification.notification == null) {
return ByteArray(0)
}
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
if (drawable == null) {
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
getNotificationAppName(service, statusBarNotification)
}
return icon?.art?.gray ?: ByteArray(0)
}
/**
* The picture attached to one active notification (BigPictureStyle: a
* photo, a screenshot, Dexcom's glucose graph), scaled to fit
* maxWidth x maxHeight as a gray packet with the tone handling (and
* light-background inversion) of ImageFileLoader.bitmapToGrayPacket;
* empty when the notification or its picture is unavailable.
*/
@JvmStatic
fun getNotificationPictureGray(
key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean,
): ByteArray {
val service = activeService
if (service == null || maxWidth <= 0 || maxHeight <= 0) {
return ByteArray(0)
}
val out = ByteArrayOutputStream(size * size)
appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.packageName)
return out.toByteArray()
val statusBarNotification = findActiveNotificationByKey(service, key)
if (statusBarNotification == null || statusBarNotification.notification == null) {
return ByteArray(0)
}
val picture = loadNotificationPicture(service, statusBarNotification.notification)
if (picture == null) {
return ByteArray(0)
}
return ImageFileLoader.bitmapToGrayPacket(
halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground)
}
@JvmStatic
@@ -267,13 +459,21 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
}
private fun emitNotificationPosted(key: String?) {
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
}
private fun emitNotificationRemoved(key: String?) {
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationRemoved(eventKey) }
}
private fun emitNotificationEvent(key: String?, deliver: (FaceclawNotificationListener, String) -> Unit) {
if (key == null || key.isEmpty() || notificationListeners.isEmpty()) {
return
}
for (listener in notificationListeners) {
mainHandler.post {
try {
listener.onNotificationPosted(key)
deliver(listener, key)
} catch (t: Throwable) {
Log.w(TAG, "notification listener failed", t)
}
@@ -281,7 +481,14 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
}
}
private fun refreshActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
/**
* Re-seed the wake keys from what is active now. After a reconnect
* this also catches the glasses up on the gap: keys that went away
* are reported removed (closing a popup for a notification
* dismissed meanwhile), and recent notifications posted since the
* disconnect are reported posted. A first connect only seeds.
*/
private fun reconcileActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
val notifications: Array<StatusBarNotification?>?
try {
notifications = service.activeNotifications
@@ -289,19 +496,40 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
Log.w(TAG, "failed to refresh active notification wake keys", t)
return
}
if (notifications == null) {
return
}
val disconnectedAtMs = lastListenerDisconnectAtMs
val alertAfterMs = Math.max(disconnectedAtMs, System.currentTimeMillis() - RECONNECT_ALERT_WINDOW_MS)
val posted = ArrayList<String>()
val removed = ArrayList<String>()
synchronized(activeNotificationWakeKeys) {
val previous = HashSet(activeNotificationWakeKeys)
activeNotificationWakeKeys.clear()
if (notifications == null) {
return
}
for (statusBarNotification in notifications) {
if (shouldShowNotificationInList(service, statusBarNotification)) {
val key: String? = statusBarNotification!!.key
if (key != null && !key.isEmpty()) {
activeNotificationWakeKeys.add(key)
}
if (!shouldShowNotificationInList(service, statusBarNotification)) {
continue
}
val key: String? = statusBarNotification!!.key
if (key == null || key.isEmpty()) {
continue
}
activeNotificationWakeKeys.add(key)
if (disconnectedAtMs > 0 && !previous.contains(key) && statusBarNotification.postTime > alertAfterMs) {
posted.add(key)
}
}
for (key in previous) {
if (!activeNotificationWakeKeys.contains(key)) {
removed.add(key)
}
}
}
for (key in removed) {
emitNotificationRemoved(key)
}
for (key in posted) {
emitNotificationPosted(key)
}
}
@@ -319,16 +547,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
return !alreadyActive || !isPersistentNotification(statusBarNotification)
}
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?) {
/** Stop tracking a notification; returns whether it was tracked (i.e. listed on the glasses). */
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?): Boolean {
if (statusBarNotification == null) {
return
return false
}
val key: String? = statusBarNotification.key
if (key == null || key.isEmpty()) {
return
return false
}
synchronized(activeNotificationWakeKeys) {
activeNotificationWakeKeys.remove(key)
return activeNotificationWakeKeys.remove(key)
}
}
@@ -397,106 +626,77 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
return groupKey
}
private fun loadNotificationIcon(service: FaceclawMediaNotificationListenerService, notification: Notification): Drawable? {
/** Whether a notification carries a picture for loadNotificationPicture, without loading it. */
@Suppress("DEPRECATION")
private fun hasNotificationPicture(extras: Bundle): Boolean {
return extras.get(Notification.EXTRA_PICTURE) != null
|| (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && extras.get(Notification.EXTRA_PICTURE_ICON) != null)
}
/**
* A notification's attached picture: BigPictureStyle's bitmap
* (EXTRA_PICTURE) or, from Android 12, its Icon (EXTRA_PICTURE_ICON).
* Newer releases swap large pictures for a content URI to a bitmap
* the system has offloaded (Dexcom's graph arrives that way); the
* offload provider lets enabled notification listeners open those.
*/
@Suppress("DEPRECATION")
private fun loadNotificationPicture(service: FaceclawMediaNotificationListenerService, notification: Notification): Bitmap? {
val extras: Bundle = notification.extras ?: return null
try {
val smallIcon: Icon? = notification.smallIcon
if (smallIcon != null) {
val drawable: Drawable? = smallIcon.loadDrawable(service)
if (drawable != null) {
// Status-bar small icons are alpha templates: the platform
// draws them tinted and ignores their color channels, which
// apps may fill with garbage (Discord ships noise there).
// Tint white so the shape comes from alpha alone, matching
// how the status bar renders them.
drawable.mutate()
drawable.setTint(Color.WHITE)
if (drawable is BitmapDrawable) {
drawable.setFilterBitmap(true)
}
return drawable
}
val picture: Bitmap? = extras.getParcelable(Notification.EXTRA_PICTURE)
if (picture != null) {
return softwareBitmap(picture)
}
} catch (t: Throwable) {
Log.w(TAG, "failed to load small notification icon", t)
Log.w(TAG, "failed to read notification picture", t)
}
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
return null
}
try {
// Large icons are real color images (avatars, album art); keep color.
val largeIcon: Icon? = notification.getLargeIcon()
if (largeIcon != null) {
return largeIcon.loadDrawable(service)
val icon: Icon? = extras.getParcelable(Notification.EXTRA_PICTURE_ICON)
val drawable: Drawable? = icon?.loadDrawable(service)
if (drawable is BitmapDrawable && drawable.bitmap != null) {
return softwareBitmap(drawable.bitmap)
}
if (drawable != null && drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) {
val bitmap = Bitmap.createBitmap(drawable.intrinsicWidth, drawable.intrinsicHeight, Bitmap.Config.ARGB_8888)
drawable.setBounds(0, 0, bitmap.width, bitmap.height)
drawable.draw(Canvas(bitmap))
return bitmap
}
} catch (t: Throwable) {
Log.w(TAG, "failed to load large notification icon", t)
Log.w(TAG, "failed to load notification picture icon", t)
}
return null
}
private fun appendIconGrayBytes(drawable: Drawable, size: Int, out: ByteArrayOutputStream,
service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?) {
val bitmap = renderIconScaled(drawable, size)
dumpIconDebugPng(service, index, packageName, bitmap)
for (y in 0 until size) {
for (x in 0 until size) {
val color = bitmap.getPixel(x, y)
val alpha = Color.alpha(color)
val grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0)
val gray = Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)).toInt()
out.write(gray and 0xff)
}
}
bitmap.recycle()
}
/**
* Render a drawable at the target size with proper downscaling. Detailed
* sources (e.g. avatar bitmaps used as notification icons) are rendered at
* native resolution and reduced by repeated halving: a single filtered pass
* from, say, 126px to 24px samples too sparsely and turns fine detail into
* speckle that reads as a garbled icon.
*/
private fun renderIconScaled(drawable: Drawable, size: Int): Bitmap {
val renderW = Math.max(size, drawable.intrinsicWidth)
val renderH = Math.max(size, drawable.intrinsicHeight)
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
drawable.setBounds(0, 0, renderW, renderH)
drawable.draw(canvas)
while (bitmap.width >= size * 2 && bitmap.height >= size * 2) {
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
bitmap.recycle()
bitmap = halved
}
if (bitmap.width != size || bitmap.height != size) {
val scaled = Bitmap.createScaledBitmap(bitmap, size, size, true)
bitmap.recycle()
bitmap = scaled
/** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */
private fun softwareBitmap(bitmap: Bitmap): Bitmap {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) {
return bitmap.copy(Bitmap.Config.ARGB_8888, false)
}
return bitmap
}
/**
* Debug aid for garbled-icon reports: saves each rendered icon to
* <externalFilesDir>/debug-icons/ (adb-pullable) so extraction problems can
* be told apart from downstream compositing/transmission problems. Cheap:
* runs at most once per icon-cache refresh on tiny bitmaps.
* Halve a picture while it is still at least twice the size it will
* be shown at, leaving the last (under 2x) step of fitting it into
* maxWidth x maxHeight to bitmapToGrayPacket. As with icons
* (NotificationIconLoader.renderArgb), one filtered pass from a much larger source
* samples too sparsely: a graph's thin lines break up into dots.
*/
private fun dumpIconDebugPng(service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?, bitmap: Bitmap) {
if (index < 0) {
return
}
try {
val dir = java.io.File(service.getExternalFilesDir(null), "debug-icons")
if (!dir.exists() && !dir.mkdirs()) {
return
private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
var bitmap = source
while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) {
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
if (bitmap !== source) {
bitmap.recycle()
}
val safeName = if (packageName == null) "unknown" else packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
val file = java.io.File(dir, "icon-$index-$safeName.png")
java.io.FileOutputStream(file).use { stream ->
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
}
} catch (t: Throwable) {
Log.w(TAG, "failed to dump debug icon", t)
bitmap = halved
}
return bitmap
}
private fun findActiveNotificationByKey(service: FaceclawMediaNotificationListenerService?, key: String?): StatusBarNotification? {
@@ -531,6 +731,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
out.put("appName", getNotificationAppName(service, statusBarNotification))
out.put("postTime", statusBarNotification.postTime)
out.put("when", notification.`when`)
out.put("foregroundService", (notification.flags and Notification.FLAG_FOREGROUND_SERVICE) != 0)
putString(out, "category", notification.category)
if (extras != null) {
putCharSequence(out, "title", firstNonEmpty(
@@ -553,8 +754,10 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
}
}
out.put("lines", lines)
out.put("hasPicture", hasNotificationPicture(extras))
} else {
out.put("lines", JSONArray())
out.put("hasPicture", false)
}
val actionsJson = JSONArray()
@@ -626,35 +829,34 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
}
}
override fun onCreate() {
super.onCreate()
activeService = this
}
override fun onDestroy() {
if (activeService === this) {
activeService = null
}
super.onDestroy()
}
override fun onListenerConnected() {
activeService = this
super.onListenerConnected()
refreshActiveNotificationWakeKeys(this)
Log.i(TAG, "notification listener connected")
resetRebindWatchdog()
reconcileActiveNotificationWakeKeys(this)
notifyListenerStateChanged()
}
// Also called from onDestroy, which is how an unbind usually arrives.
override fun onListenerDisconnected() {
if (activeService === this) {
activeService = null
lastListenerDisconnectAtMs = System.currentTimeMillis()
Log.w(TAG, "notification listener disconnected")
notifyListenerStateChanged()
}
super.onListenerDisconnected()
watchListenerBinding(applicationContext)
}
override fun onNotificationPosted(statusBarNotification: StatusBarNotification?) {
super.onNotificationPosted(statusBarNotification)
if (!shouldShowNotificationInList(this, statusBarNotification)) {
forgetActiveNotificationWakeKey(statusBarNotification)
// An update that takes a listed notification off the list removes it as far as the glasses go.
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
emitNotificationRemoved(statusBarNotification!!.key)
}
return
}
if (shouldEmitNotificationPosted(statusBarNotification!!)) {
@@ -663,7 +865,12 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
}
override fun onNotificationRemoved(statusBarNotification: StatusBarNotification?) {
forgetActiveNotificationWakeKey(statusBarNotification)
// Includes notifications their source cancels on its own, e.g. a
// short-lived foreground-service notification that was standing in
// for one posted moments later under another key.
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
emitNotificationRemoved(statusBarNotification!!.key)
}
super.onNotificationRemoved(statusBarNotification)
}
}
@@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) {
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
}
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
val composite = compositor.previewComposite() ?: return null
return PreviewBitmapUtil.fromGray(
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
cropX, cropY, cropWidth, cropHeight)
}
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
@Throws(java.io.IOException::class)
fun saveCompositePngScreenshot(): String {
@@ -0,0 +1,51 @@
package com.faceclaw.app
import android.content.Context
import android.location.Address
import android.location.Geocoder
import android.os.Handler
import android.os.Looper
import android.util.Log
import java.util.Locale
import kotlin.concurrent.thread
/**
* Place name for the Weather header where the weather source doesn't supply
* one (outside National Weather Service coverage), from Android's own
* Geocoder. Its lookup blocks on the network, so it runs on a thread of its
* own; the listener hears back on the main thread, with nulls when the phone
* has no geocoder or the lookup fails.
*/
class FaceclawReverseGeocoder(context: Context) {
interface Listener {
fun onResult(locality: String?, region: String?, country: String?)
}
companion object {
private const val TAG = "FaceclawGeocoder"
}
private val context: Context = context.applicationContext
private val mainHandler = Handler(Looper.getMainLooper())
fun lookup(latitude: Double, longitude: Double, listener: Listener) {
thread(name = "FaceclawReverseGeocoder", isDaemon = true) {
val address = lookupBlocking(latitude, longitude)
mainHandler.post { listener.onResult(address?.locality, address?.adminArea, address?.countryName) }
}
}
private fun lookupBlocking(latitude: Double, longitude: Double): Address? {
if (!Geocoder.isPresent()) {
return null
}
return try {
// The listener form needs API 33; this one is fine off the main thread.
@Suppress("DEPRECATION")
Geocoder(context, Locale.getDefault()).getFromLocation(latitude, longitude, 1)?.firstOrNull()
} catch (error: Throwable) {
Log.w(TAG, "reverse geocode failed: $error")
null
}
}
}
@@ -132,10 +132,15 @@ class ImageFileLoader private constructor() {
* round on its own, which turns smooth gradients into visible bands.
* Dithered output is already quantized: every byte is a level times 16,
* which BmpUtil re-quantizes back to exactly that level.
*
* invertLightBackground shows a picture drawn on a light background
* (a chart, a screenshot) as its negative; see GrayPacket.fromArgb.
*/
@JvmStatic
@JvmOverloads
fun bitmapToGrayPacket(
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray {
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean,
invertLightBackground: Boolean = false): ByteArray {
if (source == null || source.width <= 0 || source.height <= 0
|| maxWidth <= 0 || maxHeight <= 0) {
return ByteArray(0)
@@ -152,7 +157,7 @@ class ImageFileLoader private constructor() {
}
val pixels = IntArray(width * height)
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
return GrayPacket.fromArgb(pixels, width, height, gamma, dither, invertLightBackground)
} catch (e: Exception) {
Log.w(TAG, "bitmap conversion failed", e)
return ByteArray(0)
@@ -0,0 +1,290 @@
package com.faceclaw.app
import android.app.Notification
import android.content.Context
import android.graphics.Bitmap
import android.graphics.Canvas
import android.graphics.Color
import android.graphics.Paint
import android.graphics.Rect
import android.graphics.Typeface
import android.graphics.drawable.AdaptiveIconDrawable
import android.graphics.drawable.BitmapDrawable
import android.graphics.drawable.Drawable
import android.graphics.drawable.Icon
import android.os.Build
import android.os.SystemClock
import android.service.notification.StatusBarNotification
import android.util.Log
import com.faceclaw.app.NotificationIconArt.Kind
import java.io.File
import java.io.FileOutputStream
import kotlin.math.max
import kotlin.math.roundToInt
/**
* Picks and renders the glasses' status-bar icon for a notification (see iconFor).
* NotificationIconArt does the pixel analysis; this object owns the drawables, package lookups,
* caching and the debug dump.
*/
internal object NotificationIconLoader {
private const val TAG = "FaceclawNotifyIcon"
/** AdaptiveIconDrawable draws its layers this share of the icon's size past each edge. */
private const val ADAPTIVE_LAYER_OUTSET = 0.25f
private const val CACHE_MAX_ENTRIES = 128
/** Cached art is redone after this long, so an updated app's new icon shows up. */
private const val CACHE_MAX_AGE_MS = 30 * 60 * 1000L
/** The debug dump keeps at most this many files, deleting the oldest. */
private const val DEBUG_DUMP_MAX_FILES = 96
class Rendered(
/** The drawable it came from: small, monochrome, launcher, foreground, large or letter. */
val source: String,
val art: NotificationIconArt.Art,
)
/**
* An app's launcher icon art: its themed monochrome layer, else a glyph or shape read from
* the icon (or its adaptive foreground layer), else the icon read by brightness.
*/
private class LauncherArt(val monochrome: Rendered?, val shape: Rendered?, val picture: Rendered?)
private val smallIconCache = ArtCache<Rendered?>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
private val launcherCache = ArtCache<LauncherArt>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
/**
* The icon for a notification, size x size gray, from the first of these that works:
* 1. the small icon, when its alpha carries a shape (nearly every app);
* 2. the launcher icon's themed monochrome layer (Android 13+), which is made to be tinted;
* 3. the small icon's glyph, when it is a color tile with one background color;
* 4. the launcher icon's glyph or shape, or its adaptive foreground layer's shape;
* 5. the small icon, then the launcher icon, read by brightness;
* 6. the large icon (usually the sender's avatar: the notification's content, not its app);
* 7. the app name's initial in a rounded box.
* Null only when even the initial can't be drawn.
*/
fun iconFor(context: Context, statusBarNotification: StatusBarNotification, size: Int, appName: () -> String): Rendered? {
val packageName = statusBarNotification.packageName ?: ""
val notification = statusBarNotification.notification
val small = smallIconArt(context, packageName, notification.smallIcon, size)
val smallKind = small?.art?.kind
if (smallKind == Kind.TEMPLATE || smallKind == Kind.SOLID) return small
val launcher = launcherArt(context, packageName, size)
launcher.monochrome?.let { return it }
if (smallKind == Kind.GLYPH) return small
launcher.shape?.let { return it }
if (smallKind == Kind.PICTURE) return small
launcher.picture?.let { return it }
largeIconArt(context, packageName, notification, size)?.let { return it }
return letterArt(appName(), size)
}
private fun smallIconArt(context: Context, packageName: String, icon: Icon?, size: Int): Rendered? {
if (icon == null) return null
val key = resourceKey(icon) ?: return loadSmallIcon(context, packageName, icon, size)
return smallIconCache.getOrPut("$packageName|$key|$size") { loadSmallIcon(context, packageName, icon, size) }
}
/** Resource icons are cached by resource; bitmap and URI icons are converted every time. */
private fun resourceKey(icon: Icon): String? {
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P || icon.type != Icon.TYPE_RESOURCE) return null
return icon.resPackage + "/" + icon.resId
}
private fun loadSmallIcon(context: Context, packageName: String, icon: Icon, size: Int): Rendered? =
tryRender("small", packageName) {
val drawable = icon.loadDrawable(context) ?: return@tryRender null
// A tint (the Icon's own, or a vector's theme tint) recolors the very pixels that tell
// a template from a color tile, and can fade a template's alpha. Templates are read
// by alpha alone, so dropping it costs nothing.
drawable.mutate()
drawable.setTintList(null)
renderSource(context, packageName, "small", drawable, size)
}
private fun launcherArt(context: Context, packageName: String, size: Int): LauncherArt =
launcherCache.getOrPut("$packageName|$size") { loadLauncherArt(context, packageName, size) }
private fun loadLauncherArt(context: Context, packageName: String, size: Int): LauncherArt {
val icon: Drawable = try {
context.packageManager.getApplicationIcon(packageName)
} catch (t: Throwable) {
Log.w(TAG, "no launcher icon for $packageName", t)
return LauncherArt(null, null, null)
}
val adaptive = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) icon as? AdaptiveIconDrawable else null
if (adaptive != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
val layer = adaptive.monochrome
val monochrome = if (layer == null) null else tryRender("monochrome", packageName) {
renderSource(context, packageName, "monochrome", layer, size, ADAPTIVE_LAYER_OUTSET, template = true)
}
if (monochrome != null) return LauncherArt(monochrome, null, null)
}
val whole = tryRender("launcher", packageName) { renderSource(context, packageName, "launcher", icon, size) }
var shape = whole?.takeIf { it.art.kind == Kind.TEMPLATE || it.art.kind == Kind.GLYPH }
val foreground = adaptive?.foreground
if (shape == null && foreground != null) {
// A gradient or picture background hides the glyph; the foreground layer alone may carry it as alpha.
shape = tryRender("foreground", packageName) {
renderSource(context, packageName, "foreground", foreground, size, ADAPTIVE_LAYER_OUTSET)
}?.takeIf { it.art.kind == Kind.TEMPLATE }
}
return LauncherArt(null, shape, whole?.takeIf { it.art.kind == Kind.PICTURE })
}
private fun largeIconArt(context: Context, packageName: String, notification: Notification, size: Int): Rendered? =
tryRender("large", packageName) {
val drawable = notification.getLargeIcon()?.loadDrawable(context) ?: return@tryRender null
renderSource(context, packageName, "large", drawable, size)
}
/** The app name's first letter or digit in a rounded box, like the shell's letter window icons. */
private fun letterArt(appName: String, size: Int): Rendered? = tryRender("letter", appName) {
val letter = appName.firstOrNull { it.isLetterOrDigit() }?.uppercaseChar()?.toString() ?: "?"
val canvasSize = size * NotificationIconArt.ANALYSIS_SCALE
val bitmap = Bitmap.createBitmap(canvasSize, canvasSize, Bitmap.Config.ARGB_8888)
val canvas = Canvas(bitmap)
val stroke = NotificationIconArt.ANALYSIS_SCALE.toFloat()
val box = Paint(Paint.ANTI_ALIAS_FLAG).apply {
style = Paint.Style.STROKE
strokeWidth = stroke
color = Color.argb(160, 255, 255, 255)
}
val radius = canvasSize / 4f
canvas.drawRoundRect(stroke / 2, stroke / 2, canvasSize - stroke / 2, canvasSize - stroke / 2, radius, radius, box)
val text = Paint(Paint.ANTI_ALIAS_FLAG).apply {
color = Color.WHITE
typeface = Typeface.DEFAULT_BOLD
textSize = canvasSize * 0.6f
textAlign = Paint.Align.CENTER
}
val bounds = Rect()
text.getTextBounds(letter, 0, letter.length, bounds)
canvas.drawText(letter, canvasSize / 2f, canvasSize / 2f - bounds.exactCenterY(), text)
val argb = IntArray(canvasSize * canvasSize)
bitmap.getPixels(argb, 0, canvasSize, 0, 0, canvasSize, canvasSize)
bitmap.recycle()
val art = NotificationIconArt.template(argb, size)
if (art.kind == Kind.EMPTY) null else Rendered("letter", art)
}
private inline fun tryRender(source: String, owner: String, render: () -> Rendered?): Rendered? {
return try {
render()
} catch (t: Throwable) {
Log.w(TAG, "failed to render $source icon for $owner", t)
null
}
}
/** Render, dump and convert one source drawable; null when nothing visible came out. */
private fun renderSource(
context: Context, packageName: String, source: String, drawable: Drawable, size: Int,
outset: Float = 0f, template: Boolean = false,
): Rendered? {
val analysisSize = size * NotificationIconArt.ANALYSIS_SCALE
val argb = renderArgb(drawable, analysisSize, outset)
dumpDebugPng(context, packageName, source, argb, analysisSize)
val art = if (template) NotificationIconArt.template(argb, size) else NotificationIconArt.convert(argb, size)
return if (art.kind == Kind.EMPTY) null else Rendered(source, art)
}
/**
* Render a drawable to target x target ARGB with proper downscaling. Detailed sources (e.g.
* avatar bitmaps used as notification icons) are rendered at native resolution and reduced
* by repeated halving: a single filtered pass from, say, 126px to 24px samples too sparsely
* and turns fine detail into speckle that reads as a garbled icon. An outset draws the
* drawable that share of the canvas past each edge, as AdaptiveIconDrawable draws its layers.
*/
private fun renderArgb(drawable: Drawable, target: Int, outset: Float): IntArray {
drawable.mutate()
if (drawable is BitmapDrawable) {
drawable.setFilterBitmap(true)
}
val scale = 1f + 2f * outset
val renderW = max(target, (drawable.intrinsicWidth / scale).roundToInt())
val renderH = max(target, (drawable.intrinsicHeight / scale).roundToInt())
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
val dx = (renderW * outset).roundToInt()
val dy = (renderH * outset).roundToInt()
drawable.setBounds(-dx, -dy, renderW + dx, renderH + dy)
drawable.draw(Canvas(bitmap))
while (bitmap.width >= target * 2 && bitmap.height >= target * 2) {
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
bitmap.recycle()
bitmap = halved
}
if (bitmap.width != target || bitmap.height != target) {
val scaled = Bitmap.createScaledBitmap(bitmap, target, target, true)
bitmap.recycle()
bitmap = scaled
}
val pixels = IntArray(target * target)
bitmap.getPixels(pixels, 0, target, 0, 0, target, target)
bitmap.recycle()
return pixels
}
/**
* Debug aid for icon reports: saves each source as rendered for analysis, before any
* conversion, to <externalFilesDir>/debug-icons/<package>-<source>.png (adb-pullable), so a
* bad icon can be rerun through NotificationIconArt off-device. Runs only when a source is
* converted, not on cached icons; the oldest files past DEBUG_DUMP_MAX_FILES are deleted.
*/
private fun dumpDebugPng(context: Context, packageName: String, source: String, argb: IntArray, size: Int) {
try {
val base = context.getExternalFilesDir(null) ?: return
val dir = File(base, "debug-icons")
if (!dir.exists() && !dir.mkdirs()) {
return
}
val safeName = packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
val bitmap = Bitmap.createBitmap(argb, size, size, Bitmap.Config.ARGB_8888)
try {
FileOutputStream(File(dir, "$safeName-$source.png")).use { stream ->
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
}
} finally {
bitmap.recycle()
}
val files = dir.listFiles() ?: return
if (files.size > DEBUG_DUMP_MAX_FILES) {
files.sortBy { it.lastModified() }
for (index in 0 until files.size - DEBUG_DUMP_MAX_FILES) {
files[index].delete()
}
}
} catch (t: Throwable) {
Log.w(TAG, "failed to dump debug icon", t)
}
}
/** A small LRU of converted art with an age limit. Conversions run under its lock. */
private class ArtCache<V>(private val maxEntries: Int, private val maxAgeMs: Long) {
private class Entry<T>(val value: T, val atMs: Long)
private val entries = object : LinkedHashMap<String, Entry<V>>(16, 0.75f, true) {
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Entry<V>>): Boolean {
return size > maxEntries
}
}
@Synchronized
fun getOrPut(key: String, compute: () -> V): V {
val now = SystemClock.elapsedRealtime()
val cached = entries[key]
if (cached != null && now - cached.atMs < maxAgeMs) {
return cached.value
}
val value = compute()
entries[key] = Entry(value, now)
return value
}
}
}
@@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() {
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
}
/**
* As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the
* mirror's Scaled preview). The area may extend past the frame's edges; it is black
* there.
*/
@JvmStatic
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean,
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap {
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
throw IllegalArgumentException("invalid gray preview buffer")
}
if (cropWidth <= 0 || cropHeight <= 0) {
throw IllegalArgumentException("invalid preview crop")
}
val bytes = ByteArray(width * height)
gray.get(bytes)
val cropped = ByteArray(cropWidth * cropHeight)
val left = maxOf(cropX, 0)
val right = minOf(cropX + cropWidth, width)
if (right > left) {
for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) {
System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left)
}
}
val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green)
return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
}
/**
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
* alpha; black, which the glasses show as unlit, is transparent), for
+3 -1
View File
@@ -13,7 +13,7 @@
<key>CFBundleDisplayName</key>
<string>Faceclaw</string>
<key>CFBundleShortVersionString</key>
<string>1.0.0</string>
<string>0.8.4</string>
<key>CFBundleVersion</key>
<string>1</string>
<key>LSRequiresIPhoneOS</key>
@@ -30,6 +30,8 @@
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
<key>NSCalendarsFullAccessUsageDescription</key>
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
<key>NSAppleMusicUsageDescription</key>
<string>Faceclaw shows what is playing in Apple Music on your glasses, controls playback, and lets you browse and play your music library from the glasses.</string>
<key>NSLocationWhenInUseUsageDescription</key>
<string>Faceclaw uses your location for local weather, turn-by-turn navigation, and true north on the glasses compass.</string>
<key>NSLocationTemporaryUsageDescriptionDictionary</key>
+30
View File
@@ -0,0 +1,30 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>NSPrivacyAccessedAPITypes</key>
<array>
<dict>
<!-- Faceclaw preferences in its own NSUserDefaults domain. -->
<key>NSPrivacyAccessedAPIType</key>
<string>NSPrivacyAccessedAPICategoryUserDefaults</string>
<key>NSPrivacyAccessedAPITypeReasons</key>
<array><string>CA92.1</string></array>
</dict>
<dict>
<!-- App-container files used by Faceclaw and its NativeScript/Kotlin runtimes. -->
<key>NSPrivacyAccessedAPIType</key>
<string>NSPrivacyAccessedAPICategoryFileTimestamp</string>
<key>NSPrivacyAccessedAPITypeReasons</key>
<array><string>C617.1</string></array>
</dict>
<dict>
<!-- NativeScript's monotonic clock measures elapsed time within the app. -->
<key>NSPrivacyAccessedAPIType</key>
<string>NSPrivacyAccessedAPICategorySystemBootTime</string>
<key>NSPrivacyAccessedAPITypeReasons</key>
<array><string>35F9.1</string></array>
</dict>
</array>
</dict>
</plist>
+2 -2
View File
@@ -2,8 +2,8 @@ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
TARGETED_DEVICE_FAMILY = 1;
CODE_SIGN_STYLE = Automatic;
// Select the local signing team in Xcode or pass --teamId to the iOS scripts.
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
@@ -0,0 +1,41 @@
#import <Foundation/Foundation.h>
/**
* Apple Music (the system Music app) through the MediaPlayer framework: now-playing state
* and transport on the system player, the media library as a browse tree, and the system
* output volume. Call from the main thread; callbacks are delivered on the main queue.
* Nothing touches the player or library until media-library access is granted, so no
* permission prompt appears unasked.
*/
@interface FaceclawAppleMusic : NSObject
+ (instancetype)shared;
/** Playback state, the now-playing item or authorization may have changed. */
@property(nonatomic, copy) void (^changeHandler)(void);
- (BOOL)hasPermission;
- (NSString *)permissionStatus;
- (void)requestPermission:(void (^)(BOOL granted))completion;
/** Observe the system player (and keep observing across authorization changes). */
- (void)start;
- (void)stop;
/** The system player as a JSON object; {"access": false} until access is granted. */
- (NSString *)stateJson;
- (void)playPause;
- (void)skipNext;
- (void)skipPrevious;
/**
* The queue Faceclaw last started, as [{"id", "title", "active"}], while the player is
* still playing from it; "[]" otherwise (the system player does not expose queues that
* the Music app started).
*/
- (NSString *)queueJson;
- (void)skipToQueueIndex:(NSInteger)index;
/** [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), or nil. */
- (NSData *)albumArt:(NSInteger)maxSize;
/** System output volume 0..100, or -1 when unknown. */
- (double)volumePercent;
- (void)setVolumePercent:(double)percent;
/** One level of the library tree ("root" first) as [{"mediaId", "title", "subtitle", "browsable", "playable"}]. */
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *json, NSString *error))completion;
/** Start playback of a browsed collection, or of a song within its collection. */
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *error))completion;
@end
+512
View File
@@ -0,0 +1,512 @@
#import "FaceclawAppleMusic.h"
#import <AVFoundation/AVFoundation.h>
#import <MediaPlayer/MediaPlayer.h>
#import <UIKit/UIKit.h>
#import <errno.h>
#import <math.h>
#import <stdlib.h>
/** How long a queue Faceclaw just set may take to become the player's now-playing context. */
static const NSTimeInterval kQueueConfirmSeconds = 5.0;
static NSString *JsonString(id object, NSString *fallback) {
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
return data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : fallback;
}
/** Persistent IDs are unsigned 64-bit, beyond JavaScript's exact integers, so they cross as decimal strings. */
static NSString *IdString(MPMediaEntityPersistentID persistentId) {
return [NSString stringWithFormat:@"%llu", persistentId];
}
static BOOL ParseId(NSString *text, MPMediaEntityPersistentID *out) {
const char *chars = text.UTF8String;
if (!chars || chars[0] < '0' || chars[0] > '9') return NO;
char *end = NULL;
errno = 0;
unsigned long long value = strtoull(chars, &end, 10);
if (errno || !end || *end) return NO;
*out = value;
return YES;
}
static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *subtitle, BOOL browsable, BOOL playable) {
return @{@"mediaId": mediaId, @"title": title ?: @"", @"subtitle": subtitle ?: @"",
@"browsable": @(browsable), @"playable": @(playable)};
}
@interface FaceclawAppleMusic ()
@property(nonatomic, strong) dispatch_queue_t libraryQueue;
@property(nonatomic) BOOL started;
@property(nonatomic) BOOL observing;
/** Bumped whenever the now-playing item changes, so cached album art is refetched. */
@property(nonatomic) NSInteger artVersion;
@property(nonatomic, copy) NSString *nowPlayingKey;
/** The items of the queue Faceclaw last started, until the player leaves them. */
@property(nonatomic, copy) NSArray<MPMediaItem *> *queueItems;
@property(nonatomic) BOOL queueConfirmed;
@property(nonatomic, strong) NSDate *queueSetAt;
@property(nonatomic, strong) MPVolumeView *volumeView;
@end
@implementation FaceclawAppleMusic
+ (instancetype)shared {
static FaceclawAppleMusic *instance;
static dispatch_once_t once;
dispatch_once(&once, ^{ instance = [FaceclawAppleMusic new]; });
return instance;
}
- (instancetype)init {
if ((self = [super init])) {
_libraryQueue = dispatch_queue_create("com.faceclaw.applemusic", DISPATCH_QUEUE_SERIAL);
_nowPlayingKey = @"";
// Access may have been granted or revoked in Settings while away.
[NSNotificationCenter.defaultCenter addObserver:self selector:@selector(appBecameActive:)
name:UIApplicationDidBecomeActiveNotification object:nil];
}
return self;
}
#pragma mark - Authorization
- (BOOL)hasPermission {
return MPMediaLibrary.authorizationStatus == MPMediaLibraryAuthorizationStatusAuthorized;
}
- (NSString *)permissionStatus {
switch (MPMediaLibrary.authorizationStatus) {
case MPMediaLibraryAuthorizationStatusAuthorized: return @"granted";
case MPMediaLibraryAuthorizationStatusDenied: return @"denied";
case MPMediaLibraryAuthorizationStatusRestricted: return @"restricted";
default: return @"not-determined";
}
}
- (void)requestPermission:(void (^)(BOOL))completion {
if ([self hasPermission]) {
completion(YES);
return;
}
[MPMediaLibrary requestAuthorization:^(MPMediaLibraryAuthorizationStatus status) {
dispatch_async(dispatch_get_main_queue(), ^{
[self syncObserving];
[self notifyChanged];
completion(status == MPMediaLibraryAuthorizationStatusAuthorized);
});
}];
}
- (void)appBecameActive:(NSNotification *)notification {
[self syncObserving];
[self notifyChanged];
}
#pragma mark - Observation
- (void)start {
self.started = YES;
[self syncObserving];
}
- (void)stop {
self.started = NO;
[self syncObserving];
}
- (void)syncObserving {
BOOL want = self.started && [self hasPermission];
if (want == self.observing) return;
self.observing = want;
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
NSNotificationCenter *center = NSNotificationCenter.defaultCenter;
if (want) {
[center addObserver:self selector:@selector(playbackStateChanged:)
name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
[center addObserver:self selector:@selector(nowPlayingItemChanged:)
name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
[player beginGeneratingPlaybackNotifications];
} else {
[center removeObserver:self name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
[center removeObserver:self name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
[player endGeneratingPlaybackNotifications];
}
}
- (void)playbackStateChanged:(NSNotification *)notification {
[self notifyChanged];
}
- (void)nowPlayingItemChanged:(NSNotification *)notification {
// Streamed catalog items can share identity fields, so every change counts.
self.artVersion++;
[self notifyChanged];
}
- (void)notifyChanged {
dispatch_async(dispatch_get_main_queue(), ^{
if (self.changeHandler) self.changeHandler();
});
}
#pragma mark - Now playing
- (NSString *)stateJson {
if (![self hasPermission]) return @"{\"access\":false}";
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
MPMediaItem *item = player.nowPlayingItem;
NSString *key = item ? [NSString stringWithFormat:@"%llu|%@|%@|%@", item.persistentID,
item.playbackStoreID ?: @"", item.title ?: @"", item.artist ?: @""] : @"";
if (![key isEqualToString:self.nowPlayingKey]) {
self.nowPlayingKey = key;
self.artVersion++;
}
NSString *state;
switch (player.playbackState) {
case MPMusicPlaybackStatePlaying:
case MPMusicPlaybackStateSeekingForward:
case MPMusicPlaybackStateSeekingBackward:
state = @"playing";
break;
case MPMusicPlaybackStatePaused:
case MPMusicPlaybackStateInterrupted:
state = @"paused";
break;
default:
state = @"stopped";
break;
}
NSTimeInterval position = player.currentPlaybackTime;
NSTimeInterval duration = item.playbackDuration;
float rate = player.currentPlaybackRate;
return JsonString(@{
@"access": @YES,
@"hasItem": @(item != nil),
@"state": state,
@"title": item.title ?: @"",
@"artist": item.artist ?: @"",
@"album": item.albumTitle ?: @"",
@"positionMs": @(item && isfinite(position) && position >= 0 ? position * 1000.0 : -1),
@"durationMs": @(item && isfinite(duration) && duration > 0 ? duration * 1000.0 : -1),
@"rate": @(isfinite(rate) ? rate : 0),
@"artVersion": @(self.artVersion),
}, @"{\"access\":true}");
}
- (void)playPause {
if (![self hasPermission]) return;
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
switch (player.playbackState) {
case MPMusicPlaybackStatePlaying:
case MPMusicPlaybackStateSeekingForward:
case MPMusicPlaybackStateSeekingBackward:
[player pause];
break;
default:
[player play];
break;
}
}
- (void)skipNext {
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToNextItem];
}
- (void)skipPrevious {
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToPreviousItem];
}
- (NSData *)albumArt:(NSInteger)maxSize {
if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil;
MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork;
UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)];
CGImageRef source = image.CGImage;
if (!source) return nil;
double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source);
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
uint8_t *bytes = packet.mutableBytes;
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
uint8_t *pixels = bytes + 4;
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
CGColorSpaceRelease(space);
if (!context) return nil;
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
CGContextDrawImage(context, CGRectMake(0, 0, width, height), source);
CGContextRelease(context);
// Composited over black (the display's see-through background); now opaque.
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
return packet;
}
#pragma mark - Queue
/** Index of the now-playing item in Faceclaw's queue; forgets the queue once the player has left it. */
- (NSInteger)activeQueueIndex {
NSArray<MPMediaItem *> *items = self.queueItems;
if (!items.count || ![self hasPermission]) return -1;
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
MPMediaItem *now = player.nowPlayingItem;
NSInteger found = -1;
if (now) {
NSUInteger index = player.indexOfNowPlayingItem;
if (index < items.count && items[index].persistentID == now.persistentID) {
found = (NSInteger)index;
} else {
for (NSUInteger i = 0; i < items.count; i++) {
if (items[i].persistentID == now.persistentID) { found = (NSInteger)i; break; }
}
}
}
if (found >= 0) {
self.queueConfirmed = YES;
return found;
}
// Either the player hasn't switched over yet, or it has moved on to another queue.
if (self.queueConfirmed || -[self.queueSetAt timeIntervalSinceNow] > kQueueConfirmSeconds) {
self.queueItems = nil;
}
return -1;
}
- (NSString *)queueJson {
NSInteger active = [self activeQueueIndex];
if (active < 0) return @"[]";
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:self.queueItems.count];
[self.queueItems enumerateObjectsUsingBlock:^(MPMediaItem *item, NSUInteger index, BOOL *stop) {
[rows addObject:@{@"id": @(index), @"title": item.title ?: @"", @"active": @((NSInteger)index == active)}];
}];
return JsonString(rows, @"[]");
}
- (void)skipToQueueIndex:(NSInteger)index {
if ([self activeQueueIndex] < 0 || index < 0 || index >= (NSInteger)self.queueItems.count) return;
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
player.nowPlayingItem = self.queueItems[index];
[player play];
}
#pragma mark - Volume
- (UIWindow *)hostWindow {
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
if (![scene isKindOfClass:UIWindowScene.class]) continue;
NSArray<UIWindow *> *windows = ((UIWindowScene *)scene).windows;
for (UIWindow *window in windows) if (window.isKeyWindow) return window;
if (windows.firstObject) return windows.firstObject;
}
return nil;
}
/**
* The slider of an off-screen MPVolumeView: iOS has no public setter for the system volume,
* but the volume view's slider drives it while the view is in a window.
*/
- (UISlider *)volumeSlider:(BOOL *)attached {
if (!self.volumeView) {
self.volumeView = [[MPVolumeView alloc] initWithFrame:CGRectMake(-2000, -2000, 160, 40)];
}
if (!self.volumeView.window) {
UIWindow *window = [self hostWindow];
if (!window) return nil;
[window addSubview:self.volumeView];
if (attached) *attached = YES;
}
for (UIView *view in self.volumeView.subviews) {
if ([view isKindOfClass:UISlider.class]) return (UISlider *)view;
}
return nil;
}
- (double)volumePercent {
// The slider tracks the system volume once attached; until then (nothing has been set
// yet) the session's reading, which can lag while Faceclaw's audio session is inactive.
UISlider *slider = self.volumeView.window ? [self volumeSlider:NULL] : nil;
float volume = slider ? slider.value : AVAudioSession.sharedInstance.outputVolume;
return isfinite(volume) ? MAX(0.0, MIN(100.0, round(volume * 100.0))) : -1;
}
- (void)setVolumePercent:(double)percent {
if (!isfinite(percent)) return;
BOOL attached = NO;
UISlider *slider = [self volumeSlider:&attached];
if (!slider) return;
float value = (float)MAX(0.0, MIN(1.0, percent / 100.0));
void (^apply)(void) = ^{
[slider setValue:value animated:NO];
[slider sendActionsForControlEvents:UIControlEventValueChanged];
};
// A freshly attached volume view ignores changes until it has wired up its slider.
if (attached) dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.15 * NSEC_PER_SEC)), dispatch_get_main_queue(), apply);
else apply();
}
#pragma mark - Library
/**
* Songs of a playable collection id: "playlist:<id>", "album:<id>", "artist:<id>" (album by
* album, in track order) or "songs" (the whole library). Runs on the library queue.
*/
- (NSArray<MPMediaItem *> *)itemsForCollection:(NSString *)collectionId {
if ([collectionId isEqualToString:@"songs"]) return [MPMediaQuery songsQuery].items ?: @[];
NSRange colon = [collectionId rangeOfString:@":"];
MPMediaEntityPersistentID persistentId;
if (colon.location == NSNotFound || !ParseId([collectionId substringFromIndex:colon.location + 1], &persistentId)) return @[];
NSString *kind = [collectionId substringToIndex:colon.location];
MPMediaQuery *query;
NSString *property;
if ([kind isEqualToString:@"playlist"]) {
query = [MPMediaQuery playlistsQuery];
property = MPMediaPlaylistPropertyPersistentID;
} else if ([kind isEqualToString:@"album"]) {
query = [MPMediaQuery albumsQuery];
property = MPMediaItemPropertyAlbumPersistentID;
} else if ([kind isEqualToString:@"artist"]) {
query = [MPMediaQuery albumsQuery];
property = MPMediaItemPropertyArtistPersistentID;
} else {
return @[];
}
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(persistentId) forProperty:property]];
NSMutableArray<MPMediaItem *> *items = [NSMutableArray new];
for (MPMediaItemCollection *collection in query.collections) [items addObjectsFromArray:collection.items];
return items;
}
- (NSArray *)songRows:(NSArray<MPMediaItem *> *)items context:(NSString *)collectionId {
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:items.count];
for (MPMediaItem *item in items) {
[rows addObject:BrowseRow([NSString stringWithFormat:@"song:%@@%@", IdString(item.persistentID), collectionId],
item.title, item.artist, NO, YES)];
}
return rows;
}
- (NSArray *)albumRows:(MPMediaQuery *)query {
NSMutableArray *rows = [NSMutableArray new];
for (MPMediaItemCollection *album in query.collections) {
MPMediaItem *item = album.representativeItem;
if (!item) continue;
[rows addObject:BrowseRow([@"album:" stringByAppendingString:IdString(item.albumPersistentID)],
item.albumTitle, item.albumArtist.length ? item.albumArtist : item.artist, YES, YES)];
}
return rows;
}
- (NSArray *)childrenOf:(NSString *)parentId {
if ([parentId isEqualToString:@"root"]) {
return @[BrowseRow(@"playlists", @"Playlists", nil, YES, NO), BrowseRow(@"artists", @"Artists", nil, YES, NO),
BrowseRow(@"albums", @"Albums", nil, YES, NO), BrowseRow(@"songs", @"Songs", nil, YES, NO)];
}
if ([parentId isEqualToString:@"playlists"]) {
NSMutableArray *rows = [NSMutableArray new];
for (MPMediaPlaylist *playlist in [MPMediaQuery playlistsQuery].collections) {
NSUInteger count = playlist.count;
NSString *subtitle = count == 1 ? @"1 song" : [NSString stringWithFormat:@"%lu songs", (unsigned long)count];
[rows addObject:BrowseRow([@"playlist:" stringByAppendingString:IdString(playlist.persistentID)],
playlist.name, subtitle, YES, count > 0)];
}
return rows;
}
if ([parentId isEqualToString:@"artists"]) {
NSMutableArray *rows = [NSMutableArray new];
for (MPMediaItemCollection *artist in [MPMediaQuery artistsQuery].collections) {
MPMediaItem *item = artist.representativeItem;
if (!item) continue;
[rows addObject:BrowseRow([@"artist:" stringByAppendingString:IdString(item.artistPersistentID)],
item.artist.length ? item.artist : @"Unknown Artist", nil, YES, YES)];
}
return rows;
}
if ([parentId isEqualToString:@"albums"]) return [self albumRows:[MPMediaQuery albumsQuery]];
if ([parentId isEqualToString:@"songs"]) return [self songRows:[MPMediaQuery songsQuery].items ?: @[] context:@"songs"];
if ([parentId hasPrefix:@"artist:"]) {
MPMediaEntityPersistentID artistId;
if (!ParseId([parentId substringFromIndex:7], &artistId)) return nil;
MPMediaQuery *query = [MPMediaQuery albumsQuery];
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(artistId)
forProperty:MPMediaItemPropertyArtistPersistentID]];
return [self albumRows:query];
}
if ([parentId hasPrefix:@"playlist:"] || [parentId hasPrefix:@"album:"]) {
return [self songRows:[self itemsForCollection:parentId] context:parentId];
}
return nil;
}
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *, NSString *))completion {
// Library queries can take a while on large libraries; keep them off the main thread.
dispatch_async(self.libraryQueue, ^{
NSString *json = nil, *error = nil;
if (![self hasPermission]) {
error = @"Apple Music access is not allowed";
} else {
@try {
NSArray *rows = [self childrenOf:parentId ?: @""];
if (rows) json = JsonString(rows, nil);
if (!json) error = rows ? @"Library listing failed" : @"Unknown library item";
} @catch (NSException *exception) {
error = exception.reason ?: @"Library unavailable";
}
}
dispatch_async(dispatch_get_main_queue(), ^{ completion(json, error); });
});
}
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *))completion {
dispatch_async(self.libraryQueue, ^{
NSString *collectionId = mediaId ?: @"";
MPMediaEntityPersistentID startId = 0;
BOOL hasStart = NO;
if ([collectionId hasPrefix:@"song:"]) {
NSRange at = [collectionId rangeOfString:@"@"];
if (at.location == NSNotFound) collectionId = @"";
else {
hasStart = ParseId([collectionId substringWithRange:NSMakeRange(5, at.location - 5)], &startId);
collectionId = [collectionId substringFromIndex:at.location + 1];
}
}
NSArray<MPMediaItem *> *items = @[];
NSString *error = nil;
if (![self hasPermission]) {
error = @"Apple Music access is not allowed";
} else {
@try {
items = [self itemsForCollection:collectionId];
} @catch (NSException *exception) {
error = exception.reason ?: @"Library unavailable";
}
if (!error && !items.count) error = @"Nothing to play";
}
MPMediaItem *start = nil;
if (hasStart) {
for (MPMediaItem *item in items) {
if (item.persistentID == startId) { start = item; break; }
}
}
dispatch_async(dispatch_get_main_queue(), ^{
if (error) {
completion(error);
return;
}
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
MPMusicPlayerMediaItemQueueDescriptor *descriptor = [[MPMusicPlayerMediaItemQueueDescriptor alloc]
initWithItemCollection:[MPMediaItemCollection collectionWithItems:items]];
if (start) descriptor.startItem = start;
[player setQueueWithDescriptor:descriptor];
[player play];
self.queueItems = items;
self.queueConfirmed = NO;
self.queueSetAt = [NSDate date];
[self notifyChanged];
completion(nil);
});
});
}
@end
@@ -2,7 +2,7 @@
#import <WebKit/WebKit.h>
/** One local or URL EvenHub runtime, retained by its TS session until its window closes. */
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKURLSchemeHandler>
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKUIDelegate, WKURLSchemeHandler>
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
@property(nonatomic, copy) NSString *packageIdentifier;
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script;
+183 -34
View File
@@ -1,5 +1,6 @@
#import "FaceclawEvenHubWebView.h"
#import <UIKit/UIKit.h>
#import <SafariServices/SafariServices.h>
#import <UniformTypeIdentifiers/UniformTypeIdentifiers.h>
#import <CommonCrypto/CommonDigest.h>
@@ -7,19 +8,49 @@
@property(nonatomic) WKWebView *webView;
@property(nonatomic) UIView *host;
@property(nonatomic) UIButton *back;
@property(nonatomic) UIButton *returnButton;
@property(nonatomic) NSTimer *timer;
@property(nonatomic) NSString *directory;
@property(nonatomic) NSURL *remoteURL;
@property(nonatomic) NSString *script;
/** The app's start page. Only documents from its origin get the bridge. */
@property(nonatomic) NSURL *startURL;
/** The redirect_uri of a sign-in the app started; the provider's reply is finished inside the app. */
@property(nonatomic) NSURL *returnURL;
/** The main frame shows a page outside the app's origin (a sign-in page, a followed link). */
@property(nonatomic) BOOL away;
@property(nonatomic) BOOL committed;
@property(nonatomic) BOOL ticking;
@property(nonatomic) BOOL destroyed;
@end
static NSInteger FCDefaultPort(NSString *scheme) {
scheme = scheme.lowercaseString;
return [scheme isEqual:@"https"] ? 443 : [scheme isEqual:@"http"] ? 80 : 0;
}
static BOOL FCIsWebURL(NSURL *url) {
NSString *scheme = url.scheme.lowercaseString;
return [scheme isEqual:@"http"] || [scheme isEqual:@"https"];
}
static BOOL FCSameOrigin(NSURL *origin, NSString *scheme, NSString *host, NSInteger port) {
return origin && host.length && [scheme.lowercaseString isEqual:origin.scheme.lowercaseString] &&
[host.lowercaseString isEqual:origin.host.lowercaseString] &&
(port ?: FCDefaultPort(scheme)) == (origin.port.integerValue ?: FCDefaultPort(origin.scheme));
}
/** A newer navigation replaced this one, or the policy below cancelled it; the current page is unaffected. */
static BOOL FCIsInterruption(NSError *error) {
return ([error.domain isEqual:NSURLErrorDomain] && error.code == NSURLErrorCancelled) ||
([error.domain isEqual:@"WebKitErrorDomain"] && error.code == 102); // FrameLoadInterruptedByPolicyChange
}
@implementation FaceclawEvenHubWebView
- (void)emit:(NSDictionary *)event {
if (!_eventHandler || _destroyed) return;
NSData *data = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil];
if (data) _eventHandler([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]);
}
- (void)notice:(NSString *)message {
[self emit:@{@"kind": @"notice", @"message": message.length > 4000 ? [message substringToIndex:4000] : message}];
}
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script {
_directory = directory.stringByStandardizingPath.stringByResolvingSymlinksInPath;
NSURLComponents *url = [NSURLComponents new];
@@ -28,14 +59,14 @@
}
- (void)startURL:(NSString *)url script:(NSString *)script {
NSURL *target = [NSURL URLWithString:url];
if (!target.host.length || ![@[@"http", @"https"] containsObject:target.scheme.lowercaseString] || target.user || target.password) {
if (!target.host.length || !FCIsWebURL(target) || target.user || target.password) {
[self emit:@{@"kind": @"error", @"message": @"Invalid app URL."}]; return;
}
_remoteURL = target;
[self startRequest:target script:script];
}
- (void)startRequest:(NSURL *)url script:(NSString *)script {
if (_destroyed) return;
_startURL = url; _script = script;
WKWebViewConfiguration *config = [WKWebViewConfiguration new];
// Isolate browser storage by package, retaining it across closing/reopening.
// SDK storage also persists independently through the shared TS session.
@@ -49,10 +80,9 @@
}
[config setURLSchemeHandler:self forURLScheme:@"faceclaw-ehpk"];
[config.userContentController addScriptMessageHandler:self name:@"faceclaw"];
[config.userContentController addUserScript:[[WKUserScript alloc] initWithSource:script
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
[self installScript:config.userContentController];
_webView = [[WKWebView alloc] initWithFrame:CGRectZero configuration:config];
_webView.navigationDelegate = self;
_webView.navigationDelegate = self; _webView.UIDelegate = self;
_webView.accessibilityIdentifier = @"evenhub-webview";
UIWindow *window = nil;
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
@@ -73,13 +103,21 @@
[_back setTitle:@"Back to Faceclaw" forState:UIControlStateNormal];
_back.accessibilityIdentifier = @"evenhub-back";
[_back addTarget:self action:@selector(hideOnPhone) forControlEvents:UIControlEventTouchUpInside];
[_host addSubview:_back]; _back.translatesAutoresizingMaskIntoConstraints = NO;
// Shown only while the page is outside the app, e.g. an abandoned sign-in.
_returnButton = [UIButton buttonWithType:UIButtonTypeSystem];
[_returnButton setTitle:@"Return to app" forState:UIControlStateNormal];
_returnButton.accessibilityIdentifier = @"evenhub-return";
_returnButton.hidden = YES;
[_returnButton addTarget:self action:@selector(returnToApp) forControlEvents:UIControlEventTouchUpInside];
UIStackView *bar = [[UIStackView alloc] initWithArrangedSubviews:@[_back, _returnButton]];
bar.distribution = UIStackViewDistributionFillEqually;
[_host addSubview:bar]; bar.translatesAutoresizingMaskIntoConstraints = NO;
[NSLayoutConstraint activateConstraints:@[
[_back.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
[_back.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
[_back.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
[_back.heightAnchor constraintEqualToConstant:44],
[_webView.topAnchor constraintEqualToAnchor:_back.bottomAnchor],
[bar.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
[bar.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
[bar.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
[bar.heightAnchor constraintEqualToConstant:44],
[_webView.topAnchor constraintEqualToAnchor:bar.bottomAnchor],
[_webView.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
[_webView.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
[_webView.bottomAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.bottomAnchor]
@@ -92,7 +130,7 @@
__weak typeof(self) weakSelf = self;
_timer = [NSTimer timerWithTimeInterval:1.0 / 60 repeats:YES block:^(NSTimer *timer) {
typeof(self) strongSelf = weakSelf;
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking) return;
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking || strongSelf.away) return;
strongSelf.ticking = YES;
[strongSelf.webView evaluateJavaScript:@"window.__fcTimerTick && window.__fcTimerTick(); window.__fcRafTick && window.__fcRafTick();" completionHandler:^(id result, NSError *error) {
weakSelf.ticking = NO;
@@ -101,8 +139,18 @@
[NSRunLoop.mainRunLoop addTimer:_timer forMode:NSRunLoopCommonModes];
[_webView loadRequest:[NSURLRequest requestWithURL:url]];
}
/** The bridge scripts run only in the app's own documents, never in pages it navigates to. */
- (void)installScript:(WKUserContentController *)controller {
NSData *start = [NSJSONSerialization dataWithJSONObject:_startURL.absoluteString options:NSJSONWritingFragmentsAllowed error:nil];
NSString *source = [NSString stringWithFormat:@"if ((function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(%@))) {\n%@\n}",
[[NSString alloc] initWithData:start encoding:NSUTF8StringEncoding], _script];
[controller removeAllUserScripts];
[controller addUserScript:[[WKUserScript alloc] initWithSource:source
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
}
- (void)evaluate:(NSString *)script {
if (!_destroyed) [_webView evaluateJavaScript:script completionHandler:nil];
// Host messages are for the app's documents; a page it navigated to must not receive them.
if (!_destroyed && !_away) [_webView evaluateJavaScript:script completionHandler:nil];
}
- (void)showOnPhone {
if (_destroyed) return;
@@ -112,51 +160,152 @@
[_host endEditing:YES]; [_host.superview sendSubviewToBack:_host]; _host.userInteractionEnabled = NO; _host.accessibilityElementsHidden = YES;
[self emit:@{@"kind": @"hidden"}];
}
- (void)returnToApp {
if (_destroyed) return;
_returnURL = nil; [_webView loadRequest:[NSURLRequest requestWithURL:_startURL]];
}
- (void)destroy {
if (_destroyed) return;
_destroyed = YES; [_timer invalidate]; _timer = nil;
[_webView stopLoading]; _webView.navigationDelegate = nil;
[_webView stopLoading]; _webView.navigationDelegate = nil; _webView.UIDelegate = nil;
[_webView.configuration.userContentController removeScriptMessageHandlerForName:@"faceclaw"];
[_webView.configuration.userContentController removeAllUserScripts];
[_host removeFromSuperview]; _host = nil; _back = nil; _webView = nil; _eventHandler = nil;
[_host removeFromSuperview]; _host = nil; _back = nil; _returnButton = nil; _webView = nil; _eventHandler = nil;
}
- (BOOL)allowsURL:(NSURL *)url {
if (!_remoteURL) return [url.scheme isEqual:@"faceclaw-ehpk"] && [url.host isEqual:@"app"];
NSNumber *port = url.port ?: @([url.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
NSNumber *remotePort = _remoteURL.port ?: @([_remoteURL.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
return [url.scheme.lowercaseString isEqual:_remoteURL.scheme.lowercaseString] &&
[url.host.lowercaseString isEqual:_remoteURL.host.lowercaseString] && [port isEqual:remotePort];
- (BOOL)isAppURL:(NSURL *)url {
return FCSameOrigin(_startURL, url.scheme, url.host, url.port.integerValue);
}
- (BOOL)isAppFrame:(WKFrameInfo *)frame {
WKSecurityOrigin *origin = frame.securityOrigin;
return frame.isMainFrame && FCSameOrigin(_startURL, origin.protocol, origin.host, origin.port);
}
/** An OAuth authorization request names where the provider replies. */
- (void)rememberReturnURLFrom:(NSURL *)url {
for (NSURLQueryItem *item in [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO].queryItems) {
NSURL *target = [item.name isEqual:@"redirect_uri"] && item.value ? [NSURL URLWithString:item.value] : nil;
// A redirect_uri on the app's own origin needs no help: the app handles it itself.
if (FCIsWebURL(target) && target.host.length && ![self isAppURL:target]) _returnURL = target;
}
}
- (BOOL)isReturnURL:(NSURL *)url {
// NSURL.path drops a trailing slash, so ".../callback/" and ".../callback" match.
NSString *path = url.path.length ? url.path : @"/", *returnPath = _returnURL.path.length ? _returnURL.path : @"/";
return FCSameOrigin(_returnURL, url.scheme, url.host, url.port.integerValue) && [path isEqual:returnPath];
}
/** The app's start page carrying the provider's reply (its query and fragment). */
- (void)loadAppWithResultOf:(NSURL *)url {
NSURLComponents *result = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO];
NSURLComponents *app = [NSURLComponents componentsWithURL:_startURL resolvingAgainstBaseURL:NO];
if (result.percentEncodedQuery.length) app.percentEncodedQuery = app.percentEncodedQuery.length ?
[NSString stringWithFormat:@"%@&%@", app.percentEncodedQuery, result.percentEncodedQuery] : result.percentEncodedQuery;
if (result.percentEncodedFragment) app.percentEncodedFragment = result.percentEncodedFragment;
NSURL *target = app.URL;
_returnURL = nil;
dispatch_async(dispatch_get_main_queue(), ^{
if (!self.destroyed) [self.webView loadRequest:[NSURLRequest requestWithURL:target]];
});
}
- (void)userContentController:(WKUserContentController *)controller didReceiveScriptMessage:(WKScriptMessage *)message {
// Subframes and navigated third-party documents never receive host access.
NSURL *url = message.frameInfo.request.URL;
if (!message.frameInfo.isMainFrame || ![self allowsURL:url]) return;
// Only the app's own main frame gets host access: not subframes, nor pages the app navigated to.
if (![self isAppFrame:message.frameInfo]) return;
if ([message.body isKindOfClass:NSDictionary.class]) [self emit:message.body];
}
- (void)webView:(WKWebView *)webView decidePolicyForNavigationAction:(WKNavigationAction *)action decisionHandler:(void (^)(WKNavigationActionPolicy))decisionHandler {
NSURL *url = action.request.URL;
BOOL allowed = [self allowsURL:url] && action.targetFrame.isMainFrame;
if (!allowed && action.targetFrame.isMainFrame)
[self emit:@{@"kind": @"error", @"message": @"Navigation outside the app origin is blocked. Open the destination as a separate app URL."}];
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel);
NSString *scheme = url.scheme.lowercaseString;
// A new window (target=_blank, window.open): createWebView below opens it outside the app.
if (!action.targetFrame) { decisionHandler(WKNavigationActionPolicyAllow); return; }
// Subframes load as in a browser; they never get the bridge.
if (!action.targetFrame.isMainFrame) {
BOOL allowed = FCIsWebURL(url) || [self isAppURL:url] || [@[@"about", @"data", @"blob"] containsObject:scheme];
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel); return;
}
if ([self isAppURL:url]) { _returnURL = nil; decisionHandler(WKNavigationActionPolicyAllow); return; }
if (FCIsWebURL(url) && !_committed && !_directory.length) {
// The app URL redirected before anything loaded: the app lives where it lands. The
// script update reaches the web process ahead of the policy reply.
_startURL = url; [self installScript:webView.configuration.userContentController];
decisionHandler(WKNavigationActionPolicyAllow); return;
}
if (_returnURL && [self isReturnURL:url]) {
// A sign-in provider replying: finish in the app rather than at the page the redirect_uri names.
decisionHandler(WKNavigationActionPolicyCancel); [self loadAppWithResultOf:url]; return;
}
if (!FCIsWebURL(url)) {
[self notice:[NSString stringWithFormat:@"Blocked navigation to a %@: URL.", scheme ?: @"schemeless"]];
decisionHandler(WKNavigationActionPolicyCancel); return;
}
// Leaving the app's origin. Such pages never get the bridge; "Return to app" leads back.
if ([self isAppFrame:action.sourceFrame]) [self rememberReturnURLFrom:url];
decisionHandler(WKNavigationActionPolicyAllow);
}
- (void)webView:(WKWebView *)webView didCommitNavigation:(WKNavigation *)navigation {
_committed = YES; _away = ![self isAppURL:webView.URL]; _returnButton.hidden = !_away;
}
- (void)webView:(WKWebView *)webView didFinishNavigation:(WKNavigation *)navigation {
[self emit:@{@"kind": @"loaded"}];
if (!_away) [self emit:@{@"kind": @"loaded"}];
}
- (void)webView:(WKWebView *)webView didFailProvisionalNavigation:(WKNavigation *)navigation withError:(NSError *)error {
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
}
- (void)webView:(WKWebView *)webView didFailNavigation:(WKNavigation *)navigation withError:(NSError *)error {
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
}
- (void)webViewWebContentProcessDidTerminate:(WKWebView *)webView {
[self emit:@{@"kind": @"error", @"message": @"The app's web process ended. Close and reopen the app."}];
[self emit:@{@"kind": @"error", @"fatal": @YES, @"message": @"The app's web process ended. Close and reopen the app."}];
}
/** What to present over, only while the app's phone UI is shown and not mid-transition. */
- (UIViewController *)presenter {
if (_destroyed || !_host.userInteractionEnabled) return nil;
UIViewController *controller = _host.window.rootViewController;
while (controller.presentedViewController) controller = controller.presentedViewController;
return controller.transitionCoordinator ? nil : controller;
}
- (WKWebView *)webView:(WKWebView *)webView createWebViewWithConfiguration:(WKWebViewConfiguration *)configuration forNavigationAction:(WKNavigationAction *)action windowFeatures:(WKWindowFeatures *)windowFeatures {
NSURL *url = action.request.URL;
UIViewController *presenter = FCIsWebURL(url) ? [self presenter] : nil;
if (presenter) [presenter presentViewController:[[SFSafariViewController alloc] initWithURL:url] animated:YES completion:nil];
else [self notice:[NSString stringWithFormat:@"Did not open a window for a %@: URL.", url.scheme ?: @"schemeless"]];
return nil;
}
// JavaScript dialogs. WebKit requires each completion handler to run exactly once, so a
// dialog that cannot be shown (the app is hidden) completes at once with its default.
- (UIAlertController *)dialog:(NSString *)message {
if ([self presenter]) return [UIAlertController alertControllerWithTitle:nil message:message preferredStyle:UIAlertControllerStyleAlert];
[self notice:[@"Dialog while hidden: " stringByAppendingString:message ?: @""]];
return nil;
}
- (BOOL)presentDialog:(UIAlertController *)alert {
if (!alert) return NO;
[[self presenter] presentViewController:alert animated:YES completion:nil];
return alert.presentingViewController != nil;
}
- (void)webView:(WKWebView *)webView runJavaScriptAlertPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(void))completionHandler {
UIAlertController *alert = [self dialog:message];
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(); }]];
if (![self presentDialog:alert]) completionHandler();
}
- (void)webView:(WKWebView *)webView runJavaScriptConfirmPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(BOOL))completionHandler {
UIAlertController *alert = [self dialog:message];
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(NO); }]];
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(YES); }]];
if (![self presentDialog:alert]) completionHandler(NO);
}
- (void)webView:(WKWebView *)webView runJavaScriptTextInputPanelWithPrompt:(NSString *)prompt defaultText:(NSString *)defaultText initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(NSString *))completionHandler {
UIAlertController *alert = [self dialog:prompt];
[alert addTextFieldWithConfigurationHandler:^(UITextField *field) { field.text = defaultText; }];
__weak UIAlertController *weakAlert = alert;
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(nil); }]];
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) {
completionHandler(weakAlert.textFields.firstObject.text ?: @"");
}]];
if (![self presentDialog:alert]) completionHandler(nil);
}
- (void)webView:(WKWebView *)webView startURLSchemeTask:(id<WKURLSchemeTask>)task {
NSURL *url = task.request.URL;
NSString *relative = [url.path stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@"/"]];
NSString *file = [[_directory stringByAppendingPathComponent:relative] stringByStandardizingPath].stringByResolvingSymlinksInPath;
BOOL allowed = !_destroyed && _directory.length && !_remoteURL && [url.host isEqual:@"app"] &&
BOOL allowed = !_destroyed && _directory.length && [url.host isEqual:@"app"] &&
[file hasPrefix:[_directory stringByAppendingString:@"/"]];
NSData *data = allowed ? [NSData dataWithContentsOfFile:file options:NSDataReadingMappedIfSafe error:nil] : nil;
NSString *ext = file.pathExtension.lowercaseString;
+2
View File
@@ -2,6 +2,8 @@
@interface FaceclawGraphics : NSObject
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
/** A glasses icon as white-on-transparent (gray becomes alpha), for the dark header. */
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height;
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
+ (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height;
+ (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight;
+17
View File
@@ -76,6 +76,23 @@
CGColorSpaceRelease(space);
return result;
}
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height {
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
const uint8_t *src = gray.bytes;
uint8_t *dst = rgba.mutableBytes;
// Premultiplied white: every channel equals the alpha.
for (NSInteger i = 0; i < width * height; i++) memset(dst + i * 4, src[i], 4);
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
if (image) CGImageRelease(image);
CGDataProviderRelease(provider);
CGColorSpaceRelease(space);
return result;
}
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
if (size <= 0 || size > 1024) return nil;
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
+3
View File
@@ -2,7 +2,10 @@
/** Thread-safe event queue; each NativeScript worker drains it on its own thread. */
@interface FaceclawSocket : NSObject <NSURLSessionWebSocketDelegate>
- (instancetype)initWithURL:(NSString *)url;
/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue;
- (NSString *)takeEvents;
- (void)sendText:(NSString *)text;
- (void)sendData:(NSData *)data;
- (void)close;
@end
+20 -3
View File
@@ -8,12 +8,17 @@
@end
@implementation FaceclawSocket
- (instancetype)initWithURL:(NSString *)url {
return [self initWithURL:url headerName:nil headerValue:nil];
}
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue {
if ((self = [super init])) {
_events = [NSMutableArray array];
NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration];
config.timeoutIntervalForRequest = 20;
_session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
_task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]];
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]];
if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName];
_task = [_session webSocketTaskWithRequest:request];
_task.maximumMessageSize = 8 * 1024 * 1024;
[_task resume];
}
@@ -25,7 +30,7 @@
_queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) {
[_events removeAllObjects]; _queuedBytes = 0;
[_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}];
[_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}];
[self close];
} else [_events addObject:event];
}
@@ -56,7 +61,13 @@
[self close];
}
- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error {
if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}];
if (error) {
// A refused handshake only says "bad response"; name the HTTP status
// so callers can tell a rejected key (401/403) from a network error.
NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0;
NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription;
[self enqueue:@{@"kind": @"error", @"text": text}];
}
[self close];
}
- (void)sendText:(NSString *)text {
@@ -65,6 +76,12 @@
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
}];
}
- (void)sendData:(NSData *)data {
__weak FaceclawSocket *weakSelf = self;
[_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) {
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
}];
}
- (void)close {
@synchronized(self) {
if (_ended) return; _ended = YES;
+5
View File
@@ -2,6 +2,8 @@
/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */
@interface FaceclawSpeech : NSObject
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */
@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm);
+ (NSInteger)authorizationStatus;
+ (NSInteger)microphoneAuthorizationStatus;
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion;
@@ -9,6 +11,9 @@
/** Returns an actionable error, or an empty string when capture starts. */
- (NSString *)startWithEndpointing:(BOOL)endpointing;
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing;
/** Capture without the recognizer (for a cloud provider): audio goes to
* pcmHandler, and "ended" follows the last chunk. No transcript events. */
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone;
- (void)acceptPacket:(NSData *)packet;
- (void)finish;
- (void)cancel;
+89 -26
View File
@@ -15,6 +15,7 @@
@property(atomic) NSUInteger generation;
@property(nonatomic) NSUInteger pendingPackets;
@property(atomic) BOOL accepting;
@property(nonatomic) BOOL pcmOutput;
@property(nonatomic, strong) AVAudioEngine *phoneEngine;
@property(nonatomic) BOOL phoneTapInstalled;
@property(nonatomic) BOOL phoneSessionActive;
@@ -23,6 +24,23 @@
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
@end
/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial
* audio queue; the converter keeps its filter state between buffers. */
static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) {
AVAudioFormat *format = converter.outputFormat;
AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32;
AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity];
__block BOOL supplied = NO;
NSError *error = nil;
AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error
withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) {
// NoDataNow (not EndOfStream) leaves the stream open for the next buffer.
if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; }
supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input;
}];
if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil;
return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)];
}
@implementation FaceclawSpeech
- (instancetype)init {
if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL);
@@ -46,37 +64,51 @@
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
}
- (NSString *)startWithEndpointing:(BOOL)endpointing {
return [self startWithEndpointing:endpointing phone:NO];
return [self startWithEndpointing:endpointing phone:NO pcm:NO];
}
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing {
return [self startWithEndpointing:endpointing phone:YES];
return [self startWithEndpointing:endpointing phone:YES pcm:NO];
}
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
return [self startWithEndpointing:endpointing phone:phone pcm:YES];
}
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm {
[self cancel];
if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized)
return @"Allow Microphone access for Faceclaw in iPhone Settings.";
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US";
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
if (!self.recognizer.supportsOnDeviceRecognition)
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
if (!pcm) {
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
if (!self.recognizer.supportsOnDeviceRecognition)
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
}
if (!phone) {
self.decoder = [FaceclawLc3Decoder new];
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
}
self.pcmOutput = pcm;
self.bestText = @""; self.accepting = YES;
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
NSUInteger generation = self.generation;
__weak FaceclawSpeech *weakSelf = self;
if (pcm) {
if (phone) {
NSString *error = [self startPhoneAudio];
if (error.length) { [self cancel]; return error; }
}
// The PCM consumer reports its own status and has no session limit.
return @"";
}
self.recognizer.queue = NSOperationQueue.mainQueue;
self.request = [SFSpeechAudioBufferRecognitionRequest new];
self.request.requiresOnDeviceRecognition = YES;
self.request.shouldReportPartialResults = YES;
self.request.taskHint = SFSpeechRecognitionTaskHintDictation;
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
self.bestText = @""; self.accepting = YES;
self.transcript = [FaceclawSpeechTranscript new];
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
NSUInteger generation = self.generation;
__weak FaceclawSpeech *weakSelf = self;
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
dispatch_async(dispatch_get_main_queue(), ^{
FaceclawSpeech *owner = weakSelf;
@@ -124,6 +156,13 @@
AVAudioInputNode *input = self.phoneEngine.inputNode;
AVAudioFormat *format = [input outputFormatForBus:0];
if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again.";
AVAudioConverter *converter = nil;
if (self.pcmOutput) {
AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES];
converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat];
if (!converter) return @"Could not convert phone microphone audio.";
converter.downmix = YES;
}
NSUInteger generation = self.generation;
SFSpeechAudioBufferRecognitionRequest *request = self.request;
FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector;
@@ -141,7 +180,10 @@
memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize);
dispatch_async(owner.audioQueue, ^{ @autoreleasepool {
if (owner.generation != generation) return;
[request appendAudioPCMBuffer:copy];
if (converter) {
NSData *pcm = resampleToPcm(converter, copy);
if (pcm.length) [owner deliverPcm:pcm generation:generation];
} else [request appendAudioPCMBuffer:copy];
double sum = 0;
if (copy.floatChannelData) {
const float *samples = copy.floatChannelData[0];
@@ -188,6 +230,7 @@
FaceclawLc3Decoder *decoder = self.decoder;
SFSpeechAudioBufferRecognitionRequest *request = self.request;
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
BOOL pcmOutput = self.pcmOutput;
NSData *copy = [packet copy];
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
if (self.generation != generation) return;
@@ -195,17 +238,26 @@
double rms = 0;
BOOL speechEnded = NO;
if (pcm.length) {
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
buffer.frameLength = buffer.frameCapacity;
const int16_t *samples = pcm.bytes;
for (NSUInteger i = 0; i < buffer.frameLength; i++) {
float value = samples[i] / 32768.0f;
buffer.floatChannelData[0][i] = value; rms += value * value;
AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2);
AVAudioPCMBuffer *buffer = nil;
if (!pcmOutput) {
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames];
buffer.frameLength = frames;
}
rms = sqrt(rms / buffer.frameLength);
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
const int16_t *samples = pcm.bytes;
for (NSUInteger i = 0; i < frames; i++) {
float value = samples[i] / 32768.0f;
if (buffer) buffer.floatChannelData[0][i] = value;
rms += value * value;
}
rms = sqrt(rms / frames);
if (self.generation == generation) {
// The decoder's output is already the PCM consumer's format.
if (pcmOutput) [self deliverPcm:pcm generation:generation];
else [request appendAudioPCMBuffer:buffer];
}
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames];
}
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
dispatch_async(dispatch_get_main_queue(), ^{
@@ -223,9 +275,15 @@
[self stopPhoneAudio];
NSUInteger generation = self.generation;
SFSpeechAudioBufferRecognitionRequest *request = self.request;
BOOL pcmOutput = self.pcmOutput;
[self emit:@{@"kind":@"finishing"}];
// Drain already received audio before marking the utterance complete.
dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; });
dispatch_async(self.audioQueue, ^{
if (self.generation != generation) return;
if (!pcmOutput) { [request endAudio]; return; }
// Drained chunks are already queued for pcmHandler; "ended" follows them.
dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; });
});
__weak FaceclawSpeech *weakSelf = self;
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
FaceclawSpeech *owner = weakSelf;
@@ -239,9 +297,14 @@
if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}];
if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}];
}
- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation {
dispatch_async(dispatch_get_main_queue(), ^{
if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm);
});
}
- (void)cancel {
self.generation++;
self.accepting = NO; self.pendingPackets = 0;
self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO;
[self stopPhoneAudio];
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
+1
View File
@@ -6,6 +6,7 @@ module FaceclawGraphics {
header "FaceclawSpeech.h"
header "FaceclawLocation.h"
header "FaceclawCalendar.h"
header "FaceclawAppleMusic.h"
header "FaceclawConfigPort.h"
header "FaceclawEvenHubWebView.h"
header "FaceclawCrypto.h"