mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' into ring-health-fixes
This commit is contained in:
commit
4f8c082b52
368 files changed
+26238
-3401
No files matched your search
@@ -46,6 +46,17 @@ def faceclawNdkHostTag = {
|
||||
return "linux-x86_64"
|
||||
}
|
||||
|
||||
// Host tools carry .exe on Windows. Call the binaries themselves: the NDK's
|
||||
// target-prefixed clang wrappers are bash scripts (or .cmd files) that only add
|
||||
// --target.
|
||||
def faceclawHostExecutable = { File dir, String name ->
|
||||
return file("$dir/$name${Os.isFamily(Os.FAMILY_WINDOWS) ? ".exe" : ""}")
|
||||
}
|
||||
|
||||
def faceclawNdkTool = { File ndkRoot, String name ->
|
||||
return faceclawHostExecutable(file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin"), name)
|
||||
}
|
||||
|
||||
def faceclawFindNdkRoot = {
|
||||
def explicitNdk = System.getenv("ANDROID_NDK_HOME")
|
||||
if (explicitNdk != null && file(explicitNdk).exists()) {
|
||||
@@ -111,7 +122,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
if (ndkRoot == null || !ndkRoot.exists()) {
|
||||
throw new GradleException("Android NDK is required to build libfaceclaw_lc3.so")
|
||||
}
|
||||
def clang = file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/aarch64-linux-android24-clang")
|
||||
def clang = faceclawNdkTool(ndkRoot, "clang")
|
||||
if (!clang.exists()) {
|
||||
throw new GradleException("Could not find Android clang at $clang")
|
||||
}
|
||||
@@ -125,6 +136,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
exec {
|
||||
commandLine([
|
||||
clang.absolutePath,
|
||||
"--target=aarch64-linux-android24",
|
||||
"-shared",
|
||||
"-fPIC",
|
||||
"-O3",
|
||||
@@ -192,8 +204,8 @@ tasks.register("prepareFaceclawLlama") {
|
||||
if (cmakeDir == null) {
|
||||
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
|
||||
}
|
||||
def cmake = file("$cmakeDir/bin/cmake")
|
||||
def ninja = file("$cmakeDir/bin/ninja")
|
||||
def cmake = faceclawHostExecutable(file("$cmakeDir/bin"), "cmake")
|
||||
def ninja = faceclawHostExecutable(file("$cmakeDir/bin"), "ninja")
|
||||
def march = "-march=armv8.2-a+dotprod+fp16"
|
||||
|
||||
// Configure once; afterwards the ninja no-op build keeps this cheap while
|
||||
@@ -227,14 +239,13 @@ tasks.register("prepareFaceclawLlama") {
|
||||
}
|
||||
// The static-linked lib carries llama.cpp's full debug info (~50MB);
|
||||
// strip it so the APK only pays for the code (~8MB).
|
||||
def ndkRootForStrip = faceclawFindNdkRoot()
|
||||
if (ndkRootForStrip != null) {
|
||||
def llvmStrip = file("$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
def llvmStrip = faceclawNdkTool(ndkRoot, "llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
} else {
|
||||
logger.warn("Could not find llvm-strip at $llvmStrip; packaging an unstripped libfaceclaw_llama.so (~50MB)")
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -251,7 +262,8 @@ if (faceclawVersionFile.exists()) {
|
||||
}
|
||||
}
|
||||
def faceclawVersionCode = {
|
||||
def parts = (faceclawVersionName.tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
// Prerelease labels belong in versionName, not the numeric versionCode.
|
||||
def parts = (faceclawVersionName.split('-')[0].tokenize('.') + ["0", "0"]).take(3)*.toInteger()
|
||||
return parts[0] * 10000 + parts[1] * 100 + parts[2]
|
||||
}()
|
||||
|
||||
|
||||
@@ -13,7 +13,8 @@ import java.io.IOException
|
||||
/**
|
||||
* Transcodes a 16-bit PCM WAV conversation recording into AAC-LC in an M4A
|
||||
* container, cutting speech recordings to roughly a tenth of their WAV size.
|
||||
* Runs on its own thread; the source WAV is deleted on success when requested.
|
||||
* Runs on its own thread; when requested, the source WAV is deleted once the
|
||||
* listener's onDone has returned normally.
|
||||
*/
|
||||
class FaceclawAudioTranscoder(
|
||||
private val wavPath: String?,
|
||||
@@ -66,19 +67,38 @@ class FaceclawAudioTranscoder(
|
||||
private fun run() {
|
||||
try {
|
||||
transcode()
|
||||
val size = File(outputPath).length()
|
||||
if (deleteSourceOnSuccess) {
|
||||
File(wavPath).delete()
|
||||
}
|
||||
if (listener != null) {
|
||||
listener.onDone(outputPath, size)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "transcode failed for $wavPath", t)
|
||||
File(outputPath).delete()
|
||||
if (listener != null) {
|
||||
listener.onError(if (t.message == null) t.toString() else t.message)
|
||||
try {
|
||||
listener?.onError(if (t.message == null) t.toString() else t.message)
|
||||
} catch (listenerError: Throwable) {
|
||||
Log.w(TAG, "transcode error listener failed", listenerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
val size = File(outputPath).length()
|
||||
if (size <= 0) {
|
||||
Log.w(TAG, "transcode produced no output for $wavPath")
|
||||
File(outputPath).delete()
|
||||
try {
|
||||
listener?.onError("transcode produced no output")
|
||||
} catch (listenerError: Throwable) {
|
||||
Log.w(TAG, "transcode error listener failed", listenerError)
|
||||
}
|
||||
return
|
||||
}
|
||||
try {
|
||||
listener?.onDone(outputPath, size)
|
||||
} catch (t: Throwable) {
|
||||
// The caller never took up the new file, so whatever it recorded still names the
|
||||
// source: keep that and drop the output, rather than lose both.
|
||||
Log.w(TAG, "transcode listener failed for $wavPath", t)
|
||||
File(outputPath).delete()
|
||||
return
|
||||
}
|
||||
if (deleteSourceOnSuccess) {
|
||||
File(wavPath).delete()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -93,6 +113,7 @@ class FaceclawAudioTranscoder(
|
||||
|
||||
val codec = MediaCodec.createEncoderByType(MediaFormat.MIMETYPE_AUDIO_AAC)
|
||||
var muxer: MediaMuxer? = null
|
||||
var muxerStopped = false
|
||||
var input: FileInputStream? = null
|
||||
try {
|
||||
codec.configure(format, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE)
|
||||
@@ -159,6 +180,11 @@ class FaceclawAudioTranscoder(
|
||||
}
|
||||
}
|
||||
}
|
||||
// stop() writes the index (moov); without it the file won't play, so its failure
|
||||
// (storage full, I/O error, a muxer that never started) fails the transcode rather
|
||||
// than reporting a broken output as done.
|
||||
muxerStopped = true
|
||||
muxer.stop()
|
||||
} finally {
|
||||
if (input != null) {
|
||||
try {
|
||||
@@ -172,9 +198,12 @@ class FaceclawAudioTranscoder(
|
||||
}
|
||||
codec.release()
|
||||
if (muxer != null) {
|
||||
try {
|
||||
muxer.stop()
|
||||
} catch (ignored: Throwable) {
|
||||
if (!muxerStopped) {
|
||||
// Already failing; the output is deleted either way.
|
||||
try {
|
||||
muxer.stop()
|
||||
} catch (ignored: Throwable) {
|
||||
}
|
||||
}
|
||||
muxer.release()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
|
||||
import java.io.IOException
|
||||
import java.nio.ByteBuffer
|
||||
import java.util.concurrent.TimeUnit
|
||||
|
||||
import okhttp3.Call
|
||||
import okhttp3.Callback
|
||||
import okhttp3.OkHttpClient
|
||||
import okhttp3.Request
|
||||
import okhttp3.RequestBody.Companion.toRequestBody
|
||||
import okhttp3.Response
|
||||
|
||||
/**
|
||||
* One POST of a binary body (a Whisper server WAV upload), sent by start().
|
||||
* NativeScript's http module hands an ArrayBuffer body to Java one bridge call
|
||||
* per byte, which for a ~1 MB upload stalls the JS thread; here the body
|
||||
* arrives as a direct ByteBuffer (copyToJavaByteBuffer) and is copied in bulk.
|
||||
* headers is a flat [name, value, ...] array, as for FaceclawSseRequest, and
|
||||
* may carry the Content-Type. The response text and status reach the listener
|
||||
* on the constructing thread's Looper.
|
||||
*/
|
||||
class FaceclawBinaryPost(
|
||||
url: String?,
|
||||
body: ByteBuffer?,
|
||||
headers: Array<String?>?,
|
||||
timeoutMs: Long,
|
||||
listener: FaceclawBinaryPostListener?
|
||||
) {
|
||||
private val callbackHandler = Handler(Looper.myLooper() ?: Looper.getMainLooper())
|
||||
private val listener: FaceclawBinaryPostListener =
|
||||
listener ?: throw IllegalArgumentException("listener is required")
|
||||
private val call: Call
|
||||
|
||||
@Volatile
|
||||
private var cancelled = false
|
||||
|
||||
init {
|
||||
if (url == null || url.trim().isEmpty()) {
|
||||
throw IllegalArgumentException("url is required")
|
||||
}
|
||||
val bytes = ByteArray(body?.remaining() ?: 0)
|
||||
body?.duplicate()?.get(bytes)
|
||||
// No media type on the body, so okhttp keeps the caller's Content-Type header.
|
||||
val builder = Request.Builder().url(url.trim()).post(bytes.toRequestBody())
|
||||
for ((name, value) in SseStream.unpackHeaders(headers)) builder.addHeader(name, value)
|
||||
call = getClient().newCall(builder.build())
|
||||
call.timeout().timeout(timeoutMs, TimeUnit.MILLISECONDS)
|
||||
}
|
||||
|
||||
fun start() {
|
||||
call.enqueue(object : Callback {
|
||||
override fun onFailure(call: Call, e: IOException) {
|
||||
post { listener.onFailure(e.message ?: e.toString()) }
|
||||
}
|
||||
|
||||
override fun onResponse(call: Call, response: Response) {
|
||||
val text = try {
|
||||
response.use { it.body?.string() ?: "" }
|
||||
} catch (e: IOException) {
|
||||
post { listener.onFailure(e.message ?: e.toString()) }
|
||||
return
|
||||
}
|
||||
post { listener.onResponse(response.code, text) }
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
/** Aborts the request; no callback follows. */
|
||||
fun cancel() {
|
||||
cancelled = true
|
||||
call.cancel()
|
||||
}
|
||||
|
||||
private fun post(runnable: () -> Unit) {
|
||||
callbackHandler.post {
|
||||
if (!cancelled) runnable()
|
||||
}
|
||||
}
|
||||
|
||||
companion object {
|
||||
@Volatile
|
||||
private var sharedClient: OkHttpClient? = null
|
||||
|
||||
private fun getClient(): OkHttpClient {
|
||||
if (sharedClient == null) {
|
||||
synchronized(FaceclawBinaryPost::class.java) {
|
||||
if (sharedClient == null) {
|
||||
sharedClient = OkHttpClient.Builder()
|
||||
.connectTimeout(30, TimeUnit.SECONDS)
|
||||
.readTimeout(60, TimeUnit.SECONDS)
|
||||
.addInterceptor(FaceclawHttp.userAgentInterceptor())
|
||||
.build()
|
||||
}
|
||||
}
|
||||
}
|
||||
return sharedClient!!
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -294,7 +294,7 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
|
||||
fun removeMicStatusListener(listener: FaceclawMicStatusListener?) = core.removeMicStatusListener(listener)
|
||||
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean) =
|
||||
fun sendFaceclawMicControl(record: ByteArray?, label: String, rightTemple: Boolean, leftTemple: Boolean): Boolean =
|
||||
core.sendFaceclawMicControl(record, label, rightTemple, leftTemple)
|
||||
|
||||
fun startG2AudioForwarding(listener: FaceclawAudioPacketListener?): Boolean = core.startG2AudioForwarding(listener)
|
||||
@@ -329,6 +329,15 @@ class FaceclawBleCommunicator(context: Context, rightAddress: String?, leftAddre
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||
}
|
||||
|
||||
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||
val composite = core.previewComposite() ?: return null
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||
cropX, cropY, cropWidth, cropHeight)
|
||||
}
|
||||
|
||||
/**
|
||||
* Posted to the main looper after each step of an animation the preview is replaying (menu
|
||||
* highlight slides, scrolls), so the mirror can pull the new frame; frame metrics only cover
|
||||
|
||||
@@ -550,6 +550,7 @@ class FaceclawBleManager(context: Context) {
|
||||
override fun onConnectionStateChange(gatt: BluetoothGatt, status: Int, newState: Int) {
|
||||
val address = gatt.device.address
|
||||
Log.i(TAG, "onConnectionStateChange: address=$address status=$status newState=$newState")
|
||||
if (status != BluetoothGatt.GATT_SUCCESS) ConnectionCounters.increment("ble.gatt-status.$status")
|
||||
val link = links[address]
|
||||
val current = link?.gatt
|
||||
if (link == null || (current != null && current !== gatt)) {
|
||||
|
||||
@@ -68,6 +68,11 @@ class FaceclawCaptionEngine {
|
||||
session.stop()
|
||||
}
|
||||
|
||||
/** Stop without blocking; the listener's onStopped follows the trailing utterance. */
|
||||
fun requestStop() {
|
||||
session.requestStop()
|
||||
}
|
||||
|
||||
/** Push decoded 16 kHz mono S16LE PCM (any chunking). */
|
||||
fun acceptPcm(pcm16le: ByteArray?) {
|
||||
session.acceptPcm(pcm16le)
|
||||
|
||||
@@ -718,6 +718,94 @@ class FaceclawConversationStore private constructor(context: Context) : SQLiteOp
|
||||
}
|
||||
}
|
||||
|
||||
// ---- crash recovery ----
|
||||
|
||||
/**
|
||||
* Close out sessions a dead process left open (ended_at NULL): the end becomes the last
|
||||
* stored segment's end (or the start), and the tone average comes from the stored segments.
|
||||
* excludeIdsJson is a JSON array of session ids that are legitimately still open. Returns
|
||||
* the sessions it closed as a JSON array of {id, audioPath, audioCodec}.
|
||||
*/
|
||||
fun closeInterruptedSessions(excludeIdsJson: String?): String {
|
||||
val exclude = HashSet<Long>()
|
||||
if (excludeIdsJson != null && !excludeIdsJson.isEmpty()) {
|
||||
val ids = JSONArray(excludeIdsJson)
|
||||
for (i in 0 until ids.length()) {
|
||||
exclude.add(ids.getLong(i))
|
||||
}
|
||||
}
|
||||
val db = writableDatabase
|
||||
val closed = JSONArray()
|
||||
val closedIds = ArrayList<Long>()
|
||||
db.rawQuery("SELECT id, audio_path, audio_codec FROM sessions WHERE ended_at IS NULL", null).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
val id = c.getLong(0)
|
||||
if (id in exclude) continue
|
||||
closedIds.add(id)
|
||||
val row = JSONObject()
|
||||
row.put("id", id)
|
||||
row.put("audioPath", if (c.isNull(1)) JSONObject.NULL else c.getString(1))
|
||||
row.put("audioCodec", if (c.isNull(2)) JSONObject.NULL else c.getString(2))
|
||||
closed.put(row)
|
||||
}
|
||||
}
|
||||
for (id in closedIds) {
|
||||
db.execSQL(
|
||||
"UPDATE sessions SET"
|
||||
+ " ended_at = COALESCE((SELECT MAX(ended_at) FROM segments WHERE session_id = ?), started_at),"
|
||||
+ " avg_sentiment = COALESCE((SELECT AVG(sentiment) FROM segments WHERE session_id = ?), 0)"
|
||||
+ " WHERE id = ?",
|
||||
arrayOf<Any?>(id, id, id))
|
||||
}
|
||||
return closed.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Closed sessions whose audio is still the WAV: a transcode that never finished (the process
|
||||
* died mid-transcode, or recovery failed partway on an earlier boot). Returns a JSON array of
|
||||
* {id, audioPath}.
|
||||
*/
|
||||
fun listUntranscodedSessions(): String {
|
||||
val sessions = JSONArray()
|
||||
readableDatabase.rawQuery(
|
||||
"SELECT id, audio_path FROM sessions"
|
||||
+ " WHERE ended_at IS NOT NULL AND audio_codec = 'wav' AND audio_path IS NOT NULL",
|
||||
null,
|
||||
).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
val row = JSONObject()
|
||||
row.put("id", c.getLong(0))
|
||||
row.put("audioPath", c.getString(1))
|
||||
sessions.put(row)
|
||||
}
|
||||
}
|
||||
return sessions.toString()
|
||||
}
|
||||
|
||||
/**
|
||||
* Delete recordings in dirPath that no session's audio_path names: files a session row never
|
||||
* learned about, or partial transcoder output. Matched by file name, so a storage move that
|
||||
* changed the directory's absolute path can't make live recordings look orphaned. Only call
|
||||
* while no recording or transcode is in flight. Returns how many files were deleted.
|
||||
*/
|
||||
fun deleteUnreferencedRecordings(dirPath: String?): Int {
|
||||
if (dirPath == null) return 0
|
||||
val files = File(dirPath).listFiles() ?: return 0
|
||||
val referenced = HashSet<String>()
|
||||
readableDatabase.rawQuery("SELECT audio_path FROM sessions WHERE audio_path IS NOT NULL", null).use { c ->
|
||||
while (c.moveToNext()) {
|
||||
referenced.add(File(c.getString(0)).name)
|
||||
}
|
||||
}
|
||||
var deleted = 0
|
||||
for (file in files) {
|
||||
if (file.isFile && file.name !in referenced && file.delete()) {
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
return deleted
|
||||
}
|
||||
|
||||
fun deleteSession(sessionId: Long) {
|
||||
val db = writableDatabase
|
||||
val path = getSessionAudioPath(sessionId)
|
||||
|
||||
@@ -1,32 +1,100 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.net.Uri
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.webkit.JavascriptInterface
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import androidx.webkit.JavaScriptReplyProxy
|
||||
import androidx.webkit.WebMessageCompat
|
||||
import androidx.webkit.WebViewCompat
|
||||
import androidx.webkit.WebViewFeature
|
||||
|
||||
import org.json.JSONArray
|
||||
import org.json.JSONException
|
||||
|
||||
/**
|
||||
* The object injected into a hosted EvenHub app's WebView as
|
||||
* window.__faceclawEvenHub. The document-start shim (served inline by
|
||||
* FaceclawEvenHubWebViewClient) wraps it in a promise-returning
|
||||
* window.flutter_inappwebview.callHandler, which is what the EvenHub SDK
|
||||
* actually talks to.
|
||||
* The host end of a hosted EvenHub app's bridge: window.__faceclawEvenHubPort,
|
||||
* a WebMessageListener object that WebView injects only into frames from the
|
||||
* app's own origin. Sites the app navigates to, and other-origin iframes it
|
||||
* embeds, never see it, and messages from the app's own subframes are dropped,
|
||||
* so only the app's main document reaches the host (its SDK storage, sensors,
|
||||
* glasses display and the Faceclaw extensions, API keys included). This
|
||||
* replaced an addJavascriptInterface object, which WebView exposes to every
|
||||
* frame of every page.
|
||||
*
|
||||
* JavascriptInterface methods run on a WebView-internal thread; everything
|
||||
* is bounced to the main thread before touching the NativeScript listener.
|
||||
* The transport script (webview.ts) wraps the port as window.__faceclawEvenHub,
|
||||
* whose postMessage(name, argsJson, id) and wakeTimers() the stock bridge shim
|
||||
* and the extensions call. Messages are JSON arrays: ["call", name, argsJson,
|
||||
* id] or ["wake"].
|
||||
*/
|
||||
class FaceclawEvenHubJsBridge(private val listener: FaceclawEvenHubListener) {
|
||||
class FaceclawEvenHubJsBridge private constructor(
|
||||
private val listener: FaceclawEvenHubListener
|
||||
) : WebViewCompat.WebMessageListener {
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
@JavascriptInterface
|
||||
fun postMessage(handlerName: String?, argsJson: String?, callId: Int) {
|
||||
mainHandler.post {
|
||||
listener.onEvenAppMessage(handlerName, argsJson, callId)
|
||||
override fun onPostMessage(
|
||||
view: WebView,
|
||||
message: WebMessageCompat,
|
||||
sourceOrigin: Uri,
|
||||
isMainFrame: Boolean,
|
||||
replyProxy: JavaScriptReplyProxy
|
||||
) {
|
||||
if (!isMainFrame) return
|
||||
val parts = try {
|
||||
JSONArray(message.data ?: return)
|
||||
} catch (e: JSONException) {
|
||||
return
|
||||
}
|
||||
when (parts.optString(0)) {
|
||||
"call" -> {
|
||||
val name = parts.optString(1)
|
||||
val argsJson = parts.optString(2)
|
||||
val callId = parts.optInt(3)
|
||||
mainHandler.post { listener.onEvenAppMessage(name, argsJson, callId) }
|
||||
}
|
||||
// The page's timer shim has work due before the host's next tick.
|
||||
"wake" -> FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
|
||||
}
|
||||
}
|
||||
|
||||
/** The page's timer shim has work due before the host's next tick. */
|
||||
@JavascriptInterface
|
||||
fun wakeTimers() {
|
||||
FaceclawEvenHubWebViewHost.getInstance().wakeTimers()
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
private const val OBJECT_NAME = "__faceclawEvenHubPort"
|
||||
|
||||
/**
|
||||
* The origin rule matching exactly [url]'s origin (default port elided),
|
||||
* or null when it isn't an http(s) URL with a host.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun originRule(url: String): String? {
|
||||
val uri = Uri.parse(url)
|
||||
val scheme = uri.scheme?.lowercase()
|
||||
val host = uri.host?.lowercase()
|
||||
if ((scheme != "http" && scheme != "https") || host.isNullOrEmpty()) return null
|
||||
val defaultPort = if (scheme == "https") 443 else 80
|
||||
return if (uri.port == -1 || uri.port == defaultPort) "$scheme://$host" else "$scheme://$host:${uri.port}"
|
||||
}
|
||||
|
||||
/**
|
||||
* Expose the bridge to [webView]'s frames from [originRule]. Must run
|
||||
* before the WebView loads anything.
|
||||
*
|
||||
* @return null on success, else why the bridge could not be installed.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun install(webView: WebView, originRule: String, listener: FaceclawEvenHubListener): String? {
|
||||
if (!WebViewFeature.isFeatureSupported(WebViewFeature.WEB_MESSAGE_LISTENER)) {
|
||||
Log.w(TAG, "WebMessageListener unsupported; not exposing the EvenHub bridge")
|
||||
return "This Android System WebView is too old to run EvenHub apps safely. Update it, then reopen the app."
|
||||
}
|
||||
return try {
|
||||
WebViewCompat.addWebMessageListener(webView, OBJECT_NAME, setOf(originRule), FaceclawEvenHubJsBridge(listener))
|
||||
null
|
||||
} catch (e: IllegalArgumentException) {
|
||||
Log.w(TAG, "bridge origin rule rejected: $originRule", e)
|
||||
"The app's address ($originRule) can't be used as a web origin."
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.util.Log
|
||||
import android.webkit.WebView
|
||||
import androidx.webkit.ProfileStore
|
||||
import androidx.webkit.WebViewCompat
|
||||
import androidx.webkit.WebViewFeature
|
||||
|
||||
import java.security.MessageDigest
|
||||
|
||||
/**
|
||||
* Gives each EvenHub package its own WebView profile, so one app's browser
|
||||
* storage (cookies, localStorage, IndexedDB, caches, service workers) is out of
|
||||
* reach of every other app. That includes the storage of third-party sites an
|
||||
* app navigates to, such as a sign-in provider's session cookies, which in the
|
||||
* shared default profile any other app's WebView could reuse. The per-app fake
|
||||
* origins alone don't cover that, and `*.evenhub.invalid` hosts share a site,
|
||||
* so domain cookies would cross between packaged apps too.
|
||||
*
|
||||
* The profile persists across launches, and is shared by every instance of the
|
||||
* same package, like the iOS per-package WKWebsiteDataStore.
|
||||
*/
|
||||
class FaceclawEvenHubProfile private constructor() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
|
||||
/** Stable per package; the hash input matches the iOS data-store identifier's. */
|
||||
@JvmStatic
|
||||
fun nameFor(packageId: String): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
.digest("com.faceclaw.ehpk:$packageId".toByteArray(Charsets.UTF_8))
|
||||
return "evenhub-" + digest.take(16).joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
|
||||
/**
|
||||
* Put a new WebView in [packageId]'s profile. Must run on the main thread
|
||||
* before anything else is done with the WebView.
|
||||
*
|
||||
* @return false when the WebView provider lacks multi-profile support,
|
||||
* leaving the WebView in the shared default profile.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun apply(webView: WebView, packageId: String): Boolean {
|
||||
if (!WebViewFeature.isFeatureSupported(WebViewFeature.MULTI_PROFILE)) {
|
||||
Log.w(TAG, "multi-profile WebView unsupported; $packageId shares the default profile")
|
||||
return false
|
||||
}
|
||||
val name = nameFor(packageId)
|
||||
ProfileStore.getInstance().getOrCreateProfile(name)
|
||||
WebViewCompat.setProfile(webView, name)
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ package com.faceclaw.app
|
||||
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.webkit.RenderProcessGoneDetail
|
||||
import android.webkit.WebResourceRequest
|
||||
import android.webkit.WebResourceResponse
|
||||
import android.webkit.WebView
|
||||
@@ -33,6 +34,11 @@ import java.nio.charset.StandardCharsets
|
||||
* The shim is registered as a document-start script instead
|
||||
* (FaceclawEvenHubDocumentStart); injectOnPageStarted is the fallback for
|
||||
* WebView versions that lack that API.
|
||||
*
|
||||
* An app can crash its renderer, or the system can kill it for memory. Unless
|
||||
* every WebView that renderer served handles that, WebView aborts the whole
|
||||
* process, taking Faceclaw down with it; so onRenderProcessGone retires the
|
||||
* affected WebView and reports it instead.
|
||||
*/
|
||||
class FaceclawEvenHubWebViewClient(
|
||||
private val rootDir: String?,
|
||||
@@ -93,6 +99,15 @@ class FaceclawEvenHubWebViewClient(
|
||||
}
|
||||
}
|
||||
|
||||
override fun onRenderProcessGone(view: WebView, detail: RenderProcessGoneDetail): Boolean {
|
||||
// Called on the main thread for each WebView the renderer served. This
|
||||
// one can't be used again: drop it, and the app shows as failed.
|
||||
Log.w(TAG, "renderer gone (crashed=${detail.didCrash()}) for $host")
|
||||
FaceclawEvenHubWebViewHost.getInstance().detach(view)
|
||||
listener.onRenderProcessGone(detail.didCrash())
|
||||
return true
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "FaceclawEvenHub"
|
||||
|
||||
|
||||
@@ -59,8 +59,8 @@ import java.util.HashMap
|
||||
* requestAnimationFrame) with JS queues fired by window.__fcTick(), and this
|
||||
* host ticks those on the main thread regardless of screen state. The tick
|
||||
* returns when the page next has work, so an idle page isn't evaluated at
|
||||
* 60Hz; work scheduled between ticks wakes the host through
|
||||
* FaceclawEvenHubJsBridge.wakeTimers. The main Looper keeps running under the
|
||||
* 60Hz; work scheduled between ticks wakes the host through the bridge's
|
||||
* "wake" message (FaceclawEvenHubJsBridge). The main Looper keeps running under the
|
||||
* foreground service, so the tick survives the screen turning off. (Matches the
|
||||
* official Even app's approach.)
|
||||
*
|
||||
@@ -105,8 +105,12 @@ class FaceclawEvenHubWebViewHost {
|
||||
* the page wakes it. It also re-ticks after IDLE_POLL_MS regardless, so a
|
||||
* lost evaluateJavascript callback (e.g. mid-navigation) or a page whose
|
||||
* wake didn't reach us stalls its timers for at most that long.
|
||||
*
|
||||
* The tick only runs in the app's own documents ([appCheck], a JS
|
||||
* expression): after a navigation the WebView may show another site.
|
||||
*/
|
||||
private inner class Ticker(private val web: WebView) : Runnable {
|
||||
private inner class Ticker(private val web: WebView, appCheck: String) : Runnable {
|
||||
private val tickJs = "if ($appCheck) { $TICK_JS }"
|
||||
private var inFlight = false
|
||||
private var wakePending = false
|
||||
private var stopped = false
|
||||
@@ -114,7 +118,7 @@ class FaceclawEvenHubWebViewHost {
|
||||
override fun run() {
|
||||
if (stopped) return
|
||||
inFlight = true
|
||||
web.evaluateJavascript(TICK_JS) { result -> onTicked(result) }
|
||||
web.evaluateJavascript(tickJs) { result -> onTicked(result) }
|
||||
schedule(IDLE_POLL_MS)
|
||||
}
|
||||
|
||||
@@ -278,8 +282,11 @@ class FaceclawEvenHubWebViewHost {
|
||||
(web.context as? MutableContextWrapper)?.baseContext = activity
|
||||
}
|
||||
|
||||
/** Add a WebView to the host, full-size and rendering, hidden behind the UI. */
|
||||
fun attach(activity: Activity, web: WebView) {
|
||||
/**
|
||||
* Add a WebView to the host, full-size and rendering, hidden behind the UI.
|
||||
* [appCheck] is a JS expression that is true only in the app's own documents.
|
||||
*/
|
||||
fun attach(activity: Activity, web: WebView, appCheck: String) {
|
||||
ensureOverlay(activity)
|
||||
rebaseContext(web, activity)
|
||||
web.visibility = View.VISIBLE
|
||||
@@ -301,7 +308,7 @@ class FaceclawEvenHubWebViewHost {
|
||||
}
|
||||
if (!webViews.contains(web)) webViews.add(web)
|
||||
if (!tickers.containsKey(web)) {
|
||||
val ticker = Ticker(web)
|
||||
val ticker = Ticker(web, appCheck)
|
||||
tickers[web] = ticker
|
||||
mainHandler.postDelayed(ticker, TICK_MS)
|
||||
}
|
||||
@@ -348,9 +355,9 @@ class FaceclawEvenHubWebViewHost {
|
||||
return shown
|
||||
}
|
||||
|
||||
/** Remove and destroy a WebView (its app is closing). */
|
||||
/** Remove and destroy a WebView (its app is closing, or its renderer is gone). Idempotent. */
|
||||
fun detach(web: WebView) {
|
||||
webViews.remove(web)
|
||||
if (!webViews.remove(web)) return
|
||||
tickers.remove(web)?.stop()
|
||||
stack?.removeView(web)
|
||||
web.destroy()
|
||||
|
||||
@@ -46,6 +46,7 @@ class FaceclawMediaController(context: Context) {
|
||||
|
||||
override fun onMetadataChanged(metadata: MediaMetadata?) {
|
||||
synchronized(lock) {
|
||||
artVersion++
|
||||
emitStateLocked()
|
||||
}
|
||||
}
|
||||
@@ -92,6 +93,15 @@ class FaceclawMediaController(context: Context) {
|
||||
private var sessionsListenerRegistered = false
|
||||
private var ignoredPackages: Set<String> = HashSet()
|
||||
|
||||
/**
|
||||
* Bumped whenever the art getAlbumArtGray would return may have changed:
|
||||
* every metadata update and every switch of active session. Players
|
||||
* re-publish the same title/album with a different bitmap (art attached
|
||||
* after download, or a stale bitmap from the previous track replaced
|
||||
* after a quick double skip), so the strings can't key an art cache.
|
||||
*/
|
||||
private var artVersion = 0
|
||||
|
||||
init {
|
||||
this.listenerComponent = ComponentName(appContext, FaceclawMediaNotificationListenerService::class.java)
|
||||
this.sessionManager = appContext.getSystemService(Context.MEDIA_SESSION_SERVICE) as MediaSessionManager?
|
||||
@@ -404,6 +414,7 @@ class FaceclawMediaController(context: Context) {
|
||||
previous.unregisterCallback(controllerCallback)
|
||||
}
|
||||
activeController = controller
|
||||
artVersion++
|
||||
if (controller != null) {
|
||||
controller.registerCallback(controllerCallback, mainHandler)
|
||||
}
|
||||
@@ -503,6 +514,7 @@ class FaceclawMediaController(context: Context) {
|
||||
val finalCanSkipPrevious = canSkipPrevious
|
||||
val finalAccessEnabled = accessEnabled
|
||||
val finalStatus = status
|
||||
val finalArtVersion = artVersion
|
||||
|
||||
mainHandler.post {
|
||||
currentListener.onStateChange(
|
||||
@@ -519,7 +531,8 @@ class FaceclawMediaController(context: Context) {
|
||||
finalCanSkipNext,
|
||||
finalCanSkipPrevious,
|
||||
finalAccessEnabled,
|
||||
finalStatus
|
||||
finalStatus,
|
||||
finalArtVersion
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
+330
-123
@@ -3,15 +3,20 @@ package com.faceclaw.app
|
||||
import android.app.Notification
|
||||
import android.app.NotificationManager
|
||||
import android.app.PendingIntent
|
||||
import android.content.ComponentName
|
||||
import android.content.Context
|
||||
import android.database.ContentObserver
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.drawable.BitmapDrawable
|
||||
import android.graphics.drawable.Drawable
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.os.SystemClock
|
||||
import android.provider.Settings
|
||||
import android.service.notification.NotificationListenerService
|
||||
import android.service.notification.StatusBarNotification
|
||||
import android.util.Log
|
||||
@@ -29,15 +34,177 @@ import org.json.JSONObject
|
||||
class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
companion object {
|
||||
private const val TAG = "FaceclawNotify"
|
||||
private const val NOTIFICATION_ICON_GAMMA = 1.6
|
||||
private const val EXTRA_SUBSTITUTE_APP_NAME = "android.substName"
|
||||
/**
|
||||
* How long notification access can be granted with no listener
|
||||
* connected before the watchdog forces a rebind: longer than a
|
||||
* normal bind at process start, and than the system's own single
|
||||
* retry 10 s after a binding dies. Later attempts double from here.
|
||||
*/
|
||||
private const val REBIND_GRACE_MS = 15_000L
|
||||
private const val REBIND_MAX_DELAY_MS = 15 * 60_000L
|
||||
/** Missing this long, the listener needs the user to toggle notification access. */
|
||||
private const val LISTENER_STUCK_MS = 30_000L
|
||||
/** Notifications posted while disconnected alert on reconnect only if this recent. */
|
||||
private const val RECONNECT_ALERT_WINDOW_MS = 5 * 60_000L
|
||||
|
||||
/** The connected listener; null until onListenerConnected and after onListenerDisconnected. */
|
||||
@Volatile
|
||||
private var activeService: FaceclawMediaNotificationListenerService? = null
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
private val notificationListeners: MutableSet<FaceclawNotificationListener> = CopyOnWriteArraySet()
|
||||
private val listenerStateObservers: MutableSet<Runnable> = CopyOnWriteArraySet()
|
||||
private val activeNotificationWakeKeys: MutableSet<String> = HashSet()
|
||||
|
||||
// Binding watchdog (see watchListenerBinding); written on the main thread only.
|
||||
private var watchdogContext: Context? = null
|
||||
private var rebindScheduled = false
|
||||
private var rebindDelayMs = REBIND_GRACE_MS
|
||||
/** elapsedRealtime when the watchdog first saw access granted with no listener; 0 when not missing. */
|
||||
@Volatile
|
||||
private var listenerMissingSinceMs = 0L
|
||||
/** Wall-clock time of the last disconnect in this process (compared with postTime); 0 if none. */
|
||||
private var lastListenerDisconnectAtMs = 0L
|
||||
private val forceRebindTask = Runnable { forceRebindIfStillMissing() }
|
||||
private val accessObserver = object : ContentObserver(mainHandler) {
|
||||
override fun onChange(selfChange: Boolean) {
|
||||
checkListenerBinding()
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for the listener going missing while notification
|
||||
* access is granted. Android can drop a listener's binding, or give
|
||||
* up rebinding it after the process dies, and not bring it back;
|
||||
* every read then comes back empty for the life of the process.
|
||||
* Called at app boot; disconnects and access changes re-check.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun watchListenerBinding(context: Context?) {
|
||||
if (context == null) {
|
||||
return
|
||||
}
|
||||
val appContext = context.applicationContext
|
||||
mainHandler.post {
|
||||
if (watchdogContext == null) {
|
||||
watchdogContext = appContext
|
||||
try {
|
||||
appContext.contentResolver.registerContentObserver(
|
||||
Settings.Secure.getUriFor("enabled_notification_listeners"), false, accessObserver)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "notification access observer registration failed", t)
|
||||
}
|
||||
}
|
||||
checkListenerBinding()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* "connected"; "no-access" (notification access not granted);
|
||||
* "connecting" (granted, listener not connected yet); or "stuck"
|
||||
* (granted, but still missing past LISTENER_STUCK_MS, which forced
|
||||
* rebinds can't always fix: the user has to toggle access).
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getListenerState(context: Context?): String {
|
||||
if (activeService != null) {
|
||||
return "connected"
|
||||
}
|
||||
if (!FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
return "no-access"
|
||||
}
|
||||
val missingSince = listenerMissingSinceMs
|
||||
if (missingSince > 0 && SystemClock.elapsedRealtime() - missingSince >= LISTENER_STUCK_MS) {
|
||||
return "stuck"
|
||||
}
|
||||
return "connecting"
|
||||
}
|
||||
|
||||
/** Run on the main thread whenever getListenerState may have changed. */
|
||||
@JvmStatic
|
||||
fun addListenerStateObserver(observer: Runnable?) {
|
||||
if (observer != null) {
|
||||
listenerStateObservers.add(observer)
|
||||
}
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun removeListenerStateObserver(observer: Runnable?) {
|
||||
if (observer != null) {
|
||||
listenerStateObservers.remove(observer)
|
||||
}
|
||||
}
|
||||
|
||||
private fun notifyListenerStateChanged() {
|
||||
for (observer in listenerStateObservers) {
|
||||
mainHandler.post {
|
||||
try {
|
||||
observer.run()
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "listener state observer failed", t)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** Main thread. Starts the forced-rebind countdown when access is granted and no listener is connected. */
|
||||
private fun checkListenerBinding() {
|
||||
val context = watchdogContext ?: return
|
||||
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
resetRebindWatchdog()
|
||||
return
|
||||
}
|
||||
if (listenerMissingSinceMs == 0L) {
|
||||
listenerMissingSinceMs = SystemClock.elapsedRealtime()
|
||||
}
|
||||
scheduleForcedRebind()
|
||||
}
|
||||
|
||||
private fun scheduleForcedRebind() {
|
||||
if (!rebindScheduled) {
|
||||
rebindScheduled = true
|
||||
mainHandler.postDelayed(forceRebindTask, rebindDelayMs)
|
||||
}
|
||||
}
|
||||
|
||||
private fun resetRebindWatchdog() {
|
||||
mainHandler.removeCallbacks(forceRebindTask)
|
||||
rebindScheduled = false
|
||||
rebindDelayMs = REBIND_GRACE_MS
|
||||
listenerMissingSinceMs = 0L
|
||||
}
|
||||
|
||||
private fun forceRebindIfStillMissing() {
|
||||
rebindScheduled = false
|
||||
val context = watchdogContext ?: return
|
||||
if (activeService != null || !FaceclawEvenAppDetector.isNotificationAccessEnabled(context)) {
|
||||
resetRebindWatchdog()
|
||||
return
|
||||
}
|
||||
val missingSeconds = (SystemClock.elapsedRealtime() - listenerMissingSinceMs) / 1000
|
||||
Log.w(TAG, "notification access granted but no listener connected for ${missingSeconds}s; forcing a rebind")
|
||||
val component = ComponentName(context, FaceclawMediaNotificationListenerService::class.java)
|
||||
try {
|
||||
// requestRebind only undoes a requestUnbind, so alone it can't
|
||||
// bring back a binding the system dropped. Unbinding by
|
||||
// component first marks the listener snoozed and drops any
|
||||
// record of it; the rebind then un-snoozes it and binds afresh.
|
||||
// Below API 34 there is no public equivalent (and flipping the
|
||||
// component's enabled state can make the system revoke
|
||||
// access), so there the state goes "stuck" for the user.
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.UPSIDE_DOWN_CAKE) {
|
||||
NotificationListenerService.requestUnbind(component)
|
||||
}
|
||||
NotificationListenerService.requestRebind(component)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "forced notification listener rebind failed", t)
|
||||
}
|
||||
rebindDelayMs = Math.min(rebindDelayMs * 2, REBIND_MAX_DELAY_MS)
|
||||
scheduleForcedRebind()
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun addNotificationListener(listener: FaceclawNotificationListener?) {
|
||||
if (listener != null) {
|
||||
@@ -127,15 +294,16 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
if (dedupeGroupKey != null && emittedGroupKeys.contains(dedupeGroupKey)) {
|
||||
continue
|
||||
}
|
||||
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
|
||||
if (drawable == null) {
|
||||
Log.i(TAG, "icon skipped (no drawable): " + statusBarNotification.packageName)
|
||||
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
|
||||
getNotificationAppName(service, statusBarNotification)
|
||||
}
|
||||
if (icon == null) {
|
||||
Log.i(TAG, "icon skipped (nothing to draw): " + statusBarNotification.packageName)
|
||||
continue
|
||||
}
|
||||
Log.i(TAG, "icon[" + emitted + "] pkg=" + statusBarNotification.packageName
|
||||
+ " drawable=" + drawable.javaClass.simpleName
|
||||
+ " intrinsic=" + drawable.intrinsicWidth + "x" + drawable.intrinsicHeight)
|
||||
appendIconGrayBytes(drawable, size, out, service, emitted, statusBarNotification.packageName)
|
||||
+ " source=" + icon.source + " kind=" + icon.art.kind)
|
||||
out.write(icon.art.gray)
|
||||
keysOut?.add(statusBarNotification.key ?: "")
|
||||
if (dedupeGroupKey != null) {
|
||||
emittedGroupKeys.add(dedupeGroupKey)
|
||||
@@ -165,13 +333,37 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
if (statusBarNotification == null || statusBarNotification.notification == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val drawable = loadNotificationIcon(service, statusBarNotification.notification)
|
||||
if (drawable == null) {
|
||||
val icon = NotificationIconLoader.iconFor(service, statusBarNotification, size) {
|
||||
getNotificationAppName(service, statusBarNotification)
|
||||
}
|
||||
return icon?.art?.gray ?: ByteArray(0)
|
||||
}
|
||||
|
||||
/**
|
||||
* The picture attached to one active notification (BigPictureStyle: a
|
||||
* photo, a screenshot, Dexcom's glucose graph), scaled to fit
|
||||
* maxWidth x maxHeight as a gray packet with the tone handling (and
|
||||
* light-background inversion) of ImageFileLoader.bitmapToGrayPacket;
|
||||
* empty when the notification or its picture is unavailable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getNotificationPictureGray(
|
||||
key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean,
|
||||
): ByteArray {
|
||||
val service = activeService
|
||||
if (service == null || maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val out = ByteArrayOutputStream(size * size)
|
||||
appendIconGrayBytes(drawable, size, out, service, -1, statusBarNotification.packageName)
|
||||
return out.toByteArray()
|
||||
val statusBarNotification = findActiveNotificationByKey(service, key)
|
||||
if (statusBarNotification == null || statusBarNotification.notification == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val picture = loadNotificationPicture(service, statusBarNotification.notification)
|
||||
if (picture == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
return ImageFileLoader.bitmapToGrayPacket(
|
||||
halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
@@ -267,13 +459,21 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
|
||||
private fun emitNotificationPosted(key: String?) {
|
||||
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationPosted(eventKey) }
|
||||
}
|
||||
|
||||
private fun emitNotificationRemoved(key: String?) {
|
||||
emitNotificationEvent(key) { listener, eventKey -> listener.onNotificationRemoved(eventKey) }
|
||||
}
|
||||
|
||||
private fun emitNotificationEvent(key: String?, deliver: (FaceclawNotificationListener, String) -> Unit) {
|
||||
if (key == null || key.isEmpty() || notificationListeners.isEmpty()) {
|
||||
return
|
||||
}
|
||||
for (listener in notificationListeners) {
|
||||
mainHandler.post {
|
||||
try {
|
||||
listener.onNotificationPosted(key)
|
||||
deliver(listener, key)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "notification listener failed", t)
|
||||
}
|
||||
@@ -281,7 +481,14 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
|
||||
private fun refreshActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
|
||||
/**
|
||||
* Re-seed the wake keys from what is active now. After a reconnect
|
||||
* this also catches the glasses up on the gap: keys that went away
|
||||
* are reported removed (closing a popup for a notification
|
||||
* dismissed meanwhile), and recent notifications posted since the
|
||||
* disconnect are reported posted. A first connect only seeds.
|
||||
*/
|
||||
private fun reconcileActiveNotificationWakeKeys(service: FaceclawMediaNotificationListenerService) {
|
||||
val notifications: Array<StatusBarNotification?>?
|
||||
try {
|
||||
notifications = service.activeNotifications
|
||||
@@ -289,19 +496,40 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
Log.w(TAG, "failed to refresh active notification wake keys", t)
|
||||
return
|
||||
}
|
||||
if (notifications == null) {
|
||||
return
|
||||
}
|
||||
val disconnectedAtMs = lastListenerDisconnectAtMs
|
||||
val alertAfterMs = Math.max(disconnectedAtMs, System.currentTimeMillis() - RECONNECT_ALERT_WINDOW_MS)
|
||||
val posted = ArrayList<String>()
|
||||
val removed = ArrayList<String>()
|
||||
synchronized(activeNotificationWakeKeys) {
|
||||
val previous = HashSet(activeNotificationWakeKeys)
|
||||
activeNotificationWakeKeys.clear()
|
||||
if (notifications == null) {
|
||||
return
|
||||
}
|
||||
for (statusBarNotification in notifications) {
|
||||
if (shouldShowNotificationInList(service, statusBarNotification)) {
|
||||
val key: String? = statusBarNotification!!.key
|
||||
if (key != null && !key.isEmpty()) {
|
||||
activeNotificationWakeKeys.add(key)
|
||||
}
|
||||
if (!shouldShowNotificationInList(service, statusBarNotification)) {
|
||||
continue
|
||||
}
|
||||
val key: String? = statusBarNotification!!.key
|
||||
if (key == null || key.isEmpty()) {
|
||||
continue
|
||||
}
|
||||
activeNotificationWakeKeys.add(key)
|
||||
if (disconnectedAtMs > 0 && !previous.contains(key) && statusBarNotification.postTime > alertAfterMs) {
|
||||
posted.add(key)
|
||||
}
|
||||
}
|
||||
for (key in previous) {
|
||||
if (!activeNotificationWakeKeys.contains(key)) {
|
||||
removed.add(key)
|
||||
}
|
||||
}
|
||||
}
|
||||
for (key in removed) {
|
||||
emitNotificationRemoved(key)
|
||||
}
|
||||
for (key in posted) {
|
||||
emitNotificationPosted(key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,16 +547,17 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return !alreadyActive || !isPersistentNotification(statusBarNotification)
|
||||
}
|
||||
|
||||
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?) {
|
||||
/** Stop tracking a notification; returns whether it was tracked (i.e. listed on the glasses). */
|
||||
private fun forgetActiveNotificationWakeKey(statusBarNotification: StatusBarNotification?): Boolean {
|
||||
if (statusBarNotification == null) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
val key: String? = statusBarNotification.key
|
||||
if (key == null || key.isEmpty()) {
|
||||
return
|
||||
return false
|
||||
}
|
||||
synchronized(activeNotificationWakeKeys) {
|
||||
activeNotificationWakeKeys.remove(key)
|
||||
return activeNotificationWakeKeys.remove(key)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -397,106 +626,77 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return groupKey
|
||||
}
|
||||
|
||||
private fun loadNotificationIcon(service: FaceclawMediaNotificationListenerService, notification: Notification): Drawable? {
|
||||
/** Whether a notification carries a picture for loadNotificationPicture, without loading it. */
|
||||
@Suppress("DEPRECATION")
|
||||
private fun hasNotificationPicture(extras: Bundle): Boolean {
|
||||
return extras.get(Notification.EXTRA_PICTURE) != null
|
||||
|| (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && extras.get(Notification.EXTRA_PICTURE_ICON) != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's attached picture: BigPictureStyle's bitmap
|
||||
* (EXTRA_PICTURE) or, from Android 12, its Icon (EXTRA_PICTURE_ICON).
|
||||
* Newer releases swap large pictures for a content URI to a bitmap
|
||||
* the system has offloaded (Dexcom's graph arrives that way); the
|
||||
* offload provider lets enabled notification listeners open those.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
private fun loadNotificationPicture(service: FaceclawMediaNotificationListenerService, notification: Notification): Bitmap? {
|
||||
val extras: Bundle = notification.extras ?: return null
|
||||
try {
|
||||
val smallIcon: Icon? = notification.smallIcon
|
||||
if (smallIcon != null) {
|
||||
val drawable: Drawable? = smallIcon.loadDrawable(service)
|
||||
if (drawable != null) {
|
||||
// Status-bar small icons are alpha templates: the platform
|
||||
// draws them tinted and ignores their color channels, which
|
||||
// apps may fill with garbage (Discord ships noise there).
|
||||
// Tint white so the shape comes from alpha alone, matching
|
||||
// how the status bar renders them.
|
||||
drawable.mutate()
|
||||
drawable.setTint(Color.WHITE)
|
||||
if (drawable is BitmapDrawable) {
|
||||
drawable.setFilterBitmap(true)
|
||||
}
|
||||
return drawable
|
||||
}
|
||||
val picture: Bitmap? = extras.getParcelable(Notification.EXTRA_PICTURE)
|
||||
if (picture != null) {
|
||||
return softwareBitmap(picture)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load small notification icon", t)
|
||||
Log.w(TAG, "failed to read notification picture", t)
|
||||
}
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
// Large icons are real color images (avatars, album art); keep color.
|
||||
val largeIcon: Icon? = notification.getLargeIcon()
|
||||
if (largeIcon != null) {
|
||||
return largeIcon.loadDrawable(service)
|
||||
val icon: Icon? = extras.getParcelable(Notification.EXTRA_PICTURE_ICON)
|
||||
val drawable: Drawable? = icon?.loadDrawable(service)
|
||||
if (drawable is BitmapDrawable && drawable.bitmap != null) {
|
||||
return softwareBitmap(drawable.bitmap)
|
||||
}
|
||||
if (drawable != null && drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) {
|
||||
val bitmap = Bitmap.createBitmap(drawable.intrinsicWidth, drawable.intrinsicHeight, Bitmap.Config.ARGB_8888)
|
||||
drawable.setBounds(0, 0, bitmap.width, bitmap.height)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
return bitmap
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load large notification icon", t)
|
||||
Log.w(TAG, "failed to load notification picture icon", t)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
private fun appendIconGrayBytes(drawable: Drawable, size: Int, out: ByteArrayOutputStream,
|
||||
service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?) {
|
||||
val bitmap = renderIconScaled(drawable, size)
|
||||
dumpIconDebugPng(service, index, packageName, bitmap)
|
||||
for (y in 0 until size) {
|
||||
for (x in 0 until size) {
|
||||
val color = bitmap.getPixel(x, y)
|
||||
val alpha = Color.alpha(color)
|
||||
val grayLinear = (0.2126 * Color.red(color) + 0.7152 * Color.green(color) + 0.0722 * Color.blue(color)) * alpha / (255.0 * 255.0)
|
||||
val gray = Math.round(255.0 * Math.pow(Math.max(0.0, Math.min(1.0, grayLinear)), NOTIFICATION_ICON_GAMMA)).toInt()
|
||||
out.write(gray and 0xff)
|
||||
}
|
||||
}
|
||||
bitmap.recycle()
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a drawable at the target size with proper downscaling. Detailed
|
||||
* sources (e.g. avatar bitmaps used as notification icons) are rendered at
|
||||
* native resolution and reduced by repeated halving: a single filtered pass
|
||||
* from, say, 126px to 24px samples too sparsely and turns fine detail into
|
||||
* speckle that reads as a garbled icon.
|
||||
*/
|
||||
private fun renderIconScaled(drawable: Drawable, size: Int): Bitmap {
|
||||
val renderW = Math.max(size, drawable.intrinsicWidth)
|
||||
val renderH = Math.max(size, drawable.intrinsicHeight)
|
||||
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
drawable.setBounds(0, 0, renderW, renderH)
|
||||
drawable.draw(canvas)
|
||||
while (bitmap.width >= size * 2 && bitmap.height >= size * 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
bitmap.recycle()
|
||||
bitmap = halved
|
||||
}
|
||||
if (bitmap.width != size || bitmap.height != size) {
|
||||
val scaled = Bitmap.createScaledBitmap(bitmap, size, size, true)
|
||||
bitmap.recycle()
|
||||
bitmap = scaled
|
||||
/** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */
|
||||
private fun softwareBitmap(bitmap: Bitmap): Bitmap {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) {
|
||||
return bitmap.copy(Bitmap.Config.ARGB_8888, false)
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug aid for garbled-icon reports: saves each rendered icon to
|
||||
* <externalFilesDir>/debug-icons/ (adb-pullable) so extraction problems can
|
||||
* be told apart from downstream compositing/transmission problems. Cheap:
|
||||
* runs at most once per icon-cache refresh on tiny bitmaps.
|
||||
* Halve a picture while it is still at least twice the size it will
|
||||
* be shown at, leaving the last (under 2x) step of fitting it into
|
||||
* maxWidth x maxHeight to bitmapToGrayPacket. As with icons
|
||||
* (NotificationIconLoader.renderArgb), one filtered pass from a much larger source
|
||||
* samples too sparsely: a graph's thin lines break up into dots.
|
||||
*/
|
||||
private fun dumpIconDebugPng(service: FaceclawMediaNotificationListenerService, index: Int, packageName: String?, bitmap: Bitmap) {
|
||||
if (index < 0) {
|
||||
return
|
||||
}
|
||||
try {
|
||||
val dir = java.io.File(service.getExternalFilesDir(null), "debug-icons")
|
||||
if (!dir.exists() && !dir.mkdirs()) {
|
||||
return
|
||||
private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
|
||||
var bitmap = source
|
||||
while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
if (bitmap !== source) {
|
||||
bitmap.recycle()
|
||||
}
|
||||
val safeName = if (packageName == null) "unknown" else packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
|
||||
val file = java.io.File(dir, "icon-$index-$safeName.png")
|
||||
java.io.FileOutputStream(file).use { stream ->
|
||||
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to dump debug icon", t)
|
||||
bitmap = halved
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
private fun findActiveNotificationByKey(service: FaceclawMediaNotificationListenerService?, key: String?): StatusBarNotification? {
|
||||
@@ -531,6 +731,7 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
out.put("appName", getNotificationAppName(service, statusBarNotification))
|
||||
out.put("postTime", statusBarNotification.postTime)
|
||||
out.put("when", notification.`when`)
|
||||
out.put("foregroundService", (notification.flags and Notification.FLAG_FOREGROUND_SERVICE) != 0)
|
||||
putString(out, "category", notification.category)
|
||||
if (extras != null) {
|
||||
putCharSequence(out, "title", firstNonEmpty(
|
||||
@@ -553,8 +754,10 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
out.put("lines", lines)
|
||||
out.put("hasPicture", hasNotificationPicture(extras))
|
||||
} else {
|
||||
out.put("lines", JSONArray())
|
||||
out.put("hasPicture", false)
|
||||
}
|
||||
|
||||
val actionsJson = JSONArray()
|
||||
@@ -626,35 +829,34 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
activeService = this
|
||||
}
|
||||
|
||||
override fun onDestroy() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
}
|
||||
super.onDestroy()
|
||||
}
|
||||
|
||||
override fun onListenerConnected() {
|
||||
activeService = this
|
||||
super.onListenerConnected()
|
||||
refreshActiveNotificationWakeKeys(this)
|
||||
Log.i(TAG, "notification listener connected")
|
||||
resetRebindWatchdog()
|
||||
reconcileActiveNotificationWakeKeys(this)
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
|
||||
// Also called from onDestroy, which is how an unbind usually arrives.
|
||||
override fun onListenerDisconnected() {
|
||||
if (activeService === this) {
|
||||
activeService = null
|
||||
lastListenerDisconnectAtMs = System.currentTimeMillis()
|
||||
Log.w(TAG, "notification listener disconnected")
|
||||
notifyListenerStateChanged()
|
||||
}
|
||||
super.onListenerDisconnected()
|
||||
watchListenerBinding(applicationContext)
|
||||
}
|
||||
|
||||
override fun onNotificationPosted(statusBarNotification: StatusBarNotification?) {
|
||||
super.onNotificationPosted(statusBarNotification)
|
||||
if (!shouldShowNotificationInList(this, statusBarNotification)) {
|
||||
forgetActiveNotificationWakeKey(statusBarNotification)
|
||||
// An update that takes a listed notification off the list removes it as far as the glasses go.
|
||||
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
|
||||
emitNotificationRemoved(statusBarNotification!!.key)
|
||||
}
|
||||
return
|
||||
}
|
||||
if (shouldEmitNotificationPosted(statusBarNotification!!)) {
|
||||
@@ -663,7 +865,12 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
|
||||
override fun onNotificationRemoved(statusBarNotification: StatusBarNotification?) {
|
||||
forgetActiveNotificationWakeKey(statusBarNotification)
|
||||
// Includes notifications their source cancels on its own, e.g. a
|
||||
// short-lived foreground-service notification that was standing in
|
||||
// for one posted moments later under another key.
|
||||
if (forgetActiveNotificationWakeKey(statusBarNotification)) {
|
||||
emitNotificationRemoved(statusBarNotification!!.key)
|
||||
}
|
||||
super.onNotificationRemoved(statusBarNotification)
|
||||
}
|
||||
}
|
||||
@@ -139,6 +139,15 @@ class FaceclawPreviewCompositor(context: Context) {
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green)
|
||||
}
|
||||
|
||||
/** As above, cropped to a screen area (Settings > Phone display > Preview scaling); black past the screen's edges. */
|
||||
fun getCompositePreviewBitmap(brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): android.graphics.Bitmap? {
|
||||
val composite = compositor.previewComposite() ?: return null
|
||||
return PreviewBitmapUtil.fromGray(
|
||||
java.nio.ByteBuffer.wrap(composite.gray), composite.width, composite.height, brightenGamma, green,
|
||||
cropX, cropY, cropWidth, cropHeight)
|
||||
}
|
||||
|
||||
/** Save the current composite as a 4-bit grayscale PNG; returns the path or "". */
|
||||
@Throws(java.io.IOException::class)
|
||||
fun saveCompositePngScreenshot(): String {
|
||||
|
||||
@@ -0,0 +1,51 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.content.Context
|
||||
import android.location.Address
|
||||
import android.location.Geocoder
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import android.util.Log
|
||||
import java.util.Locale
|
||||
import kotlin.concurrent.thread
|
||||
|
||||
/**
|
||||
* Place name for the Weather header where the weather source doesn't supply
|
||||
* one (outside National Weather Service coverage), from Android's own
|
||||
* Geocoder. Its lookup blocks on the network, so it runs on a thread of its
|
||||
* own; the listener hears back on the main thread, with nulls when the phone
|
||||
* has no geocoder or the lookup fails.
|
||||
*/
|
||||
class FaceclawReverseGeocoder(context: Context) {
|
||||
interface Listener {
|
||||
fun onResult(locality: String?, region: String?, country: String?)
|
||||
}
|
||||
|
||||
companion object {
|
||||
private const val TAG = "FaceclawGeocoder"
|
||||
}
|
||||
|
||||
private val context: Context = context.applicationContext
|
||||
private val mainHandler = Handler(Looper.getMainLooper())
|
||||
|
||||
fun lookup(latitude: Double, longitude: Double, listener: Listener) {
|
||||
thread(name = "FaceclawReverseGeocoder", isDaemon = true) {
|
||||
val address = lookupBlocking(latitude, longitude)
|
||||
mainHandler.post { listener.onResult(address?.locality, address?.adminArea, address?.countryName) }
|
||||
}
|
||||
}
|
||||
|
||||
private fun lookupBlocking(latitude: Double, longitude: Double): Address? {
|
||||
if (!Geocoder.isPresent()) {
|
||||
return null
|
||||
}
|
||||
return try {
|
||||
// The listener form needs API 33; this one is fine off the main thread.
|
||||
@Suppress("DEPRECATION")
|
||||
Geocoder(context, Locale.getDefault()).getFromLocation(latitude, longitude, 1)?.firstOrNull()
|
||||
} catch (error: Throwable) {
|
||||
Log.w(TAG, "reverse geocode failed: $error")
|
||||
null
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -132,10 +132,15 @@ class ImageFileLoader private constructor() {
|
||||
* round on its own, which turns smooth gradients into visible bands.
|
||||
* Dithered output is already quantized: every byte is a level times 16,
|
||||
* which BmpUtil re-quantizes back to exactly that level.
|
||||
*
|
||||
* invertLightBackground shows a picture drawn on a light background
|
||||
* (a chart, a screenshot) as its negative; see GrayPacket.fromArgb.
|
||||
*/
|
||||
@JvmStatic
|
||||
@JvmOverloads
|
||||
fun bitmapToGrayPacket(
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean,
|
||||
invertLightBackground: Boolean = false): ByteArray {
|
||||
if (source == null || source.width <= 0 || source.height <= 0
|
||||
|| maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
@@ -152,7 +157,7 @@ class ImageFileLoader private constructor() {
|
||||
}
|
||||
val pixels = IntArray(width * height)
|
||||
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither, invertLightBackground)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "bitmap conversion failed", e)
|
||||
return ByteArray(0)
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import android.app.Notification
|
||||
import android.content.Context
|
||||
import android.graphics.Bitmap
|
||||
import android.graphics.Canvas
|
||||
import android.graphics.Color
|
||||
import android.graphics.Paint
|
||||
import android.graphics.Rect
|
||||
import android.graphics.Typeface
|
||||
import android.graphics.drawable.AdaptiveIconDrawable
|
||||
import android.graphics.drawable.BitmapDrawable
|
||||
import android.graphics.drawable.Drawable
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.SystemClock
|
||||
import android.service.notification.StatusBarNotification
|
||||
import android.util.Log
|
||||
|
||||
import com.faceclaw.app.NotificationIconArt.Kind
|
||||
|
||||
import java.io.File
|
||||
import java.io.FileOutputStream
|
||||
|
||||
import kotlin.math.max
|
||||
import kotlin.math.roundToInt
|
||||
|
||||
/**
|
||||
* Picks and renders the glasses' status-bar icon for a notification (see iconFor).
|
||||
* NotificationIconArt does the pixel analysis; this object owns the drawables, package lookups,
|
||||
* caching and the debug dump.
|
||||
*/
|
||||
internal object NotificationIconLoader {
|
||||
private const val TAG = "FaceclawNotifyIcon"
|
||||
|
||||
/** AdaptiveIconDrawable draws its layers this share of the icon's size past each edge. */
|
||||
private const val ADAPTIVE_LAYER_OUTSET = 0.25f
|
||||
|
||||
private const val CACHE_MAX_ENTRIES = 128
|
||||
/** Cached art is redone after this long, so an updated app's new icon shows up. */
|
||||
private const val CACHE_MAX_AGE_MS = 30 * 60 * 1000L
|
||||
|
||||
/** The debug dump keeps at most this many files, deleting the oldest. */
|
||||
private const val DEBUG_DUMP_MAX_FILES = 96
|
||||
|
||||
class Rendered(
|
||||
/** The drawable it came from: small, monochrome, launcher, foreground, large or letter. */
|
||||
val source: String,
|
||||
val art: NotificationIconArt.Art,
|
||||
)
|
||||
|
||||
/**
|
||||
* An app's launcher icon art: its themed monochrome layer, else a glyph or shape read from
|
||||
* the icon (or its adaptive foreground layer), else the icon read by brightness.
|
||||
*/
|
||||
private class LauncherArt(val monochrome: Rendered?, val shape: Rendered?, val picture: Rendered?)
|
||||
|
||||
private val smallIconCache = ArtCache<Rendered?>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
|
||||
private val launcherCache = ArtCache<LauncherArt>(CACHE_MAX_ENTRIES, CACHE_MAX_AGE_MS)
|
||||
|
||||
/**
|
||||
* The icon for a notification, size x size gray, from the first of these that works:
|
||||
* 1. the small icon, when its alpha carries a shape (nearly every app);
|
||||
* 2. the launcher icon's themed monochrome layer (Android 13+), which is made to be tinted;
|
||||
* 3. the small icon's glyph, when it is a color tile with one background color;
|
||||
* 4. the launcher icon's glyph or shape, or its adaptive foreground layer's shape;
|
||||
* 5. the small icon, then the launcher icon, read by brightness;
|
||||
* 6. the large icon (usually the sender's avatar: the notification's content, not its app);
|
||||
* 7. the app name's initial in a rounded box.
|
||||
* Null only when even the initial can't be drawn.
|
||||
*/
|
||||
fun iconFor(context: Context, statusBarNotification: StatusBarNotification, size: Int, appName: () -> String): Rendered? {
|
||||
val packageName = statusBarNotification.packageName ?: ""
|
||||
val notification = statusBarNotification.notification
|
||||
val small = smallIconArt(context, packageName, notification.smallIcon, size)
|
||||
val smallKind = small?.art?.kind
|
||||
if (smallKind == Kind.TEMPLATE || smallKind == Kind.SOLID) return small
|
||||
val launcher = launcherArt(context, packageName, size)
|
||||
launcher.monochrome?.let { return it }
|
||||
if (smallKind == Kind.GLYPH) return small
|
||||
launcher.shape?.let { return it }
|
||||
if (smallKind == Kind.PICTURE) return small
|
||||
launcher.picture?.let { return it }
|
||||
largeIconArt(context, packageName, notification, size)?.let { return it }
|
||||
return letterArt(appName(), size)
|
||||
}
|
||||
|
||||
private fun smallIconArt(context: Context, packageName: String, icon: Icon?, size: Int): Rendered? {
|
||||
if (icon == null) return null
|
||||
val key = resourceKey(icon) ?: return loadSmallIcon(context, packageName, icon, size)
|
||||
return smallIconCache.getOrPut("$packageName|$key|$size") { loadSmallIcon(context, packageName, icon, size) }
|
||||
}
|
||||
|
||||
/** Resource icons are cached by resource; bitmap and URI icons are converted every time. */
|
||||
private fun resourceKey(icon: Icon): String? {
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.P || icon.type != Icon.TYPE_RESOURCE) return null
|
||||
return icon.resPackage + "/" + icon.resId
|
||||
}
|
||||
|
||||
private fun loadSmallIcon(context: Context, packageName: String, icon: Icon, size: Int): Rendered? =
|
||||
tryRender("small", packageName) {
|
||||
val drawable = icon.loadDrawable(context) ?: return@tryRender null
|
||||
// A tint (the Icon's own, or a vector's theme tint) recolors the very pixels that tell
|
||||
// a template from a color tile, and can fade a template's alpha. Templates are read
|
||||
// by alpha alone, so dropping it costs nothing.
|
||||
drawable.mutate()
|
||||
drawable.setTintList(null)
|
||||
renderSource(context, packageName, "small", drawable, size)
|
||||
}
|
||||
|
||||
private fun launcherArt(context: Context, packageName: String, size: Int): LauncherArt =
|
||||
launcherCache.getOrPut("$packageName|$size") { loadLauncherArt(context, packageName, size) }
|
||||
|
||||
private fun loadLauncherArt(context: Context, packageName: String, size: Int): LauncherArt {
|
||||
val icon: Drawable = try {
|
||||
context.packageManager.getApplicationIcon(packageName)
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "no launcher icon for $packageName", t)
|
||||
return LauncherArt(null, null, null)
|
||||
}
|
||||
val adaptive = if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) icon as? AdaptiveIconDrawable else null
|
||||
if (adaptive != null && Build.VERSION.SDK_INT >= Build.VERSION_CODES.TIRAMISU) {
|
||||
val layer = adaptive.monochrome
|
||||
val monochrome = if (layer == null) null else tryRender("monochrome", packageName) {
|
||||
renderSource(context, packageName, "monochrome", layer, size, ADAPTIVE_LAYER_OUTSET, template = true)
|
||||
}
|
||||
if (monochrome != null) return LauncherArt(monochrome, null, null)
|
||||
}
|
||||
val whole = tryRender("launcher", packageName) { renderSource(context, packageName, "launcher", icon, size) }
|
||||
var shape = whole?.takeIf { it.art.kind == Kind.TEMPLATE || it.art.kind == Kind.GLYPH }
|
||||
val foreground = adaptive?.foreground
|
||||
if (shape == null && foreground != null) {
|
||||
// A gradient or picture background hides the glyph; the foreground layer alone may carry it as alpha.
|
||||
shape = tryRender("foreground", packageName) {
|
||||
renderSource(context, packageName, "foreground", foreground, size, ADAPTIVE_LAYER_OUTSET)
|
||||
}?.takeIf { it.art.kind == Kind.TEMPLATE }
|
||||
}
|
||||
return LauncherArt(null, shape, whole?.takeIf { it.art.kind == Kind.PICTURE })
|
||||
}
|
||||
|
||||
private fun largeIconArt(context: Context, packageName: String, notification: Notification, size: Int): Rendered? =
|
||||
tryRender("large", packageName) {
|
||||
val drawable = notification.getLargeIcon()?.loadDrawable(context) ?: return@tryRender null
|
||||
renderSource(context, packageName, "large", drawable, size)
|
||||
}
|
||||
|
||||
/** The app name's first letter or digit in a rounded box, like the shell's letter window icons. */
|
||||
private fun letterArt(appName: String, size: Int): Rendered? = tryRender("letter", appName) {
|
||||
val letter = appName.firstOrNull { it.isLetterOrDigit() }?.uppercaseChar()?.toString() ?: "?"
|
||||
val canvasSize = size * NotificationIconArt.ANALYSIS_SCALE
|
||||
val bitmap = Bitmap.createBitmap(canvasSize, canvasSize, Bitmap.Config.ARGB_8888)
|
||||
val canvas = Canvas(bitmap)
|
||||
val stroke = NotificationIconArt.ANALYSIS_SCALE.toFloat()
|
||||
val box = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
style = Paint.Style.STROKE
|
||||
strokeWidth = stroke
|
||||
color = Color.argb(160, 255, 255, 255)
|
||||
}
|
||||
val radius = canvasSize / 4f
|
||||
canvas.drawRoundRect(stroke / 2, stroke / 2, canvasSize - stroke / 2, canvasSize - stroke / 2, radius, radius, box)
|
||||
val text = Paint(Paint.ANTI_ALIAS_FLAG).apply {
|
||||
color = Color.WHITE
|
||||
typeface = Typeface.DEFAULT_BOLD
|
||||
textSize = canvasSize * 0.6f
|
||||
textAlign = Paint.Align.CENTER
|
||||
}
|
||||
val bounds = Rect()
|
||||
text.getTextBounds(letter, 0, letter.length, bounds)
|
||||
canvas.drawText(letter, canvasSize / 2f, canvasSize / 2f - bounds.exactCenterY(), text)
|
||||
val argb = IntArray(canvasSize * canvasSize)
|
||||
bitmap.getPixels(argb, 0, canvasSize, 0, 0, canvasSize, canvasSize)
|
||||
bitmap.recycle()
|
||||
val art = NotificationIconArt.template(argb, size)
|
||||
if (art.kind == Kind.EMPTY) null else Rendered("letter", art)
|
||||
}
|
||||
|
||||
private inline fun tryRender(source: String, owner: String, render: () -> Rendered?): Rendered? {
|
||||
return try {
|
||||
render()
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to render $source icon for $owner", t)
|
||||
null
|
||||
}
|
||||
}
|
||||
|
||||
/** Render, dump and convert one source drawable; null when nothing visible came out. */
|
||||
private fun renderSource(
|
||||
context: Context, packageName: String, source: String, drawable: Drawable, size: Int,
|
||||
outset: Float = 0f, template: Boolean = false,
|
||||
): Rendered? {
|
||||
val analysisSize = size * NotificationIconArt.ANALYSIS_SCALE
|
||||
val argb = renderArgb(drawable, analysisSize, outset)
|
||||
dumpDebugPng(context, packageName, source, argb, analysisSize)
|
||||
val art = if (template) NotificationIconArt.template(argb, size) else NotificationIconArt.convert(argb, size)
|
||||
return if (art.kind == Kind.EMPTY) null else Rendered(source, art)
|
||||
}
|
||||
|
||||
/**
|
||||
* Render a drawable to target x target ARGB with proper downscaling. Detailed sources (e.g.
|
||||
* avatar bitmaps used as notification icons) are rendered at native resolution and reduced
|
||||
* by repeated halving: a single filtered pass from, say, 126px to 24px samples too sparsely
|
||||
* and turns fine detail into speckle that reads as a garbled icon. An outset draws the
|
||||
* drawable that share of the canvas past each edge, as AdaptiveIconDrawable draws its layers.
|
||||
*/
|
||||
private fun renderArgb(drawable: Drawable, target: Int, outset: Float): IntArray {
|
||||
drawable.mutate()
|
||||
if (drawable is BitmapDrawable) {
|
||||
drawable.setFilterBitmap(true)
|
||||
}
|
||||
val scale = 1f + 2f * outset
|
||||
val renderW = max(target, (drawable.intrinsicWidth / scale).roundToInt())
|
||||
val renderH = max(target, (drawable.intrinsicHeight / scale).roundToInt())
|
||||
var bitmap = Bitmap.createBitmap(renderW, renderH, Bitmap.Config.ARGB_8888)
|
||||
val dx = (renderW * outset).roundToInt()
|
||||
val dy = (renderH * outset).roundToInt()
|
||||
drawable.setBounds(-dx, -dy, renderW + dx, renderH + dy)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
while (bitmap.width >= target * 2 && bitmap.height >= target * 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
bitmap.recycle()
|
||||
bitmap = halved
|
||||
}
|
||||
if (bitmap.width != target || bitmap.height != target) {
|
||||
val scaled = Bitmap.createScaledBitmap(bitmap, target, target, true)
|
||||
bitmap.recycle()
|
||||
bitmap = scaled
|
||||
}
|
||||
val pixels = IntArray(target * target)
|
||||
bitmap.getPixels(pixels, 0, target, 0, 0, target, target)
|
||||
bitmap.recycle()
|
||||
return pixels
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug aid for icon reports: saves each source as rendered for analysis, before any
|
||||
* conversion, to <externalFilesDir>/debug-icons/<package>-<source>.png (adb-pullable), so a
|
||||
* bad icon can be rerun through NotificationIconArt off-device. Runs only when a source is
|
||||
* converted, not on cached icons; the oldest files past DEBUG_DUMP_MAX_FILES are deleted.
|
||||
*/
|
||||
private fun dumpDebugPng(context: Context, packageName: String, source: String, argb: IntArray, size: Int) {
|
||||
try {
|
||||
val base = context.getExternalFilesDir(null) ?: return
|
||||
val dir = File(base, "debug-icons")
|
||||
if (!dir.exists() && !dir.mkdirs()) {
|
||||
return
|
||||
}
|
||||
val safeName = packageName.replace(Regex("[^A-Za-z0-9._-]"), "_")
|
||||
val bitmap = Bitmap.createBitmap(argb, size, size, Bitmap.Config.ARGB_8888)
|
||||
try {
|
||||
FileOutputStream(File(dir, "$safeName-$source.png")).use { stream ->
|
||||
bitmap.compress(Bitmap.CompressFormat.PNG, 100, stream)
|
||||
}
|
||||
} finally {
|
||||
bitmap.recycle()
|
||||
}
|
||||
val files = dir.listFiles() ?: return
|
||||
if (files.size > DEBUG_DUMP_MAX_FILES) {
|
||||
files.sortBy { it.lastModified() }
|
||||
for (index in 0 until files.size - DEBUG_DUMP_MAX_FILES) {
|
||||
files[index].delete()
|
||||
}
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to dump debug icon", t)
|
||||
}
|
||||
}
|
||||
|
||||
/** A small LRU of converted art with an age limit. Conversions run under its lock. */
|
||||
private class ArtCache<V>(private val maxEntries: Int, private val maxAgeMs: Long) {
|
||||
private class Entry<T>(val value: T, val atMs: Long)
|
||||
|
||||
private val entries = object : LinkedHashMap<String, Entry<V>>(16, 0.75f, true) {
|
||||
override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Entry<V>>): Boolean {
|
||||
return size > maxEntries
|
||||
}
|
||||
}
|
||||
|
||||
@Synchronized
|
||||
fun getOrPut(key: String, compute: () -> V): V {
|
||||
val now = SystemClock.elapsedRealtime()
|
||||
val cached = entries[key]
|
||||
if (cached != null && now - cached.atMs < maxAgeMs) {
|
||||
return cached.value
|
||||
}
|
||||
val value = compute()
|
||||
entries[key] = Entry(value, now)
|
||||
return value
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -31,6 +31,34 @@ class PreviewBitmapUtil private constructor() {
|
||||
return Bitmap.createBitmap(colors, width, height, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
|
||||
/**
|
||||
* As above, showing only the cropWidth x cropHeight area at (cropX, cropY) (the
|
||||
* mirror's Scaled preview). The area may extend past the frame's edges; it is black
|
||||
* there.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun fromGray(gray: ByteBuffer?, width: Int, height: Int, brightenGamma: Double, green: Boolean,
|
||||
cropX: Int, cropY: Int, cropWidth: Int, cropHeight: Int): Bitmap {
|
||||
if (gray == null || width <= 0 || height <= 0 || gray.remaining() < width * height) {
|
||||
throw IllegalArgumentException("invalid gray preview buffer")
|
||||
}
|
||||
if (cropWidth <= 0 || cropHeight <= 0) {
|
||||
throw IllegalArgumentException("invalid preview crop")
|
||||
}
|
||||
val bytes = ByteArray(width * height)
|
||||
gray.get(bytes)
|
||||
val cropped = ByteArray(cropWidth * cropHeight)
|
||||
val left = maxOf(cropX, 0)
|
||||
val right = minOf(cropX + cropWidth, width)
|
||||
if (right > left) {
|
||||
for (y in maxOf(cropY, 0) until minOf(cropY + cropHeight, height)) {
|
||||
System.arraycopy(bytes, y * width + left, cropped, (y - cropY) * cropWidth + (left - cropX), right - left)
|
||||
}
|
||||
}
|
||||
val colors = PreviewPalette.expand(cropped, 0, cropped.size, brightenGamma, green)
|
||||
return Bitmap.createBitmap(colors, cropWidth, cropHeight, Bitmap.Config.ARGB_8888)
|
||||
}
|
||||
|
||||
/**
|
||||
* A white-on-transparent bitmap of an 8bpp glasses icon (gray becomes
|
||||
* alpha; black, which the glasses show as unlit, is transparent), for
|
||||
|
||||
@@ -13,7 +13,7 @@
|
||||
<key>CFBundleDisplayName</key>
|
||||
<string>Faceclaw</string>
|
||||
<key>CFBundleShortVersionString</key>
|
||||
<string>1.0.0</string>
|
||||
<string>0.8.4</string>
|
||||
<key>CFBundleVersion</key>
|
||||
<string>1</string>
|
||||
<key>LSRequiresIPhoneOS</key>
|
||||
@@ -30,6 +30,8 @@
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSAppleMusicUsageDescription</key>
|
||||
<string>Faceclaw shows what is playing in Apple Music on your glasses, controls playback, and lets you browse and play your music library from the glasses.</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Faceclaw uses your location for local weather, turn-by-turn navigation, and true north on the glasses compass.</string>
|
||||
<key>NSLocationTemporaryUsageDescriptionDictionary</key>
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
||||
<plist version="1.0">
|
||||
<dict>
|
||||
<key>NSPrivacyAccessedAPITypes</key>
|
||||
<array>
|
||||
<dict>
|
||||
<!-- Faceclaw preferences in its own NSUserDefaults domain. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategoryUserDefaults</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>CA92.1</string></array>
|
||||
</dict>
|
||||
<dict>
|
||||
<!-- App-container files used by Faceclaw and its NativeScript/Kotlin runtimes. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategoryFileTimestamp</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>C617.1</string></array>
|
||||
</dict>
|
||||
<dict>
|
||||
<!-- NativeScript's monotonic clock measures elapsed time within the app. -->
|
||||
<key>NSPrivacyAccessedAPIType</key>
|
||||
<string>NSPrivacyAccessedAPICategorySystemBootTime</string>
|
||||
<key>NSPrivacyAccessedAPITypeReasons</key>
|
||||
<array><string>35F9.1</string></array>
|
||||
</dict>
|
||||
</array>
|
||||
</dict>
|
||||
</plist>
|
||||
@@ -2,8 +2,8 @@ IPHONEOS_DEPLOYMENT_TARGET = 15.1;
|
||||
TARGETED_DEVICE_FAMILY = 1;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
|
||||
// Select the local signing team in Xcode or pass --teamId to the iOS scripts.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* Apple Music (the system Music app) through the MediaPlayer framework: now-playing state
|
||||
* and transport on the system player, the media library as a browse tree, and the system
|
||||
* output volume. Call from the main thread; callbacks are delivered on the main queue.
|
||||
* Nothing touches the player or library until media-library access is granted, so no
|
||||
* permission prompt appears unasked.
|
||||
*/
|
||||
@interface FaceclawAppleMusic : NSObject
|
||||
+ (instancetype)shared;
|
||||
/** Playback state, the now-playing item or authorization may have changed. */
|
||||
@property(nonatomic, copy) void (^changeHandler)(void);
|
||||
- (BOOL)hasPermission;
|
||||
- (NSString *)permissionStatus;
|
||||
- (void)requestPermission:(void (^)(BOOL granted))completion;
|
||||
/** Observe the system player (and keep observing across authorization changes). */
|
||||
- (void)start;
|
||||
- (void)stop;
|
||||
/** The system player as a JSON object; {"access": false} until access is granted. */
|
||||
- (NSString *)stateJson;
|
||||
- (void)playPause;
|
||||
- (void)skipNext;
|
||||
- (void)skipPrevious;
|
||||
/**
|
||||
* The queue Faceclaw last started, as [{"id", "title", "active"}], while the player is
|
||||
* still playing from it; "[]" otherwise (the system player does not expose queues that
|
||||
* the Music app started).
|
||||
*/
|
||||
- (NSString *)queueJson;
|
||||
- (void)skipToQueueIndex:(NSInteger)index;
|
||||
/** [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), or nil. */
|
||||
- (NSData *)albumArt:(NSInteger)maxSize;
|
||||
/** System output volume 0..100, or -1 when unknown. */
|
||||
- (double)volumePercent;
|
||||
- (void)setVolumePercent:(double)percent;
|
||||
/** One level of the library tree ("root" first) as [{"mediaId", "title", "subtitle", "browsable", "playable"}]. */
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *json, NSString *error))completion;
|
||||
/** Start playback of a browsed collection, or of a song within its collection. */
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *error))completion;
|
||||
@end
|
||||
@@ -0,0 +1,512 @@
|
||||
#import "FaceclawAppleMusic.h"
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <MediaPlayer/MediaPlayer.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <errno.h>
|
||||
#import <math.h>
|
||||
#import <stdlib.h>
|
||||
|
||||
/** How long a queue Faceclaw just set may take to become the player's now-playing context. */
|
||||
static const NSTimeInterval kQueueConfirmSeconds = 5.0;
|
||||
|
||||
static NSString *JsonString(id object, NSString *fallback) {
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
|
||||
return data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : fallback;
|
||||
}
|
||||
|
||||
/** Persistent IDs are unsigned 64-bit, beyond JavaScript's exact integers, so they cross as decimal strings. */
|
||||
static NSString *IdString(MPMediaEntityPersistentID persistentId) {
|
||||
return [NSString stringWithFormat:@"%llu", persistentId];
|
||||
}
|
||||
|
||||
static BOOL ParseId(NSString *text, MPMediaEntityPersistentID *out) {
|
||||
const char *chars = text.UTF8String;
|
||||
if (!chars || chars[0] < '0' || chars[0] > '9') return NO;
|
||||
char *end = NULL;
|
||||
errno = 0;
|
||||
unsigned long long value = strtoull(chars, &end, 10);
|
||||
if (errno || !end || *end) return NO;
|
||||
*out = value;
|
||||
return YES;
|
||||
}
|
||||
|
||||
static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *subtitle, BOOL browsable, BOOL playable) {
|
||||
return @{@"mediaId": mediaId, @"title": title ?: @"", @"subtitle": subtitle ?: @"",
|
||||
@"browsable": @(browsable), @"playable": @(playable)};
|
||||
}
|
||||
|
||||
@interface FaceclawAppleMusic ()
|
||||
@property(nonatomic, strong) dispatch_queue_t libraryQueue;
|
||||
@property(nonatomic) BOOL started;
|
||||
@property(nonatomic) BOOL observing;
|
||||
/** Bumped whenever the now-playing item changes, so cached album art is refetched. */
|
||||
@property(nonatomic) NSInteger artVersion;
|
||||
@property(nonatomic, copy) NSString *nowPlayingKey;
|
||||
/** The items of the queue Faceclaw last started, until the player leaves them. */
|
||||
@property(nonatomic, copy) NSArray<MPMediaItem *> *queueItems;
|
||||
@property(nonatomic) BOOL queueConfirmed;
|
||||
@property(nonatomic, strong) NSDate *queueSetAt;
|
||||
@property(nonatomic, strong) MPVolumeView *volumeView;
|
||||
@end
|
||||
|
||||
@implementation FaceclawAppleMusic
|
||||
+ (instancetype)shared {
|
||||
static FaceclawAppleMusic *instance;
|
||||
static dispatch_once_t once;
|
||||
dispatch_once(&once, ^{ instance = [FaceclawAppleMusic new]; });
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) {
|
||||
_libraryQueue = dispatch_queue_create("com.faceclaw.applemusic", DISPATCH_QUEUE_SERIAL);
|
||||
_nowPlayingKey = @"";
|
||||
// Access may have been granted or revoked in Settings while away.
|
||||
[NSNotificationCenter.defaultCenter addObserver:self selector:@selector(appBecameActive:)
|
||||
name:UIApplicationDidBecomeActiveNotification object:nil];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - Authorization
|
||||
|
||||
- (BOOL)hasPermission {
|
||||
return MPMediaLibrary.authorizationStatus == MPMediaLibraryAuthorizationStatusAuthorized;
|
||||
}
|
||||
|
||||
- (NSString *)permissionStatus {
|
||||
switch (MPMediaLibrary.authorizationStatus) {
|
||||
case MPMediaLibraryAuthorizationStatusAuthorized: return @"granted";
|
||||
case MPMediaLibraryAuthorizationStatusDenied: return @"denied";
|
||||
case MPMediaLibraryAuthorizationStatusRestricted: return @"restricted";
|
||||
default: return @"not-determined";
|
||||
}
|
||||
}
|
||||
|
||||
- (void)requestPermission:(void (^)(BOOL))completion {
|
||||
if ([self hasPermission]) {
|
||||
completion(YES);
|
||||
return;
|
||||
}
|
||||
[MPMediaLibrary requestAuthorization:^(MPMediaLibraryAuthorizationStatus status) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
completion(status == MPMediaLibraryAuthorizationStatusAuthorized);
|
||||
});
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)appBecameActive:(NSNotification *)notification {
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
#pragma mark - Observation
|
||||
|
||||
- (void)start {
|
||||
self.started = YES;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
self.started = NO;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)syncObserving {
|
||||
BOOL want = self.started && [self hasPermission];
|
||||
if (want == self.observing) return;
|
||||
self.observing = want;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
NSNotificationCenter *center = NSNotificationCenter.defaultCenter;
|
||||
if (want) {
|
||||
[center addObserver:self selector:@selector(playbackStateChanged:)
|
||||
name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center addObserver:self selector:@selector(nowPlayingItemChanged:)
|
||||
name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player beginGeneratingPlaybackNotifications];
|
||||
} else {
|
||||
[center removeObserver:self name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center removeObserver:self name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player endGeneratingPlaybackNotifications];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)playbackStateChanged:(NSNotification *)notification {
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)nowPlayingItemChanged:(NSNotification *)notification {
|
||||
// Streamed catalog items can share identity fields, so every change counts.
|
||||
self.artVersion++;
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)notifyChanged {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.changeHandler) self.changeHandler();
|
||||
});
|
||||
}
|
||||
|
||||
#pragma mark - Now playing
|
||||
|
||||
- (NSString *)stateJson {
|
||||
if (![self hasPermission]) return @"{\"access\":false}";
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *item = player.nowPlayingItem;
|
||||
NSString *key = item ? [NSString stringWithFormat:@"%llu|%@|%@|%@", item.persistentID,
|
||||
item.playbackStoreID ?: @"", item.title ?: @"", item.artist ?: @""] : @"";
|
||||
if (![key isEqualToString:self.nowPlayingKey]) {
|
||||
self.nowPlayingKey = key;
|
||||
self.artVersion++;
|
||||
}
|
||||
NSString *state;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
state = @"playing";
|
||||
break;
|
||||
case MPMusicPlaybackStatePaused:
|
||||
case MPMusicPlaybackStateInterrupted:
|
||||
state = @"paused";
|
||||
break;
|
||||
default:
|
||||
state = @"stopped";
|
||||
break;
|
||||
}
|
||||
NSTimeInterval position = player.currentPlaybackTime;
|
||||
NSTimeInterval duration = item.playbackDuration;
|
||||
float rate = player.currentPlaybackRate;
|
||||
return JsonString(@{
|
||||
@"access": @YES,
|
||||
@"hasItem": @(item != nil),
|
||||
@"state": state,
|
||||
@"title": item.title ?: @"",
|
||||
@"artist": item.artist ?: @"",
|
||||
@"album": item.albumTitle ?: @"",
|
||||
@"positionMs": @(item && isfinite(position) && position >= 0 ? position * 1000.0 : -1),
|
||||
@"durationMs": @(item && isfinite(duration) && duration > 0 ? duration * 1000.0 : -1),
|
||||
@"rate": @(isfinite(rate) ? rate : 0),
|
||||
@"artVersion": @(self.artVersion),
|
||||
}, @"{\"access\":true}");
|
||||
}
|
||||
|
||||
- (void)playPause {
|
||||
if (![self hasPermission]) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
[player pause];
|
||||
break;
|
||||
default:
|
||||
[player play];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)skipNext {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToNextItem];
|
||||
}
|
||||
|
||||
- (void)skipPrevious {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToPreviousItem];
|
||||
}
|
||||
|
||||
- (NSData *)albumArt:(NSInteger)maxSize {
|
||||
if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil;
|
||||
MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork;
|
||||
UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)];
|
||||
CGImageRef source = image.CGImage;
|
||||
if (!source) return nil;
|
||||
double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
|
||||
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
|
||||
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
|
||||
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
|
||||
uint8_t *pixels = bytes + 4;
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
|
||||
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) return nil;
|
||||
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), source);
|
||||
CGContextRelease(context);
|
||||
// Composited over black (the display's see-through background); now opaque.
|
||||
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
|
||||
return packet;
|
||||
}
|
||||
|
||||
#pragma mark - Queue
|
||||
|
||||
/** Index of the now-playing item in Faceclaw's queue; forgets the queue once the player has left it. */
|
||||
- (NSInteger)activeQueueIndex {
|
||||
NSArray<MPMediaItem *> *items = self.queueItems;
|
||||
if (!items.count || ![self hasPermission]) return -1;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *now = player.nowPlayingItem;
|
||||
NSInteger found = -1;
|
||||
if (now) {
|
||||
NSUInteger index = player.indexOfNowPlayingItem;
|
||||
if (index < items.count && items[index].persistentID == now.persistentID) {
|
||||
found = (NSInteger)index;
|
||||
} else {
|
||||
for (NSUInteger i = 0; i < items.count; i++) {
|
||||
if (items[i].persistentID == now.persistentID) { found = (NSInteger)i; break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found >= 0) {
|
||||
self.queueConfirmed = YES;
|
||||
return found;
|
||||
}
|
||||
// Either the player hasn't switched over yet, or it has moved on to another queue.
|
||||
if (self.queueConfirmed || -[self.queueSetAt timeIntervalSinceNow] > kQueueConfirmSeconds) {
|
||||
self.queueItems = nil;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
- (NSString *)queueJson {
|
||||
NSInteger active = [self activeQueueIndex];
|
||||
if (active < 0) return @"[]";
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:self.queueItems.count];
|
||||
[self.queueItems enumerateObjectsUsingBlock:^(MPMediaItem *item, NSUInteger index, BOOL *stop) {
|
||||
[rows addObject:@{@"id": @(index), @"title": item.title ?: @"", @"active": @((NSInteger)index == active)}];
|
||||
}];
|
||||
return JsonString(rows, @"[]");
|
||||
}
|
||||
|
||||
- (void)skipToQueueIndex:(NSInteger)index {
|
||||
if ([self activeQueueIndex] < 0 || index < 0 || index >= (NSInteger)self.queueItems.count) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
player.nowPlayingItem = self.queueItems[index];
|
||||
[player play];
|
||||
}
|
||||
|
||||
#pragma mark - Volume
|
||||
|
||||
- (UIWindow *)hostWindow {
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
if (![scene isKindOfClass:UIWindowScene.class]) continue;
|
||||
NSArray<UIWindow *> *windows = ((UIWindowScene *)scene).windows;
|
||||
for (UIWindow *window in windows) if (window.isKeyWindow) return window;
|
||||
if (windows.firstObject) return windows.firstObject;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
/**
|
||||
* The slider of an off-screen MPVolumeView: iOS has no public setter for the system volume,
|
||||
* but the volume view's slider drives it while the view is in a window.
|
||||
*/
|
||||
- (UISlider *)volumeSlider:(BOOL *)attached {
|
||||
if (!self.volumeView) {
|
||||
self.volumeView = [[MPVolumeView alloc] initWithFrame:CGRectMake(-2000, -2000, 160, 40)];
|
||||
}
|
||||
if (!self.volumeView.window) {
|
||||
UIWindow *window = [self hostWindow];
|
||||
if (!window) return nil;
|
||||
[window addSubview:self.volumeView];
|
||||
if (attached) *attached = YES;
|
||||
}
|
||||
for (UIView *view in self.volumeView.subviews) {
|
||||
if ([view isKindOfClass:UISlider.class]) return (UISlider *)view;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (double)volumePercent {
|
||||
// The slider tracks the system volume once attached; until then (nothing has been set
|
||||
// yet) the session's reading, which can lag while Faceclaw's audio session is inactive.
|
||||
UISlider *slider = self.volumeView.window ? [self volumeSlider:NULL] : nil;
|
||||
float volume = slider ? slider.value : AVAudioSession.sharedInstance.outputVolume;
|
||||
return isfinite(volume) ? MAX(0.0, MIN(100.0, round(volume * 100.0))) : -1;
|
||||
}
|
||||
|
||||
- (void)setVolumePercent:(double)percent {
|
||||
if (!isfinite(percent)) return;
|
||||
BOOL attached = NO;
|
||||
UISlider *slider = [self volumeSlider:&attached];
|
||||
if (!slider) return;
|
||||
float value = (float)MAX(0.0, MIN(1.0, percent / 100.0));
|
||||
void (^apply)(void) = ^{
|
||||
[slider setValue:value animated:NO];
|
||||
[slider sendActionsForControlEvents:UIControlEventValueChanged];
|
||||
};
|
||||
// A freshly attached volume view ignores changes until it has wired up its slider.
|
||||
if (attached) dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.15 * NSEC_PER_SEC)), dispatch_get_main_queue(), apply);
|
||||
else apply();
|
||||
}
|
||||
|
||||
#pragma mark - Library
|
||||
|
||||
/**
|
||||
* Songs of a playable collection id: "playlist:<id>", "album:<id>", "artist:<id>" (album by
|
||||
* album, in track order) or "songs" (the whole library). Runs on the library queue.
|
||||
*/
|
||||
- (NSArray<MPMediaItem *> *)itemsForCollection:(NSString *)collectionId {
|
||||
if ([collectionId isEqualToString:@"songs"]) return [MPMediaQuery songsQuery].items ?: @[];
|
||||
NSRange colon = [collectionId rangeOfString:@":"];
|
||||
MPMediaEntityPersistentID persistentId;
|
||||
if (colon.location == NSNotFound || !ParseId([collectionId substringFromIndex:colon.location + 1], &persistentId)) return @[];
|
||||
NSString *kind = [collectionId substringToIndex:colon.location];
|
||||
MPMediaQuery *query;
|
||||
NSString *property;
|
||||
if ([kind isEqualToString:@"playlist"]) {
|
||||
query = [MPMediaQuery playlistsQuery];
|
||||
property = MPMediaPlaylistPropertyPersistentID;
|
||||
} else if ([kind isEqualToString:@"album"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyAlbumPersistentID;
|
||||
} else if ([kind isEqualToString:@"artist"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyArtistPersistentID;
|
||||
} else {
|
||||
return @[];
|
||||
}
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(persistentId) forProperty:property]];
|
||||
NSMutableArray<MPMediaItem *> *items = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *collection in query.collections) [items addObjectsFromArray:collection.items];
|
||||
return items;
|
||||
}
|
||||
|
||||
- (NSArray *)songRows:(NSArray<MPMediaItem *> *)items context:(NSString *)collectionId {
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:items.count];
|
||||
for (MPMediaItem *item in items) {
|
||||
[rows addObject:BrowseRow([NSString stringWithFormat:@"song:%@@%@", IdString(item.persistentID), collectionId],
|
||||
item.title, item.artist, NO, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)albumRows:(MPMediaQuery *)query {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *album in query.collections) {
|
||||
MPMediaItem *item = album.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"album:" stringByAppendingString:IdString(item.albumPersistentID)],
|
||||
item.albumTitle, item.albumArtist.length ? item.albumArtist : item.artist, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)childrenOf:(NSString *)parentId {
|
||||
if ([parentId isEqualToString:@"root"]) {
|
||||
return @[BrowseRow(@"playlists", @"Playlists", nil, YES, NO), BrowseRow(@"artists", @"Artists", nil, YES, NO),
|
||||
BrowseRow(@"albums", @"Albums", nil, YES, NO), BrowseRow(@"songs", @"Songs", nil, YES, NO)];
|
||||
}
|
||||
if ([parentId isEqualToString:@"playlists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaPlaylist *playlist in [MPMediaQuery playlistsQuery].collections) {
|
||||
NSUInteger count = playlist.count;
|
||||
NSString *subtitle = count == 1 ? @"1 song" : [NSString stringWithFormat:@"%lu songs", (unsigned long)count];
|
||||
[rows addObject:BrowseRow([@"playlist:" stringByAppendingString:IdString(playlist.persistentID)],
|
||||
playlist.name, subtitle, YES, count > 0)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"artists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *artist in [MPMediaQuery artistsQuery].collections) {
|
||||
MPMediaItem *item = artist.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"artist:" stringByAppendingString:IdString(item.artistPersistentID)],
|
||||
item.artist.length ? item.artist : @"Unknown Artist", nil, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"albums"]) return [self albumRows:[MPMediaQuery albumsQuery]];
|
||||
if ([parentId isEqualToString:@"songs"]) return [self songRows:[MPMediaQuery songsQuery].items ?: @[] context:@"songs"];
|
||||
if ([parentId hasPrefix:@"artist:"]) {
|
||||
MPMediaEntityPersistentID artistId;
|
||||
if (!ParseId([parentId substringFromIndex:7], &artistId)) return nil;
|
||||
MPMediaQuery *query = [MPMediaQuery albumsQuery];
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(artistId)
|
||||
forProperty:MPMediaItemPropertyArtistPersistentID]];
|
||||
return [self albumRows:query];
|
||||
}
|
||||
if ([parentId hasPrefix:@"playlist:"] || [parentId hasPrefix:@"album:"]) {
|
||||
return [self songRows:[self itemsForCollection:parentId] context:parentId];
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *, NSString *))completion {
|
||||
// Library queries can take a while on large libraries; keep them off the main thread.
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *json = nil, *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
NSArray *rows = [self childrenOf:parentId ?: @""];
|
||||
if (rows) json = JsonString(rows, nil);
|
||||
if (!json) error = rows ? @"Library listing failed" : @"Unknown library item";
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ completion(json, error); });
|
||||
});
|
||||
}
|
||||
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *))completion {
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *collectionId = mediaId ?: @"";
|
||||
MPMediaEntityPersistentID startId = 0;
|
||||
BOOL hasStart = NO;
|
||||
if ([collectionId hasPrefix:@"song:"]) {
|
||||
NSRange at = [collectionId rangeOfString:@"@"];
|
||||
if (at.location == NSNotFound) collectionId = @"";
|
||||
else {
|
||||
hasStart = ParseId([collectionId substringWithRange:NSMakeRange(5, at.location - 5)], &startId);
|
||||
collectionId = [collectionId substringFromIndex:at.location + 1];
|
||||
}
|
||||
}
|
||||
NSArray<MPMediaItem *> *items = @[];
|
||||
NSString *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
items = [self itemsForCollection:collectionId];
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
if (!error && !items.count) error = @"Nothing to play";
|
||||
}
|
||||
MPMediaItem *start = nil;
|
||||
if (hasStart) {
|
||||
for (MPMediaItem *item in items) {
|
||||
if (item.persistentID == startId) { start = item; break; }
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (error) {
|
||||
completion(error);
|
||||
return;
|
||||
}
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMusicPlayerMediaItemQueueDescriptor *descriptor = [[MPMusicPlayerMediaItemQueueDescriptor alloc]
|
||||
initWithItemCollection:[MPMediaItemCollection collectionWithItems:items]];
|
||||
if (start) descriptor.startItem = start;
|
||||
[player setQueueWithDescriptor:descriptor];
|
||||
[player play];
|
||||
self.queueItems = items;
|
||||
self.queueConfirmed = NO;
|
||||
self.queueSetAt = [NSDate date];
|
||||
[self notifyChanged];
|
||||
completion(nil);
|
||||
});
|
||||
});
|
||||
}
|
||||
@end
|
||||
@@ -2,7 +2,7 @@
|
||||
#import <WebKit/WebKit.h>
|
||||
|
||||
/** One local or URL EvenHub runtime, retained by its TS session until its window closes. */
|
||||
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKURLSchemeHandler>
|
||||
@interface FaceclawEvenHubWebView : NSObject <WKScriptMessageHandler, WKNavigationDelegate, WKUIDelegate, WKURLSchemeHandler>
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
@property(nonatomic, copy) NSString *packageIdentifier;
|
||||
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script;
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
#import "FaceclawEvenHubWebView.h"
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <SafariServices/SafariServices.h>
|
||||
#import <UniformTypeIdentifiers/UniformTypeIdentifiers.h>
|
||||
#import <CommonCrypto/CommonDigest.h>
|
||||
|
||||
@@ -7,19 +8,49 @@
|
||||
@property(nonatomic) WKWebView *webView;
|
||||
@property(nonatomic) UIView *host;
|
||||
@property(nonatomic) UIButton *back;
|
||||
@property(nonatomic) UIButton *returnButton;
|
||||
@property(nonatomic) NSTimer *timer;
|
||||
@property(nonatomic) NSString *directory;
|
||||
@property(nonatomic) NSURL *remoteURL;
|
||||
@property(nonatomic) NSString *script;
|
||||
/** The app's start page. Only documents from its origin get the bridge. */
|
||||
@property(nonatomic) NSURL *startURL;
|
||||
/** The redirect_uri of a sign-in the app started; the provider's reply is finished inside the app. */
|
||||
@property(nonatomic) NSURL *returnURL;
|
||||
/** The main frame shows a page outside the app's origin (a sign-in page, a followed link). */
|
||||
@property(nonatomic) BOOL away;
|
||||
@property(nonatomic) BOOL committed;
|
||||
@property(nonatomic) BOOL ticking;
|
||||
@property(nonatomic) BOOL destroyed;
|
||||
@end
|
||||
|
||||
static NSInteger FCDefaultPort(NSString *scheme) {
|
||||
scheme = scheme.lowercaseString;
|
||||
return [scheme isEqual:@"https"] ? 443 : [scheme isEqual:@"http"] ? 80 : 0;
|
||||
}
|
||||
static BOOL FCIsWebURL(NSURL *url) {
|
||||
NSString *scheme = url.scheme.lowercaseString;
|
||||
return [scheme isEqual:@"http"] || [scheme isEqual:@"https"];
|
||||
}
|
||||
static BOOL FCSameOrigin(NSURL *origin, NSString *scheme, NSString *host, NSInteger port) {
|
||||
return origin && host.length && [scheme.lowercaseString isEqual:origin.scheme.lowercaseString] &&
|
||||
[host.lowercaseString isEqual:origin.host.lowercaseString] &&
|
||||
(port ?: FCDefaultPort(scheme)) == (origin.port.integerValue ?: FCDefaultPort(origin.scheme));
|
||||
}
|
||||
/** A newer navigation replaced this one, or the policy below cancelled it; the current page is unaffected. */
|
||||
static BOOL FCIsInterruption(NSError *error) {
|
||||
return ([error.domain isEqual:NSURLErrorDomain] && error.code == NSURLErrorCancelled) ||
|
||||
([error.domain isEqual:@"WebKitErrorDomain"] && error.code == 102); // FrameLoadInterruptedByPolicyChange
|
||||
}
|
||||
|
||||
@implementation FaceclawEvenHubWebView
|
||||
- (void)emit:(NSDictionary *)event {
|
||||
if (!_eventHandler || _destroyed) return;
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:event options:0 error:nil];
|
||||
if (data) _eventHandler([[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (void)notice:(NSString *)message {
|
||||
[self emit:@{@"kind": @"notice", @"message": message.length > 4000 ? [message substringToIndex:4000] : message}];
|
||||
}
|
||||
- (void)start:(NSString *)directory entrypoint:(NSString *)entrypoint script:(NSString *)script {
|
||||
_directory = directory.stringByStandardizingPath.stringByResolvingSymlinksInPath;
|
||||
NSURLComponents *url = [NSURLComponents new];
|
||||
@@ -28,14 +59,14 @@
|
||||
}
|
||||
- (void)startURL:(NSString *)url script:(NSString *)script {
|
||||
NSURL *target = [NSURL URLWithString:url];
|
||||
if (!target.host.length || ![@[@"http", @"https"] containsObject:target.scheme.lowercaseString] || target.user || target.password) {
|
||||
if (!target.host.length || !FCIsWebURL(target) || target.user || target.password) {
|
||||
[self emit:@{@"kind": @"error", @"message": @"Invalid app URL."}]; return;
|
||||
}
|
||||
_remoteURL = target;
|
||||
[self startRequest:target script:script];
|
||||
}
|
||||
- (void)startRequest:(NSURL *)url script:(NSString *)script {
|
||||
if (_destroyed) return;
|
||||
_startURL = url; _script = script;
|
||||
WKWebViewConfiguration *config = [WKWebViewConfiguration new];
|
||||
// Isolate browser storage by package, retaining it across closing/reopening.
|
||||
// SDK storage also persists independently through the shared TS session.
|
||||
@@ -49,10 +80,9 @@
|
||||
}
|
||||
[config setURLSchemeHandler:self forURLScheme:@"faceclaw-ehpk"];
|
||||
[config.userContentController addScriptMessageHandler:self name:@"faceclaw"];
|
||||
[config.userContentController addUserScript:[[WKUserScript alloc] initWithSource:script
|
||||
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
|
||||
[self installScript:config.userContentController];
|
||||
_webView = [[WKWebView alloc] initWithFrame:CGRectZero configuration:config];
|
||||
_webView.navigationDelegate = self;
|
||||
_webView.navigationDelegate = self; _webView.UIDelegate = self;
|
||||
_webView.accessibilityIdentifier = @"evenhub-webview";
|
||||
UIWindow *window = nil;
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
@@ -73,13 +103,21 @@
|
||||
[_back setTitle:@"Back to Faceclaw" forState:UIControlStateNormal];
|
||||
_back.accessibilityIdentifier = @"evenhub-back";
|
||||
[_back addTarget:self action:@selector(hideOnPhone) forControlEvents:UIControlEventTouchUpInside];
|
||||
[_host addSubview:_back]; _back.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
// Shown only while the page is outside the app, e.g. an abandoned sign-in.
|
||||
_returnButton = [UIButton buttonWithType:UIButtonTypeSystem];
|
||||
[_returnButton setTitle:@"Return to app" forState:UIControlStateNormal];
|
||||
_returnButton.accessibilityIdentifier = @"evenhub-return";
|
||||
_returnButton.hidden = YES;
|
||||
[_returnButton addTarget:self action:@selector(returnToApp) forControlEvents:UIControlEventTouchUpInside];
|
||||
UIStackView *bar = [[UIStackView alloc] initWithArrangedSubviews:@[_back, _returnButton]];
|
||||
bar.distribution = UIStackViewDistributionFillEqually;
|
||||
[_host addSubview:bar]; bar.translatesAutoresizingMaskIntoConstraints = NO;
|
||||
[NSLayoutConstraint activateConstraints:@[
|
||||
[_back.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
|
||||
[_back.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[_back.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[_back.heightAnchor constraintEqualToConstant:44],
|
||||
[_webView.topAnchor constraintEqualToAnchor:_back.bottomAnchor],
|
||||
[bar.topAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.topAnchor],
|
||||
[bar.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[bar.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[bar.heightAnchor constraintEqualToConstant:44],
|
||||
[_webView.topAnchor constraintEqualToAnchor:bar.bottomAnchor],
|
||||
[_webView.leadingAnchor constraintEqualToAnchor:_host.leadingAnchor],
|
||||
[_webView.trailingAnchor constraintEqualToAnchor:_host.trailingAnchor],
|
||||
[_webView.bottomAnchor constraintEqualToAnchor:_host.safeAreaLayoutGuide.bottomAnchor]
|
||||
@@ -92,7 +130,7 @@
|
||||
__weak typeof(self) weakSelf = self;
|
||||
_timer = [NSTimer timerWithTimeInterval:1.0 / 60 repeats:YES block:^(NSTimer *timer) {
|
||||
typeof(self) strongSelf = weakSelf;
|
||||
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking) return;
|
||||
if (!strongSelf || strongSelf.destroyed || strongSelf.ticking || strongSelf.away) return;
|
||||
strongSelf.ticking = YES;
|
||||
[strongSelf.webView evaluateJavaScript:@"window.__fcTimerTick && window.__fcTimerTick(); window.__fcRafTick && window.__fcRafTick();" completionHandler:^(id result, NSError *error) {
|
||||
weakSelf.ticking = NO;
|
||||
@@ -101,8 +139,18 @@
|
||||
[NSRunLoop.mainRunLoop addTimer:_timer forMode:NSRunLoopCommonModes];
|
||||
[_webView loadRequest:[NSURLRequest requestWithURL:url]];
|
||||
}
|
||||
/** The bridge scripts run only in the app's own documents, never in pages it navigates to. */
|
||||
- (void)installScript:(WKUserContentController *)controller {
|
||||
NSData *start = [NSJSONSerialization dataWithJSONObject:_startURL.absoluteString options:NSJSONWritingFragmentsAllowed error:nil];
|
||||
NSString *source = [NSString stringWithFormat:@"if ((function (app) { return app.protocol === location.protocol && app.host === location.host; })(new URL(%@))) {\n%@\n}",
|
||||
[[NSString alloc] initWithData:start encoding:NSUTF8StringEncoding], _script];
|
||||
[controller removeAllUserScripts];
|
||||
[controller addUserScript:[[WKUserScript alloc] initWithSource:source
|
||||
injectionTime:WKUserScriptInjectionTimeAtDocumentStart forMainFrameOnly:YES]];
|
||||
}
|
||||
- (void)evaluate:(NSString *)script {
|
||||
if (!_destroyed) [_webView evaluateJavaScript:script completionHandler:nil];
|
||||
// Host messages are for the app's documents; a page it navigated to must not receive them.
|
||||
if (!_destroyed && !_away) [_webView evaluateJavaScript:script completionHandler:nil];
|
||||
}
|
||||
- (void)showOnPhone {
|
||||
if (_destroyed) return;
|
||||
@@ -112,51 +160,152 @@
|
||||
[_host endEditing:YES]; [_host.superview sendSubviewToBack:_host]; _host.userInteractionEnabled = NO; _host.accessibilityElementsHidden = YES;
|
||||
[self emit:@{@"kind": @"hidden"}];
|
||||
}
|
||||
- (void)returnToApp {
|
||||
if (_destroyed) return;
|
||||
_returnURL = nil; [_webView loadRequest:[NSURLRequest requestWithURL:_startURL]];
|
||||
}
|
||||
- (void)destroy {
|
||||
if (_destroyed) return;
|
||||
_destroyed = YES; [_timer invalidate]; _timer = nil;
|
||||
[_webView stopLoading]; _webView.navigationDelegate = nil;
|
||||
[_webView stopLoading]; _webView.navigationDelegate = nil; _webView.UIDelegate = nil;
|
||||
[_webView.configuration.userContentController removeScriptMessageHandlerForName:@"faceclaw"];
|
||||
[_webView.configuration.userContentController removeAllUserScripts];
|
||||
[_host removeFromSuperview]; _host = nil; _back = nil; _webView = nil; _eventHandler = nil;
|
||||
[_host removeFromSuperview]; _host = nil; _back = nil; _returnButton = nil; _webView = nil; _eventHandler = nil;
|
||||
}
|
||||
- (BOOL)allowsURL:(NSURL *)url {
|
||||
if (!_remoteURL) return [url.scheme isEqual:@"faceclaw-ehpk"] && [url.host isEqual:@"app"];
|
||||
NSNumber *port = url.port ?: @([url.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
|
||||
NSNumber *remotePort = _remoteURL.port ?: @([_remoteURL.scheme.lowercaseString isEqual:@"https"] ? 443 : 80);
|
||||
return [url.scheme.lowercaseString isEqual:_remoteURL.scheme.lowercaseString] &&
|
||||
[url.host.lowercaseString isEqual:_remoteURL.host.lowercaseString] && [port isEqual:remotePort];
|
||||
- (BOOL)isAppURL:(NSURL *)url {
|
||||
return FCSameOrigin(_startURL, url.scheme, url.host, url.port.integerValue);
|
||||
}
|
||||
- (BOOL)isAppFrame:(WKFrameInfo *)frame {
|
||||
WKSecurityOrigin *origin = frame.securityOrigin;
|
||||
return frame.isMainFrame && FCSameOrigin(_startURL, origin.protocol, origin.host, origin.port);
|
||||
}
|
||||
/** An OAuth authorization request names where the provider replies. */
|
||||
- (void)rememberReturnURLFrom:(NSURL *)url {
|
||||
for (NSURLQueryItem *item in [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO].queryItems) {
|
||||
NSURL *target = [item.name isEqual:@"redirect_uri"] && item.value ? [NSURL URLWithString:item.value] : nil;
|
||||
// A redirect_uri on the app's own origin needs no help: the app handles it itself.
|
||||
if (FCIsWebURL(target) && target.host.length && ![self isAppURL:target]) _returnURL = target;
|
||||
}
|
||||
}
|
||||
- (BOOL)isReturnURL:(NSURL *)url {
|
||||
// NSURL.path drops a trailing slash, so ".../callback/" and ".../callback" match.
|
||||
NSString *path = url.path.length ? url.path : @"/", *returnPath = _returnURL.path.length ? _returnURL.path : @"/";
|
||||
return FCSameOrigin(_returnURL, url.scheme, url.host, url.port.integerValue) && [path isEqual:returnPath];
|
||||
}
|
||||
/** The app's start page carrying the provider's reply (its query and fragment). */
|
||||
- (void)loadAppWithResultOf:(NSURL *)url {
|
||||
NSURLComponents *result = [NSURLComponents componentsWithURL:url resolvingAgainstBaseURL:NO];
|
||||
NSURLComponents *app = [NSURLComponents componentsWithURL:_startURL resolvingAgainstBaseURL:NO];
|
||||
if (result.percentEncodedQuery.length) app.percentEncodedQuery = app.percentEncodedQuery.length ?
|
||||
[NSString stringWithFormat:@"%@&%@", app.percentEncodedQuery, result.percentEncodedQuery] : result.percentEncodedQuery;
|
||||
if (result.percentEncodedFragment) app.percentEncodedFragment = result.percentEncodedFragment;
|
||||
NSURL *target = app.URL;
|
||||
_returnURL = nil;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (!self.destroyed) [self.webView loadRequest:[NSURLRequest requestWithURL:target]];
|
||||
});
|
||||
}
|
||||
- (void)userContentController:(WKUserContentController *)controller didReceiveScriptMessage:(WKScriptMessage *)message {
|
||||
// Subframes and navigated third-party documents never receive host access.
|
||||
NSURL *url = message.frameInfo.request.URL;
|
||||
if (!message.frameInfo.isMainFrame || ![self allowsURL:url]) return;
|
||||
// Only the app's own main frame gets host access: not subframes, nor pages the app navigated to.
|
||||
if (![self isAppFrame:message.frameInfo]) return;
|
||||
if ([message.body isKindOfClass:NSDictionary.class]) [self emit:message.body];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView decidePolicyForNavigationAction:(WKNavigationAction *)action decisionHandler:(void (^)(WKNavigationActionPolicy))decisionHandler {
|
||||
NSURL *url = action.request.URL;
|
||||
BOOL allowed = [self allowsURL:url] && action.targetFrame.isMainFrame;
|
||||
if (!allowed && action.targetFrame.isMainFrame)
|
||||
[self emit:@{@"kind": @"error", @"message": @"Navigation outside the app origin is blocked. Open the destination as a separate app URL."}];
|
||||
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel);
|
||||
NSString *scheme = url.scheme.lowercaseString;
|
||||
// A new window (target=_blank, window.open): createWebView below opens it outside the app.
|
||||
if (!action.targetFrame) { decisionHandler(WKNavigationActionPolicyAllow); return; }
|
||||
// Subframes load as in a browser; they never get the bridge.
|
||||
if (!action.targetFrame.isMainFrame) {
|
||||
BOOL allowed = FCIsWebURL(url) || [self isAppURL:url] || [@[@"about", @"data", @"blob"] containsObject:scheme];
|
||||
decisionHandler(allowed ? WKNavigationActionPolicyAllow : WKNavigationActionPolicyCancel); return;
|
||||
}
|
||||
if ([self isAppURL:url]) { _returnURL = nil; decisionHandler(WKNavigationActionPolicyAllow); return; }
|
||||
if (FCIsWebURL(url) && !_committed && !_directory.length) {
|
||||
// The app URL redirected before anything loaded: the app lives where it lands. The
|
||||
// script update reaches the web process ahead of the policy reply.
|
||||
_startURL = url; [self installScript:webView.configuration.userContentController];
|
||||
decisionHandler(WKNavigationActionPolicyAllow); return;
|
||||
}
|
||||
if (_returnURL && [self isReturnURL:url]) {
|
||||
// A sign-in provider replying: finish in the app rather than at the page the redirect_uri names.
|
||||
decisionHandler(WKNavigationActionPolicyCancel); [self loadAppWithResultOf:url]; return;
|
||||
}
|
||||
if (!FCIsWebURL(url)) {
|
||||
[self notice:[NSString stringWithFormat:@"Blocked navigation to a %@: URL.", scheme ?: @"schemeless"]];
|
||||
decisionHandler(WKNavigationActionPolicyCancel); return;
|
||||
}
|
||||
// Leaving the app's origin. Such pages never get the bridge; "Return to app" leads back.
|
||||
if ([self isAppFrame:action.sourceFrame]) [self rememberReturnURLFrom:url];
|
||||
decisionHandler(WKNavigationActionPolicyAllow);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didCommitNavigation:(WKNavigation *)navigation {
|
||||
_committed = YES; _away = ![self isAppURL:webView.URL]; _returnButton.hidden = !_away;
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFinishNavigation:(WKNavigation *)navigation {
|
||||
[self emit:@{@"kind": @"loaded"}];
|
||||
if (!_away) [self emit:@{@"kind": @"loaded"}];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFailProvisionalNavigation:(WKNavigation *)navigation withError:(NSError *)error {
|
||||
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView didFailNavigation:(WKNavigation *)navigation withError:(NSError *)error {
|
||||
[self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
if (!FCIsInterruption(error)) [self emit:@{@"kind": @"error", @"message": error.localizedDescription}];
|
||||
}
|
||||
- (void)webViewWebContentProcessDidTerminate:(WKWebView *)webView {
|
||||
[self emit:@{@"kind": @"error", @"message": @"The app's web process ended. Close and reopen the app."}];
|
||||
[self emit:@{@"kind": @"error", @"fatal": @YES, @"message": @"The app's web process ended. Close and reopen the app."}];
|
||||
}
|
||||
/** What to present over, only while the app's phone UI is shown and not mid-transition. */
|
||||
- (UIViewController *)presenter {
|
||||
if (_destroyed || !_host.userInteractionEnabled) return nil;
|
||||
UIViewController *controller = _host.window.rootViewController;
|
||||
while (controller.presentedViewController) controller = controller.presentedViewController;
|
||||
return controller.transitionCoordinator ? nil : controller;
|
||||
}
|
||||
- (WKWebView *)webView:(WKWebView *)webView createWebViewWithConfiguration:(WKWebViewConfiguration *)configuration forNavigationAction:(WKNavigationAction *)action windowFeatures:(WKWindowFeatures *)windowFeatures {
|
||||
NSURL *url = action.request.URL;
|
||||
UIViewController *presenter = FCIsWebURL(url) ? [self presenter] : nil;
|
||||
if (presenter) [presenter presentViewController:[[SFSafariViewController alloc] initWithURL:url] animated:YES completion:nil];
|
||||
else [self notice:[NSString stringWithFormat:@"Did not open a window for a %@: URL.", url.scheme ?: @"schemeless"]];
|
||||
return nil;
|
||||
}
|
||||
// JavaScript dialogs. WebKit requires each completion handler to run exactly once, so a
|
||||
// dialog that cannot be shown (the app is hidden) completes at once with its default.
|
||||
- (UIAlertController *)dialog:(NSString *)message {
|
||||
if ([self presenter]) return [UIAlertController alertControllerWithTitle:nil message:message preferredStyle:UIAlertControllerStyleAlert];
|
||||
[self notice:[@"Dialog while hidden: " stringByAppendingString:message ?: @""]];
|
||||
return nil;
|
||||
}
|
||||
- (BOOL)presentDialog:(UIAlertController *)alert {
|
||||
if (!alert) return NO;
|
||||
[[self presenter] presentViewController:alert animated:YES completion:nil];
|
||||
return alert.presentingViewController != nil;
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptAlertPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(void))completionHandler {
|
||||
UIAlertController *alert = [self dialog:message];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(); }]];
|
||||
if (![self presentDialog:alert]) completionHandler();
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptConfirmPanelWithMessage:(NSString *)message initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(BOOL))completionHandler {
|
||||
UIAlertController *alert = [self dialog:message];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(NO); }]];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) { completionHandler(YES); }]];
|
||||
if (![self presentDialog:alert]) completionHandler(NO);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView runJavaScriptTextInputPanelWithPrompt:(NSString *)prompt defaultText:(NSString *)defaultText initiatedByFrame:(WKFrameInfo *)frame completionHandler:(void (^)(NSString *))completionHandler {
|
||||
UIAlertController *alert = [self dialog:prompt];
|
||||
[alert addTextFieldWithConfigurationHandler:^(UITextField *field) { field.text = defaultText; }];
|
||||
__weak UIAlertController *weakAlert = alert;
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"Cancel" style:UIAlertActionStyleCancel handler:^(UIAlertAction *action) { completionHandler(nil); }]];
|
||||
[alert addAction:[UIAlertAction actionWithTitle:@"OK" style:UIAlertActionStyleDefault handler:^(UIAlertAction *action) {
|
||||
completionHandler(weakAlert.textFields.firstObject.text ?: @"");
|
||||
}]];
|
||||
if (![self presentDialog:alert]) completionHandler(nil);
|
||||
}
|
||||
- (void)webView:(WKWebView *)webView startURLSchemeTask:(id<WKURLSchemeTask>)task {
|
||||
NSURL *url = task.request.URL;
|
||||
NSString *relative = [url.path stringByTrimmingCharactersInSet:[NSCharacterSet characterSetWithCharactersInString:@"/"]];
|
||||
NSString *file = [[_directory stringByAppendingPathComponent:relative] stringByStandardizingPath].stringByResolvingSymlinksInPath;
|
||||
BOOL allowed = !_destroyed && _directory.length && !_remoteURL && [url.host isEqual:@"app"] &&
|
||||
BOOL allowed = !_destroyed && _directory.length && [url.host isEqual:@"app"] &&
|
||||
[file hasPrefix:[_directory stringByAppendingString:@"/"]];
|
||||
NSData *data = allowed ? [NSData dataWithContentsOfFile:file options:NSDataReadingMappedIfSafe error:nil] : nil;
|
||||
NSString *ext = file.pathExtension.lowercaseString;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
@interface FaceclawGraphics : NSObject
|
||||
+ (UIImage *)preview:(NSData *)gray width:(NSInteger)width height:(NSInteger)height green:(BOOL)green;
|
||||
/** A glasses icon as white-on-transparent (gray becomes alpha), for the dark header. */
|
||||
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height;
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size;
|
||||
+ (NSData *)decodeImage:(NSData *)data width:(NSInteger)width height:(NSInteger)height;
|
||||
+ (NSData *)decodeImageFile:(NSString *)path maxWidth:(NSInteger)maxWidth maxHeight:(NSInteger)maxHeight;
|
||||
|
||||
@@ -76,6 +76,23 @@
|
||||
CGColorSpaceRelease(space);
|
||||
return result;
|
||||
}
|
||||
+ (UIImage *)whiteIcon:(NSData *)gray width:(NSInteger)width height:(NSInteger)height {
|
||||
if (width <= 0 || height <= 0 || gray.length != width * height) return nil;
|
||||
NSMutableData *rgba = [NSMutableData dataWithLength:width * height * 4];
|
||||
const uint8_t *src = gray.bytes;
|
||||
uint8_t *dst = rgba.mutableBytes;
|
||||
// Premultiplied white: every channel equals the alpha.
|
||||
for (NSInteger i = 0; i < width * height; i++) memset(dst + i * 4, src[i], 4);
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
CGDataProviderRef provider = CGDataProviderCreateWithCFData((__bridge CFDataRef)rgba);
|
||||
CGImageRef image = CGImageCreate(width, height, 8, 32, width * 4, space,
|
||||
kCGBitmapByteOrder32Big | (CGBitmapInfo)kCGImageAlphaPremultipliedLast, provider, NULL, false, kCGRenderingIntentDefault);
|
||||
UIImage *result = image ? [UIImage imageWithCGImage:image scale:1 orientation:UIImageOrientationUp] : nil;
|
||||
if (image) CGImageRelease(image);
|
||||
CGDataProviderRelease(provider);
|
||||
CGColorSpaceRelease(space);
|
||||
return result;
|
||||
}
|
||||
+ (NSData *)renderSVG:(NSString *)svg size:(NSInteger)size {
|
||||
if (size <= 0 || size > 1024) return nil;
|
||||
SVGKImage *image = [SVGKImage imageWithData:[svg dataUsingEncoding:NSUTF8StringEncoding]];
|
||||
|
||||
@@ -2,7 +2,10 @@
|
||||
/** Thread-safe event queue; each NativeScript worker drains it on its own thread. */
|
||||
@interface FaceclawSocket : NSObject <NSURLSessionWebSocketDelegate>
|
||||
- (instancetype)initWithURL:(NSString *)url;
|
||||
/** headerName/headerValue add one handshake header (e.g. an API key); nil for none. */
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue;
|
||||
- (NSString *)takeEvents;
|
||||
- (void)sendText:(NSString *)text;
|
||||
- (void)sendData:(NSData *)data;
|
||||
- (void)close;
|
||||
@end
|
||||
@@ -8,12 +8,17 @@
|
||||
@end
|
||||
@implementation FaceclawSocket
|
||||
- (instancetype)initWithURL:(NSString *)url {
|
||||
return [self initWithURL:url headerName:nil headerValue:nil];
|
||||
}
|
||||
- (instancetype)initWithURL:(NSString *)url headerName:(NSString *)headerName headerValue:(NSString *)headerValue {
|
||||
if ((self = [super init])) {
|
||||
_events = [NSMutableArray array];
|
||||
NSURLSessionConfiguration *config = [NSURLSessionConfiguration ephemeralSessionConfiguration];
|
||||
config.timeoutIntervalForRequest = 20;
|
||||
_session = [NSURLSession sessionWithConfiguration:config delegate:self delegateQueue:nil];
|
||||
_task = [_session webSocketTaskWithURL:[NSURL URLWithString:url]];
|
||||
NSMutableURLRequest *request = [NSMutableURLRequest requestWithURL:[NSURL URLWithString:url]];
|
||||
if (headerName.length && headerValue) [request setValue:headerValue forHTTPHeaderField:headerName];
|
||||
_task = [_session webSocketTaskWithRequest:request];
|
||||
_task.maximumMessageSize = 8 * 1024 * 1024;
|
||||
[_task resume];
|
||||
}
|
||||
@@ -25,7 +30,7 @@
|
||||
_queuedBytes += [event[@"text"] lengthOfBytesUsingEncoding:NSUTF8StringEncoding];
|
||||
if (_events.count >= 256 || _queuedBytes > 16 * 1024 * 1024) {
|
||||
[_events removeAllObjects]; _queuedBytes = 0;
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"Terminal input queue overflow"}];
|
||||
[_events addObject:@{@"kind": @"error", @"text": @"WebSocket receive queue overflow"}];
|
||||
[self close];
|
||||
} else [_events addObject:event];
|
||||
}
|
||||
@@ -56,7 +61,13 @@
|
||||
[self close];
|
||||
}
|
||||
- (void)URLSession:(NSURLSession *)session task:(NSURLSessionTask *)task didCompleteWithError:(NSError *)error {
|
||||
if (error) [self enqueue:@{@"kind": @"error", @"text": error.localizedDescription}];
|
||||
if (error) {
|
||||
// A refused handshake only says "bad response"; name the HTTP status
|
||||
// so callers can tell a rejected key (401/403) from a network error.
|
||||
NSInteger status = [task.response isKindOfClass:NSHTTPURLResponse.class] ? ((NSHTTPURLResponse *)task.response).statusCode : 0;
|
||||
NSString *text = status >= 400 ? [NSString stringWithFormat:@"HTTP %ld: %@", (long)status, error.localizedDescription] : error.localizedDescription;
|
||||
[self enqueue:@{@"kind": @"error", @"text": text}];
|
||||
}
|
||||
[self close];
|
||||
}
|
||||
- (void)sendText:(NSString *)text {
|
||||
@@ -65,6 +76,12 @@
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)sendData:(NSData *)data {
|
||||
__weak FaceclawSocket *weakSelf = self;
|
||||
[_task sendMessage:[[NSURLSessionWebSocketMessage alloc] initWithData:data] completionHandler:^(NSError *error) {
|
||||
if (error) { [weakSelf enqueue:@{@"kind": @"error", @"text": error.localizedDescription}]; [weakSelf close]; }
|
||||
}];
|
||||
}
|
||||
- (void)close {
|
||||
@synchronized(self) {
|
||||
if (_ended) return; _ended = YES;
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
/** On-device speech recognition fed by G2 BLE audio or the default phone microphone. */
|
||||
@interface FaceclawSpeech : NSObject
|
||||
@property(nonatomic, copy) void (^eventHandler)(NSString *json);
|
||||
/** 16 kHz mono PCM16LE chunks from a PCM capture, on the main queue in capture order. */
|
||||
@property(nonatomic, copy) void (^pcmHandler)(NSData *pcm);
|
||||
+ (NSInteger)authorizationStatus;
|
||||
+ (NSInteger)microphoneAuthorizationStatus;
|
||||
+ (void)requestMicrophoneAuthorization:(void (^)(NSInteger status))completion;
|
||||
@@ -9,6 +11,9 @@
|
||||
/** Returns an actionable error, or an empty string when capture starts. */
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing;
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing;
|
||||
/** Capture without the recognizer (for a cloud provider): audio goes to
|
||||
* pcmHandler, and "ended" follows the last chunk. No transcript events. */
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone;
|
||||
- (void)acceptPacket:(NSData *)packet;
|
||||
- (void)finish;
|
||||
- (void)cancel;
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
@property(atomic) NSUInteger generation;
|
||||
@property(nonatomic) NSUInteger pendingPackets;
|
||||
@property(atomic) BOOL accepting;
|
||||
@property(nonatomic) BOOL pcmOutput;
|
||||
@property(nonatomic, strong) AVAudioEngine *phoneEngine;
|
||||
@property(nonatomic) BOOL phoneTapInstalled;
|
||||
@property(nonatomic) BOOL phoneSessionActive;
|
||||
@@ -23,6 +24,23 @@
|
||||
@property(nonatomic, strong) FaceclawSpeechTranscript *transcript;
|
||||
@property(nonatomic, strong) FaceclawKitVoiceEndpointDetector *endpointDetector;
|
||||
@end
|
||||
/** Resamples one tap buffer to 16 kHz mono PCM16LE. Called on the serial
|
||||
* audio queue; the converter keeps its filter state between buffers. */
|
||||
static NSData *resampleToPcm(AVAudioConverter *converter, AVAudioPCMBuffer *input) {
|
||||
AVAudioFormat *format = converter.outputFormat;
|
||||
AVAudioFrameCount capacity = (AVAudioFrameCount)ceil(input.frameLength * format.sampleRate / input.format.sampleRate) + 32;
|
||||
AVAudioPCMBuffer *output = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:capacity];
|
||||
__block BOOL supplied = NO;
|
||||
NSError *error = nil;
|
||||
AVAudioConverterOutputStatus status = [converter convertToBuffer:output error:&error
|
||||
withInputFromBlock:^AVAudioBuffer *(AVAudioPacketCount count, AVAudioConverterInputStatus *inputStatus) {
|
||||
// NoDataNow (not EndOfStream) leaves the stream open for the next buffer.
|
||||
if (supplied) { *inputStatus = AVAudioConverterInputStatus_NoDataNow; return nil; }
|
||||
supplied = YES; *inputStatus = AVAudioConverterInputStatus_HaveData; return input;
|
||||
}];
|
||||
if (status == AVAudioConverterOutputStatus_Error || !output.frameLength) return nil;
|
||||
return [NSData dataWithBytes:output.int16ChannelData[0] length:output.frameLength * sizeof(int16_t)];
|
||||
}
|
||||
@implementation FaceclawSpeech
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) _audioQueue = dispatch_queue_create("com.faceclaw.glasses-speech", DISPATCH_QUEUE_SERIAL);
|
||||
@@ -46,37 +64,51 @@
|
||||
if (json) self.eventHandler([[NSString alloc] initWithData:json encoding:NSUTF8StringEncoding]);
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:NO];
|
||||
return [self startWithEndpointing:endpointing phone:NO pcm:NO];
|
||||
}
|
||||
- (NSString *)startPhoneWithEndpointing:(BOOL)endpointing {
|
||||
return [self startWithEndpointing:endpointing phone:YES];
|
||||
return [self startWithEndpointing:endpointing phone:YES pcm:NO];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
- (NSString *)startPcmCaptureWithEndpointing:(BOOL)endpointing phone:(BOOL)phone {
|
||||
return [self startWithEndpointing:endpointing phone:phone pcm:YES];
|
||||
}
|
||||
- (NSString *)startWithEndpointing:(BOOL)endpointing phone:(BOOL)phone pcm:(BOOL)pcm {
|
||||
[self cancel];
|
||||
if (phone && [FaceclawSpeech microphoneAuthorizationStatus] != AVAuthorizationStatusAuthorized)
|
||||
return @"Allow Microphone access for Faceclaw in iPhone Settings.";
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
NSString *language = NSLocale.preferredLanguages.firstObject ?: @"en-US";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
if (!pcm) {
|
||||
if (SFSpeechRecognizer.authorizationStatus != SFSpeechRecognizerAuthorizationStatusAuthorized)
|
||||
return @"Allow Speech Recognition for Faceclaw in iPhone Settings.";
|
||||
self.recognizer = [[SFSpeechRecognizer alloc] initWithLocale:[NSLocale localeWithLocaleIdentifier:language]];
|
||||
if (!self.recognizer.supportsOnDeviceRecognition)
|
||||
return [NSString stringWithFormat:@"On-device speech is unavailable for %@. Enable Dictation for that language in iPhone Settings → General → Keyboard, then retry.", language];
|
||||
if (!self.recognizer.available) return @"Speech recognition is temporarily unavailable. Try again shortly.";
|
||||
}
|
||||
if (!phone) {
|
||||
self.decoder = [FaceclawLc3Decoder new];
|
||||
if (!self.decoder) return @"Could not initialize the glasses audio decoder.";
|
||||
}
|
||||
self.pcmOutput = pcm;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
if (pcm) {
|
||||
if (phone) {
|
||||
NSString *error = [self startPhoneAudio];
|
||||
if (error.length) { [self cancel]; return error; }
|
||||
}
|
||||
// The PCM consumer reports its own status and has no session limit.
|
||||
return @"";
|
||||
}
|
||||
self.recognizer.queue = NSOperationQueue.mainQueue;
|
||||
self.request = [SFSpeechAudioBufferRecognitionRequest new];
|
||||
self.request.requiresOnDeviceRecognition = YES;
|
||||
self.request.shouldReportPartialResults = YES;
|
||||
self.request.taskHint = SFSpeechRecognitionTaskHintDictation;
|
||||
if (@available(iOS 16.0, *)) self.request.addsPunctuation = YES;
|
||||
self.bestText = @""; self.accepting = YES;
|
||||
self.transcript = [FaceclawSpeechTranscript new];
|
||||
self.endpointDetector = endpointing ? [FaceclawKitVoiceEndpointDetector new] : nil;
|
||||
NSUInteger generation = self.generation;
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
self.task = [self.recognizer recognitionTaskWithRequest:self.request resultHandler:^(SFSpeechRecognitionResult *result, NSError *error) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -124,6 +156,13 @@
|
||||
AVAudioInputNode *input = self.phoneEngine.inputNode;
|
||||
AVAudioFormat *format = [input outputFormatForBus:0];
|
||||
if (format.sampleRate <= 0 || !format.channelCount) return @"No phone microphone is available. Check your audio input and try again.";
|
||||
AVAudioConverter *converter = nil;
|
||||
if (self.pcmOutput) {
|
||||
AVAudioFormat *pcmFormat = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatInt16 sampleRate:16000 channels:1 interleaved:YES];
|
||||
converter = [[AVAudioConverter alloc] initFromFormat:format toFormat:pcmFormat];
|
||||
if (!converter) return @"Could not convert phone microphone audio.";
|
||||
converter.downmix = YES;
|
||||
}
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *detector = self.endpointDetector;
|
||||
@@ -141,7 +180,10 @@
|
||||
memcpy(destination->mBuffers[i].mData, source->mBuffers[i].mData, source->mBuffers[i].mDataByteSize);
|
||||
dispatch_async(owner.audioQueue, ^{ @autoreleasepool {
|
||||
if (owner.generation != generation) return;
|
||||
[request appendAudioPCMBuffer:copy];
|
||||
if (converter) {
|
||||
NSData *pcm = resampleToPcm(converter, copy);
|
||||
if (pcm.length) [owner deliverPcm:pcm generation:generation];
|
||||
} else [request appendAudioPCMBuffer:copy];
|
||||
double sum = 0;
|
||||
if (copy.floatChannelData) {
|
||||
const float *samples = copy.floatChannelData[0];
|
||||
@@ -188,6 +230,7 @@
|
||||
FaceclawLc3Decoder *decoder = self.decoder;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
FaceclawKitVoiceEndpointDetector *endpointDetector = self.endpointDetector;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
NSData *copy = [packet copy];
|
||||
dispatch_async(self.audioQueue, ^{ @autoreleasepool {
|
||||
if (self.generation != generation) return;
|
||||
@@ -195,17 +238,26 @@
|
||||
double rms = 0;
|
||||
BOOL speechEnded = NO;
|
||||
if (pcm.length) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
AVAudioPCMBuffer *buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:(AVAudioFrameCount)(pcm.length / 2)];
|
||||
buffer.frameLength = buffer.frameCapacity;
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < buffer.frameLength; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
buffer.floatChannelData[0][i] = value; rms += value * value;
|
||||
AVAudioFrameCount frames = (AVAudioFrameCount)(pcm.length / 2);
|
||||
AVAudioPCMBuffer *buffer = nil;
|
||||
if (!pcmOutput) {
|
||||
AVAudioFormat *format = [[AVAudioFormat alloc] initWithCommonFormat:AVAudioPCMFormatFloat32 sampleRate:16000 channels:1 interleaved:NO];
|
||||
buffer = [[AVAudioPCMBuffer alloc] initWithPCMFormat:format frameCapacity:frames];
|
||||
buffer.frameLength = frames;
|
||||
}
|
||||
rms = sqrt(rms / buffer.frameLength);
|
||||
if (self.generation == generation) [request appendAudioPCMBuffer:buffer];
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)buffer.frameLength];
|
||||
const int16_t *samples = pcm.bytes;
|
||||
for (NSUInteger i = 0; i < frames; i++) {
|
||||
float value = samples[i] / 32768.0f;
|
||||
if (buffer) buffer.floatChannelData[0][i] = value;
|
||||
rms += value * value;
|
||||
}
|
||||
rms = sqrt(rms / frames);
|
||||
if (self.generation == generation) {
|
||||
// The decoder's output is already the PCM consumer's format.
|
||||
if (pcmOutput) [self deliverPcm:pcm generation:generation];
|
||||
else [request appendAudioPCMBuffer:buffer];
|
||||
}
|
||||
speechEnded = [endpointDetector acceptLevelRms:rms * 32768.0 sampleCount:(int32_t)frames];
|
||||
}
|
||||
NSUInteger packets = decoder.packets, errors = decoder.errors, missing = decoder.missing;
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
@@ -223,9 +275,15 @@
|
||||
[self stopPhoneAudio];
|
||||
NSUInteger generation = self.generation;
|
||||
SFSpeechAudioBufferRecognitionRequest *request = self.request;
|
||||
BOOL pcmOutput = self.pcmOutput;
|
||||
[self emit:@{@"kind":@"finishing"}];
|
||||
// Drain already received audio before marking the utterance complete.
|
||||
dispatch_async(self.audioQueue, ^{ if (self.generation == generation) [request endAudio]; });
|
||||
dispatch_async(self.audioQueue, ^{
|
||||
if (self.generation != generation) return;
|
||||
if (!pcmOutput) { [request endAudio]; return; }
|
||||
// Drained chunks are already queued for pcmHandler; "ended" follows them.
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ if (self.generation == generation) [self complete:@"Ready to send"]; });
|
||||
});
|
||||
__weak FaceclawSpeech *weakSelf = self;
|
||||
dispatch_after(dispatch_time(DISPATCH_TIME_NOW, 8 * NSEC_PER_SEC), dispatch_get_main_queue(), ^{
|
||||
FaceclawSpeech *owner = weakSelf;
|
||||
@@ -239,9 +297,14 @@
|
||||
if (text.length) [self emit:@{@"kind":@"transcript", @"text":text, @"final":@YES}];
|
||||
if (self.generation == generation) [self emit:@{@"kind":@"ended", @"message":status}];
|
||||
}
|
||||
- (void)deliverPcm:(NSData *)pcm generation:(NSUInteger)generation {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.generation == generation && self.pcmHandler) self.pcmHandler(pcm);
|
||||
});
|
||||
}
|
||||
- (void)cancel {
|
||||
self.generation++;
|
||||
self.accepting = NO; self.pendingPackets = 0;
|
||||
self.accepting = NO; self.pendingPackets = 0; self.pcmOutput = NO;
|
||||
[self stopPhoneAudio];
|
||||
[self.task cancel]; self.task = nil; self.request = nil; self.decoder = nil; self.recognizer = nil;
|
||||
self.bestText = @""; self.transcript = nil; self.endpointDetector = nil;
|
||||
|
||||
@@ -6,6 +6,7 @@ module FaceclawGraphics {
|
||||
header "FaceclawSpeech.h"
|
||||
header "FaceclawLocation.h"
|
||||
header "FaceclawCalendar.h"
|
||||
header "FaceclawAppleMusic.h"
|
||||
header "FaceclawConfigPort.h"
|
||||
header "FaceclawEvenHubWebView.h"
|
||||
header "FaceclawCrypto.h"
|
||||
|
||||
Reference in new issue
Block a user