mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge branch 'main' of https://github.com/jimrandomh/faceclaw into HEAD
This commit is contained in:
commit
ae48313b81
48 files changed
+1946
-213
No files matched your search
@@ -0,0 +1,6 @@
|
||||
# Check text out with LF everywhere, so Windows checkouts keep runnable bash
|
||||
# scripts and the LF-anchored source regexes in tests. Batch files are the
|
||||
# exception: cmd.exe can miss goto/call labels in LF-only batch files.
|
||||
* text=auto eol=lf
|
||||
*.bat text eol=crlf
|
||||
*.cmd text eol=crlf
|
||||
@@ -46,6 +46,17 @@ def faceclawNdkHostTag = {
|
||||
return "linux-x86_64"
|
||||
}
|
||||
|
||||
// Host tools carry .exe on Windows. Call the binaries themselves: the NDK's
|
||||
// target-prefixed clang wrappers are bash scripts (or .cmd files) that only add
|
||||
// --target.
|
||||
def faceclawHostExecutable = { File dir, String name ->
|
||||
return file("$dir/$name${Os.isFamily(Os.FAMILY_WINDOWS) ? ".exe" : ""}")
|
||||
}
|
||||
|
||||
def faceclawNdkTool = { File ndkRoot, String name ->
|
||||
return faceclawHostExecutable(file("$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin"), name)
|
||||
}
|
||||
|
||||
def faceclawFindNdkRoot = {
|
||||
def explicitNdk = System.getenv("ANDROID_NDK_HOME")
|
||||
if (explicitNdk != null && file(explicitNdk).exists()) {
|
||||
@@ -111,12 +122,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
if (ndkRoot == null || !ndkRoot.exists()) {
|
||||
throw new GradleException("Android NDK is required to build libfaceclaw_lc3.so")
|
||||
}
|
||||
// The NDK ships the clang driver twice: extensionless (a Unix shell script)
|
||||
// and as .cmd (the CreateProcess-launchable one Windows needs).
|
||||
def clangBin = "$ndkRoot/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/aarch64-linux-android24-clang"
|
||||
def clang = Os.isFamily(Os.FAMILY_WINDOWS)
|
||||
? file("${clangBin}.cmd")
|
||||
: file(clangBin)
|
||||
def clang = faceclawNdkTool(ndkRoot, "clang")
|
||||
if (!clang.exists()) {
|
||||
throw new GradleException("Could not find Android clang at $clang")
|
||||
}
|
||||
@@ -130,6 +136,7 @@ tasks.register("prepareFaceclawNativeLibs") {
|
||||
exec {
|
||||
commandLine([
|
||||
clang.absolutePath,
|
||||
"--target=aarch64-linux-android24",
|
||||
"-shared",
|
||||
"-fPIC",
|
||||
"-O3",
|
||||
@@ -197,8 +204,8 @@ tasks.register("prepareFaceclawLlama") {
|
||||
if (cmakeDir == null) {
|
||||
throw new GradleException("Android SDK cmake package is required to build libfaceclaw_llama.so")
|
||||
}
|
||||
def cmake = file("$cmakeDir/bin/cmake")
|
||||
def ninja = file("$cmakeDir/bin/ninja")
|
||||
def cmake = faceclawHostExecutable(file("$cmakeDir/bin"), "cmake")
|
||||
def ninja = faceclawHostExecutable(file("$cmakeDir/bin"), "ninja")
|
||||
def march = "-march=armv8.2-a+dotprod+fp16"
|
||||
|
||||
// Configure once; afterwards the ninja no-op build keeps this cheap while
|
||||
@@ -232,18 +239,13 @@ tasks.register("prepareFaceclawLlama") {
|
||||
}
|
||||
// The static-linked lib carries llama.cpp's full debug info (~50MB);
|
||||
// strip it so the APK only pays for the code (~8MB).
|
||||
def ndkRootForStrip = faceclawFindNdkRoot()
|
||||
if (ndkRootForStrip != null) {
|
||||
// Windows wants llvm-strip.exe; the extensionless file is a Unix script.
|
||||
def llvmStripBin = "$ndkRootForStrip/toolchains/llvm/prebuilt/${faceclawNdkHostTag()}/bin/llvm-strip"
|
||||
def llvmStrip = Os.isFamily(Os.FAMILY_WINDOWS) && file("${llvmStripBin}.exe").exists()
|
||||
? file("${llvmStripBin}.exe")
|
||||
: file(llvmStripBin)
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
def llvmStrip = faceclawNdkTool(ndkRoot, "llvm-strip")
|
||||
if (llvmStrip.exists()) {
|
||||
exec {
|
||||
commandLine([llvmStrip.absolutePath, "--strip-unneeded", llamaSo.absolutePath])
|
||||
}
|
||||
} else {
|
||||
logger.warn("Could not find llvm-strip at $llvmStrip; packaging an unstripped libfaceclaw_llama.so (~50MB)")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+103
@@ -9,6 +9,7 @@ import android.graphics.Color
|
||||
import android.graphics.drawable.BitmapDrawable
|
||||
import android.graphics.drawable.Drawable
|
||||
import android.graphics.drawable.Icon
|
||||
import android.os.Build
|
||||
import android.os.Bundle
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
@@ -182,6 +183,33 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return out.toByteArray()
|
||||
}
|
||||
|
||||
/**
|
||||
* The picture attached to one active notification (BigPictureStyle: a
|
||||
* photo, a screenshot, Dexcom's glucose graph), scaled to fit
|
||||
* maxWidth x maxHeight as a gray packet with the tone handling (and
|
||||
* light-background inversion) of ImageFileLoader.bitmapToGrayPacket;
|
||||
* empty when the notification or its picture is unavailable.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun getNotificationPictureGray(
|
||||
key: String?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean,
|
||||
): ByteArray {
|
||||
val service = activeService
|
||||
if (service == null || maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val statusBarNotification = findActiveNotificationByKey(service, key)
|
||||
if (statusBarNotification == null || statusBarNotification.notification == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
val picture = loadNotificationPicture(service, statusBarNotification.notification)
|
||||
if (picture == null) {
|
||||
return ByteArray(0)
|
||||
}
|
||||
return ImageFileLoader.bitmapToGrayPacket(
|
||||
halveToward(picture, maxWidth, maxHeight), maxWidth, maxHeight, gamma, dither, invertLightBackground)
|
||||
}
|
||||
|
||||
@JvmStatic
|
||||
fun getActiveNotificationsJson(maxNotifications: Int): String {
|
||||
val service = activeService
|
||||
@@ -541,6 +569,79 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
return (flags and android.content.pm.ApplicationInfo.FLAG_DEBUGGABLE) != 0
|
||||
}
|
||||
|
||||
/** Whether a notification carries a picture for loadNotificationPicture, without loading it. */
|
||||
@Suppress("DEPRECATION")
|
||||
private fun hasNotificationPicture(extras: Bundle): Boolean {
|
||||
return extras.get(Notification.EXTRA_PICTURE) != null
|
||||
|| (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S && extras.get(Notification.EXTRA_PICTURE_ICON) != null)
|
||||
}
|
||||
|
||||
/**
|
||||
* A notification's attached picture: BigPictureStyle's bitmap
|
||||
* (EXTRA_PICTURE) or, from Android 12, its Icon (EXTRA_PICTURE_ICON).
|
||||
* Newer releases swap large pictures for a content URI to a bitmap
|
||||
* the system has offloaded (Dexcom's graph arrives that way); the
|
||||
* offload provider lets enabled notification listeners open those.
|
||||
*/
|
||||
@Suppress("DEPRECATION")
|
||||
private fun loadNotificationPicture(service: FaceclawMediaNotificationListenerService, notification: Notification): Bitmap? {
|
||||
val extras: Bundle = notification.extras ?: return null
|
||||
try {
|
||||
val picture: Bitmap? = extras.getParcelable(Notification.EXTRA_PICTURE)
|
||||
if (picture != null) {
|
||||
return softwareBitmap(picture)
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to read notification picture", t)
|
||||
}
|
||||
if (Build.VERSION.SDK_INT < Build.VERSION_CODES.S) {
|
||||
return null
|
||||
}
|
||||
try {
|
||||
val icon: Icon? = extras.getParcelable(Notification.EXTRA_PICTURE_ICON)
|
||||
val drawable: Drawable? = icon?.loadDrawable(service)
|
||||
if (drawable is BitmapDrawable && drawable.bitmap != null) {
|
||||
return softwareBitmap(drawable.bitmap)
|
||||
}
|
||||
if (drawable != null && drawable.intrinsicWidth > 0 && drawable.intrinsicHeight > 0) {
|
||||
val bitmap = Bitmap.createBitmap(drawable.intrinsicWidth, drawable.intrinsicHeight, Bitmap.Config.ARGB_8888)
|
||||
drawable.setBounds(0, 0, bitmap.width, bitmap.height)
|
||||
drawable.draw(Canvas(bitmap))
|
||||
return bitmap
|
||||
}
|
||||
} catch (t: Throwable) {
|
||||
Log.w(TAG, "failed to load notification picture icon", t)
|
||||
}
|
||||
return null
|
||||
}
|
||||
|
||||
/** Hardware bitmaps can't be read back pixel by pixel or drawn on a software canvas; copy those. */
|
||||
private fun softwareBitmap(bitmap: Bitmap): Bitmap {
|
||||
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O && bitmap.config == Bitmap.Config.HARDWARE) {
|
||||
return bitmap.copy(Bitmap.Config.ARGB_8888, false)
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
/**
|
||||
* Halve a picture while it is still at least twice the size it will
|
||||
* be shown at, leaving the last (under 2x) step of fitting it into
|
||||
* maxWidth x maxHeight to bitmapToGrayPacket. As with icons
|
||||
* (renderIconScaled), one filtered pass from a much larger source
|
||||
* samples too sparsely: a graph's thin lines break up into dots.
|
||||
*/
|
||||
private fun halveToward(source: Bitmap, maxWidth: Int, maxHeight: Int): Bitmap {
|
||||
var bitmap = source
|
||||
while ((bitmap.width >= maxWidth * 2 || bitmap.height >= maxHeight * 2) && bitmap.width >= 2 && bitmap.height >= 2) {
|
||||
val halved = Bitmap.createScaledBitmap(bitmap, bitmap.width / 2, bitmap.height / 2, true)
|
||||
if (bitmap !== source) {
|
||||
bitmap.recycle()
|
||||
}
|
||||
bitmap = halved
|
||||
}
|
||||
return bitmap
|
||||
}
|
||||
|
||||
/**
|
||||
* Debug aid for garbled-icon reports: saves each rendered icon to
|
||||
* <externalFilesDir>/debug-icons/ (adb-pullable) so extraction problems can
|
||||
@@ -630,8 +731,10 @@ class FaceclawMediaNotificationListenerService : NotificationListenerService() {
|
||||
}
|
||||
}
|
||||
out.put("lines", lines)
|
||||
out.put("hasPicture", hasNotificationPicture(extras))
|
||||
} else {
|
||||
out.put("lines", JSONArray())
|
||||
out.put("hasPicture", false)
|
||||
}
|
||||
|
||||
val actionsJson = JSONArray()
|
||||
|
||||
@@ -132,10 +132,15 @@ class ImageFileLoader private constructor() {
|
||||
* round on its own, which turns smooth gradients into visible bands.
|
||||
* Dithered output is already quantized: every byte is a level times 16,
|
||||
* which BmpUtil re-quantizes back to exactly that level.
|
||||
*
|
||||
* invertLightBackground shows a picture drawn on a light background
|
||||
* (a chart, a screenshot) as its negative; see GrayPacket.fromArgb.
|
||||
*/
|
||||
@JvmStatic
|
||||
@JvmOverloads
|
||||
fun bitmapToGrayPacket(
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
source: Bitmap?, maxWidth: Int, maxHeight: Int, gamma: Float, dither: Boolean,
|
||||
invertLightBackground: Boolean = false): ByteArray {
|
||||
if (source == null || source.width <= 0 || source.height <= 0
|
||||
|| maxWidth <= 0 || maxHeight <= 0) {
|
||||
return ByteArray(0)
|
||||
@@ -152,7 +157,7 @@ class ImageFileLoader private constructor() {
|
||||
}
|
||||
val pixels = IntArray(width * height)
|
||||
scaled.getPixels(pixels, 0, width, 0, 0, width, height)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither)
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither, invertLightBackground)
|
||||
} catch (e: Exception) {
|
||||
Log.w(TAG, "bitmap conversion failed", e)
|
||||
return ByteArray(0)
|
||||
|
||||
@@ -30,6 +30,8 @@
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSCalendarsFullAccessUsageDescription</key>
|
||||
<string>Faceclaw reads your upcoming calendar events to display them in Calendar and Glanceboard on your glasses.</string>
|
||||
<key>NSAppleMusicUsageDescription</key>
|
||||
<string>Faceclaw shows what is playing in Apple Music on your glasses, controls playback, and lets you browse and play your music library from the glasses.</string>
|
||||
<key>NSLocationWhenInUseUsageDescription</key>
|
||||
<string>Faceclaw uses your location for local weather, turn-by-turn navigation, and true north on the glasses compass.</string>
|
||||
<key>NSLocationTemporaryUsageDescriptionDictionary</key>
|
||||
|
||||
@@ -3,7 +3,7 @@ TARGETED_DEVICE_FAMILY = 1;
|
||||
CODE_SIGN_STYLE = Automatic;
|
||||
|
||||
// Set IOS_TEAM_ID in build_paths.sh (or pass --teamId) to choose the signing team.
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit
|
||||
OTHER_LDFLAGS = $(inherited) -framework CoreBluetooth -framework CoreLocation -framework CoreText -framework Speech -framework AVFoundation -framework WebKit -framework SafariServices -framework UniformTypeIdentifiers -framework ImageIO -framework EventKit -framework MediaPlayer
|
||||
|
||||
ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon;
|
||||
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
#import <Foundation/Foundation.h>
|
||||
|
||||
/**
|
||||
* Apple Music (the system Music app) through the MediaPlayer framework: now-playing state
|
||||
* and transport on the system player, the media library as a browse tree, and the system
|
||||
* output volume. Call from the main thread; callbacks are delivered on the main queue.
|
||||
* Nothing touches the player or library until media-library access is granted, so no
|
||||
* permission prompt appears unasked.
|
||||
*/
|
||||
@interface FaceclawAppleMusic : NSObject
|
||||
+ (instancetype)shared;
|
||||
/** Playback state, the now-playing item or authorization may have changed. */
|
||||
@property(nonatomic, copy) void (^changeHandler)(void);
|
||||
- (BOOL)hasPermission;
|
||||
- (NSString *)permissionStatus;
|
||||
- (void)requestPermission:(void (^)(BOOL granted))completion;
|
||||
/** Observe the system player (and keep observing across authorization changes). */
|
||||
- (void)start;
|
||||
- (void)stop;
|
||||
/** The system player as a JSON object; {"access": false} until access is granted. */
|
||||
- (NSString *)stateJson;
|
||||
- (void)playPause;
|
||||
- (void)skipNext;
|
||||
- (void)skipPrevious;
|
||||
/**
|
||||
* The queue Faceclaw last started, as [{"id", "title", "active"}], while the player is
|
||||
* still playing from it; "[]" otherwise (the system player does not expose queues that
|
||||
* the Music app started).
|
||||
*/
|
||||
- (NSString *)queueJson;
|
||||
- (void)skipToQueueIndex:(NSInteger)index;
|
||||
/** [widthLo, widthHi, heightLo, heightHi, BGRA...] fitted within maxSize (never upscaled), or nil. */
|
||||
- (NSData *)albumArt:(NSInteger)maxSize;
|
||||
/** System output volume 0..100, or -1 when unknown. */
|
||||
- (double)volumePercent;
|
||||
- (void)setVolumePercent:(double)percent;
|
||||
/** One level of the library tree ("root" first) as [{"mediaId", "title", "subtitle", "browsable", "playable"}]. */
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *json, NSString *error))completion;
|
||||
/** Start playback of a browsed collection, or of a song within its collection. */
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *error))completion;
|
||||
@end
|
||||
@@ -0,0 +1,512 @@
|
||||
#import "FaceclawAppleMusic.h"
|
||||
#import <AVFoundation/AVFoundation.h>
|
||||
#import <MediaPlayer/MediaPlayer.h>
|
||||
#import <UIKit/UIKit.h>
|
||||
#import <errno.h>
|
||||
#import <math.h>
|
||||
#import <stdlib.h>
|
||||
|
||||
/** How long a queue Faceclaw just set may take to become the player's now-playing context. */
|
||||
static const NSTimeInterval kQueueConfirmSeconds = 5.0;
|
||||
|
||||
static NSString *JsonString(id object, NSString *fallback) {
|
||||
NSData *data = [NSJSONSerialization dataWithJSONObject:object options:0 error:nil];
|
||||
return data ? [[NSString alloc] initWithData:data encoding:NSUTF8StringEncoding] : fallback;
|
||||
}
|
||||
|
||||
/** Persistent IDs are unsigned 64-bit, beyond JavaScript's exact integers, so they cross as decimal strings. */
|
||||
static NSString *IdString(MPMediaEntityPersistentID persistentId) {
|
||||
return [NSString stringWithFormat:@"%llu", persistentId];
|
||||
}
|
||||
|
||||
static BOOL ParseId(NSString *text, MPMediaEntityPersistentID *out) {
|
||||
const char *chars = text.UTF8String;
|
||||
if (!chars || chars[0] < '0' || chars[0] > '9') return NO;
|
||||
char *end = NULL;
|
||||
errno = 0;
|
||||
unsigned long long value = strtoull(chars, &end, 10);
|
||||
if (errno || !end || *end) return NO;
|
||||
*out = value;
|
||||
return YES;
|
||||
}
|
||||
|
||||
static NSDictionary *BrowseRow(NSString *mediaId, NSString *title, NSString *subtitle, BOOL browsable, BOOL playable) {
|
||||
return @{@"mediaId": mediaId, @"title": title ?: @"", @"subtitle": subtitle ?: @"",
|
||||
@"browsable": @(browsable), @"playable": @(playable)};
|
||||
}
|
||||
|
||||
@interface FaceclawAppleMusic ()
|
||||
@property(nonatomic, strong) dispatch_queue_t libraryQueue;
|
||||
@property(nonatomic) BOOL started;
|
||||
@property(nonatomic) BOOL observing;
|
||||
/** Bumped whenever the now-playing item changes, so cached album art is refetched. */
|
||||
@property(nonatomic) NSInteger artVersion;
|
||||
@property(nonatomic, copy) NSString *nowPlayingKey;
|
||||
/** The items of the queue Faceclaw last started, until the player leaves them. */
|
||||
@property(nonatomic, copy) NSArray<MPMediaItem *> *queueItems;
|
||||
@property(nonatomic) BOOL queueConfirmed;
|
||||
@property(nonatomic, strong) NSDate *queueSetAt;
|
||||
@property(nonatomic, strong) MPVolumeView *volumeView;
|
||||
@end
|
||||
|
||||
@implementation FaceclawAppleMusic
|
||||
+ (instancetype)shared {
|
||||
static FaceclawAppleMusic *instance;
|
||||
static dispatch_once_t once;
|
||||
dispatch_once(&once, ^{ instance = [FaceclawAppleMusic new]; });
|
||||
return instance;
|
||||
}
|
||||
|
||||
- (instancetype)init {
|
||||
if ((self = [super init])) {
|
||||
_libraryQueue = dispatch_queue_create("com.faceclaw.applemusic", DISPATCH_QUEUE_SERIAL);
|
||||
_nowPlayingKey = @"";
|
||||
// Access may have been granted or revoked in Settings while away.
|
||||
[NSNotificationCenter.defaultCenter addObserver:self selector:@selector(appBecameActive:)
|
||||
name:UIApplicationDidBecomeActiveNotification object:nil];
|
||||
}
|
||||
return self;
|
||||
}
|
||||
|
||||
#pragma mark - Authorization
|
||||
|
||||
- (BOOL)hasPermission {
|
||||
return MPMediaLibrary.authorizationStatus == MPMediaLibraryAuthorizationStatusAuthorized;
|
||||
}
|
||||
|
||||
- (NSString *)permissionStatus {
|
||||
switch (MPMediaLibrary.authorizationStatus) {
|
||||
case MPMediaLibraryAuthorizationStatusAuthorized: return @"granted";
|
||||
case MPMediaLibraryAuthorizationStatusDenied: return @"denied";
|
||||
case MPMediaLibraryAuthorizationStatusRestricted: return @"restricted";
|
||||
default: return @"not-determined";
|
||||
}
|
||||
}
|
||||
|
||||
- (void)requestPermission:(void (^)(BOOL))completion {
|
||||
if ([self hasPermission]) {
|
||||
completion(YES);
|
||||
return;
|
||||
}
|
||||
[MPMediaLibrary requestAuthorization:^(MPMediaLibraryAuthorizationStatus status) {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
completion(status == MPMediaLibraryAuthorizationStatusAuthorized);
|
||||
});
|
||||
}];
|
||||
}
|
||||
|
||||
- (void)appBecameActive:(NSNotification *)notification {
|
||||
[self syncObserving];
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
#pragma mark - Observation
|
||||
|
||||
- (void)start {
|
||||
self.started = YES;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)stop {
|
||||
self.started = NO;
|
||||
[self syncObserving];
|
||||
}
|
||||
|
||||
- (void)syncObserving {
|
||||
BOOL want = self.started && [self hasPermission];
|
||||
if (want == self.observing) return;
|
||||
self.observing = want;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
NSNotificationCenter *center = NSNotificationCenter.defaultCenter;
|
||||
if (want) {
|
||||
[center addObserver:self selector:@selector(playbackStateChanged:)
|
||||
name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center addObserver:self selector:@selector(nowPlayingItemChanged:)
|
||||
name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player beginGeneratingPlaybackNotifications];
|
||||
} else {
|
||||
[center removeObserver:self name:MPMusicPlayerControllerPlaybackStateDidChangeNotification object:player];
|
||||
[center removeObserver:self name:MPMusicPlayerControllerNowPlayingItemDidChangeNotification object:player];
|
||||
[player endGeneratingPlaybackNotifications];
|
||||
}
|
||||
}
|
||||
|
||||
- (void)playbackStateChanged:(NSNotification *)notification {
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)nowPlayingItemChanged:(NSNotification *)notification {
|
||||
// Streamed catalog items can share identity fields, so every change counts.
|
||||
self.artVersion++;
|
||||
[self notifyChanged];
|
||||
}
|
||||
|
||||
- (void)notifyChanged {
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (self.changeHandler) self.changeHandler();
|
||||
});
|
||||
}
|
||||
|
||||
#pragma mark - Now playing
|
||||
|
||||
- (NSString *)stateJson {
|
||||
if (![self hasPermission]) return @"{\"access\":false}";
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *item = player.nowPlayingItem;
|
||||
NSString *key = item ? [NSString stringWithFormat:@"%llu|%@|%@|%@", item.persistentID,
|
||||
item.playbackStoreID ?: @"", item.title ?: @"", item.artist ?: @""] : @"";
|
||||
if (![key isEqualToString:self.nowPlayingKey]) {
|
||||
self.nowPlayingKey = key;
|
||||
self.artVersion++;
|
||||
}
|
||||
NSString *state;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
state = @"playing";
|
||||
break;
|
||||
case MPMusicPlaybackStatePaused:
|
||||
case MPMusicPlaybackStateInterrupted:
|
||||
state = @"paused";
|
||||
break;
|
||||
default:
|
||||
state = @"stopped";
|
||||
break;
|
||||
}
|
||||
NSTimeInterval position = player.currentPlaybackTime;
|
||||
NSTimeInterval duration = item.playbackDuration;
|
||||
float rate = player.currentPlaybackRate;
|
||||
return JsonString(@{
|
||||
@"access": @YES,
|
||||
@"hasItem": @(item != nil),
|
||||
@"state": state,
|
||||
@"title": item.title ?: @"",
|
||||
@"artist": item.artist ?: @"",
|
||||
@"album": item.albumTitle ?: @"",
|
||||
@"positionMs": @(item && isfinite(position) && position >= 0 ? position * 1000.0 : -1),
|
||||
@"durationMs": @(item && isfinite(duration) && duration > 0 ? duration * 1000.0 : -1),
|
||||
@"rate": @(isfinite(rate) ? rate : 0),
|
||||
@"artVersion": @(self.artVersion),
|
||||
}, @"{\"access\":true}");
|
||||
}
|
||||
|
||||
- (void)playPause {
|
||||
if (![self hasPermission]) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
switch (player.playbackState) {
|
||||
case MPMusicPlaybackStatePlaying:
|
||||
case MPMusicPlaybackStateSeekingForward:
|
||||
case MPMusicPlaybackStateSeekingBackward:
|
||||
[player pause];
|
||||
break;
|
||||
default:
|
||||
[player play];
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
- (void)skipNext {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToNextItem];
|
||||
}
|
||||
|
||||
- (void)skipPrevious {
|
||||
if ([self hasPermission]) [MPMusicPlayerController.systemMusicPlayer skipToPreviousItem];
|
||||
}
|
||||
|
||||
- (NSData *)albumArt:(NSInteger)maxSize {
|
||||
if (maxSize <= 0 || maxSize > 1024 || ![self hasPermission]) return nil;
|
||||
MPMediaItemArtwork *artwork = MPMusicPlayerController.systemMusicPlayer.nowPlayingItem.artwork;
|
||||
UIImage *image = [artwork imageWithSize:CGSizeMake(maxSize, maxSize)];
|
||||
CGImageRef source = image.CGImage;
|
||||
if (!source) return nil;
|
||||
double sourceWidth = CGImageGetWidth(source), sourceHeight = CGImageGetHeight(source);
|
||||
if (sourceWidth <= 0 || sourceHeight <= 0) return nil;
|
||||
double scale = MIN(1.0, MIN(maxSize / sourceWidth, maxSize / sourceHeight));
|
||||
NSInteger width = MAX(1, lround(sourceWidth * scale)), height = MAX(1, lround(sourceHeight * scale));
|
||||
NSMutableData *packet = [NSMutableData dataWithLength:4 + width * height * 4];
|
||||
uint8_t *bytes = packet.mutableBytes;
|
||||
bytes[0] = width & 255; bytes[1] = (width >> 8) & 255;
|
||||
bytes[2] = height & 255; bytes[3] = (height >> 8) & 255;
|
||||
uint8_t *pixels = bytes + 4;
|
||||
CGColorSpaceRef space = CGColorSpaceCreateDeviceRGB();
|
||||
// Memory order B, G, R, A: one little-endian ARGB word per pixel.
|
||||
CGContextRef context = CGBitmapContextCreate(pixels, width, height, 8, width * 4, space,
|
||||
kCGBitmapByteOrder32Little | (CGBitmapInfo)kCGImageAlphaPremultipliedFirst);
|
||||
CGColorSpaceRelease(space);
|
||||
if (!context) return nil;
|
||||
CGContextSetInterpolationQuality(context, kCGInterpolationHigh);
|
||||
CGContextDrawImage(context, CGRectMake(0, 0, width, height), source);
|
||||
CGContextRelease(context);
|
||||
// Composited over black (the display's see-through background); now opaque.
|
||||
for (NSInteger i = 0; i < width * height; i++) pixels[i * 4 + 3] = 255;
|
||||
return packet;
|
||||
}
|
||||
|
||||
#pragma mark - Queue
|
||||
|
||||
/** Index of the now-playing item in Faceclaw's queue; forgets the queue once the player has left it. */
|
||||
- (NSInteger)activeQueueIndex {
|
||||
NSArray<MPMediaItem *> *items = self.queueItems;
|
||||
if (!items.count || ![self hasPermission]) return -1;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMediaItem *now = player.nowPlayingItem;
|
||||
NSInteger found = -1;
|
||||
if (now) {
|
||||
NSUInteger index = player.indexOfNowPlayingItem;
|
||||
if (index < items.count && items[index].persistentID == now.persistentID) {
|
||||
found = (NSInteger)index;
|
||||
} else {
|
||||
for (NSUInteger i = 0; i < items.count; i++) {
|
||||
if (items[i].persistentID == now.persistentID) { found = (NSInteger)i; break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
if (found >= 0) {
|
||||
self.queueConfirmed = YES;
|
||||
return found;
|
||||
}
|
||||
// Either the player hasn't switched over yet, or it has moved on to another queue.
|
||||
if (self.queueConfirmed || -[self.queueSetAt timeIntervalSinceNow] > kQueueConfirmSeconds) {
|
||||
self.queueItems = nil;
|
||||
}
|
||||
return -1;
|
||||
}
|
||||
|
||||
- (NSString *)queueJson {
|
||||
NSInteger active = [self activeQueueIndex];
|
||||
if (active < 0) return @"[]";
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:self.queueItems.count];
|
||||
[self.queueItems enumerateObjectsUsingBlock:^(MPMediaItem *item, NSUInteger index, BOOL *stop) {
|
||||
[rows addObject:@{@"id": @(index), @"title": item.title ?: @"", @"active": @((NSInteger)index == active)}];
|
||||
}];
|
||||
return JsonString(rows, @"[]");
|
||||
}
|
||||
|
||||
- (void)skipToQueueIndex:(NSInteger)index {
|
||||
if ([self activeQueueIndex] < 0 || index < 0 || index >= (NSInteger)self.queueItems.count) return;
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
player.nowPlayingItem = self.queueItems[index];
|
||||
[player play];
|
||||
}
|
||||
|
||||
#pragma mark - Volume
|
||||
|
||||
- (UIWindow *)hostWindow {
|
||||
for (UIScene *scene in UIApplication.sharedApplication.connectedScenes) {
|
||||
if (![scene isKindOfClass:UIWindowScene.class]) continue;
|
||||
NSArray<UIWindow *> *windows = ((UIWindowScene *)scene).windows;
|
||||
for (UIWindow *window in windows) if (window.isKeyWindow) return window;
|
||||
if (windows.firstObject) return windows.firstObject;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
/**
|
||||
* The slider of an off-screen MPVolumeView: iOS has no public setter for the system volume,
|
||||
* but the volume view's slider drives it while the view is in a window.
|
||||
*/
|
||||
- (UISlider *)volumeSlider:(BOOL *)attached {
|
||||
if (!self.volumeView) {
|
||||
self.volumeView = [[MPVolumeView alloc] initWithFrame:CGRectMake(-2000, -2000, 160, 40)];
|
||||
}
|
||||
if (!self.volumeView.window) {
|
||||
UIWindow *window = [self hostWindow];
|
||||
if (!window) return nil;
|
||||
[window addSubview:self.volumeView];
|
||||
if (attached) *attached = YES;
|
||||
}
|
||||
for (UIView *view in self.volumeView.subviews) {
|
||||
if ([view isKindOfClass:UISlider.class]) return (UISlider *)view;
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (double)volumePercent {
|
||||
// The slider tracks the system volume once attached; until then (nothing has been set
|
||||
// yet) the session's reading, which can lag while Faceclaw's audio session is inactive.
|
||||
UISlider *slider = self.volumeView.window ? [self volumeSlider:NULL] : nil;
|
||||
float volume = slider ? slider.value : AVAudioSession.sharedInstance.outputVolume;
|
||||
return isfinite(volume) ? MAX(0.0, MIN(100.0, round(volume * 100.0))) : -1;
|
||||
}
|
||||
|
||||
- (void)setVolumePercent:(double)percent {
|
||||
if (!isfinite(percent)) return;
|
||||
BOOL attached = NO;
|
||||
UISlider *slider = [self volumeSlider:&attached];
|
||||
if (!slider) return;
|
||||
float value = (float)MAX(0.0, MIN(1.0, percent / 100.0));
|
||||
void (^apply)(void) = ^{
|
||||
[slider setValue:value animated:NO];
|
||||
[slider sendActionsForControlEvents:UIControlEventValueChanged];
|
||||
};
|
||||
// A freshly attached volume view ignores changes until it has wired up its slider.
|
||||
if (attached) dispatch_after(dispatch_time(DISPATCH_TIME_NOW, (int64_t)(0.15 * NSEC_PER_SEC)), dispatch_get_main_queue(), apply);
|
||||
else apply();
|
||||
}
|
||||
|
||||
#pragma mark - Library
|
||||
|
||||
/**
|
||||
* Songs of a playable collection id: "playlist:<id>", "album:<id>", "artist:<id>" (album by
|
||||
* album, in track order) or "songs" (the whole library). Runs on the library queue.
|
||||
*/
|
||||
- (NSArray<MPMediaItem *> *)itemsForCollection:(NSString *)collectionId {
|
||||
if ([collectionId isEqualToString:@"songs"]) return [MPMediaQuery songsQuery].items ?: @[];
|
||||
NSRange colon = [collectionId rangeOfString:@":"];
|
||||
MPMediaEntityPersistentID persistentId;
|
||||
if (colon.location == NSNotFound || !ParseId([collectionId substringFromIndex:colon.location + 1], &persistentId)) return @[];
|
||||
NSString *kind = [collectionId substringToIndex:colon.location];
|
||||
MPMediaQuery *query;
|
||||
NSString *property;
|
||||
if ([kind isEqualToString:@"playlist"]) {
|
||||
query = [MPMediaQuery playlistsQuery];
|
||||
property = MPMediaPlaylistPropertyPersistentID;
|
||||
} else if ([kind isEqualToString:@"album"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyAlbumPersistentID;
|
||||
} else if ([kind isEqualToString:@"artist"]) {
|
||||
query = [MPMediaQuery albumsQuery];
|
||||
property = MPMediaItemPropertyArtistPersistentID;
|
||||
} else {
|
||||
return @[];
|
||||
}
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(persistentId) forProperty:property]];
|
||||
NSMutableArray<MPMediaItem *> *items = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *collection in query.collections) [items addObjectsFromArray:collection.items];
|
||||
return items;
|
||||
}
|
||||
|
||||
- (NSArray *)songRows:(NSArray<MPMediaItem *> *)items context:(NSString *)collectionId {
|
||||
NSMutableArray *rows = [NSMutableArray arrayWithCapacity:items.count];
|
||||
for (MPMediaItem *item in items) {
|
||||
[rows addObject:BrowseRow([NSString stringWithFormat:@"song:%@@%@", IdString(item.persistentID), collectionId],
|
||||
item.title, item.artist, NO, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)albumRows:(MPMediaQuery *)query {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *album in query.collections) {
|
||||
MPMediaItem *item = album.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"album:" stringByAppendingString:IdString(item.albumPersistentID)],
|
||||
item.albumTitle, item.albumArtist.length ? item.albumArtist : item.artist, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
- (NSArray *)childrenOf:(NSString *)parentId {
|
||||
if ([parentId isEqualToString:@"root"]) {
|
||||
return @[BrowseRow(@"playlists", @"Playlists", nil, YES, NO), BrowseRow(@"artists", @"Artists", nil, YES, NO),
|
||||
BrowseRow(@"albums", @"Albums", nil, YES, NO), BrowseRow(@"songs", @"Songs", nil, YES, NO)];
|
||||
}
|
||||
if ([parentId isEqualToString:@"playlists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaPlaylist *playlist in [MPMediaQuery playlistsQuery].collections) {
|
||||
NSUInteger count = playlist.count;
|
||||
NSString *subtitle = count == 1 ? @"1 song" : [NSString stringWithFormat:@"%lu songs", (unsigned long)count];
|
||||
[rows addObject:BrowseRow([@"playlist:" stringByAppendingString:IdString(playlist.persistentID)],
|
||||
playlist.name, subtitle, YES, count > 0)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"artists"]) {
|
||||
NSMutableArray *rows = [NSMutableArray new];
|
||||
for (MPMediaItemCollection *artist in [MPMediaQuery artistsQuery].collections) {
|
||||
MPMediaItem *item = artist.representativeItem;
|
||||
if (!item) continue;
|
||||
[rows addObject:BrowseRow([@"artist:" stringByAppendingString:IdString(item.artistPersistentID)],
|
||||
item.artist.length ? item.artist : @"Unknown Artist", nil, YES, YES)];
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
if ([parentId isEqualToString:@"albums"]) return [self albumRows:[MPMediaQuery albumsQuery]];
|
||||
if ([parentId isEqualToString:@"songs"]) return [self songRows:[MPMediaQuery songsQuery].items ?: @[] context:@"songs"];
|
||||
if ([parentId hasPrefix:@"artist:"]) {
|
||||
MPMediaEntityPersistentID artistId;
|
||||
if (!ParseId([parentId substringFromIndex:7], &artistId)) return nil;
|
||||
MPMediaQuery *query = [MPMediaQuery albumsQuery];
|
||||
[query addFilterPredicate:[MPMediaPropertyPredicate predicateWithValue:@(artistId)
|
||||
forProperty:MPMediaItemPropertyArtistPersistentID]];
|
||||
return [self albumRows:query];
|
||||
}
|
||||
if ([parentId hasPrefix:@"playlist:"] || [parentId hasPrefix:@"album:"]) {
|
||||
return [self songRows:[self itemsForCollection:parentId] context:parentId];
|
||||
}
|
||||
return nil;
|
||||
}
|
||||
|
||||
- (void)browse:(NSString *)parentId completion:(void (^)(NSString *, NSString *))completion {
|
||||
// Library queries can take a while on large libraries; keep them off the main thread.
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *json = nil, *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
NSArray *rows = [self childrenOf:parentId ?: @""];
|
||||
if (rows) json = JsonString(rows, nil);
|
||||
if (!json) error = rows ? @"Library listing failed" : @"Unknown library item";
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{ completion(json, error); });
|
||||
});
|
||||
}
|
||||
|
||||
- (void)play:(NSString *)mediaId completion:(void (^)(NSString *))completion {
|
||||
dispatch_async(self.libraryQueue, ^{
|
||||
NSString *collectionId = mediaId ?: @"";
|
||||
MPMediaEntityPersistentID startId = 0;
|
||||
BOOL hasStart = NO;
|
||||
if ([collectionId hasPrefix:@"song:"]) {
|
||||
NSRange at = [collectionId rangeOfString:@"@"];
|
||||
if (at.location == NSNotFound) collectionId = @"";
|
||||
else {
|
||||
hasStart = ParseId([collectionId substringWithRange:NSMakeRange(5, at.location - 5)], &startId);
|
||||
collectionId = [collectionId substringFromIndex:at.location + 1];
|
||||
}
|
||||
}
|
||||
NSArray<MPMediaItem *> *items = @[];
|
||||
NSString *error = nil;
|
||||
if (![self hasPermission]) {
|
||||
error = @"Apple Music access is not allowed";
|
||||
} else {
|
||||
@try {
|
||||
items = [self itemsForCollection:collectionId];
|
||||
} @catch (NSException *exception) {
|
||||
error = exception.reason ?: @"Library unavailable";
|
||||
}
|
||||
if (!error && !items.count) error = @"Nothing to play";
|
||||
}
|
||||
MPMediaItem *start = nil;
|
||||
if (hasStart) {
|
||||
for (MPMediaItem *item in items) {
|
||||
if (item.persistentID == startId) { start = item; break; }
|
||||
}
|
||||
}
|
||||
dispatch_async(dispatch_get_main_queue(), ^{
|
||||
if (error) {
|
||||
completion(error);
|
||||
return;
|
||||
}
|
||||
MPMusicPlayerController *player = MPMusicPlayerController.systemMusicPlayer;
|
||||
MPMusicPlayerMediaItemQueueDescriptor *descriptor = [[MPMusicPlayerMediaItemQueueDescriptor alloc]
|
||||
initWithItemCollection:[MPMediaItemCollection collectionWithItems:items]];
|
||||
if (start) descriptor.startItem = start;
|
||||
[player setQueueWithDescriptor:descriptor];
|
||||
[player play];
|
||||
self.queueItems = items;
|
||||
self.queueConfirmed = NO;
|
||||
self.queueSetAt = [NSDate date];
|
||||
[self notifyChanged];
|
||||
completion(nil);
|
||||
});
|
||||
});
|
||||
}
|
||||
@end
|
||||
@@ -6,6 +6,7 @@ module FaceclawGraphics {
|
||||
header "FaceclawSpeech.h"
|
||||
header "FaceclawLocation.h"
|
||||
header "FaceclawCalendar.h"
|
||||
header "FaceclawAppleMusic.h"
|
||||
header "FaceclawConfigPort.h"
|
||||
header "FaceclawEvenHubWebView.h"
|
||||
header "FaceclawCrypto.h"
|
||||
|
||||
@@ -5,6 +5,8 @@
|
||||
### Features
|
||||
|
||||
* Use Open-Meteo for weather in regions that the US National Weather Service doesn't cover
|
||||
* Android: Show pictures attached to notifications (such as Dexcom's glucose graph) in the notification view
|
||||
* Add an Apple Music integration on iOS
|
||||
|
||||
### Bug fixes
|
||||
|
||||
|
||||
@@ -1055,6 +1055,12 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
);
|
||||
}
|
||||
|
||||
/** Log a problem with the app's own calls where its developer will see it: the app's console. */
|
||||
private warnApp(message: string): void {
|
||||
this.log(`evenhub: ${message}`);
|
||||
this.webViewHandle?.evaluateJs(`window.console && console.warn(${JSON.stringify(`Faceclaw: ${message}`)})`);
|
||||
}
|
||||
|
||||
/** Deliver an extension event to the app (e.g. windowLifecycle). */
|
||||
private pushExtEvent(name: string, data: unknown): void {
|
||||
this.webViewHandle?.evaluateJs(
|
||||
@@ -1266,12 +1272,16 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
for (const raw of rawSpecs) {
|
||||
const record = asRecord(raw);
|
||||
const name = readString(record, "name", "");
|
||||
if (!name) continue;
|
||||
const problem = appToolProblem(name, record.parameters);
|
||||
if (problem) {
|
||||
this.warnApp(`setAssistantTools: skipped tool ${JSON.stringify(name)}: ${problem}`);
|
||||
continue;
|
||||
}
|
||||
const availability = record.availability === "open" ? "open" : "foreground";
|
||||
specs.push({
|
||||
name,
|
||||
description: readString(record, "description", ""),
|
||||
inputSchema: asRecord(record.parameters),
|
||||
inputSchema: record.parameters == null ? { type: "object", properties: {} } : asRecord(record.parameters),
|
||||
availability,
|
||||
});
|
||||
}
|
||||
@@ -1362,13 +1372,8 @@ export class EvenHubSession implements EvenHubMicClient, EvenHubImuClient, EvenH
|
||||
if (steps.length === 0) return;
|
||||
for (let i = 0; i < steps.length; i += CFW_SEQ_MAX) {
|
||||
const chunk = steps.slice(i, i + CFW_SEQ_MAX);
|
||||
const payload = buildSoundSequencePayload(chunk);
|
||||
try {
|
||||
await this.sendBuzzerSequence(payload);
|
||||
} catch (error) {
|
||||
this.log(`evenhub: playBuzzer failed: ${error}`);
|
||||
return;
|
||||
}
|
||||
// A send failure rejects the app's promise (handleExtensionCall logs it).
|
||||
await this.sendBuzzerSequence(buildSoundSequencePayload(chunk));
|
||||
if (i + CFW_SEQ_MAX < steps.length) await delay(phraseDurationMs(chunk));
|
||||
}
|
||||
}
|
||||
@@ -1511,6 +1516,35 @@ function configuredApiKeyServices(): string[] {
|
||||
return API_KEY_SERVICES.filter((service) => service.setting.get().trim()).map((service) => service.id);
|
||||
}
|
||||
|
||||
/** App assistant-tool names: what every model provider accepts in a tool name. */
|
||||
const APP_TOOL_NAME = /^[A-Za-z0-9_-]{1,64}$/;
|
||||
|
||||
/**
|
||||
* Why an app's assistant tool can't be offered, or null if it can. Provider
|
||||
* APIs reject a whole request over one bad tool definition, so offering it
|
||||
* would break every assistant turn while the app runs.
|
||||
*/
|
||||
function appToolProblem(name: string, parameters: unknown): string | null {
|
||||
if (!APP_TOOL_NAME.test(name)) return "name must be 1-64 letters, digits, underscores or hyphens";
|
||||
if (parameters == null) return null;
|
||||
const schema = parameters as Record<string, unknown>;
|
||||
if (typeof parameters !== "object" || Array.isArray(parameters) || schema.type !== "object") {
|
||||
return 'parameters must be a JSON Schema with type "object"';
|
||||
}
|
||||
if ("oneOf" in schema || "anyOf" in schema || "allOf" in schema) {
|
||||
return "parameters can't use oneOf, anyOf or allOf at the top level";
|
||||
}
|
||||
const properties = schema.properties;
|
||||
if (properties !== undefined && (!properties || typeof properties !== "object" || Array.isArray(properties))) {
|
||||
return "parameters.properties must be an object";
|
||||
}
|
||||
const required = schema.required;
|
||||
if (required !== undefined && !(Array.isArray(required) && required.every((key) => typeof key === "string"))) {
|
||||
return "parameters.required must be an array of property names";
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Coerce an app-supplied buzzer step list into validated {freq, ms, duty?}. */
|
||||
function normalizeBuzzerSteps(raw: unknown[]): Step[] {
|
||||
const out: Step[] = [];
|
||||
|
||||
@@ -57,9 +57,9 @@ export function musicSettingsMenuItems(): MenuItem[] {
|
||||
|
||||
/**
|
||||
* Now playing: album art (unless turned off), title/artist/album and a
|
||||
* progress bar for the active Android media session, the Music app's header
|
||||
* at card size. The media bridge runs for the whole session; the widget
|
||||
* subscribes and ticks once a second only while something is playing.
|
||||
* progress bar for the active media session (Apple Music on iOS), the Music
|
||||
* app's header at card size. The media bridge runs for the whole session; the
|
||||
* widget subscribes and ticks once a second only while something is playing.
|
||||
*/
|
||||
export class MusicWidget implements GlanceWidget {
|
||||
private media: MediaControllerState = mediaControllerBridge.snapshot();
|
||||
@@ -105,7 +105,7 @@ export class MusicWidget implements GlanceWidget {
|
||||
if (!media.accessEnabled) {
|
||||
// Actionable, so it stays as text.
|
||||
image.drawText(small, PAD, PAD, "Music", 150);
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, "Notification access needed", 140);
|
||||
image.drawText(small, PAD, PAD + small.lineHeight + 6, mediaControllerBridge.accessLabel, 140);
|
||||
return;
|
||||
}
|
||||
// Nothing playing: a dim note, centred, and nothing else.
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
/** Native services still outside the current iOS app-screen milestone. */
|
||||
const unavailable: Record<string, string> = {
|
||||
music: 'Media-player integration is not available on iOS yet.',
|
||||
microphones: 'Glasses microphone capture is not available on iOS yet.',
|
||||
}
|
||||
export function iosAppUnavailableReason(appId: string): string | null {
|
||||
|
||||
+18
-18
@@ -46,10 +46,11 @@ type FocusColumn = "actions" | "playlist";
|
||||
|
||||
/**
|
||||
* Music controller app: metadata + album art + transport controls for the
|
||||
* active Android media session (any player that publishes one), plus the
|
||||
* player's queue when it exposes one (scroll to a track, click to jump).
|
||||
* With no session, the front page lists the apps whose library can be
|
||||
* browsed (click one to open its MediaBrowseLayer).
|
||||
* active Android media session (any player that publishes one) or, on iOS,
|
||||
* the Apple Music system player, plus the player's queue when it exposes one
|
||||
* (scroll to a track, click to jump). With no session, the front page lists
|
||||
* the apps whose library can be browsed (click one to open its
|
||||
* MediaBrowseLayer).
|
||||
*/
|
||||
class MusicAppLayer implements Layer {
|
||||
// Watch swipes skip tracks: right = next, left = previous.
|
||||
@@ -97,15 +98,11 @@ class MusicAppLayer implements Layer {
|
||||
const media = mediaControllerBridge.snapshot();
|
||||
|
||||
if (!media.accessEnabled) {
|
||||
const lines = wrapText(
|
||||
font,
|
||||
"Notification access is required to control Android media sessions. Tap to open settings.",
|
||||
width - 48,
|
||||
);
|
||||
const lines = wrapText(font, mediaControllerBridge.accessPrompt, width - 48);
|
||||
for (let index = 0; index < lines.length; index++) {
|
||||
image.drawText(font, 24, 16 + index * lineStep(font), lines[index]!, 180);
|
||||
}
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} open settings`, 110);
|
||||
image.drawText(font, 20, height - 16, `${GESTURE_CLICK} ${mediaControllerBridge.accessAction}`, 110);
|
||||
return image;
|
||||
}
|
||||
|
||||
@@ -113,10 +110,12 @@ class MusicAppLayer implements Layer {
|
||||
const apps = mediaBrowserBridge.listBrowsableApps(!this.libraryAppsFresh);
|
||||
this.libraryAppsFresh = true;
|
||||
this.libraryMenu.setItems(apps);
|
||||
image.drawText(font, 24, 16, "No active media session.", 180);
|
||||
const prompt = apps.length
|
||||
? "Browse a music app's library, or start playback on the phone."
|
||||
: "Start playback in another app on the phone.";
|
||||
image.drawText(font, 24, 16, global.isIOS ? "Nothing is playing in Apple Music." : "No active media session.", 180);
|
||||
const prompt = global.isIOS
|
||||
? "Browse your library, or start playback on the phone."
|
||||
: apps.length
|
||||
? "Browse a music app's library, or start playback on the phone."
|
||||
: "Start playback in another app on the phone.";
|
||||
let y = 22 + lineStep(font);
|
||||
for (const line of wrapText(font, prompt, width - 48)) {
|
||||
image.drawText(font, 24, y, line, 150);
|
||||
@@ -229,7 +228,7 @@ class MusicAppLayer implements Layer {
|
||||
}
|
||||
if (!media.accessEnabled) {
|
||||
if (event.type === "click") {
|
||||
mediaControllerBridge.openNotificationAccessSettings();
|
||||
void mediaControllerBridge.requestAccess();
|
||||
} else if (event.type === "double-click") {
|
||||
ctx.stack.pop();
|
||||
}
|
||||
@@ -268,8 +267,8 @@ class MusicAppLayer implements Layer {
|
||||
const item = this.queueMenu.selectedItem;
|
||||
if (item) {
|
||||
await mediaControllerBridge.skipToQueueItem(item.id);
|
||||
// The player rebuilds the queue with the chosen track at index 0.
|
||||
this.queueMenu.select(0);
|
||||
// Android players rebuild the queue with the chosen track at index 0.
|
||||
if (mediaControllerBridge.skipRebuildsQueue) this.queueMenu.select(0);
|
||||
}
|
||||
return;
|
||||
}
|
||||
@@ -497,7 +496,8 @@ export function createMusicAppWindow(options: InProcessAppOptions): InProcessWin
|
||||
iconLetter: "M",
|
||||
icon: "music",
|
||||
closeable: true,
|
||||
menuItems: () => [{
|
||||
// Music settings choose among Android media apps; iOS has only Apple Music.
|
||||
menuItems: () => global.isIOS ? [] : [{
|
||||
label: "Music settings",
|
||||
onSelect: (ctx) => {
|
||||
ctx.stack.pop();
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
import { temperatureUnitSetting } from "../../ui/dashboard-settings";
|
||||
|
||||
/**
|
||||
* A temperature (weather data is kept in Fahrenheit) in the unit chosen in
|
||||
* Settings > Customization > Temperature unit, rounded: "72°F" / "22°C", or
|
||||
* "72°" without the unit letter.
|
||||
*/
|
||||
export function formatTemperature(fahrenheit: number, withUnit = true): string {
|
||||
const celsius = temperatureUnitSetting.get() === "celsius";
|
||||
const value = Math.round(celsius ? (fahrenheit - 32) * 5 / 9 : fahrenheit);
|
||||
return withUnit ? `${value}°${celsius ? "C" : "F"}` : `${value}°`;
|
||||
}
|
||||
|
||||
/** The widest temperature the forecast's column is sized for. */
|
||||
export function widestTemperatureSample(): string {
|
||||
return temperatureUnitSetting.get() === "celsius" ? "-20°C" : "100°F";
|
||||
}
|
||||
@@ -1,6 +1,6 @@
|
||||
import { ensureLocationPermission, hasLocationPermission } from "../../native/location-permissions";
|
||||
import { weatherBridge } from "../../native/weather";
|
||||
import { toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { onAnySettingChanged, toggleSettingMenuItem } from "../../ui/dashboard-settings";
|
||||
import { openSettingsSubMenu } from "../../ui/dashboard/settings-panel";
|
||||
import { WeatherLayer } from "./weather";
|
||||
import { weatherShowInStatusBarSetting, weatherShowOnSystemCardSetting } from "./weather-settings";
|
||||
@@ -19,6 +19,7 @@ export function createWeatherAppWindow(options: InProcessAppOptions): InProcessW
|
||||
let closed = false;
|
||||
let requestingPermission = false;
|
||||
let unsubscribe: (() => void) | null = null;
|
||||
let unsubscribeSettings: (() => void) | null = null;
|
||||
|
||||
const requestUpdate = () => {
|
||||
if (closed || requestingPermission) return;
|
||||
@@ -81,11 +82,15 @@ export function createWeatherAppWindow(options: InProcessAppOptions): InProcessW
|
||||
closed = true;
|
||||
unsubscribe?.();
|
||||
unsubscribe = null;
|
||||
unsubscribeSettings?.();
|
||||
unsubscribeSettings = null;
|
||||
weatherBridge.stop();
|
||||
options.onClosed();
|
||||
},
|
||||
});
|
||||
unsubscribe = weatherBridge.onStateChange(() => app.requestRender());
|
||||
// Repaint for a temperature-unit change made in the Settings app.
|
||||
unsubscribeSettings = onAnySettingChanged(() => app.requestRender());
|
||||
requestUpdate();
|
||||
return app;
|
||||
}
|
||||
@@ -1,6 +1,7 @@
|
||||
import { renderSvgIcon } from "../../graphics/icons";
|
||||
import { type GrayImage, type UiFont } from "../../graphics/image";
|
||||
import { type WeatherCondition, type WeatherState } from "../../native/weather";
|
||||
import { formatTemperature } from "./temperature-format";
|
||||
|
||||
// Lucide weather icons (ISC licensed), kept verbatim so they can be diffed
|
||||
// against upstream.
|
||||
@@ -90,7 +91,7 @@ export function weatherReading(state: WeatherState, nowMs: number): WeatherReadi
|
||||
return {
|
||||
condition: current.condition,
|
||||
isDaytime: current.isDaytime,
|
||||
temperature: current.temperatureF === null ? "--°" : `${Math.round(current.temperatureF)}°`,
|
||||
temperature: current.temperatureF === null ? "--°" : formatTemperature(current.temperatureF, false),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ import { type Layer, type LayerContext } from "../../ui/layers";
|
||||
import { Menu, type MenuDrawArgs } from "../../ui/menu-core";
|
||||
import { centeredTextY, lineStep, tightRowHeight } from "../../ui/metrics";
|
||||
import { renderWeatherIcon } from "./weather-icons";
|
||||
import { formatTemperature, widestTemperatureSample } from "./temperature-format";
|
||||
|
||||
const PAGE_X = 18;
|
||||
const HEADER_Y = 8;
|
||||
@@ -115,7 +116,7 @@ export class WeatherLayer implements Layer {
|
||||
const large = getDefaultLargeFont();
|
||||
const medium = getDefaultMediumFont();
|
||||
const small = getDefaultSmallFont();
|
||||
const temperature = current.temperatureF === null ? "--°" : `${Math.round(current.temperatureF)}°F`;
|
||||
const temperature = current.temperatureF === null ? "--°" : formatTemperature(current.temperatureF);
|
||||
const temperatureY = CURRENT_TOP + 8;
|
||||
let temperatureX = PAGE_X;
|
||||
if (current.condition) {
|
||||
@@ -182,7 +183,7 @@ export class WeatherLayer implements Layer {
|
||||
...forecast.map((period) => font.measureText(period.name)),
|
||||
);
|
||||
const tempX = PAGE_X + nameWidth + FORECAST_COL_GAP;
|
||||
const precipX = tempX + Math.max(font.measureText("Temp"), font.measureText("100°F")) + FORECAST_COL_GAP;
|
||||
const precipX = tempX + Math.max(font.measureText("Temp"), font.measureText(widestTemperatureSample())) + FORECAST_COL_GAP;
|
||||
const summaryX = precipX + Math.max(font.measureText("Rain"), font.measureText("100%")) + FORECAST_COL_GAP;
|
||||
|
||||
image.drawText(font, PAGE_X, FORECAST_HEADER_Y, "Upcoming", 150);
|
||||
@@ -209,7 +210,7 @@ export class WeatherLayer implements Layer {
|
||||
const shade = selected ? 245 : 190;
|
||||
const textY = centeredTextY(font, y, height);
|
||||
image.drawText(font, x + columns.name, textY, period.name, shade);
|
||||
image.drawText(font, x + columns.temp, textY, period.temperatureF === null ? "--" : `${Math.round(period.temperatureF)}°F`, shade);
|
||||
image.drawText(font, x + columns.temp, textY, period.temperatureF === null ? "--" : formatTemperature(period.temperatureF), shade);
|
||||
image.drawText(font, x + columns.precip, textY, period.precipitationPercent === null ? "--" : `${Math.round(period.precipitationPercent)}%`, selected ? 220 : 155);
|
||||
// The summary stops PAGE_X short of the viewport's right edge, as the name starts PAGE_X in from the left.
|
||||
const summaryWidth = width - columns.summary - columns.name;
|
||||
|
||||
@@ -34,6 +34,11 @@ export type AssistantBackendConfig =
|
||||
/** Trim history from the head once it grows past this many messages. */
|
||||
const MAX_HISTORY_MESSAGES = 40;
|
||||
|
||||
/** The longest tool name every provider accepts (OpenAI allows 64; Anthropic, 128). */
|
||||
const MAX_API_TOOL_NAME_LENGTH = 64;
|
||||
/** Characters of the sanitized name kept at the start when shortening it. */
|
||||
const SHORTENED_TOOL_NAME_HEAD = 16;
|
||||
|
||||
|
||||
/** Hermes brings its own persona and tools; it only needs to know where the reply is shown. */
|
||||
const HERMES_SYSTEM_PROMPT =
|
||||
@@ -237,13 +242,20 @@ export class AssistantSession {
|
||||
/** Convert a canonical registry name to a provider-legal tool name. */
|
||||
private toApiToolName(name: string): string {
|
||||
let base = name.replace(/[^a-zA-Z0-9_-]/g, "_");
|
||||
// Disambiguate the rare case where two canonical names sanitize alike.
|
||||
if (this.toolNameMap.has(base)) {
|
||||
let i = 2;
|
||||
while (this.toolNameMap.has(`${base}_${i}`)) i++;
|
||||
base = `${base}_${i}`;
|
||||
if (base.length > MAX_API_TOOL_NAME_LENGTH) {
|
||||
// Keep the start (the namespace) and the end (the tool's own name). The
|
||||
// hash of the whole name keeps shortened names distinct and stable.
|
||||
const hash = fnv32(name).toString(16).padStart(8, "0");
|
||||
const tail = MAX_API_TOOL_NAME_LENGTH - SHORTENED_TOOL_NAME_HEAD - hash.length - 2;
|
||||
base = `${base.slice(0, SHORTENED_TOOL_NAME_HEAD)}_${hash}_${base.slice(-tail)}`;
|
||||
}
|
||||
return base;
|
||||
// Disambiguate the rare case where two canonical names sanitize alike.
|
||||
let apiName = base;
|
||||
for (let i = 2; this.toolNameMap.has(apiName); i++) {
|
||||
const suffix = `_${i}`;
|
||||
apiName = base.slice(0, MAX_API_TOOL_NAME_LENGTH - suffix.length) + suffix;
|
||||
}
|
||||
return apiName;
|
||||
}
|
||||
|
||||
private trimHistory(): void {
|
||||
@@ -266,3 +278,13 @@ function startsWithToolResult(message: LlmMessage): boolean {
|
||||
message.content[0]!.type === "tool_result"
|
||||
);
|
||||
}
|
||||
|
||||
/** FNV-1a over UTF-16 code units: a small stable hash for shortened tool names. */
|
||||
function fnv32(text: string): number {
|
||||
let hash = 2166136261;
|
||||
for (let i = 0; i < text.length; i++) {
|
||||
hash ^= text.charCodeAt(i);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
@@ -21,7 +21,7 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
registered = true;
|
||||
|
||||
// Keep a media listener warm so "what's playing" works without opening Music.
|
||||
if (!global.isIOS) void mediaControllerBridge.start();
|
||||
void mediaControllerBridge.start();
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
@@ -86,9 +86,6 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
},
|
||||
);
|
||||
|
||||
// Phone media and Android notification tools are still platform-specific.
|
||||
if (global.isIOS) return;
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "media.now_playing",
|
||||
@@ -127,6 +124,9 @@ export function registerSystemTools(registry: ToolRegistry = toolRegistry): void
|
||||
},
|
||||
);
|
||||
|
||||
// Android notification tools are still platform-specific.
|
||||
if (global.isIOS) return;
|
||||
|
||||
registry.registerSystemTool(
|
||||
{
|
||||
name: "notifications.list",
|
||||
@@ -181,7 +181,7 @@ function describeState(): string {
|
||||
|
||||
function describeNowPlaying(): string {
|
||||
const state = mediaControllerBridge.snapshot();
|
||||
if (!state.accessEnabled) return "Notification access isn't granted, so media state is unavailable.";
|
||||
if (!state.accessEnabled) return `Media state is unavailable (${mediaControllerBridge.accessLabel.toLowerCase()}).`;
|
||||
if (state.playbackState === "playing" || state.playbackState === "paused") {
|
||||
const verb = state.playbackState === "playing" ? "Playing" : "Paused";
|
||||
const title = state.title || "(unknown title)";
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
import {
|
||||
APPLE_MUSIC_APP_NAME,
|
||||
APPLE_MUSIC_PACKAGE,
|
||||
parseMediaBrowseItems,
|
||||
type MediaBrowseItem,
|
||||
type MediaBrowserApp,
|
||||
type MediaBrowserBridge,
|
||||
} from "./media-types";
|
||||
|
||||
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
|
||||
|
||||
declare const FaceclawAppleMusic: any;
|
||||
|
||||
const APPLE_MUSIC_APP: MediaBrowserApp = { packageName: APPLE_MUSIC_PACKAGE, serviceClass: "", appName: APPLE_MUSIC_APP_NAME };
|
||||
|
||||
/**
|
||||
* The Apple Music library (playlists, artists, albums, songs in the
|
||||
* on-device / iCloud music library) as a browse tree, through
|
||||
* FaceclawAppleMusic's MediaPlayer queries. Playing an item queues its
|
||||
* collection on the system Music player.
|
||||
*/
|
||||
export class AppleMusicBrowserBridge implements MediaBrowserBridge {
|
||||
private connected = false;
|
||||
/** Bumped on every connect/disconnect; answers to older requests are dropped. */
|
||||
private generation = 0;
|
||||
|
||||
listBrowsableApps(): MediaBrowserApp[] {
|
||||
return FaceclawAppleMusic.shared().hasPermission() ? [APPLE_MUSIC_APP] : [];
|
||||
}
|
||||
|
||||
connect(app: MediaBrowserApp): Promise<string> {
|
||||
this.disconnect();
|
||||
if (app.packageName !== APPLE_MUSIC_PACKAGE) return Promise.reject(new Error("Only Apple Music can be browsed on iOS"));
|
||||
if (!FaceclawAppleMusic.shared().hasPermission()) return Promise.reject(new Error("Apple Music access is not allowed"));
|
||||
this.connected = true;
|
||||
return Promise.resolve("root");
|
||||
}
|
||||
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]> {
|
||||
if (!this.connected) return Promise.reject(new Error("Not connected"));
|
||||
const generation = this.generation;
|
||||
return new Promise<MediaBrowseItem[]>((resolve, reject) => {
|
||||
FaceclawAppleMusic.shared().browseCompletion(parentId, (json: string | null, error: string | null) => {
|
||||
if (generation !== this.generation) reject(new Error("Disconnected"));
|
||||
else if (error || json == null) reject(new Error(String(error || "Browse failed")));
|
||||
else {
|
||||
try {
|
||||
resolve(parseMediaBrowseItems(String(json)));
|
||||
} catch (parseError) {
|
||||
reject(new Error(`Browse result parse failed: ${parseError}`));
|
||||
}
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
playFromMediaId(mediaId: string): void {
|
||||
if (!this.connected) return;
|
||||
FaceclawAppleMusic.shared().playCompletion(mediaId, (error: string | null) => {
|
||||
if (error) console.warn(`[apple-music] play ${mediaId} failed: ${error}`);
|
||||
});
|
||||
}
|
||||
|
||||
disconnect(): void {
|
||||
this.connected = false;
|
||||
this.generation++;
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new AppleMusicBrowserBridge();
|
||||
@@ -1,22 +1,11 @@
|
||||
import { Utils } from "@nativescript/core";
|
||||
import { ignoredMediaPackages, isMediaAppEnabled, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import { parseMediaBrowseItems, type MediaBrowseItem, type MediaBrowserApp, type MediaBrowserBridge } from "./media-types";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type MediaBrowserApp = {
|
||||
packageName: string;
|
||||
serviceClass: string;
|
||||
appName: string;
|
||||
};
|
||||
|
||||
export type MediaBrowseItem = {
|
||||
mediaId: string;
|
||||
title: string;
|
||||
subtitle: string;
|
||||
browsable: boolean;
|
||||
playable: boolean;
|
||||
};
|
||||
export type { MediaBrowseItem, MediaBrowserApp } from "./media-types";
|
||||
|
||||
type PendingRequest<T> = {
|
||||
resolve: (value: T) => void;
|
||||
@@ -29,7 +18,7 @@ type PendingRequest<T> = {
|
||||
* start playback, including starting a player that is not currently running.
|
||||
* Holds at most one connection at a time.
|
||||
*/
|
||||
export class FaceclawMediaBrowserBridge {
|
||||
export class FaceclawMediaBrowserBridge implements MediaBrowserBridge {
|
||||
private browser: any = null;
|
||||
private listenerProxy: any = null;
|
||||
private nextRequestId = 1;
|
||||
@@ -155,22 +144,7 @@ export class FaceclawMediaBrowserBridge {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
const raw = JSON.parse(json) as Array<{
|
||||
mediaId?: string;
|
||||
title?: string;
|
||||
subtitle?: string;
|
||||
browsable?: boolean;
|
||||
playable?: boolean;
|
||||
}>;
|
||||
pending.resolve(
|
||||
raw.map((item) => ({
|
||||
mediaId: String(item.mediaId ?? ""),
|
||||
title: String(item.title ?? ""),
|
||||
subtitle: String(item.subtitle ?? ""),
|
||||
browsable: Boolean(item.browsable),
|
||||
playable: Boolean(item.playable),
|
||||
})),
|
||||
);
|
||||
pending.resolve(parseMediaBrowseItems(json));
|
||||
} catch (parseError) {
|
||||
pending.reject(new Error(`Browse result parse failed: ${parseError}`));
|
||||
}
|
||||
@@ -183,4 +157,4 @@ export class FaceclawMediaBrowserBridge {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaBrowserBridge = new FaceclawMediaBrowserBridge();
|
||||
export const mediaBrowserBridge: MediaBrowserBridge = new FaceclawMediaBrowserBridge();
|
||||
@@ -0,0 +1,231 @@
|
||||
import { Dialogs, Utils } from "@nativescript/core";
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { fromData } from "./kotlin-data";
|
||||
import {
|
||||
APPLE_MUSIC_APP_NAME,
|
||||
APPLE_MUSIC_PACKAGE,
|
||||
DEFAULT_MEDIA_STATE,
|
||||
type MediaControllerBridge,
|
||||
type MediaControllerState,
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
declare const FaceclawAppleMusic: any;
|
||||
declare const FaceclawKitIosProtocol: any;
|
||||
|
||||
/**
|
||||
* Re-read the system player this often while something listens: its change
|
||||
* notifications are not delivered while Faceclaw is suspended, so a track
|
||||
* change in the background would otherwise wait for the next one.
|
||||
*/
|
||||
const POLL_MS = 2_000;
|
||||
|
||||
/** Normalize FaceclawAppleMusic.stateJson()'s object. */
|
||||
export function appleMusicState(raw: any): MediaControllerState {
|
||||
if (!raw || raw.access !== true) {
|
||||
return { ...DEFAULT_MEDIA_STATE, status: "Apple Music access is required." };
|
||||
}
|
||||
const hasItem = Boolean(raw.hasItem);
|
||||
const playbackState: MediaPlaybackState = !hasItem
|
||||
? "idle"
|
||||
: raw.state === "playing" || raw.state === "paused" || raw.state === "stopped" ? raw.state : "unknown";
|
||||
const rate = Number(raw.rate);
|
||||
return {
|
||||
available: hasItem,
|
||||
accessEnabled: true,
|
||||
packageName: APPLE_MUSIC_PACKAGE,
|
||||
appName: APPLE_MUSIC_APP_NAME,
|
||||
playbackState,
|
||||
title: String(raw.title ?? ""),
|
||||
artist: String(raw.artist ?? ""),
|
||||
album: String(raw.album ?? ""),
|
||||
positionMs: finiteOr(raw.positionMs, -1),
|
||||
durationMs: finiteOr(raw.durationMs, -1),
|
||||
// The system player reports a zero rate while it is still starting up.
|
||||
playbackSpeed: playbackState === "playing" ? (rate > 0 ? rate : 1) : 0,
|
||||
status: hasItem ? "" : "Nothing is playing in Apple Music.",
|
||||
canPlayPause: hasItem,
|
||||
canSkipNext: hasItem,
|
||||
canSkipPrevious: hasItem,
|
||||
artVersion: finiteOr(raw.artVersion, 0),
|
||||
};
|
||||
}
|
||||
|
||||
function finiteOr(value: unknown, fallback: number): number {
|
||||
const number = Number(value);
|
||||
return typeof value === "number" && Number.isFinite(number) ? number : fallback;
|
||||
}
|
||||
|
||||
/** Everything a listener would redraw for; the position advances on its own. */
|
||||
function stateSignature(state: MediaControllerState): string {
|
||||
return JSON.stringify({ ...state, positionMs: 0 });
|
||||
}
|
||||
|
||||
/**
|
||||
* Apple Music through FaceclawAppleMusic (MediaPlayer's system player): iOS
|
||||
* gives other apps' players no shared session API, so the system Music app is
|
||||
* the only player Faceclaw can follow and control. Snapshots read the player
|
||||
* directly (cheap property reads), so they are current even when a change
|
||||
* notification was missed.
|
||||
*/
|
||||
export class AppleMusicControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Faceclaw needs access to Apple Music to show and control what's playing. Tap, then allow access on your phone.";
|
||||
readonly accessLabel = "Apple Music access needed";
|
||||
readonly accessAction = "allow access";
|
||||
readonly skipRebuildsQueue = false;
|
||||
private readonly stateListeners = new Set<(state: MediaControllerState) => void>();
|
||||
private bound = false;
|
||||
private started = false;
|
||||
private pollTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private lastSignature = "";
|
||||
private pendingAccess: Promise<boolean> | null = null;
|
||||
private protocol: any = null;
|
||||
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void {
|
||||
void this.start();
|
||||
this.stateListeners.add(listener);
|
||||
this.pollTimer ??= setInterval(() => this.refresh(), POLL_MS);
|
||||
listener(this.snapshot());
|
||||
return () => {
|
||||
this.stateListeners.delete(listener);
|
||||
if (!this.stateListeners.size && this.pollTimer !== null) {
|
||||
clearInterval(this.pollTimer);
|
||||
this.pollTimer = null;
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
let raw: unknown = null;
|
||||
try {
|
||||
raw = JSON.parse(String(this.native().stateJson()));
|
||||
} catch (error) {
|
||||
console.warn(`[apple-music] state read failed: ${error}`);
|
||||
}
|
||||
return appleMusicState(raw);
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
this.native().start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (!this.started) return;
|
||||
this.started = false;
|
||||
this.native().stop();
|
||||
}
|
||||
|
||||
async playPause(): Promise<void> {
|
||||
this.native().playPause();
|
||||
}
|
||||
|
||||
async skipNext(): Promise<void> {
|
||||
this.native().skipNext();
|
||||
}
|
||||
|
||||
async skipPrevious(): Promise<void> {
|
||||
this.native().skipPrevious();
|
||||
}
|
||||
|
||||
hasAccess(): boolean {
|
||||
return Boolean(this.native().hasPermission());
|
||||
}
|
||||
|
||||
/** Concurrent requests (glasses tap, phone Permissions card) share one prompt. */
|
||||
requestAccess(): Promise<boolean> {
|
||||
if (this.hasAccess()) return Promise.resolve(true);
|
||||
this.pendingAccess ??= this.askForAccess().finally(() => {
|
||||
this.pendingAccess = null;
|
||||
this.refresh();
|
||||
});
|
||||
return this.pendingAccess;
|
||||
}
|
||||
|
||||
async skipToQueueItem(id: number): Promise<void> {
|
||||
this.native().skipToQueueIndex(Math.round(id));
|
||||
}
|
||||
|
||||
getMediaVolumePercent(): number {
|
||||
const volume = Number(this.native().volumePercent());
|
||||
return Number.isFinite(volume) ? Math.max(-1, Math.min(100, Math.round(volume))) : -1;
|
||||
}
|
||||
|
||||
setMediaVolumePercent(volumePercent: number): void {
|
||||
this.native().setVolumePercent(Math.max(0, Math.min(100, Math.round(volumePercent))));
|
||||
}
|
||||
|
||||
getAlbumArt(maxSize: number): GrayImage | null {
|
||||
const packet = this.native().albumArt(Math.round(maxSize));
|
||||
if (!packet) return null;
|
||||
// The shared Kotlin tone curve and dither, as on Android.
|
||||
this.protocol ??= FaceclawKitIosProtocol.new();
|
||||
return grayImageFromPacket(fromData(this.protocol.grayPacketBgraGammaDither(packet, PHOTO_TONE.gamma, PHOTO_TONE.dither)));
|
||||
}
|
||||
|
||||
getQueue(): MediaQueueItem[] {
|
||||
try {
|
||||
const raw = JSON.parse(String(this.native().queueJson()));
|
||||
if (!Array.isArray(raw)) return [];
|
||||
return raw.map((item: { id?: number; title?: string; active?: boolean }) => ({
|
||||
id: Number(item?.id ?? -1),
|
||||
title: String(item?.title ?? ""),
|
||||
active: Boolean(item?.active),
|
||||
}));
|
||||
} catch (error) {
|
||||
console.warn(`[apple-music] queue parse failed: ${error}`);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
private native(): any {
|
||||
const music = FaceclawAppleMusic.shared();
|
||||
if (!this.bound) {
|
||||
this.bound = true;
|
||||
music.changeHandler = () => this.refresh();
|
||||
}
|
||||
return music;
|
||||
}
|
||||
|
||||
private refresh(): void {
|
||||
if (!this.stateListeners.size) return;
|
||||
const state = this.snapshot();
|
||||
const signature = stateSignature(state);
|
||||
if (signature === this.lastSignature) return;
|
||||
this.lastSignature = signature;
|
||||
for (const listener of this.stateListeners) listener(state);
|
||||
}
|
||||
|
||||
private async askForAccess(): Promise<boolean> {
|
||||
const music = this.native();
|
||||
const status = String(music.permissionStatus());
|
||||
if (status === "denied") {
|
||||
const open = await Dialogs.confirm({
|
||||
title: "Apple Music Access",
|
||||
message: "Allow Media & Apple Music access in Settings to show and control Apple Music on your glasses.",
|
||||
okButtonText: "Open Settings",
|
||||
cancelButtonText: "Cancel",
|
||||
});
|
||||
if (open) Utils.openUrl("app-settings:");
|
||||
return false;
|
||||
}
|
||||
if (status === "restricted") {
|
||||
await Dialogs.alert({
|
||||
title: "Apple Music Access Restricted",
|
||||
message: "Media library access is restricted on this iPhone. Check Screen Time or device management settings.",
|
||||
okButtonText: "OK",
|
||||
});
|
||||
return false;
|
||||
}
|
||||
return new Promise<boolean>((resolve) => {
|
||||
music.requestPermission((granted: boolean) => resolve(Boolean(granted)));
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge: MediaControllerBridge = new AppleMusicControllerBridge();
|
||||
@@ -3,68 +3,25 @@ import { GrayImage } from "../graphics/image";
|
||||
import { grayImageFromPacket, PHOTO_TONE } from "./image-files";
|
||||
import { ignoredMediaPackages, MEDIA_APPS_KEY, rememberMediaApps } from "./media-apps";
|
||||
import { onSettingsStoreChanged } from "./settings-store";
|
||||
import { isNotificationListenerEnabled } from "./notification-access";
|
||||
import {
|
||||
DEFAULT_MEDIA_STATE,
|
||||
interpolateMediaState,
|
||||
type MediaControllerBridge,
|
||||
type MediaControllerState,
|
||||
type MediaPlaybackState,
|
||||
type MediaQueueItem,
|
||||
} from "./media-types";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
export type MediaQueueItem = {
|
||||
id: number;
|
||||
title: string;
|
||||
active: boolean;
|
||||
};
|
||||
export type { MediaControllerState, MediaPlaybackState, MediaQueueItem } from "./media-types";
|
||||
|
||||
export type MediaPlaybackState =
|
||||
| "notification-access-required"
|
||||
| "idle"
|
||||
| "playing"
|
||||
| "paused"
|
||||
| "buffering"
|
||||
| "stopped"
|
||||
| "unknown";
|
||||
|
||||
export type MediaControllerState = {
|
||||
available: boolean;
|
||||
accessEnabled: boolean;
|
||||
packageName: string;
|
||||
appName: string;
|
||||
playbackState: MediaPlaybackState;
|
||||
title: string;
|
||||
artist: string;
|
||||
album: string;
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
playbackSpeed: number;
|
||||
status: string;
|
||||
canPlayPause: boolean;
|
||||
canSkipNext: boolean;
|
||||
canSkipPrevious: boolean;
|
||||
/**
|
||||
* Changes whenever getAlbumArt's result may have changed (any metadata
|
||||
* update or session switch). Key cached art on this, not the title:
|
||||
* players re-publish the same strings with a different bitmap.
|
||||
*/
|
||||
artVersion: number;
|
||||
};
|
||||
|
||||
const DEFAULT_MEDIA_STATE: MediaControllerState = {
|
||||
available: false,
|
||||
accessEnabled: false,
|
||||
packageName: "",
|
||||
appName: "",
|
||||
playbackState: "idle",
|
||||
title: "",
|
||||
artist: "",
|
||||
album: "",
|
||||
positionMs: -1,
|
||||
durationMs: -1,
|
||||
playbackSpeed: 0,
|
||||
status: "Media controller unavailable.",
|
||||
canPlayPause: false,
|
||||
canSkipNext: false,
|
||||
canSkipPrevious: false,
|
||||
artVersion: 0,
|
||||
};
|
||||
|
||||
export class FaceclawMediaControllerBridge {
|
||||
export class FaceclawMediaControllerBridge implements MediaControllerBridge {
|
||||
readonly accessPrompt = "Notification access is required to control Android media sessions. Tap to open settings.";
|
||||
readonly accessLabel = "Notification access needed";
|
||||
readonly accessAction = "open settings";
|
||||
readonly skipRebuildsQueue = true;
|
||||
private readonly stateListeners = new Set<(state: MediaControllerState) => void>();
|
||||
private controller: any = null;
|
||||
private listenerProxy: any = null;
|
||||
@@ -79,12 +36,7 @@ export class FaceclawMediaControllerBridge {
|
||||
}
|
||||
|
||||
snapshot(): MediaControllerState {
|
||||
const state = { ...this.state };
|
||||
if (state.playbackState === "playing" && state.positionMs >= 0 && state.playbackSpeed !== 0) {
|
||||
state.positionMs += (Date.now() - this.stateUpdatedAtMs) * state.playbackSpeed;
|
||||
state.positionMs = Math.max(0, state.durationMs > 0 ? Math.min(state.positionMs, state.durationMs) : state.positionMs);
|
||||
}
|
||||
return state;
|
||||
return interpolateMediaState(this.state, this.stateUpdatedAtMs, Date.now());
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
@@ -115,9 +67,15 @@ export class FaceclawMediaControllerBridge {
|
||||
this.controller?.skipPrevious();
|
||||
}
|
||||
|
||||
openNotificationAccessSettings(): void {
|
||||
hasAccess(): boolean {
|
||||
return isNotificationListenerEnabled();
|
||||
}
|
||||
|
||||
/** Opens the notification-access settings screen; the grant is only visible once the app resumes. */
|
||||
async requestAccess(): Promise<boolean> {
|
||||
this.ensureController();
|
||||
this.controller?.openNotificationAccessSettings();
|
||||
return false;
|
||||
}
|
||||
|
||||
/** Jump playback to a queue item (see getQueue). */
|
||||
@@ -257,4 +215,4 @@ function normalizePlaybackState(value: string): MediaPlaybackState {
|
||||
}
|
||||
}
|
||||
|
||||
export const mediaControllerBridge = new FaceclawMediaControllerBridge();
|
||||
export const mediaControllerBridge: MediaControllerBridge = new FaceclawMediaControllerBridge();
|
||||
@@ -0,0 +1,161 @@
|
||||
import type { GrayImage } from "../graphics/image";
|
||||
|
||||
/**
|
||||
* Shapes shared by the media bridges: media-controller.ts / media-browser.ts
|
||||
* drive Android media sessions and MediaBrowserServices; their .ios.ts
|
||||
* counterparts drive Apple Music through the MediaPlayer framework.
|
||||
*/
|
||||
|
||||
/** The player behind the iOS bridges: the system Music app. */
|
||||
export const APPLE_MUSIC_APP_NAME = "Apple Music";
|
||||
export const APPLE_MUSIC_PACKAGE = "com.apple.Music";
|
||||
|
||||
export type MediaQueueItem = {
|
||||
id: number;
|
||||
title: string;
|
||||
active: boolean;
|
||||
};
|
||||
|
||||
export type MediaPlaybackState =
|
||||
| "notification-access-required"
|
||||
| "idle"
|
||||
| "playing"
|
||||
| "paused"
|
||||
| "buffering"
|
||||
| "stopped"
|
||||
| "unknown";
|
||||
|
||||
export type MediaControllerState = {
|
||||
available: boolean;
|
||||
accessEnabled: boolean;
|
||||
packageName: string;
|
||||
appName: string;
|
||||
playbackState: MediaPlaybackState;
|
||||
title: string;
|
||||
artist: string;
|
||||
album: string;
|
||||
positionMs: number;
|
||||
durationMs: number;
|
||||
playbackSpeed: number;
|
||||
status: string;
|
||||
canPlayPause: boolean;
|
||||
canSkipNext: boolean;
|
||||
canSkipPrevious: boolean;
|
||||
/**
|
||||
* Changes whenever getAlbumArt's result may have changed (any metadata
|
||||
* update or session switch). Key cached art on this, not the title:
|
||||
* players re-publish the same strings with a different bitmap.
|
||||
*/
|
||||
artVersion: number;
|
||||
};
|
||||
|
||||
export const DEFAULT_MEDIA_STATE: MediaControllerState = {
|
||||
available: false,
|
||||
accessEnabled: false,
|
||||
packageName: "",
|
||||
appName: "",
|
||||
playbackState: "idle",
|
||||
title: "",
|
||||
artist: "",
|
||||
album: "",
|
||||
positionMs: -1,
|
||||
durationMs: -1,
|
||||
playbackSpeed: 0,
|
||||
status: "Media controller unavailable.",
|
||||
canPlayPause: false,
|
||||
canSkipNext: false,
|
||||
canSkipPrevious: false,
|
||||
artVersion: 0,
|
||||
};
|
||||
|
||||
/** Interpolate the position of a playing item from the last state update. */
|
||||
export function interpolateMediaState(state: MediaControllerState, updatedAtMs: number, nowMs: number): MediaControllerState {
|
||||
const result = { ...state };
|
||||
if (result.playbackState === "playing" && result.positionMs >= 0 && result.playbackSpeed !== 0) {
|
||||
result.positionMs += (nowMs - updatedAtMs) * result.playbackSpeed;
|
||||
result.positionMs = Math.max(0, result.durationMs > 0 ? Math.min(result.positionMs, result.durationMs) : result.positionMs);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
export interface MediaControllerBridge {
|
||||
/** Explains, on the glasses, what access is missing and that a tap asks for it. */
|
||||
readonly accessPrompt: string;
|
||||
/** The missing access as a short card label. */
|
||||
readonly accessLabel: string;
|
||||
/** What clicking the access prompt does, for its gesture hint. */
|
||||
readonly accessAction: string;
|
||||
/**
|
||||
* Whether jumping to a queue item rebuilds the queue with that item first
|
||||
* (Android players do); otherwise the queue keeps its order.
|
||||
*/
|
||||
readonly skipRebuildsQueue: boolean;
|
||||
onStateChange(listener: (state: MediaControllerState) => void): () => void;
|
||||
snapshot(): MediaControllerState;
|
||||
start(): Promise<void>;
|
||||
stop(): Promise<void>;
|
||||
playPause(): Promise<void>;
|
||||
skipNext(): Promise<void>;
|
||||
skipPrevious(): Promise<void>;
|
||||
hasAccess(): boolean;
|
||||
/** Show the grant UI: a system settings screen or a permission prompt on the phone. */
|
||||
requestAccess(): Promise<boolean>;
|
||||
/** Jump playback to a queue item (see getQueue). */
|
||||
skipToQueueItem(id: number): Promise<void>;
|
||||
/** Media volume normalized to 0..100; -1 when unavailable. */
|
||||
getMediaVolumePercent(): number;
|
||||
setMediaVolumePercent(volumePercent: number): void;
|
||||
/**
|
||||
* Album art for the current item, grayscale with the shared photo tone
|
||||
* (gamma-corrected and dithered onto the display's 16 levels), scaled to
|
||||
* fit maxSize; null when the player provides none.
|
||||
*/
|
||||
getAlbumArt(maxSize: number): GrayImage | null;
|
||||
/** The player's queue (playlist), empty when the player exposes none. */
|
||||
getQueue(): MediaQueueItem[];
|
||||
}
|
||||
|
||||
export type MediaBrowserApp = {
|
||||
packageName: string;
|
||||
serviceClass: string;
|
||||
appName: string;
|
||||
};
|
||||
|
||||
export type MediaBrowseItem = {
|
||||
mediaId: string;
|
||||
title: string;
|
||||
subtitle: string;
|
||||
browsable: boolean;
|
||||
playable: boolean;
|
||||
};
|
||||
|
||||
export interface MediaBrowserBridge {
|
||||
/** Apps whose library can be browsed; cached after the first call. */
|
||||
listBrowsableApps(refresh?: boolean): MediaBrowserApp[];
|
||||
/** Connect to an app's library, replacing any existing connection. Resolves to the browse-tree root id. */
|
||||
connect(app: MediaBrowserApp): Promise<string>;
|
||||
/** Fetch one level of the connected library's tree. */
|
||||
browse(parentId: string): Promise<MediaBrowseItem[]>;
|
||||
/** Start playback of a browsed item. */
|
||||
playFromMediaId(mediaId: string): void;
|
||||
/** Drop the connection; in-flight requests are rejected. */
|
||||
disconnect(): void;
|
||||
}
|
||||
|
||||
/** Parse a browse listing's JSON rows. */
|
||||
export function parseMediaBrowseItems(json: string): MediaBrowseItem[] {
|
||||
const raw = JSON.parse(json) as Array<{
|
||||
mediaId?: string;
|
||||
title?: string;
|
||||
subtitle?: string;
|
||||
browsable?: boolean;
|
||||
playable?: boolean;
|
||||
}>;
|
||||
return raw.map((item) => ({
|
||||
mediaId: String(item.mediaId ?? ""),
|
||||
title: String(item.title ?? ""),
|
||||
subtitle: String(item.subtitle ?? ""),
|
||||
browsable: Boolean(item.browsable),
|
||||
playable: Boolean(item.playable),
|
||||
}));
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { AncsClient, ANCS_CONNECT_MESSAGE, type AncsState } from '../g2/ancs-cli
|
||||
import { rememberNotificationSources } from './notification-sources'
|
||||
import type { AndroidNotification } from './notification-types'
|
||||
export type { AndroidNotification, AndroidNotificationAction } from './notification-types'
|
||||
export type { NotificationIconsResult, NotificationIconResult } from './notification-icons'
|
||||
export type { NotificationIconsResult, NotificationIconResult, NotificationPictureResult } from './notification-icons'
|
||||
|
||||
export const ALL_NOTIFICATIONS = 0x7fffffff
|
||||
let active: AncsClient | null = null
|
||||
@@ -61,3 +61,7 @@ export function readNotificationIconByKey(key: string, _allowStale: boolean) {
|
||||
const notification = readActiveNotifications(ALL_NOTIFICATIONS).find(n => n.key === key)
|
||||
return {icon: notification ? categoryIcon(notification.category) : null,stale:false}
|
||||
}
|
||||
// ANCS carries no attachments.
|
||||
export function readNotificationPicture(_notification: AndroidNotification, _maxWidth: number, _maxHeight: number, _allowStale: boolean) {
|
||||
return {picture: null,stale:false}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ import { GrayImage } from "../graphics/image";
|
||||
import { logCurrent, spanCurrent } from "./frame-timings";
|
||||
import { toUint8Array } from "../util/array-util";
|
||||
import { rememberNotificationSources } from "./notification-sources";
|
||||
import { PHOTO_TONE, grayImageFromPacket } from "./image-files";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
@@ -22,6 +23,11 @@ let cachedKeys: string[] = [];
|
||||
let cachedAtMs = 0;
|
||||
const keyedIconCache = new Map<string, { icon: GrayImage | null; atMs: number }>();
|
||||
const KEYED_ICON_CACHE_MAX = 128;
|
||||
/** Pictures are the size of a detail view, not 24px icons: keep just the recently viewed few. */
|
||||
const pictureCache = new Map<string, {
|
||||
postTime: number; maxWidth: number; maxHeight: number; picture: GrayImage | null; atMs: number;
|
||||
}>();
|
||||
const PICTURE_CACHE_MAX = 8;
|
||||
let notificationListenerProxy: any = null;
|
||||
const notificationPostedListeners = new Set<(notificationKey: string) => void>();
|
||||
const notificationRemovedListeners = new Set<(notificationKey: string) => void>();
|
||||
@@ -151,6 +157,71 @@ export function readNotificationIconByKey(key: string, allowStale: boolean): Not
|
||||
return { icon: icon?.clone() ?? null, stale: false };
|
||||
}
|
||||
|
||||
export type NotificationPictureResult = {
|
||||
/** Shared with the cache: blit it, don't draw on it. */
|
||||
picture: GrayImage | null;
|
||||
/** True when allowStale served an earlier fetch (or nothing) in place of a fetch for these arguments. */
|
||||
stale: boolean;
|
||||
};
|
||||
|
||||
/**
|
||||
* The picture attached to a notification (hasPicture), scaled to fit
|
||||
* maxWidth x maxHeight with the photo tone and inverted if it is drawn on a
|
||||
* light background (GrayPacket.hasLightBackground), under the allow-stale contract of
|
||||
* readNotificationIconByKey. Cached per post of the notification: an ongoing
|
||||
* notification updated in place (Dexcom redraws its graph every few minutes)
|
||||
* fires no posted event, but its postTime moves. ICON_CACHE_MS is the
|
||||
* backstop here too. Under allowStale an earlier fetch stands in, if it fits.
|
||||
*/
|
||||
export function readNotificationPicture(
|
||||
notification: AndroidNotification,
|
||||
maxWidth: number,
|
||||
maxHeight: number,
|
||||
allowStale: boolean,
|
||||
): NotificationPictureResult {
|
||||
const width = Math.round(maxWidth);
|
||||
const height = Math.round(maxHeight);
|
||||
if (!global.isAndroid || !notification.hasPicture || width <= 0 || height <= 0) {
|
||||
return { picture: null, stale: false };
|
||||
}
|
||||
|
||||
const now = Date.now();
|
||||
const cached = pictureCache.get(notification.key);
|
||||
if (
|
||||
cached && cached.postTime === notification.postTime && cached.maxWidth === width
|
||||
&& cached.maxHeight === height && now - cached.atMs < ICON_CACHE_MS
|
||||
) {
|
||||
return { picture: cached.picture, stale: false };
|
||||
}
|
||||
if (allowStale) {
|
||||
const fits = cached?.picture && cached.picture.width <= width && cached.picture.height <= height;
|
||||
return { picture: fits ? cached!.picture : null, stale: true };
|
||||
}
|
||||
|
||||
const picture = spanCurrent("fetch-notification-picture", () =>
|
||||
grayImageFromPacket(
|
||||
com.faceclaw.app.FaceclawMediaNotificationListenerService.getNotificationPictureGray(
|
||||
notification.key,
|
||||
width,
|
||||
height,
|
||||
PHOTO_TONE.gamma,
|
||||
PHOTO_TONE.dither,
|
||||
// A chart or screenshot made for a light theme would be a glaring
|
||||
// lit panel on the glasses: show it light-on-dark.
|
||||
true,
|
||||
),
|
||||
),
|
||||
);
|
||||
pictureCache.delete(notification.key);
|
||||
pictureCache.set(notification.key, { postTime: notification.postTime, maxWidth: width, maxHeight: height, picture, atMs: now });
|
||||
while (pictureCache.size > PICTURE_CACHE_MAX) {
|
||||
const oldestKey = pictureCache.keys().next().value;
|
||||
if (oldestKey === undefined) break;
|
||||
pictureCache.delete(oldestKey);
|
||||
}
|
||||
return { picture, stale: false };
|
||||
}
|
||||
|
||||
export function readActiveNotifications(maxNotifications = 50): AndroidNotification[] {
|
||||
if (!global.isAndroid || maxNotifications <= 0) return [];
|
||||
try {
|
||||
@@ -250,6 +321,7 @@ function normalizeNotification(value: any): AndroidNotification | null {
|
||||
when: Number(value.when) || 0,
|
||||
actions,
|
||||
foregroundService: Boolean(value.foregroundService),
|
||||
hasPicture: Boolean(value.hasPicture),
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ export type AndroidNotification = {
|
||||
actions: AndroidNotificationAction[]; dismissLabel?: string;
|
||||
/** Posted by a foreground service (Android): a "this app is working" status notification. */
|
||||
foregroundService?: boolean;
|
||||
/** Carries a picture (Android BigPictureStyle), fetched separately by readNotificationPicture. */
|
||||
hasPicture?: boolean;
|
||||
/** Full message size reported by iOS, even when the fetched text is bounded. */
|
||||
messageSize?: number;
|
||||
};
|
||||
@@ -15,6 +15,7 @@ import { ensureCalendarPermission, hasCalendarPermission } from "../native/calen
|
||||
import { isIgnoringBatteryOptimizations, requestIgnoreBatteryOptimizations } from "../native/battery-optimization";
|
||||
import { hasAllFilesAccess, requestAllFilesAccess } from "../native/file-access";
|
||||
import { isNotificationListenerEnabled, requestNotificationListenerAccess } from "../native/notification-access";
|
||||
import { mediaControllerBridge } from "../native/media-controller";
|
||||
|
||||
type PermissionDefinition = {
|
||||
id: string;
|
||||
@@ -112,6 +113,12 @@ const PERMISSIONS: PermissionDefinition[] = global.isIOS ? [{
|
||||
optional: true,
|
||||
isGranted: hasCalendarPermission,
|
||||
request: ensureCalendarPermission,
|
||||
}, {
|
||||
id: "apple-music", title: "Apple Music",
|
||||
description: "Optional: show and control what's playing in Apple Music, and play your music library, from the glasses.",
|
||||
optional: true,
|
||||
isGranted: () => mediaControllerBridge.hasAccess(),
|
||||
request: () => mediaControllerBridge.requestAccess(),
|
||||
}] : ANDROID_PERMISSIONS;
|
||||
|
||||
/** One Repeater card: the permission's copy plus the display-only fields the XML binds. */
|
||||
@@ -168,7 +175,7 @@ export class PermissionsViewModel extends Observable {
|
||||
}
|
||||
|
||||
get instructions(): string {
|
||||
if (global.isIOS) return "Faceclaw uses Bluetooth to communicate with your smart glasses. Calendar access is optional. Tap a card to allow access; a checkmark means it is ready.";
|
||||
if (global.isIOS) return "Faceclaw uses Bluetooth to communicate with your smart glasses. Calendar and Apple Music access are optional. Tap a card to allow access; a checkmark means it is ready.";
|
||||
return this.onboarding
|
||||
? `Faceclaw uses these ${global.isIOS ? "iOS" : "Android"} permissions. Tap a card to grant one; a checkmark means it is already granted. ` +
|
||||
"Optional permissions can also be granted later, from the Permissions item in the main-screen menu."
|
||||
|
||||
@@ -35,6 +35,7 @@ export type BatteryDisplayMode = "icon" | "percentage" | "stacked" | "stacked-pe
|
||||
/** When a top-bar battery indicator is shown: always, only below 50%, or never. */
|
||||
export type BatteryIndicatorVisibility = "always" | "low" | "never";
|
||||
export type TimeFormat = "24h" | "12h";
|
||||
export type TemperatureUnit = "fahrenheit" | "celsius";
|
||||
export type ScreenTimeoutSetting = "15s" | "30s" | "1m" | "3m" | "never";
|
||||
// Auto runs Faceclaw's light curve. Numeric values use the calibrated firmware
|
||||
// scale, clamped to 2–100; 2 is also the endpoint for screen fades.
|
||||
@@ -330,6 +331,16 @@ export const timeFormatSetting = new ConfigSettingEnum<TimeFormat>({
|
||||
description: "Whether the top-bar clock shows 24-hour or 12-hour time.",
|
||||
});
|
||||
|
||||
export const temperatureUnitSetting = new ConfigSettingEnum<TemperatureUnit>({
|
||||
id: "temperatureUnit",
|
||||
label: "Temperature unit",
|
||||
storageKey: "display.temperatureUnit",
|
||||
defaultValue: "fahrenheit",
|
||||
values: ["fahrenheit", "celsius"],
|
||||
formatValue: (value) => value === "celsius" ? "Celsius" : "Fahrenheit",
|
||||
description: "Whether the Weather app, its top-bar indicator, and the Glanceboard's system card show temperatures in Fahrenheit or Celsius.",
|
||||
});
|
||||
|
||||
/**
|
||||
* Where the app switcher (the open-window icons) sits: a strip along the
|
||||
* left or right edge of the display, a row under the window, or a popup box
|
||||
|
||||
@@ -76,6 +76,7 @@ import {
|
||||
saveVoiceRecordingsSetting,
|
||||
showBleBandwidthSetting,
|
||||
suspendEvenHubWhenScreenOffSetting,
|
||||
temperatureUnitSetting,
|
||||
textSettingMenuItem,
|
||||
timeFormatSetting,
|
||||
toggleSettingMenuItem,
|
||||
@@ -289,6 +290,8 @@ function customizationRows(): () => MenuItem[] {
|
||||
animationsMenuItem(),
|
||||
// Controls the top-bar clock (24-hour vs 12-hour).
|
||||
enumSettingMenuItem(timeFormatSetting),
|
||||
// Fahrenheit or Celsius, wherever weather temperatures show.
|
||||
enumSettingMenuItem(temperatureUnitSetting),
|
||||
];
|
||||
const bottomRows = [font, position, depth, statusBar, ...rest];
|
||||
const popupRows = [font, position, depth, statusBar, statusBarVisibility, windowBorder, ...rest];
|
||||
|
||||
+45
-4
@@ -10,6 +10,7 @@ import {
|
||||
invokeNotificationAction,
|
||||
readActiveNotifications,
|
||||
readNotificationIconByKey,
|
||||
readNotificationPicture,
|
||||
type AndroidNotification,
|
||||
type AndroidNotificationAction,
|
||||
} from "../native/notification-icons";
|
||||
@@ -44,6 +45,13 @@ const DETAIL_TITLE_Y = 21;
|
||||
/** Gaps below the title's line box and below the icon/app-name header row. */
|
||||
const DETAIL_HEADER_GAP = 3;
|
||||
const DETAIL_BODY_GAP = 4;
|
||||
/** Text lines an attached picture leaves room for above it (when there are that many); it takes the rest. */
|
||||
const DETAIL_PICTURE_TEXT_LINES = 2;
|
||||
/** Gap between the text and an attached picture, and the space kept below the picture. */
|
||||
const DETAIL_PICTURE_GAP = 6;
|
||||
const DETAIL_PICTURE_BOTTOM_MARGIN = 12;
|
||||
/** An attached picture with less height than this to fit in (a large UI font) is left out. */
|
||||
const DETAIL_PICTURE_MIN_HEIGHT = 32;
|
||||
// Disable-source confirmation: actions on top, the explanation below them.
|
||||
const CONFIRM_X = 24;
|
||||
const CONFIRM_Y = 8;
|
||||
@@ -59,6 +67,16 @@ function iconForNotification(key: string): GrayImage | null {
|
||||
return icon;
|
||||
}
|
||||
|
||||
/** Attached picture for a paint pass, fitting maxWidth x maxHeight; allow-stale like iconForNotification. */
|
||||
function pictureForNotification(notification: AndroidNotification, maxWidth: number, maxHeight: number): GrayImage | null {
|
||||
if (!notification.hasPicture || maxHeight < DETAIL_PICTURE_MIN_HEIGHT) return null;
|
||||
const { picture, stale } = readNotificationPicture(notification, maxWidth, maxHeight, renderPassAllowsStaleData());
|
||||
if (stale) {
|
||||
noteStaleDataUsed();
|
||||
}
|
||||
return picture;
|
||||
}
|
||||
|
||||
type CardLayout = {
|
||||
notification: AndroidNotification;
|
||||
height: number;
|
||||
@@ -422,23 +440,46 @@ function drawDetailContent(
|
||||
const bodyTop = drawNotificationHeader(image, font, icon, `${notification.appName || notification.packageName} ${formatRelativeTime(notification.postTime)}`);
|
||||
|
||||
const lines: string[] = [];
|
||||
lines.push(...wrapText(font, notification.title || "(untitled)", contentWidth));
|
||||
// A picture can be the whole notification (Dexcom's has no text), so it
|
||||
// stands in for the "(untitled)" placeholder.
|
||||
const hasTitle = Boolean(notification.title) || !notification.hasPicture;
|
||||
if (hasTitle) {
|
||||
lines.push(...wrapText(font, notification.title || "(untitled)", contentWidth));
|
||||
}
|
||||
const body = detailNotificationBody(notification);
|
||||
if (body) {
|
||||
lines.push("");
|
||||
if (lines.length) lines.push("");
|
||||
lines.push(...wrapText(font, body, contentWidth));
|
||||
}
|
||||
const meta = [notification.subText, notification.infoText, notification.summaryText].filter(Boolean).join(" ");
|
||||
if (meta) {
|
||||
lines.push("");
|
||||
if (lines.length) lines.push("");
|
||||
lines.push(...wrapText(font, meta, contentWidth));
|
||||
}
|
||||
|
||||
const step = lineStep(font);
|
||||
const textValue = (index: number) => index === 0 && hasTitle ? 230 : 190;
|
||||
const reservedTextHeight = Math.min(lines.length, DETAIL_PICTURE_TEXT_LINES) * step;
|
||||
const picture = pictureForNotification(notification, contentWidth,
|
||||
height - DETAIL_PICTURE_BOTTOM_MARGIN - bodyTop - reservedTextHeight - (reservedTextHeight ? DETAIL_PICTURE_GAP : 0));
|
||||
if (picture) {
|
||||
// The text gets whatever the picture leaves, above it, with its last
|
||||
// shown line marked when some is cut off.
|
||||
const roomForText = height - DETAIL_PICTURE_BOTTOM_MARGIN - picture.height - DETAIL_PICTURE_GAP - bodyTop;
|
||||
const shownLines = Math.min(lines.length, Math.max(0, (roomForText / step) | 0));
|
||||
for (let index = 0; index < shownLines; index++) {
|
||||
const line = index === shownLines - 1 && lines.length > shownLines
|
||||
? truncateText(font, lines[index] + "...", contentWidth) : lines[index]!;
|
||||
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
|
||||
}
|
||||
image.bitBlt(picture, contentX, bodyTop + shownLines * step + (shownLines ? DETAIL_PICTURE_GAP : 0));
|
||||
return;
|
||||
}
|
||||
|
||||
const maxLines = Math.max(1, ((height - bodyTop - step) / step) | 0);
|
||||
for (let index = 0; index < Math.min(lines.length, maxLines); index++) {
|
||||
const line = lines[index]!;
|
||||
image.drawText(font, contentX, bodyTop + index * step, line, index === 0 ? 230 : 190);
|
||||
image.drawText(font, contentX, bodyTop + index * step, line, textValue(index));
|
||||
}
|
||||
if (lines.length > maxLines) {
|
||||
image.drawText(font, contentX, height - 24, "...", 140);
|
||||
|
||||
@@ -51,8 +51,9 @@ if (!fc) {
|
||||
}
|
||||
```
|
||||
|
||||
`getFaceclawExtensions()` returns `null` everywhere except inside Faceclaw:
|
||||
in the stock Even app, in the EvenHub simulator, in a browser, and in Node.
|
||||
`getFaceclawExtensions()` returns `null` everywhere except inside Faceclaw
|
||||
0.8.3 or later: in the stock Even app, in the EvenHub simulator, in a browser,
|
||||
and in Node.
|
||||
Faceclaw installs the API before any page script runs, so you can call it at
|
||||
startup. You don't need to wait for the bridge first.
|
||||
|
||||
@@ -81,7 +82,12 @@ if (fc && fc.apiVersion >= 2) {
|
||||
}
|
||||
```
|
||||
|
||||
Everything in this release is API level 1.
|
||||
Everything in this release is API level 1, which Faceclaw 0.8.3 and later
|
||||
implement. Faceclaw 0.8.0–0.8.2 injected an incomplete, unversioned preview of
|
||||
the API; `getFaceclawExtensions()` returns `null` there, so whenever it returns
|
||||
the API, every level-1 member is present. If you use
|
||||
`window.getFaceclawExtensions` directly, check that `apiVersion` is a number
|
||||
before relying on it.
|
||||
|
||||
## TypeScript
|
||||
|
||||
@@ -284,9 +290,16 @@ fc.setAssistantTools([
|
||||
|
||||
The assistant sees each tool under your package name, as
|
||||
`app.<packageName>.set_bpm`. Whatever the handler returns is reported back to
|
||||
the assistant. If it throws, the assistant gets the error message. Tools are
|
||||
removed when the app closes; call `setAssistantTools([])` to remove them
|
||||
sooner.
|
||||
the assistant. If it throws, the assistant gets the error message. A handler
|
||||
has 10 seconds to finish before the assistant is told the tool timed out.
|
||||
Tools are removed when the app closes; call `setAssistantTools([])` to remove
|
||||
them sooner.
|
||||
|
||||
Tool names are 1–64 letters, digits, underscores or hyphens. `parameters`
|
||||
must be an object schema (`type: "object"`) without a top-level `oneOf`,
|
||||
`anyOf` or `allOf`, since that is what the assistant's model providers accept.
|
||||
Faceclaw skips a tool that breaks these rules and logs a warning to the app's
|
||||
console.
|
||||
|
||||
### API keys
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ carry them out.
|
||||
|
||||
| Property | Modifier | Type | Description |
|
||||
| ------ | ------ | ------ | ------ |
|
||||
| <a id="property-apiversion"></a> `apiVersion` | `readonly` | `number` | The extension API level the host implements. It is `1` for the API described here, and increases by one each time members are added. Members added later are documented with the level that introduced them, so an app can check for them before calling: `if (fc.apiVersion >= 2) fc.someNewerMethod();` |
|
||||
| <a id="property-apiversion"></a> `apiVersion` | `readonly` | `number` | The extension API level the host implements. It is `1` for the API described here (Faceclaw 0.8.3 and later), and increases by one each time members are added. Members added later are documented with the level that introduced them, so an app can check for them before calling: `if (fc.apiVersion >= 2) fc.someNewerMethod();` |
|
||||
|
||||
#### Methods
|
||||
|
||||
@@ -165,7 +165,9 @@ firmware's per-message limit of 48 steps are split and paced automatically.
|
||||
`Promise`\<`void`\>
|
||||
|
||||
A promise that resolves once every part of the sequence has been
|
||||
queued on the glasses, which can be before it finishes playing.
|
||||
queued on the glasses, which can be before it finishes playing. It
|
||||
rejects if sending to the glasses fails. If the glasses aren't
|
||||
connected, the sequence is dropped and the promise still resolves.
|
||||
|
||||
##### addCompassListener()
|
||||
|
||||
@@ -251,6 +253,9 @@ setAssistantTools(tools): void;
|
||||
Declare the app's voice-assistant tools, replacing any set earlier. Each
|
||||
tool's `handler` runs in the app when the assistant calls it.
|
||||
|
||||
Faceclaw skips a tool whose `name` or `parameters` break the rules in
|
||||
[AssistantTool](#assistanttool), and logs a warning to the app's console saying why.
|
||||
|
||||
###### Parameters
|
||||
|
||||
| Parameter | Type | Description |
|
||||
@@ -436,6 +441,10 @@ Return the Faceclaw extension API if the app is running inside Faceclaw, or
|
||||
Safe to call anywhere, at any time; Faceclaw installs the API before any page
|
||||
script runs.
|
||||
|
||||
It also returns `null` in Faceclaw 0.8.0–0.8.2, which had an incomplete,
|
||||
unversioned preview of the API. Whenever it returns the API, every member
|
||||
of API level 1 is present.
|
||||
|
||||
#### Returns
|
||||
|
||||
[`FaceclawExtensions`](#faceclawextensions) \| `null`
|
||||
@@ -878,8 +887,8 @@ The assistant sees the tool namespaced by the app's package name, as
|
||||
|
||||
| Property | Type | Default value | Description |
|
||||
| ------ | ------ | ------ | ------ |
|
||||
| <a id="property-name"></a> `name` | `string` | `undefined` | Bare tool name (no package prefix), for example `"set_color"`. |
|
||||
| <a id="property-name"></a> `name` | `string` | `undefined` | Bare tool name (no package prefix), for example `"set_color"`: 1–64 letters, digits, underscores or hyphens. |
|
||||
| <a id="property-description"></a> `description` | `string` | `undefined` | What the tool does. The assistant reads this to decide when to call it. |
|
||||
| <a id="property-parameters"></a> `parameters?` | [`JsonSchema`](#jsonschema) | `{ type: "object", properties: {} }` | JSON Schema for the arguments object the assistant will pass to `handler`. |
|
||||
| <a id="property-parameters"></a> `parameters?` | [`JsonSchema`](#jsonschema) | `{ type: "object", properties: {} }` | JSON Schema for the arguments object the assistant will pass to `handler`. It must describe an object (`type: "object"`), with any `properties` an object and any `required` an array of names, and must not use `oneOf`, `anyOf` or `allOf` at the top level. These are the schemas the assistant's model providers accept. |
|
||||
| <a id="property-availability"></a> `availability?` | `"open"` \| `"foreground"` | `"foreground"` | When the tool can be called: `"foreground"` only while the app owns the visible window, or `"open"` any time the app is running, including in the background. |
|
||||
| <a id="property-handler"></a> `handler` | (`args`) => `unknown` | `undefined` | Called when the assistant invokes the tool. Return a string, or any JSON-serializable value, to report back to the assistant. Throw (or reject) to report an error. May be async. |
|
||||
| <a id="property-handler"></a> `handler` | (`args`) => `unknown` | `undefined` | Called when the assistant invokes the tool. Return a string, or any JSON-serializable value, to report back to the assistant. Throw (or reject) to report an error. May be async, but must finish within 10 seconds, or the assistant is told the tool timed out. |
|
||||
@@ -489,12 +489,19 @@ export type JsonSchema = Record<string, unknown>;
|
||||
* @group Assistant tools
|
||||
*/
|
||||
export interface AssistantTool<Args = any> {
|
||||
/** Bare tool name (no package prefix), for example `"set_color"`. */
|
||||
/**
|
||||
* Bare tool name (no package prefix), for example `"set_color"`: 1–64
|
||||
* letters, digits, underscores or hyphens.
|
||||
*/
|
||||
name: string;
|
||||
/** What the tool does. The assistant reads this to decide when to call it. */
|
||||
description: string;
|
||||
/**
|
||||
* JSON Schema for the arguments object the assistant will pass to `handler`.
|
||||
* It must describe an object (`type: "object"`), with any `properties` an
|
||||
* object and any `required` an array of names, and must not use `oneOf`,
|
||||
* `anyOf` or `allOf` at the top level. These are the schemas the
|
||||
* assistant's model providers accept.
|
||||
* @defaultValue { type: "object", properties: {} }
|
||||
*/
|
||||
parameters?: JsonSchema;
|
||||
@@ -508,7 +515,8 @@ export interface AssistantTool<Args = any> {
|
||||
/**
|
||||
* Called when the assistant invokes the tool. Return a string, or any
|
||||
* JSON-serializable value, to report back to the assistant. Throw (or reject)
|
||||
* to report an error. May be async.
|
||||
* to report an error. May be async, but must finish within 10 seconds, or
|
||||
* the assistant is told the tool timed out.
|
||||
*/
|
||||
handler: (args: Args) => unknown;
|
||||
}
|
||||
@@ -525,9 +533,9 @@ export interface AssistantTool<Args = any> {
|
||||
export interface FaceclawExtensions {
|
||||
/**
|
||||
* The extension API level the host implements. It is `1` for the API
|
||||
* described here, and increases by one each time members are added. Members
|
||||
* added later are documented with the level that introduced them, so an app
|
||||
* can check for them before calling:
|
||||
* described here (Faceclaw 0.8.3 and later), and increases by one each time
|
||||
* members are added. Members added later are documented with the level that
|
||||
* introduced them, so an app can check for them before calling:
|
||||
*
|
||||
* ```ts
|
||||
* if (fc.apiVersion >= 2) fc.someNewerMethod();
|
||||
@@ -587,7 +595,9 @@ export interface FaceclawExtensions {
|
||||
* firmware's per-message limit of 48 steps are split and paced automatically.
|
||||
*
|
||||
* @returns A promise that resolves once every part of the sequence has been
|
||||
* queued on the glasses, which can be before it finishes playing.
|
||||
* queued on the glasses, which can be before it finishes playing. It
|
||||
* rejects if sending to the glasses fails. If the glasses aren't
|
||||
* connected, the sequence is dropped and the promise still resolves.
|
||||
*/
|
||||
playBuzzer(steps: BuzzerStep[]): Promise<void>;
|
||||
|
||||
@@ -632,6 +642,9 @@ export interface FaceclawExtensions {
|
||||
* Declare the app's voice-assistant tools, replacing any set earlier. Each
|
||||
* tool's `handler` runs in the app when the assistant calls it.
|
||||
*
|
||||
* Faceclaw skips a tool whose `name` or `parameters` break the rules in
|
||||
* {@link AssistantTool}, and logs a warning to the app's console saying why.
|
||||
*
|
||||
* @param tools - The tools to offer. Pass `[]` to remove them all.
|
||||
*/
|
||||
setAssistantTools(tools: AssistantTool[]): void;
|
||||
@@ -718,7 +731,9 @@ declare global {
|
||||
/**
|
||||
* Injected by Faceclaw before any page script runs. `undefined` in the
|
||||
* stock Even app and other hosts. Prefer {@link getFaceclawExtensions},
|
||||
* which checks for it.
|
||||
* which checks for it. If you call it directly, check that the result's
|
||||
* `apiVersion` is a number: Faceclaw 0.8.0–0.8.2 returned an incomplete
|
||||
* preview without one.
|
||||
*/
|
||||
getFaceclawExtensions?: (() => FaceclawExtensions) | null;
|
||||
}
|
||||
@@ -730,9 +745,17 @@ declare global {
|
||||
* Safe to call anywhere, at any time; Faceclaw installs the API before any page
|
||||
* script runs.
|
||||
*
|
||||
* It also returns `null` in Faceclaw 0.8.0–0.8.2, which had an incomplete,
|
||||
* unversioned preview of the API. Whenever it returns the API, every member
|
||||
* of API level 1 is present.
|
||||
*
|
||||
* @group Entry point
|
||||
*/
|
||||
export function getFaceclawExtensions(): FaceclawExtensions | null {
|
||||
const getter = typeof window !== "undefined" ? window.getFaceclawExtensions : null;
|
||||
return typeof getter === "function" ? getter() : null;
|
||||
if (typeof getter !== "function") return null;
|
||||
const extensions = getter();
|
||||
// Faceclaw 0.8.0–0.8.2 injected an unversioned preview with only some of
|
||||
// the level-1 members; treat it as absent.
|
||||
return extensions && typeof extensions.apiVersion === "number" ? extensions : null;
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
package com.faceclaw.app
|
||||
|
||||
import kotlin.jvm.JvmOverloads
|
||||
import kotlin.jvm.JvmStatic
|
||||
import kotlin.math.pow
|
||||
import kotlin.math.roundToInt
|
||||
@@ -23,20 +24,28 @@ object GrayPacket {
|
||||
* (BmpUtil.GRAY_TO_NIBBLE) instead of leaving each pixel to round on its own, which turns
|
||||
* smooth gradients into visible bands. Dithered output is already quantized: every byte is
|
||||
* a level times 16, which BmpUtil re-quantizes back to exactly that level.
|
||||
*
|
||||
* invertLightBackground converts a picture drawn on a light background (hasLightBackground)
|
||||
* as its negative, so a chart or screenshot made for a light theme shows light-on-dark: the
|
||||
* display lights what is light, so a light background is a glaring panel, while black is
|
||||
* see-through. The colors are inverted, not their alpha, before the tone curve, where the
|
||||
* sRGB-encoded values are roughly perceptually even; inverting the curve's output instead
|
||||
* would turn dim strokes into ones barely darker than the background.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun fromArgb(pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean): ByteArray {
|
||||
@JvmOverloads
|
||||
fun fromArgb(
|
||||
pixels: IntArray, width: Int, height: Int, gamma: Float, dither: Boolean, invertLightBackground: Boolean = false,
|
||||
): ByteArray {
|
||||
require(width > 0 && height > 0 && pixels.size >= width * height) { "bad pixel plane" }
|
||||
val out = ByteArray(HEADER_SIZE + width * height)
|
||||
writeHeader(out, width, height)
|
||||
val tone = toneCurve(gamma)
|
||||
val invert = invertLightBackground && hasLightBackground(pixels, width, height)
|
||||
for (i in 0 until width * height) {
|
||||
val p = pixels[i]
|
||||
val a = (p ushr 24) and 0xff
|
||||
val r = (p shr 16) and 0xff
|
||||
val g = (p shr 8) and 0xff
|
||||
val b = p and 0xff
|
||||
out[HEADER_SIZE + i] = tone[((r * 299 + g * 587 + b * 114) / 1000) * a / 255].toByte()
|
||||
val luma = luma(p)
|
||||
out[HEADER_SIZE + i] = tone[(if (invert) 255 - luma else luma) * alpha(p) / 255].toByte()
|
||||
}
|
||||
if (dither) {
|
||||
ditherToDisplayLevels(out, HEADER_SIZE, width, height)
|
||||
@@ -44,6 +53,53 @@ object GrayPacket {
|
||||
return out
|
||||
}
|
||||
|
||||
/** Grays within this span of each other count as one background (compression noise, faint shading). */
|
||||
private const val BACKGROUND_SPAN = 16
|
||||
/** The share of the pixels a background covers; a photo's tones mostly spread wider than this. */
|
||||
private const val BACKGROUND_MIN_SHARE = 0.4f
|
||||
/** A background at least this light (source gray, before the tone curve) counts as light. */
|
||||
private const val LIGHT_BACKGROUND_MIN_GRAY = 160
|
||||
|
||||
/**
|
||||
* Whether a picture is drawn on a light background: the most populous span of
|
||||
* BACKGROUND_SPAN adjacent grays (as displayed: composited over black, before the tone
|
||||
* curve) covers BACKGROUND_MIN_SHARE of the picture and is light. The flat backgrounds of
|
||||
* charts, screenshots and documents qualify; photographs mostly don't, though a large
|
||||
* blown-out sky can.
|
||||
*/
|
||||
@JvmStatic
|
||||
fun hasLightBackground(pixels: IntArray, width: Int, height: Int): Boolean {
|
||||
val count = width * height
|
||||
val histogram = IntArray(256)
|
||||
for (i in 0 until count) {
|
||||
val p = pixels[i]
|
||||
histogram[luma(p) * alpha(p) / 255] += 1
|
||||
}
|
||||
var window = 0
|
||||
var best = 0
|
||||
var bestStart = 0
|
||||
for (v in 0 until 256) {
|
||||
window += histogram[v]
|
||||
if (v >= BACKGROUND_SPAN) {
|
||||
window -= histogram[v - BACKGROUND_SPAN]
|
||||
}
|
||||
if (window > best) {
|
||||
best = window
|
||||
bestStart = maxOf(0, v - BACKGROUND_SPAN + 1)
|
||||
}
|
||||
}
|
||||
return best >= BACKGROUND_MIN_SHARE * count && bestStart + BACKGROUND_SPAN / 2 >= LIGHT_BACKGROUND_MIN_GRAY
|
||||
}
|
||||
|
||||
private fun luma(p: Int): Int {
|
||||
val r = (p shr 16) and 0xff
|
||||
val g = (p shr 8) and 0xff
|
||||
val b = p and 0xff
|
||||
return (r * 299 + g * 587 + b * 114) / 1000
|
||||
}
|
||||
|
||||
private fun alpha(p: Int): Int = (p ushr 24) and 0xff
|
||||
|
||||
@JvmStatic
|
||||
fun writeHeader(out: ByteArray, width: Int, height: Int) {
|
||||
out[0] = (width and 0xff).toByte()
|
||||
|
||||
@@ -195,6 +195,28 @@ class IosProtocol {
|
||||
|
||||
fun png(data: NSData, width: Int, height: Int): NSData =
|
||||
SharedScreenshots.encode4BitGrayPng(data.byteArray(), width, height).data()
|
||||
|
||||
/**
|
||||
* A [widthLo, widthHi, heightLo, heightHi, B, G, R, A...] packet (little-endian ARGB words,
|
||||
* as CoreGraphics draws them) to the gray packet Android's ImageFileLoader returns, with the
|
||||
* same tone curve and dither. Malformed packets give empty data (no image) rather than an
|
||||
* exception, which would terminate the app on its way into Objective-C.
|
||||
*/
|
||||
fun grayPacket(bgra: NSData, gamma: Float, dither: Boolean): NSData {
|
||||
val bytes = bgra.byteArray()
|
||||
if (bytes.size < GrayPacket.HEADER_SIZE) return NSData()
|
||||
val width = (bytes[0].toInt() and 0xff) or ((bytes[1].toInt() and 0xff) shl 8)
|
||||
val height = (bytes[2].toInt() and 0xff) or ((bytes[3].toInt() and 0xff) shl 8)
|
||||
if (width <= 0 || height <= 0 || bytes.size != GrayPacket.HEADER_SIZE + width * height * 4) return NSData()
|
||||
val pixels = IntArray(width * height) { i ->
|
||||
val at = GrayPacket.HEADER_SIZE + i * 4
|
||||
(bytes[at].toInt() and 0xff) or
|
||||
((bytes[at + 1].toInt() and 0xff) shl 8) or
|
||||
((bytes[at + 2].toInt() and 0xff) shl 16) or
|
||||
((bytes[at + 3].toInt() and 0xff) shl 24)
|
||||
}
|
||||
return GrayPacket.fromArgb(pixels, width, height, gamma, dither).data()
|
||||
}
|
||||
}
|
||||
|
||||
/** iOS retains/coalesces surface updates in the same compositor Android uses. */
|
||||
|
||||
+12
-11
@@ -30,19 +30,20 @@ async function buildKotlin(platform, configuration = 'debug') {
|
||||
if (!['debug', 'release'].includes(configuration)) throw new Error(`Unsupported Kotlin configuration: ${configuration}`);
|
||||
const variant = configuration === 'release' ? 'Release' : 'Debug';
|
||||
const task = platform === 'android' ? ':shared:assembleAndroidMain' : `:shared:assembleFaceclawKit${variant}XCFramework`;
|
||||
const wrapper = path.join(root, 'wear', process.platform === 'win32' ? 'gradlew.bat' : 'gradlew');
|
||||
const windows = process.platform === 'win32';
|
||||
const wrapper = path.join(root, 'wear', windows ? 'gradlew.bat' : 'gradlew');
|
||||
// Through cmd.exe a missing wrapper is only "not recognized" and exit 1.
|
||||
if (!existsSync(wrapper)) throw new Error(`Gradle wrapper not found: ${wrapper}`);
|
||||
const args = [task, '--console=plain'];
|
||||
console.log(`[kotlin] Building ${platform} (${configuration})`);
|
||||
await new Promise((resolve, reject) => {
|
||||
// A .bat cannot be exec'd directly: Node passes it to CreateProcess, which
|
||||
// rejects it with EINVAL. cmd.exe has to run it, so the Windows path goes
|
||||
// through a shell, quoted because the checkout may sit under a path with
|
||||
// spaces in it.
|
||||
const windows = process.platform === 'win32';
|
||||
const child = spawn(windows ? `"${wrapper}"` : wrapper, [task, '--console=plain'], {
|
||||
cwd: kotlin,
|
||||
stdio: 'inherit',
|
||||
shell: windows,
|
||||
});
|
||||
// Node refuses to spawn a .bat directly (EINVAL, since the CVE-2024-27980
|
||||
// fix), so cmd.exe runs it. It gets one command string because args
|
||||
// alongside shell: true are deprecated (DEP0190); the wrapper is quoted for
|
||||
// checkouts under paths with spaces.
|
||||
const child = windows
|
||||
? spawn(`"${wrapper}" ${args.join(' ')}`, { cwd: kotlin, stdio: 'inherit', shell: true })
|
||||
: spawn(wrapper, args, { cwd: kotlin, stdio: 'inherit' });
|
||||
child.on('error', reject);
|
||||
child.on('exit', (code, signal) => {
|
||||
if (code === 0) resolve();
|
||||
|
||||
@@ -117,6 +117,27 @@ test('cancelled tools and late callbacks cannot mutate the next turn or its hist
|
||||
assert.equal(session.isTurnActive(), false);
|
||||
});
|
||||
|
||||
test('tool names are sanitized, shortened to 64 characters, kept distinct, and mapped back', () => {
|
||||
const env = sessions();
|
||||
const pkg = 'com.example.some-long-developer-name.evenhub';
|
||||
const names = ['glasses.get_state', `app.${pkg}.set_metronome_tempo_and_signature`,
|
||||
`app.${pkg}.set_metronome_tempo_and_signature_v2`, 'a.b', 'a_b'];
|
||||
const registry = { listTools: () => names.map(name => ({ name, description: '', inputSchema: { type: 'object' } })) };
|
||||
const session = new env.AssistantSession(config(), registry);
|
||||
session.sendUtterance('hi', {}, callbacks());
|
||||
const turn = env.turns.at(-1);
|
||||
const apiNames = turn.buildTools().map(tool => tool.name);
|
||||
assert.equal(apiNames[0], 'glasses_get_state');
|
||||
assert.ok(apiNames.every(name => /^[a-zA-Z0-9_-]{1,64}$/.test(name)), apiNames.join(', '));
|
||||
assert.equal(new Set(apiNames).size, names.length);
|
||||
assert.match(apiNames[1], /^app_com_example_/);
|
||||
assert.match(apiNames[1], /tempo_and_signature$/);
|
||||
assert.deepEqual(apiNames.slice(3), ['a_b', 'a_b_2']);
|
||||
assert.deepEqual(apiNames.map(turn.resolveToolName), names);
|
||||
// The same canonical name shortens the same way on the next turn.
|
||||
assert.deepEqual(turn.buildTools().map(tool => tool.name), apiNames);
|
||||
});
|
||||
|
||||
test('synchronous provider errors do not leave a permanently active turn', () => {
|
||||
const env = sessions(); env.setError();
|
||||
const session = new env.AssistantSession(config());
|
||||
|
||||
@@ -7,7 +7,7 @@ const vm = require('node:vm');
|
||||
// The injected shim (app/apps/evenhub/session.ts) and the published types
|
||||
// (evenhub-extensions/src/index.ts) describe the same API from two sides.
|
||||
const source = fs.readFileSync(path.join(__dirname, '../app/apps/evenhub/session.ts'), 'utf8');
|
||||
const template = source.match(/export function buildFaceclawExtensionsScript[^`]*`([\s\S]*?)`;\r?\n}/)[1];
|
||||
const template = source.match(/export function buildFaceclawExtensionsScript[^`]*`([\s\S]*?)`;\n}/)[1];
|
||||
const shim = template.replace('${JSON.stringify(versionString)}', JSON.stringify('Faceclaw/9.9.9'));
|
||||
const types = fs.readFileSync(path.join(__dirname, '../evenhub-extensions/src/index.ts'), 'utf8');
|
||||
|
||||
@@ -30,6 +30,16 @@ function page() {
|
||||
return { win, fc, posted };
|
||||
}
|
||||
|
||||
test('the package helper returns the shim, and treats unversioned 0.8.0-0.8.2 hosts as absent', () => {
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const helper = (window) => loader(window === undefined ? {} : { window })('evenhub-extensions/src/index.ts').getFaceclawExtensions();
|
||||
const { win, fc } = page();
|
||||
assert.equal(helper(win), fc);
|
||||
assert.equal(helper({ getFaceclawExtensions: () => ({ getVersion: () => 'Faceclaw/0.8.2' }) }), null);
|
||||
assert.equal(helper({}), null);
|
||||
assert.equal(helper(undefined), null);
|
||||
});
|
||||
|
||||
test('the shim implements exactly the members the package declares', () => {
|
||||
const { fc } = page();
|
||||
assert.deepEqual(Object.keys(fc).sort(), interfaceMembers('FaceclawExtensions'));
|
||||
|
||||
@@ -0,0 +1,196 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const fs = require('node:fs');
|
||||
const path = require('node:path');
|
||||
const vm = require('node:vm');
|
||||
const ts = require('typescript');
|
||||
|
||||
function load(file, modules = {}, globals = {}) {
|
||||
const sandbox = { exports: {}, console, JSON, Promise, require: id => {
|
||||
if (!(id in modules)) throw new Error(`Unstubbed import: ${id}`);
|
||||
return modules[id];
|
||||
}, ...globals };
|
||||
vm.runInNewContext(ts.transpileModule(fs.readFileSync(path.join(__dirname, '..', file), 'utf8'), {
|
||||
compilerOptions: { module: ts.ModuleKind.CommonJS, target: ts.ScriptTarget.ES2020 },
|
||||
}).outputText, sandbox);
|
||||
return sandbox.exports;
|
||||
}
|
||||
|
||||
const types = load('app/native/media-types.ts');
|
||||
|
||||
/** A FaceclawAppleMusic stand-in: state is plain data the tests mutate. */
|
||||
function fakeMusic() {
|
||||
const music = {
|
||||
permission: 'granted', state: { access: true, hasItem: true, state: 'playing', title: 'Song', artist: 'Band',
|
||||
album: 'Record', positionMs: 1000, durationMs: 200000, rate: 1, artVersion: 3 },
|
||||
queue: '[]', volume: 40, art: null, calls: [], permissionRequests: [], browses: [], plays: [],
|
||||
changeHandler: null,
|
||||
hasPermission() { return this.permission === 'granted'; },
|
||||
permissionStatus() { return this.permission; },
|
||||
requestPermission(done) { this.permissionRequests.push(done); },
|
||||
start() { this.calls.push('start'); }, stop() { this.calls.push('stop'); },
|
||||
stateJson() { return JSON.stringify(this.hasPermission() ? this.state : { access: false }); },
|
||||
playPause() { this.calls.push('playPause'); }, skipNext() { this.calls.push('skipNext'); },
|
||||
skipPrevious() { this.calls.push('skipPrevious'); }, skipToQueueIndex(i) { this.calls.push(`skipTo:${i}`); },
|
||||
queueJson() { return this.queue; },
|
||||
albumArt(size) { this.calls.push(`art:${size}`); return this.art; },
|
||||
volumePercent() { return this.volume; }, setVolumePercent(v) { this.calls.push(`volume:${v}`); },
|
||||
browseCompletion(parentId, done) { this.browses.push({ parentId, done }); },
|
||||
playCompletion(mediaId, done) { this.plays.push({ mediaId, done }); },
|
||||
};
|
||||
return music;
|
||||
}
|
||||
|
||||
function controller(music = fakeMusic()) {
|
||||
const timers = new Map(); let nextTimer = 1;
|
||||
const dialogs = { confirms: [], alerts: [], confirmResult: true, opened: [] };
|
||||
const protocolCalls = [];
|
||||
const api = load('app/native/media-controller.ios.ts', {
|
||||
'@nativescript/core': {
|
||||
Dialogs: {
|
||||
confirm: async options => { dialogs.confirms.push(options); return dialogs.confirmResult; },
|
||||
alert: async options => { dialogs.alerts.push(options); },
|
||||
},
|
||||
Utils: { openUrl: url => dialogs.opened.push(url) },
|
||||
},
|
||||
'./image-files': { grayImageFromPacket: bytes => (bytes && bytes.length ? { bytes } : null), PHOTO_TONE: { gamma: 2.2, dither: true } },
|
||||
'./kotlin-data': { fromData: data => data },
|
||||
'./media-types': types,
|
||||
}, {
|
||||
FaceclawAppleMusic: { shared: () => music },
|
||||
FaceclawKitIosProtocol: { new: () => ({ grayPacketBgraGammaDither: (packet, gamma, dither) => {
|
||||
protocolCalls.push({ packet, gamma, dither }); return [1, 0, 1, 0, 200];
|
||||
} }) },
|
||||
setInterval: fn => { const id = nextTimer++; timers.set(id, fn); return id; },
|
||||
clearInterval: id => timers.delete(id),
|
||||
});
|
||||
return { api, bridge: api.mediaControllerBridge, music, timers, dialogs, protocolCalls,
|
||||
tick: () => { for (const fn of timers.values()) fn(); } };
|
||||
}
|
||||
|
||||
test('Apple Music state normalizes the native player, access and missing items', () => {
|
||||
const { appleMusicState } = controller().api;
|
||||
const denied = appleMusicState({ access: false });
|
||||
assert.equal(denied.accessEnabled, false); assert.equal(denied.available, false);
|
||||
assert.equal(appleMusicState(null).accessEnabled, false);
|
||||
|
||||
const playing = appleMusicState({ access: true, hasItem: true, state: 'playing', title: 'A', artist: 'B', album: 'C',
|
||||
positionMs: 5000, durationMs: 60000, rate: 0, artVersion: 7 });
|
||||
assert.equal(playing.available, true); assert.equal(playing.appName, 'Apple Music');
|
||||
assert.equal(playing.packageName, 'com.apple.Music');
|
||||
assert.equal(playing.playbackSpeed, 1, 'a starting player reports rate 0 but is advancing');
|
||||
assert.equal(playing.positionMs, 5000); assert.equal(playing.artVersion, 7);
|
||||
assert.ok(playing.canPlayPause && playing.canSkipNext && playing.canSkipPrevious);
|
||||
|
||||
const paused = appleMusicState({ access: true, hasItem: true, state: 'paused', rate: 1, positionMs: 'x', durationMs: null });
|
||||
assert.equal(paused.playbackSpeed, 0); assert.equal(paused.positionMs, -1); assert.equal(paused.durationMs, -1);
|
||||
assert.equal(appleMusicState({ access: true, hasItem: true, state: 'odd' }).playbackState, 'unknown');
|
||||
|
||||
const empty = appleMusicState({ access: true, hasItem: false, state: 'playing' });
|
||||
assert.equal(empty.accessEnabled, true); assert.equal(empty.available, false);
|
||||
assert.equal(empty.playbackState, 'idle'); assert.equal(empty.canPlayPause, false);
|
||||
});
|
||||
|
||||
test('listeners start observation, are told only of real changes, and the poll runs while anyone listens', () => {
|
||||
const f = controller(); const seen = [];
|
||||
const off = f.bridge.onStateChange(state => seen.push(state));
|
||||
assert.deepEqual(f.music.calls, ['start']); assert.equal(seen.length, 1); assert.equal(f.timers.size, 1);
|
||||
assert.equal(typeof f.music.changeHandler, 'function');
|
||||
|
||||
f.music.changeHandler(); // first refresh records the baseline
|
||||
const baseline = seen.length;
|
||||
f.music.state.positionMs = 9000; f.tick(); f.music.changeHandler();
|
||||
assert.equal(seen.length, baseline, 'playback progress alone does not notify');
|
||||
f.music.state.title = 'Next song'; f.music.state.artVersion = 4; f.tick();
|
||||
assert.equal(seen.length, baseline + 1); assert.equal(seen.at(-1).title, 'Next song');
|
||||
f.tick(); assert.equal(seen.length, baseline + 1);
|
||||
|
||||
off(); assert.equal(f.timers.size, 0);
|
||||
f.music.state.title = 'Unheard'; f.music.changeHandler(); assert.equal(seen.length, baseline + 1);
|
||||
f.bridge.onStateChange(() => {}); assert.equal(f.music.calls.filter(c => c === 'start').length, 1);
|
||||
});
|
||||
|
||||
test('transport, queue, volume and album art go through the native player', async () => {
|
||||
const f = controller();
|
||||
await f.bridge.playPause(); await f.bridge.skipNext(); await f.bridge.skipPrevious(); await f.bridge.skipToQueueItem(2.4);
|
||||
assert.deepEqual(f.music.calls, ['playPause', 'skipNext', 'skipPrevious', 'skipTo:2']);
|
||||
assert.equal(f.bridge.skipRebuildsQueue, false);
|
||||
|
||||
f.music.queue = JSON.stringify([{ id: 0, title: 'One', active: false }, { id: 1, title: 'Two', active: true }, null]);
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(f.bridge.getQueue())),
|
||||
[{ id: 0, title: 'One', active: false }, { id: 1, title: 'Two', active: true }, { id: -1, title: '', active: false }]);
|
||||
f.music.queue = 'not json'; assert.equal(f.bridge.getQueue().length, 0);
|
||||
|
||||
assert.equal(f.bridge.getMediaVolumePercent(), 40);
|
||||
f.music.volume = -1; assert.equal(f.bridge.getMediaVolumePercent(), -1);
|
||||
f.music.volume = NaN; assert.equal(f.bridge.getMediaVolumePercent(), -1);
|
||||
f.bridge.setMediaVolumePercent(140); assert.equal(f.music.calls.at(-1), 'volume:100');
|
||||
|
||||
assert.equal(f.bridge.getAlbumArt(112), null); assert.equal(f.protocolCalls.length, 0);
|
||||
f.music.art = 'bgra-packet';
|
||||
assert.deepEqual(Array.from(f.bridge.getAlbumArt(111.6).bytes), [1, 0, 1, 0, 200]);
|
||||
assert.equal(f.music.calls.at(-1), 'art:112');
|
||||
assert.deepEqual(f.protocolCalls[0], { packet: 'bgra-packet', gamma: 2.2, dither: true });
|
||||
});
|
||||
|
||||
test('access requests share one prompt, and denied or restricted access points to Settings', async () => {
|
||||
const f = controller(); f.music.permission = 'not-determined';
|
||||
const seen = []; f.bridge.onStateChange(state => seen.push(state.accessEnabled));
|
||||
assert.equal(f.bridge.hasAccess(), false); assert.equal(f.bridge.snapshot().accessEnabled, false);
|
||||
const first = f.bridge.requestAccess(), second = f.bridge.requestAccess();
|
||||
assert.equal(f.music.permissionRequests.length, 1);
|
||||
f.music.permission = 'granted'; f.music.permissionRequests[0](true);
|
||||
assert.equal(await first, true); assert.equal(await second, true);
|
||||
assert.equal(seen.at(-1), true, 'listeners hear about the grant');
|
||||
assert.equal(await f.bridge.requestAccess(), true); assert.equal(f.music.permissionRequests.length, 1);
|
||||
|
||||
f.music.permission = 'denied';
|
||||
assert.equal(await f.bridge.requestAccess(), false);
|
||||
assert.equal(f.dialogs.confirms.length, 1); assert.deepEqual(f.dialogs.opened, ['app-settings:']);
|
||||
f.dialogs.confirmResult = false; await f.bridge.requestAccess(); assert.equal(f.dialogs.opened.length, 1);
|
||||
|
||||
f.music.permission = 'restricted';
|
||||
assert.equal(await f.bridge.requestAccess(), false); assert.equal(f.dialogs.alerts.length, 1);
|
||||
assert.equal(f.music.permissionRequests.length, 1);
|
||||
});
|
||||
|
||||
function browser(music = fakeMusic()) {
|
||||
const api = load('app/native/media-browser.ios.ts', { './media-types': types }, {
|
||||
FaceclawAppleMusic: { shared: () => music },
|
||||
});
|
||||
return { bridge: api.mediaBrowserBridge, music };
|
||||
}
|
||||
|
||||
test('the Apple Music library browses only with access and drops answers after disconnect', async () => {
|
||||
const f = browser();
|
||||
const [app] = f.bridge.listBrowsableApps();
|
||||
assert.equal(app.appName, 'Apple Music');
|
||||
await assert.rejects(f.bridge.connect({ ...app, packageName: 'other' }), /Only Apple Music/);
|
||||
await assert.rejects(f.bridge.browse('root'), /Not connected/);
|
||||
|
||||
assert.equal(await f.bridge.connect(app), 'root');
|
||||
const listing = f.bridge.browse('root');
|
||||
assert.equal(f.music.browses[0].parentId, 'root');
|
||||
f.music.browses[0].done(JSON.stringify([{ mediaId: 'playlists', title: 'Playlists', browsable: true }]), null);
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(await listing)),
|
||||
[{ mediaId: 'playlists', title: 'Playlists', subtitle: '', browsable: true, playable: false }]);
|
||||
|
||||
const failed = f.bridge.browse('album:1');
|
||||
f.music.browses[1].done(null, 'Unknown library item');
|
||||
await assert.rejects(failed, /Unknown library item/);
|
||||
const garbled = f.bridge.browse('songs');
|
||||
f.music.browses[2].done('{', null);
|
||||
await assert.rejects(garbled, /parse failed/);
|
||||
|
||||
f.bridge.playFromMediaId('song:5@album:1');
|
||||
assert.equal(f.music.plays[0].mediaId, 'song:5@album:1');
|
||||
const stale = f.bridge.browse('playlists');
|
||||
f.bridge.disconnect();
|
||||
f.music.browses[3].done('[]', null);
|
||||
await assert.rejects(stale, /Disconnected/);
|
||||
f.bridge.playFromMediaId('playlist:2'); assert.equal(f.music.plays.length, 1);
|
||||
|
||||
f.music.permission = 'denied';
|
||||
assert.equal(f.bridge.listBrowsableApps().length, 0);
|
||||
await assert.rejects(f.bridge.connect(app), /not allowed/);
|
||||
});
|
||||
@@ -228,10 +228,12 @@ test('iOS assistant calendar tool awaits events, reports denied access and failu
|
||||
const f = provider(), registered = new Map();
|
||||
const { registerSystemTools } = load('app/assistant/system-tools.ts', {
|
||||
'../native/calendar': f.api, '../native/calendar-permissions': f.permissions,
|
||||
'../native/media-controller': {}, '../native/notification-icons': {}, '../ui/shell/shell': {}, './tool-registry': {},
|
||||
'../native/media-controller': { mediaControllerBridge: { start: async () => {} } },
|
||||
'../native/notification-icons': {}, '../ui/shell/shell': {}, './tool-registry': {},
|
||||
}, { global: { isIOS: true } });
|
||||
registerSystemTools({ registerSystemTool: (definition, handler) => registered.set(definition.name, handler) });
|
||||
assert.deepEqual([...registered.keys()], ['glasses.get_state', 'glasses.show_alert', 'calendar.list_events']);
|
||||
assert.deepEqual([...registered.keys()], ['glasses.get_state', 'glasses.show_alert', 'calendar.list_events',
|
||||
'media.now_playing', 'media.play_pause', 'media.next']);
|
||||
const list = registered.get('calendar.list_events');
|
||||
const result = list({ within_hours: 2, max_events: 3 });
|
||||
assert.equal(f.reads[0].max, 3); assert.equal(f.reads[0].window, 7200000);
|
||||
|
||||
@@ -260,3 +260,40 @@ test('EvenHub buzzer RPC uses the host action on iOS and chunks and paces long s
|
||||
assert.equal(played.length, 2);
|
||||
h.session.close();
|
||||
});
|
||||
|
||||
test('EvenHub buzzer RPC rejects the app promise when sending fails', async () => {
|
||||
const sounds = loader()('app/ui/sound-effects.ts');
|
||||
const h = sessionHarness({ '../../ui/sound-effects': sounds }, () => Promise.reject(new Error('link lost')));
|
||||
await assert.rejects(h.session.dispatchExtension('playBuzzer', [[{ freq: 880, ms: 50 }]]), /link lost/);
|
||||
h.session.close();
|
||||
});
|
||||
|
||||
test('EvenHub assistant tools skip malformed names and schemas with a warning in the app console', () => {
|
||||
const offered = [];
|
||||
const h = sessionHarness({ '../../assistant/tool-registry': { toolRegistry: { setAppTools: provider => offered.push(provider), removeAppTools() {} } } });
|
||||
const warnings = [];
|
||||
h.web.console = { warn: message => warnings.push(message) };
|
||||
const object = { type: 'object', properties: { n: { type: 'number' } }, required: ['n'] };
|
||||
h.session.dispatchExtension('setAssistantTools', [[
|
||||
{ name: 'ok', description: 'fine', parameters: object },
|
||||
{ name: 'no_params', description: 'defaults' },
|
||||
{ name: 'has space', parameters: object },
|
||||
{ name: 'x'.repeat(65), parameters: object },
|
||||
{ name: 'untyped', parameters: { properties: {} } },
|
||||
{ name: 'string', parameters: { type: 'string' } },
|
||||
{ name: 'union', parameters: { type: 'object', anyOf: [object] } },
|
||||
{ name: 'bad_props', parameters: { type: 'object', properties: [] } },
|
||||
{ name: 'bad_required', parameters: { type: 'object', required: 'n' } },
|
||||
]]);
|
||||
// Through JSON: the default schema is built in the session's realm.
|
||||
assert.deepEqual(JSON.parse(JSON.stringify(offered[0].specs.map(spec => [spec.name, spec.inputSchema]))), [
|
||||
['ok', object],
|
||||
['no_params', { type: 'object', properties: {} }],
|
||||
]);
|
||||
assert.equal(warnings.length, 7);
|
||||
assert.ok(warnings.every(message => message.startsWith('Faceclaw: setAssistantTools: skipped tool')));
|
||||
assert.match(warnings[0], /"has space": name must be/);
|
||||
assert.match(warnings[2], /"untyped": parameters must be a JSON Schema with type "object"/);
|
||||
assert.match(warnings[4], /"union": parameters can't use oneOf, anyOf or allOf/);
|
||||
h.session.close();
|
||||
});
|
||||
@@ -118,7 +118,7 @@ test('closing Weather during permission prompt cannot restart polling', async ()
|
||||
'../../native/weather': { weatherBridge: { start() { starts++; }, stop() { stops++; }, onStateChange: () => () => {}, snapshot() {} } },
|
||||
'./weather': { WeatherLayer: class {} },
|
||||
'./weather-settings': {},
|
||||
'../../ui/dashboard-settings': { toggleSettingMenuItem: () => ({}) },
|
||||
'../../ui/dashboard-settings': { toggleSettingMenuItem: () => ({}), onAnySettingChanged: () => () => {} },
|
||||
'../../ui/dashboard/settings-panel': { openSettingsSubMenu() {} },
|
||||
'../../ui/shell/in-process-window': { YieldAtRootLayer: class {}, createInProcessWindow: opts => { options = opts; return { requestRender() {} }; } },
|
||||
})('app/apps/weather/weather-app.ts');
|
||||
|
||||
@@ -33,6 +33,41 @@ class GraphicsCoresTest {
|
||||
assertContentEquals(byteArrayOf(0x2c, 1, 2, 0), wide.copyOfRange(0, 4))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun lightBackgroundsInvertBeforeTheToneCurve() {
|
||||
val white = 0xffffffff.toInt()
|
||||
val black = 0xff000000.toInt()
|
||||
// A white chart with a black stroke and a fully transparent corner.
|
||||
val chart = IntArray(10 * 10) { white }
|
||||
chart[55] = black
|
||||
chart[0] = 0x00ffffff
|
||||
assertTrue(GrayPacket.hasLightBackground(chart, 10, 10))
|
||||
val inverted = GrayPacket.fromArgb(chart, 10, 10, 2.2f, false, true)
|
||||
assertEquals(0, inverted[4 + 1].toInt() and 0xff) // background goes black
|
||||
assertEquals(255, inverted[4 + 55].toInt() and 0xff) // stroke goes white
|
||||
assertEquals(0, inverted[4 + 0].toInt() and 0xff) // transparent stays black
|
||||
// Mid-gray inverts to its sRGB complement, then takes the tone curve.
|
||||
chart[56] = 0xff404040.toInt()
|
||||
val curve = GrayPacket.toneCurve(2.2f)
|
||||
assertEquals(curve[255 - 0x40], GrayPacket.fromArgb(chart, 10, 10, 2.2f, false, true)[4 + 56].toInt() and 0xff)
|
||||
// Without the flag the conversion is unchanged.
|
||||
assertContentEquals(GrayPacket.fromArgb(chart, 10, 10, 2.2f, false), GrayPacket.fromArgb(chart, 10, 10, 2.2f, false, false))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun darkBackgroundsAndPhotosKeepTheirTones() {
|
||||
// A black chart with a white stroke: already light-on-dark.
|
||||
val dark = IntArray(10 * 10) { 0xff000000.toInt() }
|
||||
dark[55] = 0xffffffff.toInt()
|
||||
assertTrue(!GrayPacket.hasLightBackground(dark, 10, 10))
|
||||
assertContentEquals(GrayPacket.fromArgb(dark, 10, 10, 1f, false), GrayPacket.fromArgb(dark, 10, 10, 1f, false, true))
|
||||
// A photo-like spread of tones, even a bright one, has no background to invert.
|
||||
val ramp = IntArray(16 * 16) { i -> val v = 64 + (i * 191) / 255; 0xff000000.toInt() or (v shl 16) or (v shl 8) or v }
|
||||
assertTrue(!GrayPacket.hasLightBackground(ramp, 16, 16))
|
||||
// A mid-gray background is not light enough to be worth inverting.
|
||||
assertTrue(!GrayPacket.hasLightBackground(IntArray(8 * 8) { 0xff808080.toInt() }, 8, 8))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun toneCurveDarkensMidtones() {
|
||||
val identity = GrayPacket.toneCurve(1f)
|
||||
|
||||
@@ -22,6 +22,21 @@ class IosNativeApiTest {
|
||||
assertContentEquals(auth.byteArray(), result.payload.byteArray())
|
||||
}
|
||||
|
||||
@Test
|
||||
fun bgraPacketsConvertWithTheSharedGrayTone() {
|
||||
// 2x1: opaque orange, half-transparent white (B, G, R, A in memory).
|
||||
val bgra = byteArrayOf(2, 0, 1, 0, 0x20, 0x80.toByte(), 0xff.toByte(), 0xff.toByte(),
|
||||
0xff.toByte(), 0xff.toByte(), 0xff.toByte(), 0x80.toByte())
|
||||
val argb = intArrayOf(0xffff8020.toInt(), 0x80ffffff.toInt())
|
||||
val protocol = IosProtocol()
|
||||
for (dither in listOf(false, true)) {
|
||||
assertContentEquals(GrayPacket.fromArgb(argb, 2, 1, 2.2f, dither),
|
||||
protocol.grayPacket(bgra.data(), 2.2f, dither).byteArray())
|
||||
}
|
||||
assertEquals(0UL, protocol.grayPacket(bgra.copyOf(10).data(), 1f, false).length)
|
||||
assertEquals(0UL, protocol.grayPacket(byteArrayOf(0, 0, 1, 0).data(), 1f, false).length)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun compassFacadePreservesTheEndOfTheCrcFreePayload() {
|
||||
val protocol = IosProtocol()
|
||||
|
||||
@@ -89,6 +89,8 @@ function bridges() {
|
||||
'./settings-store': storage,
|
||||
'../graphics/image': {},
|
||||
'./image-files': {},
|
||||
'./notification-access': { isNotificationListenerEnabled: () => true },
|
||||
'./media-types': load('app/native/media-types.ts', () => { throw new Error('media-types has no imports'); }),
|
||||
};
|
||||
const globals = { global: { isAndroid: true }, com: { faceclaw: { app: {
|
||||
FaceclawMediaBrowser: NativeBrowser,
|
||||
|
||||
@@ -273,6 +273,7 @@ function indicatorHarness() {
|
||||
} },
|
||||
'../../ui/dashboard-settings': {
|
||||
ConfigSettingBoolean: Setting,
|
||||
temperatureUnitSetting: new Setting({ storageKey: 'display.temperatureUnit', defaultValue: 'fahrenheit' }),
|
||||
onAnySettingChanged: listener => { settingListeners.add(listener); return () => settingListeners.delete(listener); },
|
||||
},
|
||||
'../../ui/shell/shell': { shell: { setTrayIcon: (id, icon) => trays.push([id, icon]) } },
|
||||
@@ -308,6 +309,9 @@ test('Weather tray icon follows the status-bar setting and hides a stale reading
|
||||
assert.equal(icon.height, 24);
|
||||
assert.deepEqual(icon.draws.map(draw => draw[0]), ['image', 'text']);
|
||||
assert.equal(icon.draws[1][3], '72°');
|
||||
h.setSetting('display.temperatureUnit', 'celsius');
|
||||
assert.equal(h.trays.at(-1)[1].draws[1][3], '22°', 'the temperature unit setting rebuilds the icon');
|
||||
h.setSetting('display.temperatureUnit', 'fahrenheit');
|
||||
|
||||
// An unchanged reading does not rebuild the icon.
|
||||
const count = h.trays.length;
|
||||
|
||||
Reference in new issue
Block a user