From cb6dddd12066328cfb2d85a2146214b26cd5cdcf Mon Sep 17 00:00:00 2001 From: James Babcock Date: Mon, 28 Sep 2026 14:11:04 -0700 Subject: [PATCH] Add a video player, launched by selecting a video file from the Files app --- .../com/faceclaw/app/FaceclawVideoPlayer.kt | 791 ++++++++++++++++++ app/apps/files/files-app.ts | 20 +- app/apps/files/index.ts | 13 + app/apps/files/video-layout.ts | 86 ++ app/apps/files/video-window.ts | 443 ++++++++++ app/native/video-player.ts | 148 ++++ app/ui/shell/in-process-window.ts | 6 + app/ui/shell/shell.ts | 11 +- .../faceclaw/app/graphics/VideoFrameScaler.kt | 133 +++ tests/ios-evenhub.test.cjs | 3 + .../com/faceclaw/app/VideoFrameScalerTest.kt | 75 ++ tests/tsconfig.json | 1 + tests/video-layout.test.cjs | 57 ++ 13 files changed, 1784 insertions(+), 3 deletions(-) create mode 100644 App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt create mode 100644 app/apps/files/video-layout.ts create mode 100644 app/apps/files/video-window.ts create mode 100644 app/native/video-player.ts create mode 100644 native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/VideoFrameScaler.kt create mode 100644 tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VideoFrameScalerTest.kt create mode 100644 tests/video-layout.test.cjs diff --git a/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt new file mode 100644 index 00000000..382655a0 --- /dev/null +++ b/App_Resources/Android/src/main/java/com/faceclaw/app/FaceclawVideoPlayer.kt @@ -0,0 +1,791 @@ +package com.faceclaw.app + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.media.MediaCodec +import android.media.MediaCodecInfo +import android.media.MediaExtractor +import android.media.MediaFormat +import android.media.MediaPlayer +import android.os.Build +import android.os.Handler +import android.os.Looper +import android.util.Log + +import org.json.JSONObject + +import java.io.IOException +import java.nio.ByteBuffer +import java.util.concurrent.TimeUnit +import java.util.concurrent.locks.ReentrantLock +import java.util.zip.CRC32 +import kotlin.concurrent.withLock + +/** + * Plays a video file onto the glasses. A background thread decodes the video track + * (MediaExtractor + MediaCodec), scales each due frame's luma to gray (VideoFrameScaler) and + * submits it straight into a rect of a window's compositor surface, through the same + * submitSurfaceFrame entry point app workers use, so no JS runs per frame. When wanted, the + * audio track plays through a MediaPlayer on the phone's default media route, and the video + * clock follows its position. + * + * The TS side (app/native/video-player.ts) owns the window: it paints the rest of the surface + * and repaints the video rect from [copyLastFrame] whenever it renders. Frames are submitted + * only while playing, and [pause] returns only once no further frame can be submitted, so a + * repaint right after it (the paused overlay) is never overwritten. + * + * Control methods are for the main thread; listener callbacks are posted to it. + */ +class FaceclawVideoPlayer(context: Context) { + interface Listener { + /** + * "playing", "paused" or "ended", with the media position. Also sent as "paused" when a + * seek made while paused has decoded its new frame (see [copyLastFrame]). + */ + fun onStateChanged(state: String?, positionMs: Long) + + /** Playback stopped on an error (unreadable file, no decoder, ...). */ + fun onError(message: String?) + } + + companion object { + private const val TAG = "FaceclawVideo" + private const val CODEC_TIMEOUT_US = 10_000L + /** Longest single sleep while waiting for a frame to come due, so commands stay responsive. */ + private const val MAX_WAIT_US = 20_000L + /** A frame this far behind the clock is dropped undrawn... */ + private const val LATE_DROP_US = 50_000L + /** ...unless nothing has been drawn for this long, so a slow phone still shows progress. */ + private const val MAX_DROP_GAP_NANOS = 250_000_000L + /** + * Decoded frames closer together than this are thinned: past ~30fps the glasses link + * could never show them, and each one still costs a scale and a composite. + */ + private const val MIN_FRAME_INTERVAL_US = 30_000L + /** Audio position is ignored this long after the clock starts (output startup latency). */ + private const val AUDIO_SETTLE_NANOS = 500_000_000L + /** Clock drift from the audio position that makes the video clock jump to it. */ + private const val AUDIO_RESYNC_US = 60_000L + /** How often the audio position is read (a binder call into the media server). */ + private const val AUDIO_CHECK_INTERVAL_NANOS = 250_000_000L + /** A seek never lands closer than this to the end, so it always shows a frame. */ + private const val SEEK_END_MARGIN_US = 500_000L + + private val AUDIO_ATTRIBUTES: AudioAttributes = AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_MEDIA) + .setContentType(AudioAttributes.CONTENT_TYPE_MOVIE) + .build() + + private fun MediaFormat.intOr(key: String, fallback: Int): Int = + if (containsKey(key)) getInteger(key) else fallback + + private fun MediaFormat.longOr(key: String, fallback: Long): Long = + if (containsKey(key)) getLong(key) else fallback + + private fun findTrack(extractor: MediaExtractor, mimePrefix: String): Int { + for (i in 0 until extractor.trackCount) { + val mime = extractor.getTrackFormat(i).getString(MediaFormat.KEY_MIME) ?: continue + if (mime.startsWith(mimePrefix)) return i + } + return -1 + } + + /** + * Describe a video file for the settings panel, as JSON: {width, height} in display + * orientation (rotation applied), rotation, durationMs, hasAudio, mime; or {error}. + */ + @JvmStatic + fun probe(path: String?): String { + val result = JSONObject() + val extractor = MediaExtractor() + try { + extractor.setDataSource(path ?: throw IOException("no path")) + val videoTrack = findTrack(extractor, "video/") + if (videoTrack < 0) { + result.put("error", "No video track") + return result.toString() + } + var durationUs = 0L + for (i in 0 until extractor.trackCount) { + durationUs = Math.max(durationUs, extractor.getTrackFormat(i).longOr(MediaFormat.KEY_DURATION, 0L)) + } + val format = extractor.getTrackFormat(videoTrack) + val rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0)) + val codedWidth = format.intOr(MediaFormat.KEY_WIDTH, 0) + val codedHeight = format.intOr(MediaFormat.KEY_HEIGHT, 0) + val swap = rotation == 90 || rotation == 270 + result.put("width", if (swap) codedHeight else codedWidth) + result.put("height", if (swap) codedWidth else codedHeight) + result.put("rotation", rotation) + result.put("durationMs", durationUs / 1000) + result.put("hasAudio", findTrack(extractor, "audio/") >= 0) + result.put("mime", format.getString(MediaFormat.KEY_MIME) ?: "") + } catch (e: Exception) { + Log.w(TAG, "probe failed for $path", e) + result.put("error", e.message ?: e.toString()) + } finally { + extractor.release() + } + return result.toString() + } + } + + /** What one start() call asked for; fixed for that playback's lifetime. */ + private class Params( + val path: String, + val surfaceId: String, + val x: Int, + val y: Int, + val width: Int, + val height: Int, + val monochrome: Boolean, + val gamma: Float, + val audio: Boolean, + ) + + private enum class Wait { DUE, INTERRUPTED, STOP } + + private val appContext: Context = context.applicationContext + private val mainHandler = Handler(Looper.getMainLooper()) + private val audioManager = appContext.getSystemService(Context.AUDIO_SERVICE) as AudioManager + @Volatile private var listener: Listener? = null + + private val lock = ReentrantLock() + private val wake = lock.newCondition() + + // Everything below is guarded by lock. + /** Bumped by every start/stop; a playback thread whose generation is stale exits. */ + private var generation = 0 + private var params: Params? = null + /** The user's intent: playing or paused. */ + private var playing = false + private var ended = false + /** A seek (or the initial start) is decoding toward its target; the clock waits for its frame. */ + private var seeking = false + private var pendingSeekUs = -1L + private var durationUs = 0L + private var clockRunning = false + /** Media time while the clock is stopped. */ + private var heldUs = 0L + private var clockBaseUs = 0L + private var clockBaseNanos = 0L + private var clockStartedNanos = 0L + private var lastAudioCheckNanos = 0L + private var audio: MediaPlayer? = null + private var hasAudioFocus = false + private var focusRequest: AudioFocusRequest? = null + /** The most recently decoded frame (params.width x params.height), shown or previewed. */ + private var lastFrame: ByteArray? = null + /** Checksum of the frame last submitted, so an unchanged frame is not recomposited; -1 = none. */ + private var lastSubmittedChecksum = -1L + private var submitFailures = 0 + + private val focusListener = AudioManager.OnAudioFocusChangeListener { change -> + when (change) { + AudioManager.AUDIOFOCUS_LOSS, AudioManager.AUDIOFOCUS_LOSS_TRANSIENT -> { + if (change == AudioManager.AUDIOFOCUS_LOSS) lock.withLock { hasAudioFocus = false } + // Another app (a call, a music player) took the audio: pause like any player would. + if (isPlaying()) { + pause() + notifyState(generationNow(), "paused", getPositionMs()) + } + } + } + } + + fun setListener(listener: Listener?) { + this.listener = listener + } + + /** + * Start playing [path] from [startMs] into the (rectX, rectY, width, height) rect of + * compositor surface [surfaceId], replacing any current playback. Decoding and audio setup + * happen on the playback thread; failures arrive as [Listener.onError]. + */ + fun start( + path: String, + surfaceId: String, + rectX: Int, + rectY: Int, + width: Int, + height: Int, + monochrome: Boolean, + gamma: Float, + withAudio: Boolean, + startMs: Long, + ) { + require(width > 0 && height > 0) { "bad video rect ${width}x$height" } + val p = Params(path, surfaceId, rectX, rectY, width, height, monochrome, gamma, withAudio) + val gen = lock.withLock { + endPlaybackLocked() + params = p + playing = true + ended = false + seeking = true + pendingSeekUs = Math.max(0L, startMs) * 1000 + heldUs = pendingSeekUs + // The old frame may not match the new settings (size, monochrome). + lastFrame = null + lastSubmittedChecksum = -1 + submitFailures = 0 + generation + } + val thread = Thread({ Playback(gen, p).run() }, "FaceclawVideo") + thread.priority = Thread.NORM_PRIORITY + 1 + thread.start() + } + + /** Pause; once this returns no further frame is submitted until [resume]. */ + fun pause() { + lock.withLock { + if (!playing) return + playing = false + holdClockLocked(clockUsLocked()) + wake.signalAll() + } + } + + fun resume() { + lock.withLock { + if (playing || ended || params == null) return + playing = true + // A fresh frame must go out even if it matches the last one: the TS side painted over + // the surface while paused. + lastSubmittedChecksum = -1 + if (!seeking) startClockLocked() + wake.signalAll() + } + } + + /** Seek to [positionMs]; while paused, the new frame is decoded and announced as "paused". */ + fun seekTo(positionMs: Long) { + lock.withLock { + if (params == null || ended) return + var targetUs = Math.max(0L, positionMs) * 1000 + if (durationUs > 0) targetUs = Math.min(targetUs, Math.max(0L, durationUs - SEEK_END_MARGIN_US)) + pendingSeekUs = targetUs + seeking = true + holdClockLocked(targetUs) + lastSubmittedChecksum = -1 + wake.signalAll() + } + } + + fun getPositionMs(): Long = lock.withLock { clockUsLocked() / 1000 } + + fun getDurationMs(): Long = lock.withLock { durationUs / 1000 } + + fun isPlaying(): Boolean = lock.withLock { playing && !ended && params != null } + + /** + * Copy the most recent frame into [out], which must hold exactly the video rect's + * width*height bytes (row-major 8bpp gray). False when there is no frame of that size yet. + */ + fun copyLastFrame(out: ByteBuffer): Boolean { + lock.withLock { + val frame = lastFrame ?: return false + if (out.capacity() != frame.size) return false + out.clear() + out.put(frame) + out.clear() + return true + } + } + + /** End playback and free the decoder and audio; the last frame stays readable. */ + fun stop() { + lock.withLock { endPlaybackLocked() } + } + + /** stop(), and drop the listener and frame. */ + fun release() { + lock.withLock { + endPlaybackLocked() + lastFrame = null + } + listener = null + } + + private fun generationNow(): Int = lock.withLock { generation } + + private fun endPlaybackLocked() { + generation++ + params = null + playing = false + seeking = false + pendingSeekUs = -1 + // Keep the position readable (an error's caller resumes from it). + heldUs = clockUsLocked() + clockRunning = false + releaseAudioLocked() + wake.signalAll() + } + + // ---- clock (all under lock) ---- + + private fun clockUsLocked(): Long = + if (!clockRunning) heldUs else clockBaseUs + (System.nanoTime() - clockBaseNanos) / 1000 + + private fun startClockLocked() { + if (clockRunning) return + clockRunning = true + clockBaseUs = heldUs + clockBaseNanos = System.nanoTime() + clockStartedNanos = clockBaseNanos + val player = audio ?: return + requestAudioFocusLocked() + try { + player.start() + } catch (e: IllegalStateException) { + Log.w(TAG, "audio start failed", e) + } + } + + private fun holdClockLocked(atUs: Long) { + heldUs = atUs + if (!clockRunning) return + clockRunning = false + try { + audio?.let { if (it.isPlaying) it.pause() } + } catch (e: IllegalStateException) { + Log.w(TAG, "audio pause failed", e) + } + } + + /** + * Keep the video clock on the audio position: output latency and seeks make the two drift, + * and a picture ahead of or behind its sound is the one sync error people notice. + */ + private fun followAudioLocked() { + val player = audio ?: return + val now = System.nanoTime() + if (!clockRunning || now - clockStartedNanos < AUDIO_SETTLE_NANOS) return + if (now - lastAudioCheckNanos < AUDIO_CHECK_INTERVAL_NANOS) return + lastAudioCheckNanos = now + val audioUs = try { + if (!player.isPlaying) return // finished (audio shorter than video) or not started + player.currentPosition * 1000L + } catch (e: IllegalStateException) { + return + } + if (Math.abs(audioUs - clockUsLocked()) > AUDIO_RESYNC_US) { + clockBaseUs = audioUs + clockBaseNanos = System.nanoTime() + } + } + + private fun seekAudioLocked(targetUs: Long) { + val player = audio ?: return + try { + if (Build.VERSION.SDK_INT >= 26) { + player.seekTo(targetUs / 1000, MediaPlayer.SEEK_CLOSEST) + } else { + player.seekTo((targetUs / 1000).toInt()) + } + } catch (e: IllegalStateException) { + Log.w(TAG, "audio seek failed", e) + } + } + + private fun requestAudioFocusLocked() { + if (hasAudioFocus) return + val result = if (Build.VERSION.SDK_INT >= 26) { + val request = focusRequest ?: AudioFocusRequest.Builder(AudioManager.AUDIOFOCUS_GAIN) + .setAudioAttributes(AUDIO_ATTRIBUTES) + .setOnAudioFocusChangeListener(focusListener, mainHandler) + .build() + .also { focusRequest = it } + audioManager.requestAudioFocus(request) + } else { + @Suppress("DEPRECATION") + audioManager.requestAudioFocus(focusListener, AudioManager.STREAM_MUSIC, AudioManager.AUDIOFOCUS_GAIN) + } + // Play regardless: the sound was asked for, and a refusal is transient (a call ending). + hasAudioFocus = result == AudioManager.AUDIOFOCUS_REQUEST_GRANTED + } + + private fun releaseAudioLocked() { + val player = audio ?: return + audio = null + try { + player.release() + } catch (e: Exception) { + Log.w(TAG, "audio release failed", e) + } + if (hasAudioFocus) { + hasAudioFocus = false + if (Build.VERSION.SDK_INT >= 26) { + focusRequest?.let { audioManager.abandonAudioFocusRequest(it) } + } else { + @Suppress("DEPRECATION") + audioManager.abandonAudioFocus(focusListener) + } + } + } + + // ---- notifications ---- + + private fun notifyState(gen: Int, state: String, positionMs: Long) { + mainHandler.post { + if (lock.withLock { gen == generation }) listener?.onStateChanged(state, positionMs) + } + } + + private fun notifyError(gen: Int, message: String) { + mainHandler.post { + if (lock.withLock { gen == generation }) listener?.onError(message) + } + } + + // ---- frame output (under lock) ---- + + private fun submitLocked(p: Params, pixels: ByteArray, checksum: Long) { + val buffer = ByteBuffer.wrap(pixels) + // Frame id 0: per-frame FrameTimings entries at video rate would crowd every real + // input/render frame out of the timing export. + val fingerprint = "video:$generation:" + java.lang.Long.toHexString(checksum) + try { + val communicator = FaceclawBleCommunicator.getActive() + if (communicator != null) { + communicator.submitSurfaceFrame(buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null) + } else { + FaceclawPreviewCompositor.getActive()?.submitSurfaceFrame( + buffer, p.surfaceId, p.x, p.y, p.width, p.height, fingerprint, 0, 0, null) + } + submitFailures = 0 + } catch (e: RuntimeException) { + // The window's surface went away or shrank under us (closed, relaid out): drop the frame. + if (submitFailures++ == 0) Log.w(TAG, "video frame submit failed: ${e.message}") + } + } + + /** + * One start() call's decode thread. Owns the extractor and decoder; everything shared with + * the control methods goes through the player's lock. + */ + private inner class Playback(private val gen: Int, private val p: Params) { + private var extractor: MediaExtractor? = null + private var codec: MediaCodec? = null + private val info = MediaCodec.BufferInfo() + private var inputDone = false + private var outputDone = false + private var trackEndUs = 0L + /** Frames before this are decoded only on the way to a seek target. */ + private var skipUntilUs = 0L + private var lastDrawnPtsUs = Long.MIN_VALUE / 2 + private var lastDrawnNanos = 0L + private var rotation = 0 + private var lut = ByteArray(256) + private var scaler: VideoFrameScaler? = null + private var luma = ByteArray(0) + /** The frame being built; swapped with lastFrame when shown. */ + private var frame = ByteArray(p.width * p.height) + private val crc = CRC32() + + fun run() { + try { + open() + loop() + } catch (t: Throwable) { + Log.w(TAG, "playback failed for ${p.path}", t) + val current = lock.withLock { + val current = gen == generation + if (current) endPlaybackLocked() + current + } + // endPlaybackLocked bumped the generation; report under the new one. + if (current) notifyError(generationNow(), t.message ?: t.toString()) + } finally { + try { + codec?.stop() + } catch (ignored: Throwable) { + } + codec?.release() + extractor?.release() + } + } + + private fun current(): Boolean = lock.withLock { gen == generation } + + private fun open() { + val ex = MediaExtractor() + extractor = ex + ex.setDataSource(p.path) + val track = findTrack(ex, "video/") + if (track < 0) throw IOException("No video track") + ex.selectTrack(track) + val format = ex.getTrackFormat(track) + rotation = VideoFrameScaler.normalizeRotation(format.intOr(MediaFormat.KEY_ROTATION, 0)) + trackEndUs = format.longOr(MediaFormat.KEY_DURATION, 0L) + // Most video is limited ("TV") range; only an explicit full-range tag says otherwise. + val limited = format.intOr(MediaFormat.KEY_COLOR_RANGE, MediaFormat.COLOR_RANGE_LIMITED) != + MediaFormat.COLOR_RANGE_FULL + lut = VideoFrameScaler.toneLut(limited, p.gamma, p.monochrome) + val mime = format.getString(MediaFormat.KEY_MIME) ?: throw IOException("Unknown video format") + format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible) + val decoder = try { + MediaCodec.createDecoderByType(mime) + } catch (e: Exception) { + throw IOException("No decoder for $mime") + } + codec = decoder + decoder.configure(format, null, null, 0) + decoder.start() + val player = if (p.audio && findTrack(ex, "audio/") >= 0) openAudio() else null + lock.withLock { + if (gen != generation) { + player?.release() + return + } + durationUs = trackEndUs + // Positioned by the start seek, which is the first command loop() handles. + audio = player + } + } + + /** A prepared MediaPlayer for the file's audio, or null (played silent) if it won't open. */ + private fun openAudio(): MediaPlayer? { + val player = MediaPlayer() + return try { + player.setAudioAttributes(AUDIO_ATTRIBUTES) + player.setDataSource(p.path) + player.prepare() + player + } catch (e: Exception) { + Log.w(TAG, "audio unavailable for ${p.path}; playing silent", e) + player.release() + null + } + } + + private fun loop() { + while (true) { + // Commands: stop, seek, or wait out a pause. + var seekUs = -1L + lock.withLock { + while (true) { + if (gen != generation) return + if (pendingSeekUs >= 0) { + seekUs = pendingSeekUs + pendingSeekUs = -1 + break + } + if (playing || seeking) break + wake.await() + } + } + if (seekUs >= 0) { + seek(seekUs) + continue + } + if (outputDone) { + if (awaitEnd()) return + continue + } + feedInput() + val index = codec!!.dequeueOutputBuffer(info, CODEC_TIMEOUT_US) + if (index < 0) continue // try again later / format or buffers changed + val endOfStream = (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0 + if (info.size > 0) { + handleFrame(index, info.presentationTimeUs) + } else { + codec!!.releaseOutputBuffer(index, false) + } + if (endOfStream) outputDone = true + } + } + + private fun seek(targetUs: Long) { + lock.withLock { + if (gen != generation) return + seekAudioLocked(targetUs) + } + extractor!!.seekTo(targetUs, MediaExtractor.SEEK_TO_PREVIOUS_SYNC) + codec!!.flush() + inputDone = false + outputDone = false + skipUntilUs = targetUs + lastDrawnPtsUs = Long.MIN_VALUE / 2 + } + + private fun feedInput() { + if (inputDone) return + val decoder = codec!! + val ex = extractor!! + while (true) { + val index = decoder.dequeueInputBuffer(0) + if (index < 0) return + val buffer = decoder.getInputBuffer(index) ?: return + val size = ex.readSampleData(buffer, 0) + if (size < 0) { + decoder.queueInputBuffer(index, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM) + inputDone = true + return + } + decoder.queueInputBuffer(index, 0, size, ex.sampleTime, 0) + ex.advance() + } + } + + /** Decide a decoded frame's fate (skip, preview, drop, draw); always releases it. */ + private fun handleFrame(index: Int, ptsUs: Long) { + try { + if (ptsUs < skipUntilUs) return + val landingSeek = lock.withLock { seeking } + if (landingSeek) { + landSeek(index, ptsUs) + return + } + if (ptsUs - lastDrawnPtsUs < MIN_FRAME_INTERVAL_US) return + if (waitUntilDue(ptsUs) != Wait.DUE) return + val lateUs = lock.withLock { clockUsLocked() } - ptsUs + if (lateUs > LATE_DROP_US && System.nanoTime() - lastDrawnNanos < MAX_DROP_GAP_NANOS) return + val checksum = convert(index) + present(ptsUs, checksum) + } finally { + codec!!.releaseOutputBuffer(index, false) + } + } + + /** + * The first frame at or past a seek target (or the start position): drawn and the clock + * started when playing, kept as the paused preview otherwise. + */ + private fun landSeek(index: Int, ptsUs: Long) { + val checksum = convert(index) + var previewed = false + lock.withLock { + if (gen != generation || pendingSeekUs >= 0) return + seeking = false + heldUs = ptsUs + swapInFrameLocked() + if (playing) { + submitLocked(p, lastFrame!!, checksum) + lastSubmittedChecksum = checksum + startClockLocked() + } else { + previewed = true + } + } + lastDrawnPtsUs = ptsUs + lastDrawnNanos = System.nanoTime() + notifyState(gen, if (previewed) "paused" else "playing", ptsUs / 1000) + } + + /** Show a due frame, unless a pause or seek got in first. */ + private fun present(ptsUs: Long, checksum: Long) { + lock.withLock { + if (gen != generation || !playing || pendingSeekUs >= 0) return + swapInFrameLocked() + if (checksum != lastSubmittedChecksum) { + submitLocked(p, lastFrame!!, checksum) + lastSubmittedChecksum = checksum + } + } + lastDrawnPtsUs = ptsUs + lastDrawnNanos = System.nanoTime() + } + + private fun swapInFrameLocked() { + val shown = frame + val previous = lastFrame + frame = if (previous != null && previous.size == shown.size) previous else ByteArray(shown.size) + lastFrame = shown + } + + /** Block until the clock reaches [ptsUs]; paused time doesn't count. */ + private fun waitUntilDue(ptsUs: Long): Wait { + lock.withLock { + while (true) { + if (gen != generation) return Wait.STOP + if (pendingSeekUs >= 0) return Wait.INTERRUPTED + if (!playing) { + wake.await() + continue + } + followAudioLocked() + val now = clockUsLocked() + if (now >= ptsUs) return Wait.DUE + wake.await(Math.min(ptsUs - now, MAX_WAIT_US), TimeUnit.MICROSECONDS) + } + } + } + + /** + * The decoder is drained: hold the last frame until the clock reaches the end, then + * report "ended". False when a seek interrupted the wait (playback continues). + */ + private fun awaitEnd(): Boolean { + val endUs = Math.max(trackEndUs, lastDrawnPtsUs) + val landing = lock.withLock { seeking } + if (!landing && waitUntilDue(endUs) != Wait.DUE) return !current() + lock.withLock { + if (gen != generation) return true + if (pendingSeekUs >= 0) return false + playing = false + seeking = false + ended = true + holdClockLocked(endUs) + releaseAudioLocked() + } + notifyState(gen, "ended", endUs / 1000) + return true + } + + /** Scale the output buffer's frame into [frame]; returns its checksum. */ + private fun convert(index: Int): Long { + val decoder = codec!! + val image = try { + decoder.getOutputImage(index) + } catch (e: Exception) { + null + } + if (image != null) { + try { + val plane = image.planes[0] + val crop = image.cropRect + val offset = copyLuma(plane.buffer) + crop.top * plane.rowStride + crop.left * plane.pixelStride + scaleInto(crop.width(), crop.height(), offset, plane.rowStride, plane.pixelStride) + } finally { + image.close() + } + } else { + // No Image access: every YUV420 layout starts with the Y plane at the buffer start. + val buffer = decoder.getOutputBuffer(index) ?: throw IOException("Decoder returned no frame") + buffer.position(info.offset) + buffer.limit(info.offset + info.size) + val format = decoder.outputFormat + val width = format.getInteger(MediaFormat.KEY_WIDTH) + val stride = format.intOr(MediaFormat.KEY_STRIDE, width) + val left = format.intOr("crop-left", 0) + val top = format.intOr("crop-top", 0) + val right = format.intOr("crop-right", width - 1) + val bottom = format.intOr("crop-bottom", format.getInteger(MediaFormat.KEY_HEIGHT) - 1) + copyLuma(buffer) + scaleInto(right - left + 1, bottom - top + 1, top * stride + left, stride, 1) + } + crc.reset() + crc.update(frame, 0, frame.size) + return crc.value + } + + /** Bulk-copy the plane (one memcpy; per-byte reads from a direct buffer are slow). Returns 0. */ + private fun copyLuma(buffer: ByteBuffer): Int { + val bytes = buffer.remaining() + if (luma.size < bytes) luma = ByteArray(bytes) + buffer.get(luma, 0, bytes) + return 0 + } + + private fun scaleInto(width: Int, height: Int, offset: Int, rowStride: Int, pixelStride: Int) { + var s = scaler + if (s == null || s.sourceWidth != width || s.sourceHeight != height) { + s = VideoFrameScaler(width, height, rotation, p.width, p.height) + scaler = s + } + s.scale(luma, offset, rowStride, pixelStride, lut, frame) + } + } +} diff --git a/app/apps/files/files-app.ts b/app/apps/files/files-app.ts index 615c8f9a..d8872e58 100644 --- a/app/apps/files/files-app.ts +++ b/app/apps/files/files-app.ts @@ -2,6 +2,7 @@ import { readTextFile, type DirectoryEntry } from "../../native/file-access"; import { isFontFile } from "../../native/font-files"; import { installFontFile, isFontInstalled } from "../../graphics/installed-fonts"; import { isDecodableImageFile } from "../../native/image-files"; +import { isPlayableVideoFile } from "../../native/video-player"; import { readEvenHubPackageManifest } from "../evenhub/installed-apps"; import { EvenHubPermissionDialogLayer } from "../evenhub/permission-dialog"; import { FileBrowserLayer } from "./file-browser"; @@ -35,17 +36,21 @@ export type FilesAppOptions = InProcessAppOptions & { installEhpkApp: (path: string) => Promise | void; /** Open a font file's previewer as its own shell window. */ openFontWindow: (title: string, path: string) => void; + /** Open a video file's player (settings panel first) as its own shell window. */ + openVideoWindow: (title: string, path: string) => void; }; /** * The Files app's launcher-opened window: a file browser over Places * (bookmarks and storage roots). Picking any file opens an info dialog with - * metadata plus, for viewable types (text, images), the open actions. + * metadata plus, for viewable types (text, images, video), the open actions. */ export function createFilesAppWindow(options: FilesAppOptions): InProcessWindow { let created: InProcessWindow | null = null; const browser = new FileBrowserLayer({ - isSupportedFile: (name) => TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || (!global.isIOS && isDecodableImageFile(name)), + isSupportedFile: (name) => + TEXT_FILE.test(name) || isFontFile(name) || EHPK_FILE.test(name) || + (!global.isIOS && (isDecodableImageFile(name) || isPlayableVideoFile(name))), // The browser handles double-click itself (up a level), so it is not // wrapped in YieldAtRootLayer; it yields explicitly from the top level. onLeave: () => shell.yieldFocusToSidebar(), @@ -230,6 +235,17 @@ function fileOpenActions(entry: DirectoryEntry, options: FilesAppOptions): FileI }, ]; } + if (isPlayableVideoFile(entry.name)) { + return [ + { + label: "Play video", + onSelect: (ctx) => { + ctx.stack.pop(); + options.openVideoWindow(entry.name, entry.path); + }, + }, + ]; + } if (isFontFile(entry.name)) { // Install copies into the app-internal fonts directory, making the face // available in the Settings font pickers. The row relabels itself with diff --git a/app/apps/files/index.ts b/app/apps/files/index.ts index 92301cca..65f9430f 100644 --- a/app/apps/files/index.ts +++ b/app/apps/files/index.ts @@ -13,6 +13,7 @@ import { FILES_SURFACE_ID, FILES_WINDOW_ID, } from "./files-app"; +import { createVideoPlayerWindow } from "./video-window"; // Document windows are closeable and non-singleton; the serial keeps their // windowIds unique. @@ -51,6 +52,17 @@ function openFontDocumentWindow(ctx: AppContext, title: string, path: string): v }); } +function openVideoWindow(ctx: AppContext, title: string, path: string): void { + const windowId = `files:doc:${nextDocumentSerial++}`; + void ctx + .launchInProcessApp(windowId, `window:${windowId}`, (options) => + createVideoPlayerWindow(windowId, title, path, options), + ) + .catch((error) => { + ctx.appendLog(`video window failed: ${error}`); + }); +} + const filesApp: AppDefinition = { appId: "files", title: "Files", @@ -78,6 +90,7 @@ const filesApp: AppDefinition = { } }, openFontWindow: (title, path) => openFontDocumentWindow(ctx, title, path), + openVideoWindow: (title, path) => openVideoWindow(ctx, title, path), }), ), // A document arriving via Android's Share intent opens as its own window. diff --git a/app/apps/files/video-layout.ts b/app/apps/files/video-layout.ts new file mode 100644 index 00000000..e8cc1f15 --- /dev/null +++ b/app/apps/files/video-layout.ts @@ -0,0 +1,86 @@ +/** + * Where a video goes in the player window, for each size choice. Pure + * geometry, so it is unit-tested without the NativeScript runtime. + */ + +export type VideoSize = "small" | "medium" | "large" | "full"; + +export const VIDEO_SIZES: readonly VideoSize[] = ["small", "medium", "large", "full"]; + +export type Size = { width: number; height: number }; +export type Rect = Size & { x: number; y: number }; + +/** + * Bounding boxes for the fixed sizes; a video is fitted inside, keeping its + * aspect. Large and full fit the whole viewport instead (full in a tall + * window). Bigger pictures cost the glasses link more per frame, so smaller + * sizes play at a higher frame rate. + */ +const FIXED_BOXES: Partial> = { + small: { width: 160, height: 120 }, + medium: { width: 240, height: 180 }, +}; + +export function videoSizeLabel(size: VideoSize): string { + switch (size) { + case "small": + return "Small"; + case "medium": + return "Medium"; + case "large": + return "Large"; + case "full": + return "Full screen"; + } +} + +/** Full screen needs the tall window band; the rest play in the standard one. */ +export function videoHeightMode(size: VideoSize): "min" | "max" { + return size === "full" ? "max" : "min"; +} + +/** + * The largest size with the video's aspect ratio that fits in box, with even + * dimensions (the glasses' delta encoder works in 2-pixel rows and 4-pixel + * columns) of at least 2. Upscales small videos too: the size is the + * viewer's choice. + */ +export function fitVideo(video: Size, box: Size): Size { + if (video.width <= 0 || video.height <= 0) return { width: evenFloor(box.width), height: evenFloor(box.height) }; + const scale = Math.min(box.width / video.width, box.height / video.height); + return { + width: evenFloor(Math.min(box.width, Math.round(video.width * scale))), + height: evenFloor(Math.min(box.height, Math.round(video.height * scale))), + }; +} + +/** + * The video's rect within a viewport, centered. x snaps to a multiple of 4 + * and y to a multiple of 2 so frame deltas don't straddle an extra encoder + * block at each edge. + */ +export function videoRect(size: VideoSize, video: Size, viewport: Size): Rect { + const fixed = FIXED_BOXES[size]; + const box = fixed + ? { width: Math.min(fixed.width, viewport.width), height: Math.min(fixed.height, viewport.height) } + : viewport; + const fitted = fitVideo(video, box); + return { + x: Math.max(0, Math.floor((viewport.width - fitted.width) / 8) * 4), + y: Math.max(0, Math.floor((viewport.height - fitted.height) / 4) * 2), + ...fitted, + }; +} + +/** "3:52", or "1:02:03" past an hour. */ +export function formatMediaTime(ms: number): string { + const total = Math.max(0, Math.floor(ms / 1000)); + const hours = Math.floor(total / 3600); + const minutes = Math.floor((total % 3600) / 60); + const seconds = String(total % 60).padStart(2, "0"); + return hours > 0 ? `${hours}:${String(minutes).padStart(2, "0")}:${seconds}` : `${minutes}:${seconds}`; +} + +function evenFloor(value: number): number { + return Math.max(2, Math.floor(value / 2) * 2); +} diff --git a/app/apps/files/video-window.ts b/app/apps/files/video-window.ts new file mode 100644 index 00000000..2310e117 --- /dev/null +++ b/app/apps/files/video-window.ts @@ -0,0 +1,443 @@ +import { getDefaultSmallFont } from "../../graphics/ui-fonts"; +import { GrayImage } from "../../graphics/image"; +import { truncateText, wrapText } from "../../graphics/textwrap"; +import { PHOTO_GAMMA } from "../../native/image-files"; +import { probeVideo, VideoPlayer, type VideoInfo, type VideoPlaybackState } from "../../native/video-player"; +import { ConfigSettingBoolean, ConfigSettingEnum } from "../../ui/dashboard-settings"; +import { type InputEvent } from "../../ui/gestures"; +import { type Layer, type LayerContext } from "../../ui/layers"; +import { drawRightValueMenuItem, drawToggleMenuItem } from "../../ui/menu"; +import { Menu, type MenuDrawArgs } from "../../ui/menu-core"; +import { LIST_ROW_TEXT_INSET, lineStep, listRowHeight } from "../../ui/metrics"; +import { + createInProcessWindow, + YieldAtRootLayer, + type InProcessAppOptions, + type InProcessWindow, +} from "../../ui/shell/in-process-window"; +import { appViewportSize } from "../../ui/shell/geometry"; +import { + formatMediaTime, + VIDEO_SIZES, + videoHeightMode, + videoRect, + videoSizeLabel, + type Rect, + type VideoSize, +} from "./video-layout"; + +export const videoSizeSetting = new ConfigSettingEnum({ + id: "video-size", + label: "Size", + storageKey: "video.size", + defaultValue: "large", + values: VIDEO_SIZES, + formatValue: videoSizeLabel, +}); + +export const videoAudioSetting = new ConfigSettingBoolean({ + id: "video-audio", + label: "Audio", + storageKey: "video.audio", + defaultValue: true, + description: "Play the video's sound on the phone's current audio output.", +}); + +export const videoMonochromeSetting = new ConfigSettingBoolean({ + id: "video-monochrome", + label: "Monochrome", + storageKey: "video.monochrome", + defaultValue: false, + description: "Show only black and white: far less to send per frame, so a higher frame rate.", +}); + +/** How far one ring scroll seeks. */ +const SEEK_STEP_MS = 10_000; +const MARGIN_X = 18; +const TITLE_Y = 10; +const PANEL_WIDTH = 340; + +/** + * One opened video: its probe result, the native player, and the window + * glue. The window's base layer is the settings panel; playing pushes the + * playback layer, and backing out of it (or the video ending) returns to the + * panel with the position kept for Resume. + */ +class VideoSession { + readonly probe: VideoInfo | { error: string }; + private player: VideoPlayer | null = null; + private window: InProcessWindow | null = null; + private playback: VideoPlaybackLayer | null = null; + /** Where Resume starts; 0 = from the beginning. */ + resumeMs = 0; + /** The last playback error, shown on the panel until the next Play. */ + error: string | null = null; + + constructor( + readonly path: string, + readonly title: string, + readonly surfaceId: string, + ) { + this.probe = probeVideo(path); + } + + get info(): VideoInfo | null { + return "error" in this.probe ? null : this.probe; + } + + attach(window: InProcessWindow): void { + this.window = window; + } + + isPlaying(): boolean { + return this.player?.isPlaying() ?? false; + } + + positionMs(): number { + return this.player?.positionMs() ?? this.resumeMs; + } + + /** Current rect for a size choice, in the viewport of the band that size plays in. */ + rectFor(size: VideoSize, viewport: { width: number; height: number }): Rect { + const info = this.info; + return videoRect(size, info ? { width: info.width, height: info.height } : viewport, viewport); + } + + play(ctx: LayerContext, fromStart: boolean): void { + const window = this.window; + const info = this.info; + if (!window || !info) return; + this.error = null; + const size = videoSizeSetting.get(); + window.setHeightMode(videoHeightMode(size)); + const viewport = ctx.stack.getBaseSize(); + const rect = this.rectFor(size, viewport); + const player = this.ensurePlayer(); + this.playback = new VideoPlaybackLayer(this, rect, viewport); + ctx.stack.push(this.playback); + player.start({ + path: this.path, + surfaceId: this.surfaceId, + rect, + monochrome: videoMonochromeSetting.get(), + gamma: PHOTO_GAMMA, + audio: info.hasAudio && videoAudioSetting.get(), + startMs: fromStart ? 0 : this.resumeMs, + }); + } + + togglePause(): void { + const player = this.player; + if (!player) return; + if (player.isPlaying()) player.pause(); + else player.resume(); + } + + seekBy(deltaMs: number): void { + const player = this.player; + if (!player) return; + const duration = this.info?.durationMs ?? 0; + const target = Math.max(0, player.positionMs() + deltaMs); + player.seekTo(duration > 0 ? Math.min(target, duration) : target); + } + + /** The window went to the background or the screen went off: stop, keep the place. */ + pauseForHost(): void { + if (!this.player?.isPlaying()) return; + this.player.pause(); + // Paint the paused overlay now, so the retained surface shows it on return. + this.window?.requestRender(); + } + + /** Leave the playback layer for the panel, remembering where to resume. */ + returnToPanel(resumeMs: number): void { + const player = this.player; + if (player) { + player.pause(); + player.stop(); + } + this.resumeMs = resumeMs; + const window = this.window; + const playback = this.playback; + this.playback = null; + if (window && playback) window.stack.popThrough(playback); + window?.setHeightMode("min"); + window?.requestRender(); + } + + lastFrame(rect: Rect): GrayImage | null { + return this.player?.lastFrame(rect.width, rect.height) ?? null; + } + + release(): void { + this.player?.release(); + this.player = null; + } + + private ensurePlayer(): VideoPlayer { + if (!this.player) { + this.player = new VideoPlayer({ + onState: (state, positionMs) => this.onPlayerState(state, positionMs), + onError: (message) => { + this.error = message; + this.returnToPanel(this.positionMs()); + }, + }); + } + return this.player; + } + + private onPlayerState(state: VideoPlaybackState, _positionMs: number): void { + if (state === "ended") { + this.returnToPanel(0); + return; + } + // A paused seek decoded its new frame: repaint it under the overlay. + if (state === "paused") this.window?.requestRender(); + } +} + +type PanelRow = { kind: "size" } | { kind: "audio" } | { kind: "monochrome" } | { kind: "play"; fromStart: boolean }; + +/** + * The pre-play settings panel (the window's root): picture size, audio and + * monochrome, all remembered for the next video, then Play (or Resume plus + * Play from start once a position is kept). + */ +class VideoSettingsLayer implements Layer { + private readonly menu = new Menu({ + wrap: false, + rowGap: 1, + highlight: { radius: 6 }, + getHeight: () => listRowHeight(getDefaultSmallFont()), + draw: (args) => this.drawRow(args), + }); + constructor(private readonly session: VideoSession) {} + + private rows(): PanelRow[] { + if (!this.session.info) return []; + const rows: PanelRow[] = [{ kind: "size" }, { kind: "audio" }, { kind: "monochrome" }]; + rows.push({ kind: "play", fromStart: false }); + if (this.session.resumeMs > 0) rows.push({ kind: "play", fromStart: true }); + return rows; + } + + paint(ctx: LayerContext): GrayImage { + const font = getDefaultSmallFont(); + const { width, height } = ctx.stack.getBaseSize(); + const image = new GrayImage(width, height, 0); + image.drawText(font, MARGIN_X, TITLE_Y, truncateText(font, this.session.title, width - 2 * MARGIN_X), 220); + const step = lineStep(font); + let y = TITLE_Y + step; + + const info = this.session.info; + const errors: string[] = []; + if (!info) errors.push(`Can't play this video: ${(this.session.probe as { error: string }).error}`); + if (this.session.error) errors.push(`Playback stopped: ${this.session.error}`); + if (info) { + const facts = [`${info.width}x${info.height}`, formatMediaTime(info.durationMs), info.hasAudio ? "sound" : "no sound"]; + image.drawText(font, MARGIN_X, y, facts.join(" ยท "), 130); + y += step; + } + for (const error of errors) { + for (const line of wrapText(font, error, width - 2 * MARGIN_X, { breakLongWords: true }).slice(0, 3)) { + image.drawText(font, MARGIN_X, y, line, 200); + y += step; + } + } + y += 6; + + const rows = this.rows(); + const selected = this.menu.selectedItem; + const index = selected ? rows.findIndex((row) => sameRow(row, selected)) : -1; + // Land on Resume/Play by default: the settings usually stay as they were. + const playIndex = rows.findIndex((row) => row.kind === "play"); + this.menu.setItems(rows, index >= 0 ? index : playIndex >= 0 ? playIndex : undefined); + const panelWidth = Math.min(PANEL_WIDTH, width - 2 * (MARGIN_X - 6)); + this.menu.paint(image, { x: MARGIN_X - 6, y, width: panelWidth, height: Math.max(0, height - y - 4) }, + ctx.stack.isFocused()); + return image; + } + + async handleInput(event: InputEvent, ctx: LayerContext): Promise { + switch (event.type) { + case "scroll-up": + case "scroll-down": + await this.menu.handleInput(event); + return; + case "click": { + const row = this.menu.selectedItem; + if (row) this.activate(row, ctx); + return; + } + default: + return; + } + } + + private activate(row: PanelRow, ctx: LayerContext): void { + switch (row.kind) { + case "size": + videoSizeSetting.set(videoSizeSetting.next()); + return; + case "audio": + if (this.session.info?.hasAudio) videoAudioSetting.toggle(); + return; + case "monochrome": + videoMonochromeSetting.toggle(); + return; + case "play": + this.session.play(ctx, row.fromStart); + return; + } + } + + private drawRow({ image, item, x, y, width, selected }: MenuDrawArgs): void { + const font = getDefaultSmallFont(); + const textX = x + 10; + const textWidth = width - 20; + switch (item.kind) { + case "size": { + const size = videoSizeSetting.get(); + // The picture size this choice gets in its own band (full screen plays in the tall window). + const rect = this.session.rectFor(size, appViewportSize(videoHeightMode(size))); + drawRightValueMenuItem(image, font, textX, y, textWidth, "Size", + `${videoSizeLabel(size)} ${rect.width}x${rect.height}`); + return; + } + case "audio": { + const hasAudio = this.session.info?.hasAudio ?? false; + drawToggleMenuItem(image, font, textX, y, textWidth, hasAudio ? "Audio" : "Audio (none in file)", + hasAudio && videoAudioSetting.get(), selected); + return; + } + case "monochrome": + drawToggleMenuItem(image, font, textX, y, textWidth, "Monochrome", videoMonochromeSetting.get(), selected); + return; + case "play": { + const resumeMs = this.session.resumeMs; + const label = item.fromStart ? "Play from start" : resumeMs > 0 ? `Resume from ${formatMediaTime(resumeMs)}` : "Play"; + image.drawText(font, textX, y + LIST_ROW_TEXT_INSET, label, selected ? 255 : 200); + return; + } + } + } +} + +function sameRow(a: PanelRow, b: PanelRow): boolean { + return a.kind === b.kind && (a.kind !== "play" || b.kind !== "play" || a.fromStart === b.fromStart); +} + +/** + * The playing video. The native player draws frames into the video rect on + * its own; this layer paints only when the window renders (input, pause, + * focus): black around the latest frame, plus a status box while paused. + * Click pauses/resumes, scroll seeks 10s, double-click returns to the panel. + */ +class VideoPlaybackLayer implements Layer { + private leaving = false; + + constructor( + private readonly session: VideoSession, + private readonly rect: Rect, + /** The viewport the rect was laid out in. */ + private readonly viewport: { width: number; height: number }, + ) {} + + paint(ctx: LayerContext): GrayImage { + const { width, height } = ctx.stack.getBaseSize(); + if ((width !== this.viewport.width || height !== this.viewport.height) && !this.leaving) { + // The window was relaid out (display mode change) and the native player's rect no + // longer fits the surface: back out to the panel, which lays out afresh on Play. + this.leaving = true; + setTimeout(() => this.session.returnToPanel(this.session.positionMs()), 0); + } + const image = new GrayImage(width, height, 0); + const frame = this.session.lastFrame(this.rect); + if (frame) image.bitBlt(frame, this.rect.x, this.rect.y); + if (!this.session.isPlaying()) this.drawPausedBox(image, frame === null); + return image; + } + + async handleInput(event: InputEvent): Promise { + switch (event.type) { + case "click": + this.session.togglePause(); + return; + case "scroll-up": + this.session.seekBy(-SEEK_STEP_MS); + return; + case "scroll-down": + this.session.seekBy(SEEK_STEP_MS); + return; + case "double-click": + this.session.returnToPanel(this.session.positionMs()); + return; + default: + return; + } + } + + /** + * "Paused 1:23 / 3:52" over a progress bar, below the picture when there is + * room (small sizes), else over its bottom edge. + */ + private drawPausedBox(image: GrayImage, loading: boolean): void { + const font = getDefaultSmallFont(); + const duration = this.session.info?.durationMs ?? 0; + const position = this.session.positionMs(); + const text = loading ? "Loading..." : `Paused ${formatMediaTime(position)} / ${formatMediaTime(duration)}`; + const boxWidth = Math.min(image.width - 8, font.measureText(text) + 24); + const boxHeight = font.lineHeight + 14; + const rect = this.rect; + const x = Math.round(rect.x + (rect.width - boxWidth) / 2); + const below = rect.y + rect.height + 4; + const y = below + boxHeight <= image.height ? below : Math.max(0, rect.y + rect.height - boxHeight - 6); + image.fillRoundedRect(x, y, boxWidth, boxHeight, 1, 6); + image.drawRoundedRect(x, y, boxWidth, boxHeight, 72, 6); + image.drawText(font, x + 12, y + 5, text, 220); + if (!loading && duration > 0) { + const barWidth = boxWidth - 24; + image.fillRect(x + 12, y + boxHeight - 5, barWidth, 2, 40); + image.fillRect(x + 12, y + boxHeight - 5, Math.round((barWidth * Math.min(position, duration)) / duration), 2, 200); + } + } +} + +/** A video file's player as its own window, opened from the Files app. */ +export function createVideoPlayerWindow( + windowId: string, + title: string, + path: string, + options: InProcessAppOptions, +): InProcessWindow { + const session = new VideoSession(path, title, `window:${windowId}`); + const window = createInProcessWindow({ + appId: "files", + windowId, + title, + iconLetter: "V", + icon: "film", + closeable: true, + actions: options.actions, + baseLayer: new YieldAtRootLayer(new VideoSettingsLayer(session)), + // A playing video is the thing being watched: no idle screen-off. + keepsScreenOn: () => session.isPlaying(), + onForegroundChanged: (foreground) => { + if (!foreground) session.pauseForHost(); + }, + setScreenOn: (on) => { + if (!on) session.pauseForHost(); + }, + submitFrame: options.submitFrame, + setSurfaceVisible: options.setSurfaceVisible, + removeSurface: options.removeSurface, + reconfigureSurface: options.reconfigureSurface, + onClosed: () => { + session.release(); + options.onClosed(); + }, + }); + session.attach(window); + return window; +} diff --git a/app/native/video-player.ts b/app/native/video-player.ts new file mode 100644 index 00000000..c7b0b165 --- /dev/null +++ b/app/native/video-player.ts @@ -0,0 +1,148 @@ +/** + * Bridge to the Android FaceclawVideoPlayer: decodes a video file and + * streams its frames, scaled to gray, straight into a rect of a window's + * compositor surface, with the audio on the phone's default media route. + * Android only; on iOS probeVideo reports an error and nothing plays. + */ +import { Utils } from "@nativescript/core"; +import { GrayImage } from "../graphics/image"; + +declare const com: any; +declare const java: any; +declare const global: any; + +/** Containers MediaExtractor opens (AVI is not one of them). */ +const PLAYABLE_VIDEO = /\.(mp4|m4v|mkv|webm|3gp|mov)$/i; + +export function isPlayableVideoFile(name: string): boolean { + return PLAYABLE_VIDEO.test(name); +} + +export type VideoInfo = { + /** Display size, rotation already applied. */ + width: number; + height: number; + durationMs: number; + hasAudio: boolean; +}; + +/** Read a video's size, duration and tracks; an error string when it can't be played. */ +export function probeVideo(path: string): VideoInfo | { error: string } { + if (!global.isAndroid) return { error: "Video playback is not supported on this phone" }; + try { + const raw = JSON.parse(String(com.faceclaw.app.FaceclawVideoPlayer.probe(path))); + if (raw.error) return { error: String(raw.error) }; + return { + width: Number(raw.width) || 0, + height: Number(raw.height) || 0, + durationMs: Number(raw.durationMs) || 0, + hasAudio: Boolean(raw.hasAudio), + }; + } catch (error) { + return { error: String(error) }; + } +} + +export type VideoPlaybackState = "playing" | "paused" | "ended"; + +export type VideoPlayerCallbacks = { + /** State change, or a paused seek that landed on a new frame (state "paused"). */ + onState: (state: VideoPlaybackState, positionMs: number) => void; + onError: (message: string) => void; +}; + +export type VideoStartOptions = { + path: string; + surfaceId: string; + /** The video's rect in surface coordinates. */ + rect: { x: number; y: number; width: number; height: number }; + monochrome: boolean; + /** Tone curve for gray output (ignored for monochrome); see PHOTO_GAMMA. */ + gamma: number; + audio: boolean; + startMs: number; +}; + +/** One player; frames go to whatever surface the latest start() named. Main thread only. */ +export class VideoPlayer { + private readonly player: any; + // The Java listener proxy must stay referenced or it gets GC'd. + private readonly listenerProxy: any; + // Reused direct buffer (and its JS view) for copyLastFrame; see java-direct-buffer.ts + // for why frames cross as a Java-allocated buffer. + private frameBuffer: any = null; + private frameView: Uint8Array | null = null; + + constructor(callbacks: VideoPlayerCallbacks) { + this.player = new com.faceclaw.app.FaceclawVideoPlayer(Utils.android.getApplicationContext()); + this.listenerProxy = new com.faceclaw.app.FaceclawVideoPlayer.Listener({ + onStateChanged: (state: string, positionMs: number) => { + callbacks.onState(String(state) as VideoPlaybackState, Number(positionMs)); + }, + onError: (message: string) => callbacks.onError(String(message ?? "Playback failed")), + }); + this.player.setListener(this.listenerProxy); + } + + start(options: VideoStartOptions): void { + const { rect } = options; + this.player.start( + options.path, + options.surfaceId, + Math.round(rect.x), + Math.round(rect.y), + Math.round(rect.width), + Math.round(rect.height), + options.monochrome, + options.gamma, + options.audio, + Math.max(0, Math.round(options.startMs)), + ); + } + + /** Once this returns, no more frames reach the surface until resume(). */ + pause(): void { + this.player.pause(); + } + + resume(): void { + this.player.resume(); + } + + seekTo(positionMs: number): void { + this.player.seekTo(Math.max(0, Math.round(positionMs))); + } + + positionMs(): number { + return Number(this.player.getPositionMs()); + } + + isPlaying(): boolean { + return Boolean(this.player.isPlaying()); + } + + /** The latest decoded frame at the given size, or null before one exists. */ + lastFrame(width: number, height: number): GrayImage | null { + const bytes = width * height; + if (bytes <= 0) return null; + if (!this.frameView || this.frameView.length !== bytes) { + this.frameBuffer = java.nio.ByteBuffer.allocateDirect(bytes); + this.frameView = new Uint8Array((ArrayBuffer as any).from(this.frameBuffer)); + } + if (!this.player.copyLastFrame(this.frameBuffer)) return null; + const image = new GrayImage(width, height, 0); + image.pixels.set(this.frameView); + return image; + } + + /** End playback and free the decoder and audio (the last frame stays readable). */ + stop(): void { + this.player.stop(); + } + + release(): void { + this.player.release(); + this.frameBuffer = null; + this.frameView = null; + } +} diff --git a/app/ui/shell/in-process-window.ts b/app/ui/shell/in-process-window.ts index a3ec71d5..b372593a 100644 --- a/app/ui/shell/in-process-window.ts +++ b/app/ui/shell/in-process-window.ts @@ -44,6 +44,10 @@ export type InProcessWindowOptions = { onSystemMenuOpened?: () => void; onAppMenuOpened?: () => void; setScreenOn?: ShellWindow["setScreenOn"]; + /** See ShellWindow.keepsScreenOn. */ + keepsScreenOn?: ShellWindow["keepsScreenOn"]; + /** The window came to or left the foreground (after its surface visibility follows). */ + onForegroundChanged?: (foreground: boolean) => void; /** Shared actions; requestRender is rebound to this window's render. */ actions: LayerActions; /** @@ -162,6 +166,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces acceptsDirectional: true, holdToTalk: options.holdToTalk, isVoiceCapturing: options.isVoiceCapturing, + keepsScreenOn: options.keepsScreenOn, setScreenOn: options.setScreenOn, heightMode, hasAppMenu: () => appMenuItems().length > 0, @@ -218,6 +223,7 @@ export function createInProcessWindow(options: InProcessWindowOptions): InProces onFocus: options.onFocus, setForeground: (foreground) => { options.setSurfaceVisible(foreground); + options.onForegroundChanged?.(foreground); }, }; const setHeightMode = (mode: WindowHeightMode) => { diff --git a/app/ui/shell/shell.ts b/app/ui/shell/shell.ts index 1eeb5243..451a5e27 100644 --- a/app/ui/shell/shell.ts +++ b/app/ui/shell/shell.ts @@ -101,6 +101,12 @@ export type ShellWindow = { holdToTalk?: boolean; /** A window owns microphone capture outside the shell voice dialog. */ isVoiceCapturing?: () => boolean; + /** + * True while the window shows something the wearer watches without + * touching anything (a playing video): the idle screen timeout waits, as + * it does for voice capture, while this window is in the foreground. + */ + keepsScreenOn?: () => boolean; /** * True when the window gives swipe-left / swipe-right (watch directional * input) a meaning; otherwise the shell forwards directionalFallback(event). @@ -647,7 +653,10 @@ class Shell { // ring. An in-flight assistant turn suspends it for the same reason (a // tool loop can run for a while with no input); once the turn ends and // the Done/Follow-up menu is showing, the normal idle timeout resumes. - if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || this.foregroundWindow()?.isVoiceCapturing?.()) { + // A foreground window that is playing video holds it off the same way. + const foreground = this.foregroundWindow(); + if (this.activeVoiceLayer || this.activeKeyboardLayer || this.assistantSession?.isTurnActive() || + foreground?.isVoiceCapturing?.() || foreground?.keepsScreenOn?.()) { this.lastInputAtMs = nowMs; return false; } diff --git a/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/VideoFrameScaler.kt b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/VideoFrameScaler.kt new file mode 100644 index 00000000..76a45c27 --- /dev/null +++ b/native/kotlin/shared/src/commonMain/kotlin/com/faceclaw/app/graphics/VideoFrameScaler.kt @@ -0,0 +1,133 @@ +package com.faceclaw.app + +import kotlin.jvm.JvmStatic + +/** + * Scales one decoded video frame's luma plane into an 8bpp gray rect for a compositor surface: + * a box filter (averaging up to [MAX_TAPS] x [MAX_TAPS] evenly spaced samples per output pixel, + * nearest-neighbor when upscaling), then the container's display rotation, then a tone lookup. + * + * Only luma is read: the glasses show gray, and Y is exactly the gray a decoder hands back + * without any color conversion. Platform code owns decoding and copies the plane into a + * ByteArray; this class owns the pixel math. One instance serves one (source size, rotation, + * output size) combination and reuses its scratch buffers across frames; not thread-safe. + */ +class VideoFrameScaler( + /** Visible luma size in coded (unrotated) orientation. */ + val sourceWidth: Int, + val sourceHeight: Int, + /** Clockwise display rotation in degrees: 0, 90, 180 or 270. */ + val rotation: Int, + /** Output size in display orientation. */ + val outputWidth: Int, + val outputHeight: Int, +) { + companion object { + /** Most samples averaged along each axis per output pixel; bounds work for big downscales. */ + const val MAX_TAPS = 4 + + /** Snap any angle to the nearest of 0/90/180/270, the only rotations containers use. */ + @JvmStatic + fun normalizeRotation(degrees: Int): Int = (((((degrees % 360) + 360) % 360 + 45) / 90) % 4) * 90 + + /** + * Luma to display gray. Limited ("TV") range video codes black at 16 and white at 235, so + * it is stretched to 0..255 first. Monochrome thresholds at mid-gray to 0 or 255 (a + * 1-bit image compresses and delta-encodes far better than 16 levels); otherwise the + * gamma curve applies, as for photos (see GrayPacket.fromArgb). + */ + @JvmStatic + fun toneLut(limitedRange: Boolean, gamma: Float, monochrome: Boolean): ByteArray { + val curve = GrayPacket.toneCurve(gamma) + val lut = ByteArray(256) + for (y in 0 until 256) { + val full = if (limitedRange) ((y - 16) * 255 + 109) / 219 else y + val v = full.coerceIn(0, 255) + lut[y] = (if (monochrome) (if (v >= 128) 255 else 0) else curve[v]).toByte() + } + return lut + } + + /** + * Sample positions for each output index along one axis: [count] entries per output, + * evenly spaced inside that output's box of source pixels. + */ + private fun taps(source: Int, output: Int, count: Int): IntArray { + val positions = IntArray(output * count) + for (i in 0 until output) { + for (j in 0 until count) { + // Center of sub-box j of output i, in source pixels: (i + (j + 0.5)/count) * source / output. + val numerator = (i.toLong() * count + j) * 2 + 1 + val position = (numerator * source / (2L * count * output)).toInt() + positions[i * count + j] = position.coerceIn(0, source - 1) + } + } + return positions + } + } + + private val swapsAxes = rotation == 90 || rotation == 270 + /** Scaled size before rotation (coded orientation). */ + private val scaledWidth = if (swapsAxes) outputHeight else outputWidth + private val scaledHeight = if (swapsAxes) outputWidth else outputHeight + private val tapsX = ((sourceWidth + scaledWidth - 1) / scaledWidth).coerceIn(1, MAX_TAPS) + private val tapsY = ((sourceHeight + scaledHeight - 1) / scaledHeight).coerceIn(1, MAX_TAPS) + private val xs = taps(sourceWidth, scaledWidth, tapsX) + private val ys = taps(sourceHeight, scaledHeight, tapsY) + private val sums = IntArray(scaledWidth) + /** Unrotated scaled frame, needed only when a rotation follows. */ + private val scaled: ByteArray? = if (rotation == 0) null else ByteArray(scaledWidth * scaledHeight) + + init { + require(sourceWidth > 0 && sourceHeight > 0) { "bad source size ${sourceWidth}x$sourceHeight" } + require(outputWidth > 0 && outputHeight > 0) { "bad output size ${outputWidth}x$outputHeight" } + require(rotation == 0 || rotation == 90 || rotation == 180 || rotation == 270) { "bad rotation $rotation" } + } + + /** + * Scale the luma plane into [out] (outputWidth * outputHeight bytes, row-major). The visible + * pixel (x, y) is at luma[offset + y * rowStride + x * pixelStride]; [lut] is a 256-entry + * table from [toneLut]. + */ + fun scale(luma: ByteArray, offset: Int, rowStride: Int, pixelStride: Int, lut: ByteArray, out: ByteArray) { + require(out.size >= outputWidth * outputHeight) { "output buffer too small" } + val target = scaled ?: out + val count = tapsX * tapsY + val half = count / 2 + for (oy in 0 until scaledHeight) { + sums.fill(0) + for (ty in 0 until tapsY) { + val row = offset + ys[oy * tapsY + ty] * rowStride + var tap = 0 + for (ox in 0 until scaledWidth) { + var sum = 0 + for (tx in 0 until tapsX) { + sum += luma[row + xs[tap++] * pixelStride].toInt() and 0xff + } + sums[ox] += sum + } + } + val base = oy * scaledWidth + for (ox in 0 until scaledWidth) { + target[base + ox] = lut[(sums[ox] + half) / count] + } + } + if (scaled != null) rotate(scaled, out) + } + + /** Rotate the coded-orientation frame clockwise by [rotation] into display orientation. */ + private fun rotate(source: ByteArray, out: ByteArray) { + val w = scaledWidth + val h = scaledHeight + for (y in 0 until outputHeight) { + val base = y * outputWidth + for (x in 0 until outputWidth) { + out[base + x] = when (rotation) { + 90 -> source[(h - 1 - x) * w + y] + 180 -> source[(h - 1 - y) * w + (w - 1 - x)] + else -> source[x * w + (w - 1 - y)] + } + } + } + } +} diff --git a/tests/ios-evenhub.test.cjs b/tests/ios-evenhub.test.cjs index 7b03403c..a1c114d3 100644 --- a/tests/ios-evenhub.test.cjs +++ b/tests/ios-evenhub.test.cjs @@ -56,6 +56,7 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path' const load = loader({ global: { isIOS: true } }, { '../../native/file-access': {}, '../../native/font-files': { isFontFile: () => false }, '../../graphics/installed-fonts': {}, '../../native/image-files': { isDecodableImageFile: () => false }, + '../../native/video-player': { isPlayableVideoFile: () => true }, '../evenhub/installed-apps': { readEvenHubPackageManifest: () => null }, '../evenhub/permission-dialog': {}, './file-browser': { FileBrowserLayer: class { constructor(options) { browserOptions = options; } } }, './file-info-dialog': { FileInfoDialogLayer: class { constructor(entry, value) { actions = value; } } }, @@ -64,6 +65,8 @@ test('Files on iOS exposes Run app for EHPKs and passes the selected local path' }); load('app/apps/files/files-app.ts').createFilesAppWindow({ openEhpkApp: path => { opened = path; }, installEhpkApp: path => { installed = path; } }); assert.equal(browserOptions.isSupportedFile('test.EHPK'), true); + // Video playback is Android-only for now. + assert.equal(browserOptions.isSupportedFile('clip.mp4'), false); const ctx = { stack: { push() {}, pop() {} } }; browserOptions.onFilePicked({ name: 'test.ehpk', path: '/Documents/test.ehpk' }, ctx); assert.deepEqual(Array.from(actions, a => a.label), ['Run app', 'Install']); diff --git a/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VideoFrameScalerTest.kt b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VideoFrameScalerTest.kt new file mode 100644 index 00000000..144369e4 --- /dev/null +++ b/tests/kotlin/src/commonTest/kotlin/com/faceclaw/app/VideoFrameScalerTest.kt @@ -0,0 +1,75 @@ +package com.faceclaw.app + +import kotlin.test.* + +class VideoFrameScalerTest { + private val identity = ByteArray(256) { it.toByte() } + + private fun scale(scaler: VideoFrameScaler, luma: ByteArray, lut: ByteArray = identity): IntArray { + val out = ByteArray(scaler.outputWidth * scaler.outputHeight) + scaler.scale(luma, 0, scaler.sourceWidth, 1, lut, out) + return IntArray(out.size) { out[it].toInt() and 0xff } + } + + @Test fun twoToOneDownscaleAveragesEachBox() { + // 4x2 source -> 2x1: each output is the mean of a 2x2 box. + val luma = byteArrayOf(0, 100, 200.toByte(), 200.toByte(), 20, 80, 100, 100) + val scaler = VideoFrameScaler(4, 2, 0, 2, 1) + assertContentEquals(intArrayOf(50, 150), scale(scaler, luma)) + } + + @Test fun largeDownscaleStaysWithinTheTapBudget() { + // 100:1 horizontally still averages only MAX_TAPS samples, spread across the box. + val luma = ByteArray(200) { (if (it % 100 < 50) 0 else 200).toByte() } + val scaler = VideoFrameScaler(200, 1, 0, 2, 1) + assertContentEquals(intArrayOf(100, 100), scale(scaler, luma)) + } + + @Test fun upscaleIsNearestNeighbor() { + val scaler = VideoFrameScaler(2, 1, 0, 4, 2) + assertContentEquals(intArrayOf(10, 10, 90, 90, 10, 10, 90, 90), scale(scaler, byteArrayOf(10, 90))) + } + + @Test fun offsetAndStridesSelectTheVisiblePlane() { + // A 2x2 visible crop at (1, 1) of a 4-wide plane, with interleaved samples (pixelStride 2). + val plane = ByteArray(40) + fun put(x: Int, y: Int, v: Int) { plane[(1 + y) * 10 + 1 + x * 2] = v.toByte() } + put(0, 0, 11); put(1, 0, 22); put(0, 1, 33); put(1, 1, 44) + val scaler = VideoFrameScaler(2, 2, 0, 2, 2) + val out = ByteArray(4) + scaler.scale(plane, 11, 10, 2, identity, out) + assertContentEquals(byteArrayOf(11, 22, 33, 44), out) + } + + @Test fun rotationsMatchTheContainerClockwiseConvention() { + // Coded 3x2: 1 2 3 + // 4 5 6 + val luma = byteArrayOf(1, 2, 3, 4, 5, 6) + assertContentEquals(intArrayOf(4, 1, 5, 2, 6, 3), scale(VideoFrameScaler(3, 2, 90, 2, 3), luma)) + assertContentEquals(intArrayOf(6, 5, 4, 3, 2, 1), scale(VideoFrameScaler(3, 2, 180, 3, 2), luma)) + assertContentEquals(intArrayOf(3, 6, 2, 5, 1, 4), scale(VideoFrameScaler(3, 2, 270, 2, 3), luma)) + } + + @Test fun normalizeRotationSnapsToQuarterTurns() { + assertEquals(0, VideoFrameScaler.normalizeRotation(0)) + assertEquals(90, VideoFrameScaler.normalizeRotation(90)) + assertEquals(270, VideoFrameScaler.normalizeRotation(-90)) + assertEquals(180, VideoFrameScaler.normalizeRotation(540)) + assertEquals(0, VideoFrameScaler.normalizeRotation(359)) + } + + @Test fun limitedRangeStretchesAndMonochromeThresholds() { + val gray = VideoFrameScaler.toneLut(limitedRange = true, gamma = 1f, monochrome = false) + assertEquals(0, gray[16].toInt() and 0xff) + assertEquals(0, gray[0].toInt() and 0xff) + assertEquals(255, gray[235].toInt() and 0xff) + assertEquals(255, gray[255].toInt() and 0xff) + val mono = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = true) + assertEquals(0, mono[127].toInt() and 0xff) + assertEquals(255, mono[128].toInt() and 0xff) + // Gamma darkens midtones but leaves the ends alone. + val photo = VideoFrameScaler.toneLut(limitedRange = false, gamma = 2.2f, monochrome = false) + assertTrue((photo[128].toInt() and 0xff) < 64) + assertEquals(255, photo[255].toInt() and 0xff) + } +} diff --git a/tests/tsconfig.json b/tests/tsconfig.json index d743aa4b..bfc76199 100644 --- a/tests/tsconfig.json +++ b/tests/tsconfig.json @@ -36,6 +36,7 @@ "../app/g2/glasses-display-state.ts", "../app/g2/glance-state.ts", "../app/apps/glanceboard/layout.ts", + "../app/apps/files/video-layout.ts", "../app/g2/pairing-candidates.ts", "../app/apps/calculator/math/*.ts", "../app/apps/microphones/mic-protocol.ts", diff --git a/tests/video-layout.test.cjs b/tests/video-layout.test.cjs new file mode 100644 index 00000000..764df76f --- /dev/null +++ b/tests/video-layout.test.cjs @@ -0,0 +1,57 @@ +const test = require("node:test"); +const assert = require("node:assert/strict"); + +const { + fitVideo, + formatMediaTime, + videoHeightMode, + videoRect, +} = require("../.test-build/app/apps/files/video-layout.js"); + +const WINDOW = { width: 576, height: 260 }; +const TALL_WINDOW = { width: 576, height: 452 }; +// ../g2flash/demos/badapple.mp4 +const BAD_APPLE = { width: 1444, height: 1080 }; + +test("fixed sizes fit their box and keep the aspect ratio", () => { + assert.deepEqual(videoRect("small", BAD_APPLE, WINDOW), { x: 208, y: 70, width: 160, height: 120 }); + assert.deepEqual(videoRect("medium", BAD_APPLE, WINDOW), { x: 168, y: 40, width: 240, height: 180 }); + const wide = videoRect("small", { width: 1920, height: 1080 }, WINDOW); + assert.equal(wide.width, 160); + assert.equal(wide.height, 90); +}); + +test("large fills the window height; full screen fills the tall window's width", () => { + assert.deepEqual(videoRect("large", BAD_APPLE, WINDOW), { x: 112, y: 0, width: 348, height: 260 }); + assert.deepEqual(videoRect("full", BAD_APPLE, TALL_WINDOW), { x: 0, y: 10, width: 576, height: 430 }); + assert.equal(videoHeightMode("full"), "max"); + assert.equal(videoHeightMode("large"), "min"); +}); + +test("rects are aligned for the delta encoder and stay inside the viewport", () => { + for (const size of ["small", "medium", "large", "full"]) { + for (const video of [BAD_APPLE, { width: 1080, height: 1920 }, { width: 100, height: 20 }, { width: 7, height: 5 }]) { + for (const viewport of [WINDOW, TALL_WINDOW, { width: 150, height: 100 }]) { + const rect = videoRect(size, video, viewport); + const where = `${size} ${video.width}x${video.height} in ${viewport.width}x${viewport.height}`; + assert.equal(rect.x % 4, 0, where); + assert.equal(rect.y % 2, 0, where); + assert.equal(rect.width % 2, 0, where); + assert.equal(rect.height % 2, 0, where); + assert.ok(rect.width >= 2 && rect.height >= 2, where); + assert.ok(rect.x + rect.width <= viewport.width, where); + assert.ok(rect.y + rect.height <= viewport.height, where); + } + } + } +}); + +test("a tiny video is scaled up to the chosen size", () => { + assert.deepEqual(fitVideo({ width: 80, height: 60 }, { width: 240, height: 180 }), { width: 240, height: 180 }); +}); + +test("media times", () => { + assert.equal(formatMediaTime(0), "0:00"); + assert.equal(formatMediaTime(232_130), "3:52"); + assert.equal(formatMediaTime(3_723_000), "1:02:03"); +});