mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Stop leaking every submitted frame's pixel buffer in native memory
Passing a JS ArrayBuffer to a Java ByteBuffer parameter leaks it in @nativescript/android 9.0.4 through 9.1.1: JsArgConverter wraps the backing store with NewDirectByteBuffer and takes a JNI global ref that it never deletes. The ref keeps the ByteBuffer reachable, so makeInstanceWeakAndCheckIfAlive always answers "alive", the JS finalizer re-arms, and V8 never frees the ArrayBuffer. submitSurfaceFrame passed a fresh copy of each frame this way, so every frame leaked a full surface (576x260, 640x452 or 640x480 bytes) plus one global ref, including frames that Java then discards as unchanged. The draw-list buffer and buzzer payloads leaked the same way. Each payload now goes through a reused Java direct ByteBuffer (app/native/java-direct-buffer.ts): JS writes into it through an ArrayBuffer.from view, and the Java object itself is passed, which takes NativeScript's ordinary object path. Every consumer (SurfaceCompositor applyAndComposite, playBuzzerSequence) copies before returning, so one buffer per JS context is safe. Covered: the communicator bridge (frames, draw lists, buzzer, which also carries EvenHub tones), the preview-only display, and all eight worker apps. Workers submit through the Android getActiveDisplay() (frames) and worker-buzzer (tones), so the reused buffers live there rather than in each worker; the worker sources are shared with iOS, whose active display takes the ArrayBuffer itself. The smaller one-off crossings (glyph and image atlas registration, the gray preview helper, firmware and file writes) are left as they are. tests/java-direct-buffer.test.cjs covers the helper against a fake ByteBuffer, and the Android worker-buzzer test stubs it. (Rebased onto main by James Babcock: worker changes moved into active-display.ts and worker-buzzer.ts.) Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
833f9d97f0
commit
54eae0c6b9
8 files changed
+283
-17
No files matched your search
@@ -1,18 +1,54 @@
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
// Reused Java-side buffers for this isolate's frames: passing a JS ArrayBuffer
|
||||
// to Java leaks it (java-direct-buffer.ts). Module state is per isolate, so
|
||||
// each worker gets its own pair.
|
||||
const framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
const frameDrawsBuffer = new JavaDirectBuffer();
|
||||
|
||||
/**
|
||||
* The Java object a worker isolate submits surface frames to: the live BLE
|
||||
* The display a worker isolate submits surface frames to: the live BLE
|
||||
* communicator, or the preview-only compositor standing in when no glasses
|
||||
* are paired (both expose the same submitSurfaceFrame signature). Located
|
||||
* through Java statics because those are shared across isolates, unlike any
|
||||
* JS-side state.
|
||||
* JS-side state. Null when neither exists.
|
||||
*
|
||||
* Frame submission only. Buzzer effects use worker-buzzer, which routes to
|
||||
* the platform BLE session and skips playback when disconnected.
|
||||
*/
|
||||
export function getActiveDisplay(): any {
|
||||
export function getActiveDisplay(): typeof display | null {
|
||||
return activeJavaDisplay() ? display : null;
|
||||
}
|
||||
|
||||
function activeJavaDisplay(): any {
|
||||
return (
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive() ??
|
||||
com.faceclaw.app.FaceclawPreviewCompositor.getActive()
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Takes the same ArrayBuffer arguments as the iOS active display and copies
|
||||
* them into the reused Java buffers right before the synchronous Java call.
|
||||
*/
|
||||
const display = {
|
||||
submitSurfaceFrame(buffer: ArrayBuffer, surfaceId: string, x: number, y: number, width: number, height: number,
|
||||
fingerprint: string, paintMs: number, frameId: number, draws: ArrayBuffer | null = null): void {
|
||||
const target = activeJavaDisplay();
|
||||
if (!target) return;
|
||||
target.submitSurfaceFrame(
|
||||
framePixelsBuffer.load(new Uint8Array(buffer)),
|
||||
surfaceId,
|
||||
x,
|
||||
y,
|
||||
width,
|
||||
height,
|
||||
fingerprint,
|
||||
paintMs,
|
||||
frameId,
|
||||
frameDrawsBuffer.loadOptional(draws),
|
||||
);
|
||||
},
|
||||
};
|
||||
@@ -1,6 +1,7 @@
|
||||
import type { RingInput } from "../g2/ring-input";
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import * as frameTimings from "./frame-timings";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
@@ -132,6 +133,12 @@ export class FaceclawCommunicatorBridge {
|
||||
private javaCallQueue: Promise<void> = Promise.resolve();
|
||||
/** Calls waiting on javaCallQueue; 0 means enqueueJavaCall's fast path is safe. */
|
||||
private queuedJavaCalls = 0;
|
||||
// Reused Java-side buffers for byte payloads; passing a JS ArrayBuffer to
|
||||
// Java leaks it (see java-direct-buffer.ts). Loaded inside the queued call,
|
||||
// immediately before the synchronous Java call that consumes them.
|
||||
private readonly framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
private readonly frameDrawsBuffer = new JavaDirectBuffer();
|
||||
private readonly buzzerBuffer = new JavaDirectBuffer();
|
||||
private readonly frameMetricWaiters = new Set<(metrics: FrameMetrics) => void>();
|
||||
// Recent frame-finished outcomes from the Java side, so waitForFrameFinished
|
||||
// does not race against finishes that land before the wait starts.
|
||||
@@ -594,15 +601,17 @@ export class FaceclawCommunicatorBridge {
|
||||
*/
|
||||
glyphs: ArrayBuffer | null = null,
|
||||
): Promise<void> {
|
||||
// Snapshot because the Java call is deferred; the buffer is passed as an
|
||||
// ArrayBuffer, which NativeScript marshals to a ByteBuffer without the
|
||||
// ~150ms per-element copy a byte[] parameter would need.
|
||||
// Snapshot because the Java call is deferred. The bytes reach Java as a
|
||||
// ByteBuffer (no ~150ms per-element copy, as a byte[] parameter would
|
||||
// need), but through a reused Java direct buffer rather than by passing
|
||||
// the ArrayBuffer itself, which NativeScript never frees
|
||||
// (java-direct-buffer.ts).
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
// inlineWhenIdle: this is the frame path, the Java side of it measures
|
||||
// ~5ms (composite + pack), and the caller awaits it either way.
|
||||
await this.enqueueJavaCall(() => {
|
||||
this.communicator.submitSurfaceFrame(
|
||||
snapshot.buffer,
|
||||
this.framePixelsBuffer.load(snapshot),
|
||||
surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
@@ -611,7 +620,7 @@ export class FaceclawCommunicatorBridge {
|
||||
fingerprint,
|
||||
Math.round(nonNegativeNumber(paintMs)),
|
||||
Math.round(nonNegativeNumber(frameId)),
|
||||
glyphs,
|
||||
this.frameDrawsBuffer.loadOptional(glyphs),
|
||||
);
|
||||
}, true);
|
||||
}
|
||||
@@ -673,7 +682,7 @@ export class FaceclawCommunicatorBridge {
|
||||
async playBuzzerSequence(payload: Uint8Array): Promise<void> {
|
||||
const snapshot = new Uint8Array(payload);
|
||||
await this.enqueueJavaCall(() => {
|
||||
this.communicator.playBuzzerSequence(snapshot.buffer);
|
||||
this.communicator.playBuzzerSequence(this.buzzerBuffer.load(snapshot));
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,95 @@
|
||||
/**
|
||||
* Hands bytes to a Java `java.nio.ByteBuffer` parameter without leaking them.
|
||||
*
|
||||
* Why this exists: NativeScript Android (9.0.4 through at least 9.1.1)
|
||||
* converts a JS ArrayBuffer or typed array passed to a Java method by wrapping
|
||||
* its backing store in `NewDirectByteBuffer` and taking a JNI global ref that is
|
||||
* never deleted (JsArgConverter.cpp, `buffer = env.NewGlobalRef(buffer)`). The
|
||||
* ByteBuffer is also linked to the JS ArrayBuffer as its native counterpart.
|
||||
* The leaked global ref keeps the ByteBuffer reachable forever, so the
|
||||
* runtime's GC handshake (Runtime.makeInstanceWeakAndCheckIfAlive) always says
|
||||
* "alive", the JS finalizer re-arms, and V8 never frees the ArrayBuffer. Every
|
||||
* call leaked its whole buffer plus one global ref.
|
||||
*
|
||||
* On the frame path that meant a full surface (576x260, 640x452 or 640x480
|
||||
* bytes) per submitted frame, including frames the Java side then discards as
|
||||
* unchanged. Measured on a Galaxy Z Fold7, native-heap growth matched frame
|
||||
* bytes submitted to within a few percent, and Samsung's Heimdall memory
|
||||
* manager killed the app at about 6.4 GB. ART also aborts the process once
|
||||
* 51,200 global refs are held, so the ref leak is a second ceiling behind the
|
||||
* memory one.
|
||||
*
|
||||
* The workaround keeps the conversion out of the call: the bytes are copied
|
||||
* into a Java-allocated direct ByteBuffer that is reused, through an
|
||||
* ArrayBuffer view made once with `ArrayBuffer.from` (an external backing
|
||||
* store over the Java memory, no global ref), and the Java ByteBuffer object
|
||||
* itself is passed. A JS proxy for an existing Java object takes the ordinary
|
||||
* object path, which does not leak.
|
||||
*
|
||||
* Contract: the buffer returned by load() is only valid until the next load()
|
||||
* on the same instance, so the Java callee must consume it synchronously and
|
||||
* not keep a reference. The Kotlin consumers (SurfaceCompositor via
|
||||
* AndroidByteReader, playBuzzerSequence, ShellScene.decode) copy or parse
|
||||
* before returning, which satisfies this. One instance belongs to one JS
|
||||
* context (main thread or one worker); instances are not shared across
|
||||
* isolates.
|
||||
*/
|
||||
|
||||
declare const java: any;
|
||||
|
||||
/** Minimal surface of java.nio.ByteBuffer this module uses. */
|
||||
export interface JavaByteBufferLike {
|
||||
clear(): unknown;
|
||||
limit(newLimit: number): unknown;
|
||||
}
|
||||
|
||||
/** How to allocate a direct buffer and view its memory; injectable for tests. */
|
||||
export interface DirectBufferBackend {
|
||||
allocateDirect(capacity: number): JavaByteBufferLike;
|
||||
viewOf(buffer: JavaByteBufferLike): Uint8Array;
|
||||
}
|
||||
|
||||
const nativeScriptBackend: DirectBufferBackend = {
|
||||
allocateDirect: (capacity) => java.nio.ByteBuffer.allocateDirect(capacity),
|
||||
viewOf: (buffer) => new Uint8Array((ArrayBuffer as any).from(buffer)),
|
||||
};
|
||||
|
||||
/** Capacity grows in steps of this many bytes, so a slowly growing payload does not reallocate every call. */
|
||||
const CAPACITY_STEP_BYTES = 4096;
|
||||
|
||||
export class JavaDirectBuffer {
|
||||
private javaBuffer: JavaByteBufferLike | null = null;
|
||||
private view: Uint8Array | null = null;
|
||||
/** Number of Java buffers allocated so far (a test and diagnostics hook). */
|
||||
allocations = 0;
|
||||
|
||||
constructor(
|
||||
private readonly initialCapacity = 0,
|
||||
private readonly backend: DirectBufferBackend = nativeScriptBackend,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Copy `bytes` into the reused Java buffer and return it with position 0 and
|
||||
* limit `bytes.length`, so `remaining()` equals the byte count. Valid until
|
||||
* the next load() on this instance.
|
||||
*/
|
||||
load(bytes: Uint8Array): JavaByteBufferLike {
|
||||
if (!this.javaBuffer || !this.view || this.view.length < bytes.length) {
|
||||
const wanted = Math.max(bytes.length, this.initialCapacity, 1);
|
||||
const capacity = Math.ceil(wanted / CAPACITY_STEP_BYTES) * CAPACITY_STEP_BYTES;
|
||||
this.javaBuffer = this.backend.allocateDirect(capacity);
|
||||
this.view = this.backend.viewOf(this.javaBuffer);
|
||||
this.allocations++;
|
||||
}
|
||||
this.view.set(bytes);
|
||||
this.javaBuffer.clear();
|
||||
this.javaBuffer.limit(bytes.length);
|
||||
return this.javaBuffer;
|
||||
}
|
||||
|
||||
/** As load(), for an optional ArrayBuffer (e.g. a frame's draw list); null stays null. */
|
||||
loadOptional(buffer: ArrayBuffer | null | undefined): JavaByteBufferLike | null {
|
||||
if (!buffer || buffer.byteLength === 0) return null;
|
||||
return this.load(new Uint8Array(buffer));
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,6 @@
|
||||
import { ImageSource, Utils } from "@nativescript/core";
|
||||
import { dimFactor256, type FaceclawCommunicatorBridge, type SurfaceOptions } from "./faceclaw-communicator";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
declare const java: any;
|
||||
@@ -49,6 +50,10 @@ function nonNegativeNumber(value: number): number {
|
||||
*/
|
||||
export class PreviewDisplayTarget implements DisplayTarget {
|
||||
private readonly compositor: any;
|
||||
// Reused Java-side buffers: passing a JS ArrayBuffer to Java leaks it
|
||||
// (java-direct-buffer.ts).
|
||||
private readonly framePixelsBuffer = new JavaDirectBuffer(640 * 480);
|
||||
private readonly frameDrawsBuffer = new JavaDirectBuffer();
|
||||
|
||||
constructor() {
|
||||
const context = Utils.android.getApplicationContext();
|
||||
@@ -124,11 +129,11 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
frameId = 0,
|
||||
glyphs: ArrayBuffer | null = null,
|
||||
): Promise<void> {
|
||||
// Copy for the same reason the bridge does: the source may be a view on a
|
||||
// larger buffer, and Java receives the backing ArrayBuffer.
|
||||
const snapshot = new Uint8Array(pixels8bpp);
|
||||
// Copied into a reused Java direct buffer, sized to exactly this view's
|
||||
// bytes; passing the ArrayBuffer itself would leak it
|
||||
// (java-direct-buffer.ts).
|
||||
this.compositor.submitSurfaceFrame(
|
||||
snapshot.buffer,
|
||||
this.framePixelsBuffer.load(pixels8bpp),
|
||||
surfaceId,
|
||||
Math.round(rect.x),
|
||||
Math.round(rect.y),
|
||||
@@ -137,7 +142,7 @@ export class PreviewDisplayTarget implements DisplayTarget {
|
||||
fingerprint,
|
||||
Math.round(nonNegativeNumber(paintMs)),
|
||||
Math.round(nonNegativeNumber(frameId)),
|
||||
glyphs,
|
||||
this.frameDrawsBuffer.loadOptional(glyphs),
|
||||
);
|
||||
// The Java side finishes the frame: the composite is the end of the line,
|
||||
// and worker isolates submit to the same object without coming through here.
|
||||
|
||||
@@ -1,12 +1,17 @@
|
||||
import type { WorkerAppReply } from "../ui/shell/worker-window";
|
||||
import { JavaDirectBuffer } from "./java-direct-buffer";
|
||||
|
||||
declare const com: any;
|
||||
|
||||
// Reused Java-side buffer for this isolate's tone payloads: passing a JS
|
||||
// ArrayBuffer to Java leaks it (java-direct-buffer.ts).
|
||||
const buzzerBuffer = new JavaDirectBuffer();
|
||||
|
||||
/** Workers use the main-thread BLE session on iOS and the native queue on Android. */
|
||||
export function playWorkerBuzzerSequence(payload: Uint8Array): void {
|
||||
if (global.isIOS) {
|
||||
global.postMessage({ type: "buzzer-sequence", payload: Array.from(payload) } satisfies WorkerAppReply);
|
||||
} else {
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(new Uint8Array(payload).buffer);
|
||||
com.faceclaw.app.FaceclawBleCommunicator.getActive()?.playBuzzerSequence(buzzerBuffer.load(payload));
|
||||
}
|
||||
}
|
||||
@@ -30,10 +30,14 @@ test('iOS game worker forwards an immutable byte payload through its host and ig
|
||||
test('Android game bridge retains direct native playback and skips a missing communicator', () => {
|
||||
let active = null;
|
||||
const played = [];
|
||||
// Stands in for the reused Java direct buffer: the payload reaches Java as a
|
||||
// copy, never as the caller's own ArrayBuffer.
|
||||
const directBuffer = { JavaDirectBuffer: class { load(bytes) { return { bytes: bytes.slice() }; } } };
|
||||
const bridge = loader({ global: { isIOS: false },
|
||||
com: { faceclaw: { app: { FaceclawBleCommunicator: { getActive: () => active } } } } })('app/native/worker-buzzer.ts');
|
||||
com: { faceclaw: { app: { FaceclawBleCommunicator: { getActive: () => active } } } } },
|
||||
{ './java-direct-buffer': directBuffer })('app/native/worker-buzzer.ts');
|
||||
bridge.playWorkerBuzzerSequence(payload);
|
||||
active = { playBuzzerSequence: buffer => played.push(new Uint8Array(buffer)) };
|
||||
active = { playBuzzerSequence: buffer => played.push(buffer.bytes) };
|
||||
const backing = new Uint8Array(payload.length + 2); backing.set(payload, 1);
|
||||
bridge.playWorkerBuzzerSequence(backing.subarray(1, -1)); backing.fill(0);
|
||||
assert.deepEqual(played, [payload]);
|
||||
|
||||
@@ -0,0 +1,111 @@
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
|
||||
const { JavaDirectBuffer } = require("../.test-build/app/native/java-direct-buffer.js");
|
||||
|
||||
/**
|
||||
* A stand-in for java.nio.ByteBuffer.allocateDirect: fixed capacity, a
|
||||
* position and a limit, and the relative bulk get() SurfaceCompositor uses.
|
||||
* viewOf() plays ArrayBuffer.from: a view over the same memory, not a copy.
|
||||
*/
|
||||
function fakeBackend() {
|
||||
const allocated = [];
|
||||
return {
|
||||
allocated,
|
||||
allocateDirect(capacity) {
|
||||
const memory = new Uint8Array(capacity);
|
||||
const buffer = {
|
||||
memory,
|
||||
position: 0,
|
||||
limitValue: capacity,
|
||||
clear() {
|
||||
this.position = 0;
|
||||
this.limitValue = capacity;
|
||||
return this;
|
||||
},
|
||||
limit(newLimit) {
|
||||
if (newLimit > capacity) throw new Error("limit past capacity");
|
||||
this.limitValue = newLimit;
|
||||
if (this.position > newLimit) this.position = newLimit;
|
||||
return this;
|
||||
},
|
||||
remaining() {
|
||||
return this.limitValue - this.position;
|
||||
},
|
||||
get(dst, offset, length) {
|
||||
if (length > this.remaining()) throw new Error("BufferUnderflowException");
|
||||
dst.set(memory.subarray(this.position, this.position + length), offset);
|
||||
this.position += length;
|
||||
return this;
|
||||
},
|
||||
};
|
||||
allocated.push(buffer);
|
||||
return buffer;
|
||||
},
|
||||
viewOf(buffer) {
|
||||
return buffer.memory;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Consume the way SurfaceCompositor.applyAndComposite does: exact remaining(), then drain. */
|
||||
function consume(buffer, expectedBytes) {
|
||||
assert.equal(buffer.remaining(), expectedBytes);
|
||||
const out = new Uint8Array(expectedBytes);
|
||||
buffer.get(out, 0, expectedBytes);
|
||||
return out;
|
||||
}
|
||||
|
||||
function frame(length, seed) {
|
||||
const bytes = new Uint8Array(length);
|
||||
for (let i = 0; i < length; i++) bytes[i] = (i * 31 + seed) & 0xff;
|
||||
return bytes;
|
||||
}
|
||||
|
||||
test("a loaded buffer exposes exactly the payload bytes to a Java-style reader", () => {
|
||||
const backend = fakeBackend();
|
||||
const pool = new JavaDirectBuffer(640 * 480, backend);
|
||||
const shell = frame(640 * 480, 1);
|
||||
assert.deepEqual(consume(pool.load(shell), shell.length), shell);
|
||||
});
|
||||
|
||||
test("successive frames reuse one Java buffer, including smaller ones", () => {
|
||||
const backend = fakeBackend();
|
||||
const pool = new JavaDirectBuffer(640 * 480, backend);
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const size = i % 2 === 0 ? 640 * 480 : 640 * 452;
|
||||
const bytes = frame(size, i);
|
||||
assert.deepEqual(consume(pool.load(bytes), size), bytes);
|
||||
}
|
||||
assert.equal(pool.allocations, 1);
|
||||
assert.equal(backend.allocated.length, 1);
|
||||
});
|
||||
|
||||
test("a payload larger than the buffer allocates once more, then reuses", () => {
|
||||
const backend = fakeBackend();
|
||||
const pool = new JavaDirectBuffer(0, backend);
|
||||
consume(pool.load(frame(10, 0)), 10);
|
||||
consume(pool.load(frame(9000, 0)), 9000);
|
||||
consume(pool.load(frame(8000, 0)), 8000);
|
||||
consume(pool.load(frame(9000, 0)), 9000);
|
||||
assert.equal(pool.allocations, 2);
|
||||
});
|
||||
|
||||
test("loadOptional passes null through and loads a draw list", () => {
|
||||
const backend = fakeBackend();
|
||||
const pool = new JavaDirectBuffer(0, backend);
|
||||
assert.equal(pool.loadOptional(null), null);
|
||||
assert.equal(pool.loadOptional(undefined), null);
|
||||
assert.equal(pool.loadOptional(new ArrayBuffer(0)), null);
|
||||
assert.equal(backend.allocated.length, 0);
|
||||
const draws = frame(36, 7);
|
||||
assert.deepEqual(consume(pool.loadOptional(draws.buffer), 36), draws);
|
||||
});
|
||||
|
||||
test("a view on a larger buffer contributes only its own bytes", () => {
|
||||
const backend = fakeBackend();
|
||||
const pool = new JavaDirectBuffer(0, backend);
|
||||
const whole = frame(100, 3);
|
||||
const view = whole.subarray(10, 30);
|
||||
assert.deepEqual(consume(pool.load(view), 20), whole.slice(10, 30));
|
||||
});
|
||||
@@ -27,6 +27,7 @@
|
||||
"../app/ui/metrics.ts",
|
||||
"../app/ui/menu-core.ts",
|
||||
"../app/phone-ui/phone-gestures.ts",
|
||||
"../app/native/java-direct-buffer.ts",
|
||||
"../app/g2/firmware-compat.ts",
|
||||
"../app/g2/glasses-hardware-identity.ts",
|
||||
"../app/g2/ble-proximity.ts",
|
||||
|
||||
Reference in new issue
Block a user