Files
faceclaw/app/apps/microphones/translate.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

135 lines
4.9 KiB
TypeScript

declare const com: any;
declare const java: any;
/**
* On-device language identification and translation for captions, over ML
* Kit (bundled language-id, per-pair translation models downloaded on
* demand). Foreign-language captions show the original line with the
* translation to the phone's default language beneath it.
*/
let languageIdentifier: any = null;
// Translators are expensive to build (model download on first use); cache by
// "src>dst" and keep the proxies referenced so they aren't GC'd mid-flight.
const translators = new Map<string, any>();
const downloadedPairs = new Set<string>();
/** The phone's default language as a two-letter code (the translation target). */
export function deviceLanguage(): string {
if (!global.isAndroid) return "en";
try {
// toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit
// uses; getLanguage() returns the legacy "iw", "in", "ji".
const tag = String(java.util.Locale.getDefault().toLanguageTag() || "");
return tag.split("-")[0] || "en";
} catch {
return "en";
}
}
function successListener(resolve: (value: any) => void): any {
return new com.google.android.gms.tasks.OnSuccessListener({
onSuccess: (result: any) => resolve(result),
});
}
function failureListener(reject: (reason: Error) => void): any {
return new com.google.android.gms.tasks.OnFailureListener({
onFailure: (error: any) => reject(new Error(String(error?.getMessage?.() ?? error))),
});
}
/**
* Identify the language of a caption line. Resolves to a BCP-47 code, or
* "und" when ML Kit isn't confident. Short lines are noisy; callers should
* apply hysteresis across consecutive finals before switching languages.
*/
export function identifyLanguage(text: string): Promise<string> {
if (!global.isAndroid || !text.trim()) return Promise.resolve("und");
return new Promise((resolve) => {
try {
if (!languageIdentifier) {
languageIdentifier = com.google.mlkit.nl.languageid.LanguageIdentification.getClient();
}
languageIdentifier
.identifyLanguage(text)
.addOnSuccessListener(successListener((code: any) => resolve(String(code))))
.addOnFailureListener(failureListener(() => resolve("und")));
} catch (error) {
console.warn(`language id failed: ${error}`);
resolve("und");
}
});
}
function translatorFor(sourceLang: string, targetLang: string): any {
const key = `${sourceLang}>${targetLang}`;
const cached = translators.get(key);
if (cached) return cached;
try {
const TranslateLanguage = com.google.mlkit.nl.translate.TranslateLanguage;
const source = TranslateLanguage.fromLanguageTag(sourceLang);
const target = TranslateLanguage.fromLanguageTag(targetLang);
if (source == null || target == null) return null;
const options = new com.google.mlkit.nl.translate.TranslatorOptions.Builder()
.setSourceLanguage(source)
.setTargetLanguage(target)
.build();
const translator = com.google.mlkit.nl.translate.Translation.getClient(options);
translators.set(key, translator);
return translator;
} catch (error) {
console.warn(`translator setup failed (${key}): ${error}`);
return null;
}
}
/**
* Translate one caption line. Resolves to the translated text, or "" when the
* pair is unsupported or the model download hasn't finished (translation of
* later lines picks up once it has; downloads are WiFi-unrestricted since
* caption sessions are live).
*/
export function translateText(text: string, sourceLang: string, targetLang: string): Promise<string> {
if (!global.isAndroid || !text.trim() || sourceLang === targetLang) {
return Promise.resolve("");
}
const translator = translatorFor(sourceLang, targetLang);
if (!translator) return Promise.resolve("");
const key = `${sourceLang}>${targetLang}`;
return new Promise((resolve) => {
// Also runs from the download listener, where a throw would escape into
// the Play Services callback and leave this promise unsettled.
const runTranslate = () => {
try {
translator
.translate(text)
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
.addOnFailureListener(failureListener(() => resolve("")));
} catch (error) {
console.warn(`translation failed (${key}): ${error}`);
resolve("");
}
};
if (downloadedPairs.has(key)) {
runTranslate();
return;
}
try {
const conditions = new com.google.mlkit.common.model.DownloadConditions.Builder().build();
translator
.downloadModelIfNeeded(conditions)
.addOnSuccessListener(
successListener(() => {
downloadedPairs.add(key);
runTranslate();
}),
)
.addOnFailureListener(failureListener(() => resolve("")));
} catch (error) {
console.warn(`translation model download failed (${key}): ${error}`);
resolve("");
}
});
}