mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
135 lines
4.9 KiB
TypeScript
135 lines
4.9 KiB
TypeScript
declare const com: any;
|
|
declare const java: any;
|
|
|
|
/**
|
|
* On-device language identification and translation for captions, over ML
|
|
* Kit (bundled language-id, per-pair translation models downloaded on
|
|
* demand). Foreign-language captions show the original line with the
|
|
* translation to the phone's default language beneath it.
|
|
*/
|
|
|
|
let languageIdentifier: any = null;
|
|
// Translators are expensive to build (model download on first use); cache by
|
|
// "src>dst" and keep the proxies referenced so they aren't GC'd mid-flight.
|
|
const translators = new Map<string, any>();
|
|
const downloadedPairs = new Set<string>();
|
|
|
|
/** The phone's default language as a two-letter code (the translation target). */
|
|
export function deviceLanguage(): string {
|
|
if (!global.isAndroid) return "en";
|
|
try {
|
|
// toLanguageTag() gives the modern code ("he", "id", "yi") that ML Kit
|
|
// uses; getLanguage() returns the legacy "iw", "in", "ji".
|
|
const tag = String(java.util.Locale.getDefault().toLanguageTag() || "");
|
|
return tag.split("-")[0] || "en";
|
|
} catch {
|
|
return "en";
|
|
}
|
|
}
|
|
|
|
function successListener(resolve: (value: any) => void): any {
|
|
return new com.google.android.gms.tasks.OnSuccessListener({
|
|
onSuccess: (result: any) => resolve(result),
|
|
});
|
|
}
|
|
|
|
function failureListener(reject: (reason: Error) => void): any {
|
|
return new com.google.android.gms.tasks.OnFailureListener({
|
|
onFailure: (error: any) => reject(new Error(String(error?.getMessage?.() ?? error))),
|
|
});
|
|
}
|
|
|
|
/**
|
|
* Identify the language of a caption line. Resolves to a BCP-47 code, or
|
|
* "und" when ML Kit isn't confident. Short lines are noisy; callers should
|
|
* apply hysteresis across consecutive finals before switching languages.
|
|
*/
|
|
export function identifyLanguage(text: string): Promise<string> {
|
|
if (!global.isAndroid || !text.trim()) return Promise.resolve("und");
|
|
return new Promise((resolve) => {
|
|
try {
|
|
if (!languageIdentifier) {
|
|
languageIdentifier = com.google.mlkit.nl.languageid.LanguageIdentification.getClient();
|
|
}
|
|
languageIdentifier
|
|
.identifyLanguage(text)
|
|
.addOnSuccessListener(successListener((code: any) => resolve(String(code))))
|
|
.addOnFailureListener(failureListener(() => resolve("und")));
|
|
} catch (error) {
|
|
console.warn(`language id failed: ${error}`);
|
|
resolve("und");
|
|
}
|
|
});
|
|
}
|
|
|
|
function translatorFor(sourceLang: string, targetLang: string): any {
|
|
const key = `${sourceLang}>${targetLang}`;
|
|
const cached = translators.get(key);
|
|
if (cached) return cached;
|
|
try {
|
|
const TranslateLanguage = com.google.mlkit.nl.translate.TranslateLanguage;
|
|
const source = TranslateLanguage.fromLanguageTag(sourceLang);
|
|
const target = TranslateLanguage.fromLanguageTag(targetLang);
|
|
if (source == null || target == null) return null;
|
|
const options = new com.google.mlkit.nl.translate.TranslatorOptions.Builder()
|
|
.setSourceLanguage(source)
|
|
.setTargetLanguage(target)
|
|
.build();
|
|
const translator = com.google.mlkit.nl.translate.Translation.getClient(options);
|
|
translators.set(key, translator);
|
|
return translator;
|
|
} catch (error) {
|
|
console.warn(`translator setup failed (${key}): ${error}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Translate one caption line. Resolves to the translated text, or "" when the
|
|
* pair is unsupported or the model download hasn't finished (translation of
|
|
* later lines picks up once it has; downloads are WiFi-unrestricted since
|
|
* caption sessions are live).
|
|
*/
|
|
export function translateText(text: string, sourceLang: string, targetLang: string): Promise<string> {
|
|
if (!global.isAndroid || !text.trim() || sourceLang === targetLang) {
|
|
return Promise.resolve("");
|
|
}
|
|
const translator = translatorFor(sourceLang, targetLang);
|
|
if (!translator) return Promise.resolve("");
|
|
const key = `${sourceLang}>${targetLang}`;
|
|
return new Promise((resolve) => {
|
|
// Also runs from the download listener, where a throw would escape into
|
|
// the Play Services callback and leave this promise unsettled.
|
|
const runTranslate = () => {
|
|
try {
|
|
translator
|
|
.translate(text)
|
|
.addOnSuccessListener(successListener((result: any) => resolve(String(result))))
|
|
.addOnFailureListener(failureListener(() => resolve("")));
|
|
} catch (error) {
|
|
console.warn(`translation failed (${key}): ${error}`);
|
|
resolve("");
|
|
}
|
|
};
|
|
if (downloadedPairs.has(key)) {
|
|
runTranslate();
|
|
return;
|
|
}
|
|
try {
|
|
const conditions = new com.google.mlkit.common.model.DownloadConditions.Builder().build();
|
|
translator
|
|
.downloadModelIfNeeded(conditions)
|
|
.addOnSuccessListener(
|
|
successListener(() => {
|
|
downloadedPairs.add(key);
|
|
runTranslate();
|
|
}),
|
|
)
|
|
.addOnFailureListener(failureListener(() => resolve("")));
|
|
} catch (error) {
|
|
console.warn(`translation model download failed (${key}): ${error}`);
|
|
resolve("");
|
|
}
|
|
});
|
|
}
|