mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
194 lines
6.6 KiB
TypeScript
194 lines
6.6 KiB
TypeScript
import { CLOUD_STT_SAMPLE_RATE, type CloudSttClient, type CloudSttOptions } from "./cloud-stt";
|
|
import { ReconnectingSttClient } from "./reconnecting-stt";
|
|
import { TRANSCRIPT_PAUSE_MS } from "./speech-pause";
|
|
|
|
declare const FaceclawSpeechAnalyzer: any;
|
|
|
|
/**
|
|
* How long finish() waits for the last results: the model only has to settle
|
|
* the few seconds of audio it hadn't finalized yet.
|
|
*/
|
|
export const SPEECH_ANALYZER_FINAL_TIMEOUT_MS = 10000;
|
|
|
|
/**
|
|
* A pause is detected TRANSCRIPT_PAUSE_MS after the last loud audio. A final
|
|
* whose last word ends within this much before that (the model's word ends
|
|
* against the detector's level threshold) leaves nothing before the pause
|
|
* unfinalized, so its paragraph is complete.
|
|
*/
|
|
const PARAGRAPH_TAIL_S = TRANSCRIPT_PAUSE_MS / 1000 + 0.25;
|
|
|
|
type AnalyzerEvent = {
|
|
kind: string;
|
|
message?: string;
|
|
locale?: string;
|
|
text?: string;
|
|
final?: boolean;
|
|
start?: number | null;
|
|
end?: number | null;
|
|
speechEnd?: number | null;
|
|
};
|
|
|
|
/** Whether Apple's SpeechAnalyzer (iOS 26+, supported hardware) can run continuous capture. */
|
|
export function speechAnalyzerSupported(): boolean {
|
|
try {
|
|
return typeof FaceclawSpeechAnalyzer !== "undefined" && Boolean(FaceclawSpeechAnalyzer.isSupported());
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* On-device long-form transcription for continuous capture, wrapped like a
|
|
* cloud provider: it buffers audio until the model is ready (downloading it on
|
|
* first use) and gets pause commits.
|
|
*/
|
|
export function createSpeechAnalyzerClient(
|
|
options: Omit<CloudSttOptions, "apiKey">,
|
|
continuous: () => boolean,
|
|
): ReconnectingSttClient {
|
|
return new ReconnectingSttClient(
|
|
(clientOptions) => new SpeechAnalyzerSttClient(clientOptions),
|
|
{ ...options, apiKey: "", language: (options.language ?? "").trim() },
|
|
continuous,
|
|
SPEECH_ANALYZER_FINAL_TIMEOUT_MS,
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Transcribes PCM with FaceclawSpeechAnalyzer (FaceclawSpeechAnalyzer.swift).
|
|
* The model finalizes text in its own pieces, a few seconds behind, and can't
|
|
* be made to settle at a pause; this gathers its pieces into paragraphs ended
|
|
* by pauses, like the cloud providers' segments. A paragraph is shown as
|
|
* provisional text and sent as one final (paragraphBreakAfter) once a final
|
|
* reaches its pause, and the last one as a plain final when the audio ends.
|
|
*
|
|
* Only finals place breaks, by when their last word ends (the model's
|
|
* word-end times are accurate; first words' starts reach back into the
|
|
* silence before them, and volatile results only carry the range of all audio
|
|
* not finalized yet).
|
|
*/
|
|
export class SpeechAnalyzerSttClient implements CloudSttClient {
|
|
private native: any = null;
|
|
private samples = 0;
|
|
private finishing = false;
|
|
// Audio times (s) of pauses whose paragraph hasn't ended yet.
|
|
private pauses: number[] = [];
|
|
// Finalized text since the last paragraph break, when its last word ends,
|
|
// and the model's provisional rest.
|
|
private paragraph: string[] = [];
|
|
private paragraphEnd: number | null = null;
|
|
private volatile = "";
|
|
|
|
constructor(private readonly options: CloudSttOptions) {}
|
|
|
|
start(): void {
|
|
if (this.native) return;
|
|
const native = FaceclawSpeechAnalyzer.new();
|
|
native.eventHandler = (json: string) => {
|
|
if (this.native === native) this.receive(JSON.parse(json));
|
|
};
|
|
this.native = native;
|
|
native.startWithLanguage(this.options.language ?? "");
|
|
}
|
|
|
|
acceptPcm(pcm: Uint8Array): void {
|
|
if (!this.native || this.finishing || !pcm.length) return;
|
|
this.samples += pcm.length >> 1;
|
|
const copy = pcm.slice();
|
|
this.native.acceptPcm(NSData.dataWithBytesLength(interop.handleof(copy.buffer), copy.byteLength));
|
|
}
|
|
|
|
/** A pause: it ends the paragraph once the words before it are final. */
|
|
commitSegment(): void {
|
|
if (!this.native || this.finishing) return;
|
|
this.pauses.push(this.samples / CLOUD_STT_SAMPLE_RATE);
|
|
if (this.endCompleteParagraph() && this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
|
}
|
|
|
|
finish(): void {
|
|
if (!this.native || this.finishing) return;
|
|
this.finishing = true;
|
|
this.native.finish();
|
|
}
|
|
|
|
stop(): void {
|
|
const native = this.native;
|
|
this.native = null;
|
|
native?.cancel();
|
|
}
|
|
|
|
private receive(event: AnalyzerEvent): void {
|
|
switch (event.kind) {
|
|
case "status":
|
|
this.options.onStatus(event.message ?? "");
|
|
break;
|
|
case "ready":
|
|
this.options.onStatus("Listening (on-device)...");
|
|
this.options.onReady?.();
|
|
break;
|
|
case "transcript":
|
|
this.transcript(event);
|
|
break;
|
|
case "finished": {
|
|
const text = this.liveText();
|
|
this.paragraph = [];
|
|
this.volatile = "";
|
|
this.stop();
|
|
this.options.onTranscript({ text, isFinal: true });
|
|
break;
|
|
}
|
|
case "error":
|
|
this.stop();
|
|
this.options.onError(event.message ?? "On-device transcription failed.");
|
|
break;
|
|
}
|
|
}
|
|
|
|
private transcript(event: AnalyzerEvent): void {
|
|
const text = (event.text ?? "").trim();
|
|
let ended = false;
|
|
if (!event.final) {
|
|
this.volatile = text;
|
|
} else {
|
|
this.volatile = "";
|
|
if (text) {
|
|
const end = event.speechEnd ?? event.end ?? null;
|
|
// Words ending after a pause come after it. (One whose words run
|
|
// across it, if the model didn't split there, breaks a little early.)
|
|
if (end !== null && this.pauses.some((pause) => end > pause)) {
|
|
this.pauses = this.pauses.filter((pause) => pause >= end);
|
|
this.endParagraph();
|
|
}
|
|
this.paragraph.push(text);
|
|
this.paragraphEnd = end;
|
|
ended = this.endCompleteParagraph();
|
|
}
|
|
}
|
|
// The paragraph's final already cleared the provisional text.
|
|
if (!ended || this.volatile) this.options.onTranscript({ text: this.liveText(), isFinal: false });
|
|
}
|
|
|
|
/** End the paragraph if its last word ends just before a pause. */
|
|
private endCompleteParagraph(): boolean {
|
|
const end = this.paragraphEnd;
|
|
if (!this.paragraph.length || end === null) return false;
|
|
const reached = (pause: number) => end >= pause - PARAGRAPH_TAIL_S;
|
|
if (!this.pauses.some(reached)) return false;
|
|
this.pauses = this.pauses.filter((pause) => !reached(pause));
|
|
this.endParagraph();
|
|
return true;
|
|
}
|
|
|
|
private endParagraph(): void {
|
|
const text = this.paragraph.join(" ");
|
|
this.paragraph = [];
|
|
this.paragraphEnd = null;
|
|
if (text) this.options.onTranscript({ text, isFinal: true, paragraphBreakAfter: true });
|
|
}
|
|
|
|
private liveText(): string {
|
|
return [...this.paragraph, this.volatile].filter(Boolean).join(" ");
|
|
}
|
|
}
|