mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
261 lines
8.8 KiB
TypeScript
261 lines
8.8 KiB
TypeScript
import { CLOUD_STT_SAMPLE_RATE, CloudSttClient, CloudSttOptions, encodeBase64 } from "./cloud-stt";
|
|
import { openSocket, type SocketConnection } from "./socket";
|
|
|
|
/**
|
|
* OpenAI realtime speech-to-text over WebSocket (a transcription-only session
|
|
* on the Realtime API, with the streaming gpt-realtime-whisper model).
|
|
* https://developers.openai.com/api/docs/guides/realtime-transcription
|
|
*
|
|
* Like the ElevenLabs client, push-to-talk defines the utterance boundary:
|
|
* turn detection is disabled and input_audio_buffer.commit is sent on button
|
|
* release. Two impedance mismatches with our pipeline, both handled here:
|
|
* the endpoint only accepts 24 kHz PCM (the G2 mic path is 16 kHz, so chunks
|
|
* pass through a 2:3 linear-interpolation upsampler), and transcripts arrive
|
|
* as DELTAS, which are accumulated into the full text our replace-semantics
|
|
* listeners expect.
|
|
*/
|
|
|
|
const WS_URL = "wss://api.openai.com/v1/realtime?intent=transcription";
|
|
const MODEL_ID = "gpt-realtime-whisper";
|
|
const TARGET_SAMPLE_RATE = 24000;
|
|
|
|
export type OpenAiSttOptions = CloudSttOptions;
|
|
|
|
export class OpenAiRealtimeSttClient implements CloudSttClient {
|
|
private ws: SocketConnection | null = null;
|
|
private open = false;
|
|
private closed = false;
|
|
// Base64 audio (or the commit sentinel) queued until the socket opens.
|
|
private readonly pendingChunks: string[] = [];
|
|
private readonly upsampler = new PcmUpsampler();
|
|
private readonly items = new Map<string, { text: string; final: boolean; paragraphBreakAfter: boolean }>();
|
|
private readonly commits: boolean[] = [];
|
|
private committed = false;
|
|
|
|
constructor(private readonly options: OpenAiSttOptions) {}
|
|
|
|
start(): void {
|
|
if (this.closed || this.ws) return;
|
|
const listener = {
|
|
onOpen: () => {
|
|
if (this.closed) return;
|
|
this.open = true;
|
|
this.trySend(JSON.stringify(sessionConfig(this.options.language ?? "")));
|
|
for (const chunk of this.pendingChunks.splice(0)) {
|
|
this.sendChunk(chunk);
|
|
}
|
|
if (!this.closed) this.options.onReady?.();
|
|
},
|
|
onTextMessage: (message: string) => {
|
|
if (!this.closed) this.handleMessage(String(message));
|
|
},
|
|
onClosed: () => {
|
|
this.open = false;
|
|
if (!this.closed) this.options.onDisconnected?.("OpenAI connection closed.");
|
|
},
|
|
onFailure: (message: string) => {
|
|
if (this.closed) return;
|
|
this.options.onDisconnected?.(`OpenAI connection failed: ${String(message)}`);
|
|
},
|
|
};
|
|
try {
|
|
this.ws = openSocket(WS_URL, listener, { name: "Authorization", value: `Bearer ${this.options.apiKey}` });
|
|
this.options.onStatus("Connecting to OpenAI...");
|
|
} catch (error) {
|
|
this.options.onDisconnected?.(`OpenAI connection failed: ${String((error as Error)?.message ?? error)}`);
|
|
}
|
|
}
|
|
|
|
/** Feed PCM (16 kHz signed-16-bit LE); upsampled to 24 kHz for the API. */
|
|
acceptPcm(pcm: Uint8Array): void {
|
|
if (this.closed || pcm.length === 0) return;
|
|
const resampled = this.upsampler.process(pcm);
|
|
if (resampled.length === 0) return;
|
|
const base64 = encodeBase64(resampled);
|
|
if (this.open) {
|
|
this.sendChunk(base64);
|
|
} else {
|
|
this.pendingChunks.push(base64);
|
|
}
|
|
}
|
|
|
|
commitSegment(): void {
|
|
this.commit(true);
|
|
}
|
|
|
|
/** End of utterance: commit the buffer for a final transcript. */
|
|
finish(): void {
|
|
this.committed = true;
|
|
this.commit(false);
|
|
}
|
|
|
|
private commit(paragraphBreakAfter: boolean): void {
|
|
if (this.closed) return;
|
|
this.commits.push(paragraphBreakAfter);
|
|
if (this.open) {
|
|
this.sendChunk("__commit__");
|
|
} else {
|
|
this.pendingChunks.push("__commit__");
|
|
}
|
|
}
|
|
|
|
stop(): void {
|
|
this.closed = true;
|
|
this.open = false;
|
|
if (this.ws) {
|
|
try {
|
|
this.ws.close(1000, "bye");
|
|
} catch {
|
|
// ignore
|
|
}
|
|
this.ws = null;
|
|
}
|
|
this.pendingChunks.length = 0;
|
|
}
|
|
|
|
private sendChunk(base64OrCommit: string): void {
|
|
if (base64OrCommit === "__commit__") {
|
|
this.trySend(JSON.stringify({ type: "input_audio_buffer.commit" }));
|
|
return;
|
|
}
|
|
this.trySend(JSON.stringify({ type: "input_audio_buffer.append", audio: base64OrCommit }));
|
|
}
|
|
|
|
private trySend(message: string): void {
|
|
try {
|
|
if (this.ws?.sendText(message) === false) {
|
|
this.options.onDisconnected?.("OpenAI send failed.");
|
|
}
|
|
} catch (error) {
|
|
this.options.onDisconnected?.(`OpenAI send failed: ${String(error)}`);
|
|
}
|
|
}
|
|
|
|
private handleMessage(text: string): void {
|
|
let message: any;
|
|
try {
|
|
message = JSON.parse(text);
|
|
} catch {
|
|
return;
|
|
}
|
|
switch (message?.type) {
|
|
case "session.created":
|
|
case "session.updated":
|
|
this.options.onStatus("Listening (OpenAI)...");
|
|
return;
|
|
case "input_audio_buffer.committed": {
|
|
this.item(String(message.item_id)).paragraphBreakAfter = this.commits.shift() ?? false;
|
|
return;
|
|
}
|
|
case "conversation.item.input_audio_transcription.delta": {
|
|
this.item(String(message.item_id)).text += String(message.delta ?? "");
|
|
this.emitItems();
|
|
return;
|
|
}
|
|
case "conversation.item.input_audio_transcription.completed": {
|
|
const item = this.item(String(message.item_id));
|
|
item.text = String(message.transcript ?? item.text);
|
|
item.final = true;
|
|
this.emitItems();
|
|
if (this.committed && !this.items.size && !this.commits.length) this.stop();
|
|
return;
|
|
}
|
|
case "error":
|
|
if (message.error?.code === "rate_limit_exceeded" || message.error?.type === "server_error") {
|
|
this.options.onDisconnected?.(`OpenAI: ${String(message.error?.message)}`);
|
|
return;
|
|
}
|
|
this.options.onError(`OpenAI: ${String(message.error?.message ?? "unknown error")}`);
|
|
return;
|
|
default:
|
|
return;
|
|
}
|
|
}
|
|
|
|
private item(id: string): { text: string; final: boolean; paragraphBreakAfter: boolean } {
|
|
let item = this.items.get(id);
|
|
if (!item) {
|
|
item = { text: "", final: false, paragraphBreakAfter: false };
|
|
this.items.set(id, item);
|
|
}
|
|
return item;
|
|
}
|
|
|
|
private emitItems(): void {
|
|
// Committed notifications precede transcription events, so Map insertion
|
|
// order is audio order even when completion events arrive out of order.
|
|
for (const [id, item] of this.items) {
|
|
if (!item.final) break;
|
|
this.items.delete(id);
|
|
this.options.onTranscript({ text: item.text, isFinal: true, paragraphBreakAfter: item.paragraphBreakAfter });
|
|
}
|
|
if (this.items.size) {
|
|
const items = [...this.items.values()];
|
|
const text = items.map((item) => item.text.trim()).filter(Boolean).join(" ");
|
|
const transcribeText = items.map((item, index) =>
|
|
(index && items[index - 1]!.paragraphBreakAfter ? "\n" : index ? " " : "") + item.text.trim(),
|
|
).join("");
|
|
this.options.onTranscript({ text, transcribeText, isFinal: false });
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
function sessionConfig(language: string): object {
|
|
return {
|
|
type: "session.update",
|
|
session: {
|
|
type: "transcription",
|
|
audio: {
|
|
input: {
|
|
format: { type: "audio/pcm", rate: TARGET_SAMPLE_RATE },
|
|
transcription: { model: MODEL_ID, ...(language ? { language } : {}) },
|
|
// Push-to-talk owns the utterance boundary; no server VAD.
|
|
turn_detection: null,
|
|
},
|
|
},
|
|
},
|
|
};
|
|
}
|
|
|
|
/**
|
|
* Streaming 16 kHz -> 24 kHz PCM16LE upsampler (linear interpolation, exact
|
|
* 2:3 ratio). Keeps the last sample and fractional phase across chunks so
|
|
* chunk boundaries don't glitch. Linear interpolation adds a little imaging
|
|
* noise above 8 kHz, which is irrelevant to speech recognition.
|
|
*/
|
|
class PcmUpsampler {
|
|
private prev = 0;
|
|
private havePrev = false;
|
|
// Output position between prev and the next input sample, in input-sample
|
|
// units; advances by inputRate/outputRate per emitted sample.
|
|
private phase = 0;
|
|
|
|
process(bytes: Uint8Array): Uint8Array {
|
|
const sampleCount = bytes.length >> 1;
|
|
const step = CLOUD_STT_SAMPLE_RATE / TARGET_SAMPLE_RATE;
|
|
// Phase starts each input interval at < 1 and advances by 2/3, so the
|
|
// inner loop emits at most 2 outputs per input sample.
|
|
const out = new Uint8Array(sampleCount * 2 * 2);
|
|
let outBytes = 0;
|
|
for (let i = 0; i < sampleCount; i++) {
|
|
let sample = bytes[2 * i]! | (bytes[2 * i + 1]! << 8);
|
|
if (sample >= 0x8000) sample -= 0x10000;
|
|
if (!this.havePrev) {
|
|
this.havePrev = true;
|
|
this.prev = sample;
|
|
continue;
|
|
}
|
|
while (this.phase < 1) {
|
|
const value = Math.round(this.prev + this.phase * (sample - this.prev));
|
|
out[outBytes++] = value & 0xff;
|
|
out[outBytes++] = (value >> 8) & 0xff;
|
|
this.phase += step;
|
|
}
|
|
this.phase -= 1;
|
|
this.prev = sample;
|
|
}
|
|
return out.subarray(0, outBytes);
|
|
}
|
|
}
|