Files
faceclaw/app/native/reconnecting-stt.ts
FilipAlexanderandClaude Opus 5.5 e2caf28392 Voice: self-hosted Whisper server provider; Transcribe on iOS
- "Whisper server (self-hosted)" transcription provider for OpenAI-compatible
  Whisper servers (vLLM, speaches, LocalAI) over plain HTTP batch requests,
  with URL, model, language and key settings (Voice). Pause-ended spans are
  one request each, finalized in audio order, with provisional text while a
  span is open. See notes/whisper-server.md.
- Transcribe (and other continuous capture) works on iOS with a cloud or
  Whisper-server provider; push-to-talk shares a running capture.
- Transcribe saves each session's audio as transcribe-<time>.wav in Downloads
  (setting "Save Transcribe audio", on by default), written as it is captured
  so a crash leaves a playable file; a saved transcript uses the same stamp.
- Calculator stops continuous listening when its window closes.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-17 16:43:18 +02:00

144 lines
4.9 KiB
TypeScript

import type { CloudSttClient, CloudSttOptions, CloudSttTranscriptEvent } from "./cloud-stt";
import { SpeechPauseDetector } from "./speech-pause";
// Ten seconds of unsent PCM. Never replay audio already sent: that would
// duplicate words already shown. An extended outage drops the oldest audio.
const MAX_PENDING_BYTES = 16000 * 2 * 10;
/** Owns retries and audio buffering independently of the provider protocol. */
export class ReconnectingSttClient implements CloudSttClient {
private client: CloudSttClient | null = null;
private generation = 0;
private stopped = false;
private finishing = false;
private ready = false;
private retry: ReturnType<typeof setTimeout> | null = null;
private finishTimeout: ReturnType<typeof setTimeout> | null = null;
private attempt = 0;
private pending: Uint8Array[] = [];
private pendingBytes = 0;
private latest: CloudSttTranscriptEvent | null = null;
private readonly pause = new SpeechPauseDetector();
constructor(
private readonly create: (options: CloudSttOptions) => CloudSttClient,
private readonly options: CloudSttOptions,
private readonly continuous: () => boolean,
/** How long finish() waits for the provider's final transcript. */
readonly finalTimeoutMs = 5000,
) {}
start(): void {
if (this.stopped || this.client || this.retry !== null) return;
const generation = ++this.generation;
const current = () => !this.stopped && generation === this.generation;
this.client = this.create({
...this.options,
onReady: () => {
if (!current()) return;
this.ready = true;
const chunks = this.pending;
this.pending = [];
this.pendingBytes = 0;
for (const chunk of chunks) this.acceptPcm(chunk);
if (this.finishing) this.client?.finish();
},
onTranscript: (event) => {
if (!current()) return;
// A handshake alone can immediately be followed by another failure.
this.attempt = 0;
this.latest = event.isFinal ? null : event;
this.options.onTranscript(event);
},
onStatus: (status) => { if (current()) this.options.onStatus(status); },
onError: (message) => {
if (!current()) return;
this.stop();
this.options.onError(message);
},
onDisconnected: (message) => {
if (current()) this.disconnected(message);
},
});
try {
this.client.start();
} catch (error) {
this.disconnected(`Transcription connection failed: ${String(error)}`);
}
}
acceptPcm(pcm: Uint8Array): void {
if (this.stopped || !pcm.length) return;
if (!this.ready) {
const copy = pcm.slice(-MAX_PENDING_BYTES);
this.pending.push(copy);
this.pendingBytes += copy.length;
while (this.pendingBytes > MAX_PENDING_BYTES) this.pendingBytes -= this.pending.shift()!.length;
return;
}
this.client?.acceptPcm(pcm);
// A rejected send synchronously reports disconnection. Keep that packet
// with the unsent audio instead of losing it at the outage boundary.
if (!this.ready && !this.stopped) this.acceptPcm(pcm);
if (this.ready && !this.finishing && this.continuous() && this.pause.accept(pcm)) this.client?.commitSegment?.();
}
finish(): void {
if (this.stopped || this.finishing) return;
this.finishing = true;
// Releasing the last mic holder must not leave a reconnect loop alive.
if (this.retry !== null) {
this.preservePartial();
this.stop();
return;
}
this.finishTimeout = setTimeout(() => this.stop(), this.finalTimeoutMs);
if (this.ready) this.client?.finish();
}
stop(): void {
this.stopped = true;
++this.generation;
if (this.retry !== null) clearTimeout(this.retry);
if (this.finishTimeout !== null) clearTimeout(this.finishTimeout);
this.retry = this.finishTimeout = null;
this.client?.stop();
this.client = null;
this.ready = false;
this.pending = [];
this.pendingBytes = 0;
this.latest = null;
}
private preservePartial(): void {
if (this.latest?.text.trim()) this.options.onTranscript({ ...this.latest, isFinal: true });
this.latest = null;
}
private disconnected(message: string): void {
if (this.stopped) return;
++this.generation; // Ignore late finals / closes from the abandoned socket.
this.ready = false;
this.client?.stop();
this.client = null;
this.pause.reset();
this.preservePartial();
if (this.stopped) return; // A transcript listener can release capture.
if (/\b(401|402|403)\b/.test(message)) {
this.stop();
this.options.onError(message);
return;
}
if (this.finishing) {
this.stop();
return;
}
const delay = Math.min(1000 * 2 ** Math.min(this.attempt++, 5), 30000);
this.options.onStatus(`Connection lost. Reconnecting in ${delay / 1000}s...`);
this.retry = setTimeout(() => {
this.retry = null;
this.start();
}, delay);
}
}