Files
faceclaw/app/apps/microphones/conversation-qa.ts
T
James Babcock b50898ea02 Fix a bunch of microphone-related bugs
> Most serious fixes
> - Recordings kept forever: a recording made with captions off, or with "Save captions"
>   off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so
>   the phone UI never showed it and the retention settings never deleted it. Recording
>   now only happens alongside a saved caption session, and turning captions off ends
>   that session's recording properly.
> - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio
>   forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic
>   array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends
>   the config when it has been lost. The normal single-stream mode already recovered on
>   its own.
> - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream
>   but never switched to the normal stream. Setup changes now take effect live,
>   including switching between the two modes.
> - Setup changes were undone: after the assistant closed, the app re-sent the config it
>   loaded at launch to the glasses, reverting any changes you'd made since.
> - Startup failures: if the caption engine failed to start, the session got stuck
>   half-running. If the glasses weren't connected, listeners piled up and an empty
>   conversation could be left open. Startup now brings up the microphone first and
>   cleans up properly on failure.
>
> Other fixes
> - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms
>   frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test
>   that fails on the old code.
> - Stale direction data: an arm that stopped sending kept steering the direction
>   estimate and could hold the caption feed on a silent arm.
> - Compass: closing Microphones switched off the Compass app's compass, because both
>   used the same owner name.
> - Speaker database failures:
>   - A failed speaker reload wiped the in-memory list, so the next voice became a
>     duplicate "Speaker N".
>   - A failed speaker insert crashed caption handling; in re-diarize it reassigned
>     transcript lines to a nonexistent speaker.
> - Translation:
>   - A crash path inside the model-download callback is now caught.
>   - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of
>     old-style language codes.
> - Ask questions: once the speaker list grew past the size limit (every unknown voice
>   adds a "Speaker N"), it pushed all transcript lines out of the context sent to the
>   model. When trimming is needed, the oldest lines now go first.
> - Smaller fixes:
>   - The captions view stopped jumping back to live speech after 60 lines.
>   - "This is Dana calling…" was treated as introducing someone else.
>   - A model download that failed to start was stuck at "downloading" until restart.
>   - The developer mic monitor grew without limit while hidden.
2026-10-14 00:56:54 -07:00

271 lines
10 KiB
TypeScript

import { Utils } from "@nativescript/core";
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
import { parseConversationQuery } from "../../phone-ui/conversation-search";
declare const com: any;
/**
* Question answering over saved conversations with the on-phone LLM (the
* assistant's Qwen model). Retrieval-then-read: the question is parsed for
* date/emotion filters and keywords, matching transcript segments and
* session/speaker metadata are pulled from the conversation store, and the
* model answers strictly from that context. Everything stays on-device.
*/
// Qwen runs with a 4096-token context; leave room for the answer.
const MAX_CONTEXT_CHARS = 7000;
// The speaker roster grows with every voice ever heard; cap it and the
// conversation list so they can't crowd the transcript excerpts out.
const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2;
const EXCERPTS_HEADING = "# Transcript excerpts\n";
const MAX_KEYWORDS = 6;
const SEGMENTS_PER_KEYWORD = 30;
const RECENT_SEGMENT_LIMIT = 120;
const SESSION_LIMIT = 20;
const KEYWORD_STOPWORDS = new Set(
("the a an and or but of in on at to from with about for was were is are am be been did do does have has had " +
"what when where who whom whose which why how tell show me my i we us our you your they them their it its " +
"that this these those there anyone someone something anything talk talked talking say said saying speak spoke " +
"conversation conversations chat chats discussion discussions transcript transcripts recording recordings").split(" "),
);
export type ConversationAnswerCallbacks = {
onDelta?: (textSoFar: string) => void;
onDone: (answer: string, contextNote: string) => void;
onError: (message: string) => void;
};
export function conversationQaAvailable(): boolean {
return global.isAndroid && isLocalModelReady();
}
function store(): any {
const context = Utils.android.getApplicationContext();
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
}
type SegmentRow = {
id: number;
sessionId: number;
speakerId: number | null;
startedAt: number;
text: string;
translation: string;
emotion: string;
sentiment: number;
};
function parseSegments(json: string): SegmentRow[] {
return (JSON.parse(json) as Array<Record<string, unknown>>).map((row) => ({
id: Number(row.id),
sessionId: Number(row.sessionId ?? 0),
speakerId: row.speakerId === null ? null : Number(row.speakerId),
startedAt: Number(row.startedAt),
text: String(row.text ?? ""),
translation: String(row.translation ?? ""),
emotion: String(row.emotion ?? ""),
sentiment: Number(row.sentiment ?? 0),
}));
}
function keywords(text: string): string[] {
const seen = new Set<string>();
for (const raw of text.toLowerCase().split(/[^a-z0-9']+/)) {
const word = raw.trim();
if (word.length < 3 || KEYWORD_STOPWORDS.has(word)) continue;
seen.add(word);
if (seen.size >= MAX_KEYWORDS) break;
}
return [...seen];
}
function formatWhen(ms: number): string {
const date = new Date(ms);
const pad = (value: number) => String(value).padStart(2, "0");
return (
`${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` +
`${pad(date.getHours())}:${pad(date.getMinutes())}`
);
}
/**
* Assemble the retrieval context: speaker roster (names, tags, wearer flag),
* session metadata, and the transcript lines most relevant to the question.
* Returns the context text plus a short human-readable note about coverage.
*/
function buildContext(question: string): { context: string; note: string } {
const filters = parseConversationQuery(question);
const filterJson: Record<string, unknown> = {};
if (filters.sinceMs !== null) filterJson.sinceMs = filters.sinceMs;
if (filters.untilMs !== null) filterJson.untilMs = filters.untilMs;
if (filters.emotion !== null) filterJson.emotion = filters.emotion;
const speakerNames = new Map<number, string>();
const speakerLines: string[] = [];
for (const row of JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>) {
const id = Number(row.id);
const name = String(row.name ?? "");
speakerNames.set(id, name);
const parts = [name];
if (row.isWearer) parts.push("(the wearer/user)");
if (row.tag) parts.push(`tag: ${String(row.tag)}`);
parts.push(`${Number(row.segmentCount ?? 0)} lines heard`);
speakerLines.push(`- ${parts.join(", ")}`);
}
const sessions = JSON.parse(
String(store().querySessions(JSON.stringify({ ...filterJson, limit: SESSION_LIMIT }))),
) as Array<Record<string, unknown>>;
const sessionTitles = new Map<number, string>();
const sessionLines: string[] = [];
for (const row of sessions) {
const id = Number(row.id);
const title = String(row.title ?? "") || `Conversation ${formatWhen(Number(row.startedAt))}`;
sessionTitles.set(id, title);
const who = ((row.speakers as Array<{ name?: unknown }>) ?? [])
.map((speaker) => String(speaker.name ?? ""))
.filter(Boolean)
.join(", ");
sessionLines.push(
`- [#${id}] ${title} — ${formatWhen(Number(row.startedAt))}, ${Number(row.segmentCount ?? 0)} lines` +
(who ? `, speakers: ${who}` : "") +
`, overall sentiment ${Number(row.avgSentiment ?? 0).toFixed(2)}`,
);
}
const terms = keywords(filters.text);
const segmentMap = new Map<number, SegmentRow>();
if (terms.length > 0) {
for (const term of terms) {
const rows = parseSegments(
String(
store().searchSegments(
JSON.stringify({ ...filterJson, query: term, limit: SEGMENTS_PER_KEYWORD }),
),
),
);
for (const row of rows) segmentMap.set(row.id, row);
}
}
if (segmentMap.size === 0) {
// No keyword hits (or no keywords): fall back to the filtered recents so
// summary-style questions still have material to read.
for (const row of parseSegments(
String(store().searchSegments(JSON.stringify({ ...filterJson, limit: RECENT_SEGMENT_LIMIT }))),
)) {
segmentMap.set(row.id, row);
}
}
let header =
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
`${sessionLines.join("\n") || "(none in range)"}\n\n`;
let truncated = false;
if (header.length > MAX_HEADER_CHARS) {
header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`;
truncated = true;
}
const sessionHeading = (sessionId: number) =>
`\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`;
const segmentText = (segment: SegmentRow) => {
const speaker =
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`;
return segment.translation ? `${line}\n (translated: ${segment.translation})` : line;
};
// Fill the remaining budget newest-first, so an overflowing history drops
// its oldest excerpts rather than its most recent ones.
const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length;
const segments: SegmentRow[] = [];
let runSession = -1;
for (let index = sorted.length - 1; index >= 0; index--) {
const segment = sorted[index]!;
let cost = segmentText(segment).length + 1;
if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1;
if (cost > budget) {
truncated = true;
break;
}
budget -= cost;
runSession = segment.sessionId;
segments.push(segment);
}
segments.reverse();
const excerptLines: string[] = [];
let currentSession = -1;
for (const segment of segments) {
if (segment.sessionId !== currentSession) {
currentSession = segment.sessionId;
excerptLines.push(sessionHeading(currentSession));
}
excerptLines.push(segmentText(segment));
}
const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`;
const note =
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
(terms.length ? ` (keywords: ${terms.join(", ")})` : "") +
(truncated ? "; context truncated to fit the on-device model" : "");
return { context, note };
}
/**
* Ask the on-device model a question about saved conversations. Returns a
* cancel handle; the answer streams via onDelta and lands in onDone with a
* note describing what data was consulted.
*/
export function askConversations(
question: string,
callbacks: ConversationAnswerCallbacks,
): LlmStreamHandle {
if (!conversationQaAvailable()) {
setTimeout(
() =>
callbacks.onError(
"The on-phone model isn't downloaded yet — enable it under Settings > Assistant on the glasses.",
),
0,
);
return { cancel: () => {} };
}
let context: { context: string; note: string };
try {
context = buildContext(question);
} catch (error) {
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
return { cancel: () => {} };
}
return streamLocalQwen({
apiKey: "",
model: "local",
system:
"You answer questions about the user's saved conversations, captured by their smart glasses. " +
"Use ONLY the provided speaker roster, conversation list, and transcript excerpts. " +
"Cite conversations by their title and date when referencing them. " +
"If the provided data does not contain the answer, say so plainly rather than guessing. " +
"Transcript sentiment scores range -1 (very negative) to 1 (very positive); " +
"bracketed words after a speaker's name are detected emotional tones. " +
`Today's date is ${formatWhen(Date.now())}.`,
messages: [
{
role: "user",
content: `${context.context}\n\n# Question\n${question.trim()}`,
},
],
maxTokens: 400,
onTextDelta: (_delta, textSoFar) => callbacks.onDelta?.(textSoFar),
onDone: (result) => callbacks.onDone(result.text.trim(), context.note),
onError: (message) => callbacks.onError(message),
});
}