mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
> Most serious fixes > - Recordings kept forever: a recording made with captions off, or with "Save captions" > off, was transcoded to .m4a and kept indefinitely. No conversation linked to it, so > the phone UI never showed it and the retention settings never deleted it. Recording > now only happens alongside a saved caption session, and turning captions off ends > that session's recording properly. > - 4-mic mode went silent after a reconnect: a disconnect clears the phone's audio > forwarding, and nothing turned it back on, so the app kept showing "Listening (4-mic > array)" while hearing nothing. A check every 3 s now restores forwarding and re-sends > the config when it has been lost. The normal single-stream mode already recovered on > its own. > - Turning off "4-mic capture" mid-session went silent: it stopped the glasses' stream > but never switched to the normal stream. Setup changes now take effect live, > including switching between the two modes. > - Setup changes were undone: after the assistant closed, the app re-sent the config it > loaded at launch to the glasses, reverting any changes you'd made since. > - Startup failures: if the caption engine failed to start, the session got stuck > half-running. If the glasses weren't connected, listeners piled up and an empty > conversation could be left open. Startup now brings up the microphone first and > cleans up properly on failure. > > Other fixes > - Audio quality: the 4-mic beamformer put a small dropout at the start of every 25 ms > frame (an audible 40 Hz artifact in the audio feeding captions). Fixed, with a test > that fails on the old code. > - Stale direction data: an arm that stopped sending kept steering the direction > estimate and could hold the caption feed on a silent arm. > - Compass: closing Microphones switched off the Compass app's compass, because both > used the same owner name. > - Speaker database failures: > - A failed speaker reload wiped the in-memory list, so the next voice became a > duplicate "Speaker N". > - A failed speaker insert crashed caption handling; in re-diarize it reassigned > transcript lines to a nonexistent speaker. > - Translation: > - A crash path inside the model-download callback is now caught. > - Phones set to Hebrew, Indonesian or Yiddish never got translations, because of > old-style language codes. > - Ask questions: once the speaker list grew past the size limit (every unknown voice > adds a "Speaker N"), it pushed all transcript lines out of the context sent to the > model. When trimming is needed, the oldest lines now go first. > - Smaller fixes: > - The captions view stopped jumping back to live speech after 60 lines. > - "This is Dana calling…" was treated as introducing someone else. > - A model download that failed to start was stuck at "downloading" until restart. > - The developer mic monitor grew without limit while hidden.
271 lines
10 KiB
TypeScript
271 lines
10 KiB
TypeScript
import { Utils } from "@nativescript/core";
|
|
|
|
import { isLocalModelReady, streamLocalQwen } from "../../native/llama";
|
|
import type { LlmStreamHandle } from "../../assistant/llm-protocol";
|
|
import { parseConversationQuery } from "../../phone-ui/conversation-search";
|
|
|
|
declare const com: any;
|
|
|
|
/**
|
|
* Question answering over saved conversations with the on-phone LLM (the
|
|
* assistant's Qwen model). Retrieval-then-read: the question is parsed for
|
|
* date/emotion filters and keywords, matching transcript segments and
|
|
* session/speaker metadata are pulled from the conversation store, and the
|
|
* model answers strictly from that context. Everything stays on-device.
|
|
*/
|
|
|
|
// Qwen runs with a 4096-token context; leave room for the answer.
|
|
const MAX_CONTEXT_CHARS = 7000;
|
|
// The speaker roster grows with every voice ever heard; cap it and the
|
|
// conversation list so they can't crowd the transcript excerpts out.
|
|
const MAX_HEADER_CHARS = MAX_CONTEXT_CHARS / 2;
|
|
const EXCERPTS_HEADING = "# Transcript excerpts\n";
|
|
const MAX_KEYWORDS = 6;
|
|
const SEGMENTS_PER_KEYWORD = 30;
|
|
const RECENT_SEGMENT_LIMIT = 120;
|
|
const SESSION_LIMIT = 20;
|
|
|
|
const KEYWORD_STOPWORDS = new Set(
|
|
("the a an and or but of in on at to from with about for was were is are am be been did do does have has had " +
|
|
"what when where who whom whose which why how tell show me my i we us our you your they them their it its " +
|
|
"that this these those there anyone someone something anything talk talked talking say said saying speak spoke " +
|
|
"conversation conversations chat chats discussion discussions transcript transcripts recording recordings").split(" "),
|
|
);
|
|
|
|
export type ConversationAnswerCallbacks = {
|
|
onDelta?: (textSoFar: string) => void;
|
|
onDone: (answer: string, contextNote: string) => void;
|
|
onError: (message: string) => void;
|
|
};
|
|
|
|
export function conversationQaAvailable(): boolean {
|
|
return global.isAndroid && isLocalModelReady();
|
|
}
|
|
|
|
function store(): any {
|
|
const context = Utils.android.getApplicationContext();
|
|
return com.faceclaw.app.FaceclawConversationStore.getInstance(context);
|
|
}
|
|
|
|
type SegmentRow = {
|
|
id: number;
|
|
sessionId: number;
|
|
speakerId: number | null;
|
|
startedAt: number;
|
|
text: string;
|
|
translation: string;
|
|
emotion: string;
|
|
sentiment: number;
|
|
};
|
|
|
|
function parseSegments(json: string): SegmentRow[] {
|
|
return (JSON.parse(json) as Array<Record<string, unknown>>).map((row) => ({
|
|
id: Number(row.id),
|
|
sessionId: Number(row.sessionId ?? 0),
|
|
speakerId: row.speakerId === null ? null : Number(row.speakerId),
|
|
startedAt: Number(row.startedAt),
|
|
text: String(row.text ?? ""),
|
|
translation: String(row.translation ?? ""),
|
|
emotion: String(row.emotion ?? ""),
|
|
sentiment: Number(row.sentiment ?? 0),
|
|
}));
|
|
}
|
|
|
|
function keywords(text: string): string[] {
|
|
const seen = new Set<string>();
|
|
for (const raw of text.toLowerCase().split(/[^a-z0-9']+/)) {
|
|
const word = raw.trim();
|
|
if (word.length < 3 || KEYWORD_STOPWORDS.has(word)) continue;
|
|
seen.add(word);
|
|
if (seen.size >= MAX_KEYWORDS) break;
|
|
}
|
|
return [...seen];
|
|
}
|
|
|
|
function formatWhen(ms: number): string {
|
|
const date = new Date(ms);
|
|
const pad = (value: number) => String(value).padStart(2, "0");
|
|
return (
|
|
`${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())} ` +
|
|
`${pad(date.getHours())}:${pad(date.getMinutes())}`
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Assemble the retrieval context: speaker roster (names, tags, wearer flag),
|
|
* session metadata, and the transcript lines most relevant to the question.
|
|
* Returns the context text plus a short human-readable note about coverage.
|
|
*/
|
|
function buildContext(question: string): { context: string; note: string } {
|
|
const filters = parseConversationQuery(question);
|
|
const filterJson: Record<string, unknown> = {};
|
|
if (filters.sinceMs !== null) filterJson.sinceMs = filters.sinceMs;
|
|
if (filters.untilMs !== null) filterJson.untilMs = filters.untilMs;
|
|
if (filters.emotion !== null) filterJson.emotion = filters.emotion;
|
|
|
|
const speakerNames = new Map<number, string>();
|
|
const speakerLines: string[] = [];
|
|
for (const row of JSON.parse(String(store().querySpeakers())) as Array<Record<string, unknown>>) {
|
|
const id = Number(row.id);
|
|
const name = String(row.name ?? "");
|
|
speakerNames.set(id, name);
|
|
const parts = [name];
|
|
if (row.isWearer) parts.push("(the wearer/user)");
|
|
if (row.tag) parts.push(`tag: ${String(row.tag)}`);
|
|
parts.push(`${Number(row.segmentCount ?? 0)} lines heard`);
|
|
speakerLines.push(`- ${parts.join(", ")}`);
|
|
}
|
|
|
|
const sessions = JSON.parse(
|
|
String(store().querySessions(JSON.stringify({ ...filterJson, limit: SESSION_LIMIT }))),
|
|
) as Array<Record<string, unknown>>;
|
|
const sessionTitles = new Map<number, string>();
|
|
const sessionLines: string[] = [];
|
|
for (const row of sessions) {
|
|
const id = Number(row.id);
|
|
const title = String(row.title ?? "") || `Conversation ${formatWhen(Number(row.startedAt))}`;
|
|
sessionTitles.set(id, title);
|
|
const who = ((row.speakers as Array<{ name?: unknown }>) ?? [])
|
|
.map((speaker) => String(speaker.name ?? ""))
|
|
.filter(Boolean)
|
|
.join(", ");
|
|
sessionLines.push(
|
|
`- [#${id}] ${title} — ${formatWhen(Number(row.startedAt))}, ${Number(row.segmentCount ?? 0)} lines` +
|
|
(who ? `, speakers: ${who}` : "") +
|
|
`, overall sentiment ${Number(row.avgSentiment ?? 0).toFixed(2)}`,
|
|
);
|
|
}
|
|
|
|
const terms = keywords(filters.text);
|
|
const segmentMap = new Map<number, SegmentRow>();
|
|
if (terms.length > 0) {
|
|
for (const term of terms) {
|
|
const rows = parseSegments(
|
|
String(
|
|
store().searchSegments(
|
|
JSON.stringify({ ...filterJson, query: term, limit: SEGMENTS_PER_KEYWORD }),
|
|
),
|
|
),
|
|
);
|
|
for (const row of rows) segmentMap.set(row.id, row);
|
|
}
|
|
}
|
|
if (segmentMap.size === 0) {
|
|
// No keyword hits (or no keywords): fall back to the filtered recents so
|
|
// summary-style questions still have material to read.
|
|
for (const row of parseSegments(
|
|
String(store().searchSegments(JSON.stringify({ ...filterJson, limit: RECENT_SEGMENT_LIMIT }))),
|
|
)) {
|
|
segmentMap.set(row.id, row);
|
|
}
|
|
}
|
|
|
|
let header =
|
|
`# Known speakers\n${speakerLines.join("\n") || "(none yet)"}\n\n` +
|
|
`# Conversations${filters.sinceMs !== null || filters.emotion !== null ? " (filtered by the question)" : ""}\n` +
|
|
`${sessionLines.join("\n") || "(none in range)"}\n\n`;
|
|
let truncated = false;
|
|
if (header.length > MAX_HEADER_CHARS) {
|
|
header = `${header.slice(0, MAX_HEADER_CHARS)}\n(...)\n\n`;
|
|
truncated = true;
|
|
}
|
|
|
|
const sessionHeading = (sessionId: number) =>
|
|
`\n## ${sessionTitles.get(sessionId) ?? `Conversation #${sessionId}`}`;
|
|
const segmentText = (segment: SegmentRow) => {
|
|
const speaker =
|
|
segment.speakerId !== null ? (speakerNames.get(segment.speakerId) ?? "Speaker") : "Speaker";
|
|
const emotion = segment.emotion && segment.emotion !== "neutral" ? ` [${segment.emotion}]` : "";
|
|
const line = `[${formatWhen(segment.startedAt)}] ${speaker}${emotion}: ${segment.text}`;
|
|
return segment.translation ? `${line}\n (translated: ${segment.translation})` : line;
|
|
};
|
|
|
|
// Fill the remaining budget newest-first, so an overflowing history drops
|
|
// its oldest excerpts rather than its most recent ones.
|
|
const sorted = [...segmentMap.values()].sort((a, b) => a.startedAt - b.startedAt);
|
|
let budget = MAX_CONTEXT_CHARS - header.length - EXCERPTS_HEADING.length;
|
|
const segments: SegmentRow[] = [];
|
|
let runSession = -1;
|
|
for (let index = sorted.length - 1; index >= 0; index--) {
|
|
const segment = sorted[index]!;
|
|
let cost = segmentText(segment).length + 1;
|
|
if (segment.sessionId !== runSession) cost += sessionHeading(segment.sessionId).length + 1;
|
|
if (cost > budget) {
|
|
truncated = true;
|
|
break;
|
|
}
|
|
budget -= cost;
|
|
runSession = segment.sessionId;
|
|
segments.push(segment);
|
|
}
|
|
segments.reverse();
|
|
|
|
const excerptLines: string[] = [];
|
|
let currentSession = -1;
|
|
for (const segment of segments) {
|
|
if (segment.sessionId !== currentSession) {
|
|
currentSession = segment.sessionId;
|
|
excerptLines.push(sessionHeading(currentSession));
|
|
}
|
|
excerptLines.push(segmentText(segment));
|
|
}
|
|
|
|
const context = `${header}${EXCERPTS_HEADING}${excerptLines.join("\n") || "(no matching lines)"}`;
|
|
const note =
|
|
`${sessions.length} conversation${sessions.length === 1 ? "" : "s"}, ` +
|
|
`${segments.length} transcript line${segments.length === 1 ? "" : "s"} consulted` +
|
|
(terms.length ? ` (keywords: ${terms.join(", ")})` : "") +
|
|
(truncated ? "; context truncated to fit the on-device model" : "");
|
|
return { context, note };
|
|
}
|
|
|
|
/**
|
|
* Ask the on-device model a question about saved conversations. Returns a
|
|
* cancel handle; the answer streams via onDelta and lands in onDone with a
|
|
* note describing what data was consulted.
|
|
*/
|
|
export function askConversations(
|
|
question: string,
|
|
callbacks: ConversationAnswerCallbacks,
|
|
): LlmStreamHandle {
|
|
if (!conversationQaAvailable()) {
|
|
setTimeout(
|
|
() =>
|
|
callbacks.onError(
|
|
"The on-phone model isn't downloaded yet — enable it under Settings > Assistant on the glasses.",
|
|
),
|
|
0,
|
|
);
|
|
return { cancel: () => {} };
|
|
}
|
|
let context: { context: string; note: string };
|
|
try {
|
|
context = buildContext(question);
|
|
} catch (error) {
|
|
setTimeout(() => callbacks.onError(`Could not read the conversation store: ${String(error)}`), 0);
|
|
return { cancel: () => {} };
|
|
}
|
|
return streamLocalQwen({
|
|
apiKey: "",
|
|
model: "local",
|
|
system:
|
|
"You answer questions about the user's saved conversations, captured by their smart glasses. " +
|
|
"Use ONLY the provided speaker roster, conversation list, and transcript excerpts. " +
|
|
"Cite conversations by their title and date when referencing them. " +
|
|
"If the provided data does not contain the answer, say so plainly rather than guessing. " +
|
|
"Transcript sentiment scores range -1 (very negative) to 1 (very positive); " +
|
|
"bracketed words after a speaker's name are detected emotional tones. " +
|
|
`Today's date is ${formatWhen(Date.now())}.`,
|
|
messages: [
|
|
{
|
|
role: "user",
|
|
content: `${context.context}\n\n# Question\n${question.trim()}`,
|
|
},
|
|
],
|
|
maxTokens: 400,
|
|
onTextDelta: (_delta, textSoFar) => callbacks.onDelta?.(textSoFar),
|
|
onDone: (result) => callbacks.onDone(result.text.trim(), context.note),
|
|
onError: (message) => callbacks.onError(message),
|
|
});
|
|
}
|