mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Merge detailed heart-rate collection and export
# Conflicts: # docs/validation-health-export.md
This commit is contained in:
8 files changed
+218
-18
No files matched your search
@@ -48,6 +48,7 @@
|
||||
|
||||
import {
|
||||
HOUR_MS,
|
||||
HEART_RATE_MEASUREMENT_SPAN_MS,
|
||||
TEN_MINUTES_MS,
|
||||
type HealthSample,
|
||||
type SleepSession,
|
||||
@@ -65,6 +66,8 @@ export type WireHourlyRecord = {
|
||||
* zero bytes instead.
|
||||
*/
|
||||
anchorMs: number | null;
|
||||
/** Latest reading, only from live pages; raw journal replay does not guess its clock. */
|
||||
latest?: { timestampSeconds: number; value: number; receivedAtMs: number; clockOffsetMs: number };
|
||||
groups: readonly { hourIndex: number; avg: number; max: number; min: number }[];
|
||||
};
|
||||
|
||||
@@ -146,6 +149,22 @@ export function convertRecords(records: readonly WireRecord[]): ConversionResult
|
||||
* guesses. Dropping loses a page; guessing corrupts the history silently.
|
||||
*/
|
||||
function convertHourly(record: WireHourlyRecord, result: ConversionResult): void {
|
||||
if (record.metric === "heartRate" && record.latest) {
|
||||
const { timestampSeconds, value, receivedAtMs, clockOffsetMs } = record.latest;
|
||||
const startMs = timestampSeconds * 1000 - clockOffsetMs;
|
||||
// Invert the existing handshake clock shift, never replace measurement time
|
||||
// with reception time or snap it onto an invented ten-minute bucket.
|
||||
// Older pages can mix clock domains: only accept a recent live reading.
|
||||
if (Number.isSafeInteger(timestampSeconds) && Number.isSafeInteger(receivedAtMs)
|
||||
&& Number.isFinite(clockOffsetMs) && Number.isInteger(value) && value > 0 && value <= 300
|
||||
&& startMs >= 946684800000 && startMs <= receivedAtMs + 120_000
|
||||
&& receivedAtMs - startMs <= 20 * 60_000) {
|
||||
result.samples.push({ metric: "heartRate", startMs, spanMs: HEART_RATE_MEASUREMENT_SPAN_MS,
|
||||
min: value, max: value, avg: value, total: value });
|
||||
} else {
|
||||
result.skipped.push({ what: "latest heart rate", why: "invalid, stale or unresolved measurement timestamp/value" });
|
||||
}
|
||||
}
|
||||
if (record.anchorMs === null) {
|
||||
result.skipped.push({
|
||||
what: `${record.metric} page, ${record.groups.length} groups`,
|
||||
|
||||
+17
-14
@@ -240,7 +240,7 @@ function accumulateStepBuckets(
|
||||
return merged;
|
||||
}
|
||||
|
||||
function toWire(record: any): WireRecord | null {
|
||||
function toWire(record: any, includeLatest: boolean): WireRecord | null {
|
||||
const cmdHi = Number(record.cmdHi);
|
||||
|
||||
const metric = HOURLY_METRIC[cmdHi];
|
||||
@@ -256,7 +256,12 @@ function toWire(record: any): WireRecord | null {
|
||||
min: Number(group.min),
|
||||
});
|
||||
}
|
||||
return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups };
|
||||
const receivedAtMs = Number(record.receivedAtMs);
|
||||
const latest = includeLatest && metric === "heartRate" && Number(record.tagRaw) > 0 ? {
|
||||
timestampSeconds: Number(record.tagRaw), value: Number(record.current), receivedAtMs,
|
||||
clockOffsetMs: ringClockOffsetMs(receivedAtMs),
|
||||
} : undefined;
|
||||
return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups, latest };
|
||||
}
|
||||
|
||||
if (cmdHi === CMD_STEPS) {
|
||||
@@ -391,12 +396,12 @@ type StoreResult = Omit<LiveSyncResult, "seen">;
|
||||
* fails, so the caller keeps whatever it would otherwise mark as consumed.
|
||||
* Shared by the live batch and the journal replay.
|
||||
*/
|
||||
function storeRecords(records: JavaRecordList): StoreResult {
|
||||
function storeRecords(records: JavaRecordList, includeLatest = true): StoreResult {
|
||||
const wire: WireRecord[] = [];
|
||||
const size = records.size();
|
||||
for (let i = 0; i < size; i++) {
|
||||
try {
|
||||
const converted = toWire(records.get(i));
|
||||
const converted = toWire(records.get(i), includeLatest);
|
||||
if (converted) wire.push(converted);
|
||||
} catch (error) {
|
||||
console.warn("health live: skipped an undecodable record", error);
|
||||
@@ -480,7 +485,7 @@ function replayFrameJournalOnce(): void {
|
||||
const records: JavaRecordList = ringProtocol.decodeFrameJournal(File.fromPath(journalPath).readTextSync(), after);
|
||||
const size = records.size();
|
||||
if (size > 0) {
|
||||
const result = storeRecords(records);
|
||||
const result = storeRecords(records, false);
|
||||
let through = after;
|
||||
for (let i = 0; i < size; i++) through = Math.max(through, Number(records.get(i).receivedAtMs));
|
||||
const next: ReplayCursor = { version: JOURNAL_DECODER_VERSION, throughReceivedAtMs: through };
|
||||
@@ -543,7 +548,7 @@ export function syncLiveRecords(): LiveSyncResult {
|
||||
|
||||
/**
|
||||
* Ask the ring for a fresh pull now, rather than waiting out the automatic
|
||||
* 30-minute cycle. Returns immediately; the pull itself takes ~15s on the
|
||||
* ten-minute cycle. Returns immediately; the pull itself takes ~15s on the
|
||||
* communicator's worker thread and lands through the next `syncLiveRecords()`.
|
||||
*
|
||||
* Throttled communicator-side to one a minute — see
|
||||
@@ -584,19 +589,17 @@ export function startLiveHealthSync(): void {
|
||||
}
|
||||
|
||||
/**
|
||||
* Pull the ring on a wall-clock-aligned cadence: :01 and :31, every hour.
|
||||
* Pull the ring every ten minutes, one minute after each bucket boundary.
|
||||
*
|
||||
* Chris, 2026-09-12: *"I think we should be polling the ring on the 30 minute
|
||||
* cycle (that was my intent, not a floor honestly)."* The 30-minute value in
|
||||
* `RING_HEALTH_MIN_PULL_INTERVAL_MS` was always meant as a collection rhythm
|
||||
* and got built as a throttle, so a connected, stable ring pulled never —
|
||||
* nothing in the system drives a pull on a timer at all.
|
||||
* This collects the latest heart-rate observation at roughly the cadence
|
||||
* the stock app documents, as well as the existing historical buckets.
|
||||
* It does not force the sensor to take a new measurement.
|
||||
*
|
||||
* Two reasons this is aligned to the wall clock rather than a plain interval:
|
||||
*
|
||||
* 1. **The ring's own step buckets close on 10-minute boundaries.** Pulling on
|
||||
* the boundary collects a just-closed bucket instead of a half-formed one.
|
||||
* 2. **An elapsed timer drifts.** A pull at :07 sets the next at :37, then :07,
|
||||
* 2. **An elapsed timer drifts.** A pull at :07 sets the next at :17, then :27,
|
||||
* wandering away from the boundary it is supposed to track. Re-arming
|
||||
* against the clock each time cannot drift.
|
||||
*
|
||||
@@ -606,7 +609,7 @@ export function startLiveHealthSync(): void {
|
||||
* This only *requests* a pull. The communicator still applies its own anti-spam
|
||||
* floor, so an extra call here can never hammer the ring.
|
||||
*/
|
||||
const ALIGNED_PULL_MINUTES = [1, 31];
|
||||
const ALIGNED_PULL_MINUTES = [1, 11, 21, 31, 41, 51];
|
||||
let alignedPullTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
function msUntilNextAlignedPull(now: Date): number {
|
||||
|
||||
@@ -59,6 +59,7 @@
|
||||
|
||||
import {
|
||||
type HealthSample,
|
||||
isHeartRateMeasurement,
|
||||
type Rollup,
|
||||
type SampleMetric,
|
||||
type SleepSession,
|
||||
@@ -265,12 +266,14 @@ export class HealthStore {
|
||||
return { cursor: next, samples, sleep, scanned };
|
||||
}
|
||||
|
||||
/** Every stored sample overlapping [startMs, endMs). Exact - reads shards. */
|
||||
samplesInRange(startMs: number, endMs: number): HealthSample[] {
|
||||
/** Bucket samples in [startMs, endMs). Point observations are opt-in so
|
||||
* existing charts and daily averages never combine two overlapping series. */
|
||||
samplesInRange(startMs: number, endMs: number, includeMeasurements = false): HealthSample[] {
|
||||
const out: HealthSample[] = [];
|
||||
for (const name of this.shardNamesInRange(startMs, endMs)) {
|
||||
for (const sample of this.loadShard(name).values()) {
|
||||
if (sample.startMs >= startMs && sample.startMs < endMs) out.push(sample);
|
||||
if (sample.startMs >= startMs && sample.startMs < endMs
|
||||
&& (includeMeasurements || !isHeartRateMeasurement(sample))) out.push(sample);
|
||||
}
|
||||
}
|
||||
out.sort((a, b) => a.startMs - b.startMs);
|
||||
@@ -334,6 +337,7 @@ export class HealthStore {
|
||||
for (const name of this.backend.list()) {
|
||||
if (!name.startsWith(SAMPLE_PREFIX) || !name.endsWith(SAMPLE_SUFFIX)) continue;
|
||||
for (const sample of this.loadShard(name).values()) {
|
||||
if (isHeartRateMeasurement(sample)) continue;
|
||||
const key = dayKey(sample.startMs);
|
||||
let byMetric = perDay.get(key);
|
||||
if (!byMetric) {
|
||||
@@ -360,7 +364,9 @@ export class HealthStore {
|
||||
private updateRollupsFor(samples: readonly HealthSample[]): void {
|
||||
const file = this.loadRollups();
|
||||
const touched = new Set<string>();
|
||||
for (const sample of samples) touched.add(dayKey(sample.startMs));
|
||||
for (const sample of samples) {
|
||||
if (!isHeartRateMeasurement(sample)) touched.add(dayKey(sample.startMs));
|
||||
}
|
||||
for (const key of touched) {
|
||||
const parts = key.split("-").map(Number);
|
||||
const dayStart = new Date(parts[0]!, parts[1]! - 1, parts[2]!).getTime();
|
||||
|
||||
@@ -36,6 +36,12 @@ export function isCumulative(metric: SampleMetric): boolean {
|
||||
|
||||
export const HOUR_MS = 3_600_000;
|
||||
export const TEN_MINUTES_MS = 600_000;
|
||||
/** Point observation, not a ten-minute average. Distinct from hourly buckets. */
|
||||
export const HEART_RATE_MEASUREMENT_SPAN_MS = 1;
|
||||
|
||||
export function isHeartRateMeasurement(sample: HealthSample): boolean {
|
||||
return sample.metric === "heartRate" && sample.spanMs === HEART_RATE_MEASUREMENT_SPAN_MS;
|
||||
}
|
||||
export const DAY_MS = 86_400_000;
|
||||
|
||||
/**
|
||||
|
||||
@@ -36,3 +36,42 @@ Companion PRs: [Faceclaw #11](https://git.undefined-dev.fr/Blendman974/faceclaw/
|
||||
and [dashboards #6](https://git.undefined-dev.fr/Infrastructure/GrafanaDashboards/pulls/6).
|
||||
Combined validation: 1137 JS tests passed, 2 skipped; TypeScript and Android
|
||||
release build passed, including all seven native libraries' 16 KiB page check.
|
||||
|
||||
## Latest heart-rate observations
|
||||
|
||||
Live HR pages also carry a last-measured timestamp and value. These are now
|
||||
stored as `heartRate` with `spanMs: 1`, at the exact measurement timestamp:
|
||||
this denotes a point observation, **not** a ten-minute average. Existing hourly
|
||||
records retain `spanMs: 3600000`. The existing local and PostgreSQL keys include
|
||||
span, so readings and summaries coexist without collisions. Equal BPM at a new
|
||||
timestamp is a new measurement; repeated timestamps are deduplicated across
|
||||
restarts and corrected values update the same observation. Local bucket charts
|
||||
and daily rollups exclude points, including when the cache is rebuilt.
|
||||
|
||||
The existing aligned ring pull changes from :01/:31 to :01/:11/:21/:31/:41/:51.
|
||||
This is a collection cadence, not a sensor setting or a guarantee of six readings
|
||||
per hour. It uses the same ring worker, connection windows and anti-spam guards;
|
||||
it makes more collection attempts than before. Offline gaps remain gaps and
|
||||
there is no new background service. The exporter still sends at most one batch
|
||||
per minute while the Android process runs.
|
||||
|
||||
Timestamp handling deliberately retains the current handshake's legacy clock
|
||||
convention, inverting its existing phone-timezone offset once. Only a live point
|
||||
within 20 minutes of page reception and no more than two minutes in the future
|
||||
is accepted; invalid/missing timestamps and BPM outside 1–300 are skipped.
|
||||
Reception time is never substituted for measurement time. Old frame-journal
|
||||
replay continues restoring hourly history but does not reconstruct point readings:
|
||||
old pages may mix clock domains. No firmware/time-sync migration is attempted.
|
||||
A timezone change around measurement or an older value can therefore leave a gap;
|
||||
correct placement across such transitions still requires hardware validation.
|
||||
|
||||
The deployed v1 API already accepts span 1 and upserts on device/metric/time/span.
|
||||
No API image, schema migration, Infrastructure change or key rotation is needed.
|
||||
The Grafana overview shows separate hourly-average and latest-reading series,
|
||||
with point markers for readings, not interpolated ten-minute averages.
|
||||
|
||||
Software checks cover latest-page conversion, invalid/stale/future readings,
|
||||
timezone offsets, restart/deduplication, unchanged hourly rollups, export and
|
||||
all six scheduling slots. Hardware checks remain pending: record raw/current
|
||||
measurement times across several pulls, compare BPM/time with the stock app,
|
||||
and check reconnect, timezone/DST transitions and collection battery impact.
|
||||
@@ -94,3 +94,29 @@ test('large sleep histories split into bounded bodies without losing sessions',
|
||||
assert.ok(h.requests.every(r=>r.options.body.length<=240000));
|
||||
assert.equal(h.requests.reduce((n,r)=>n+JSON.parse(r.options.body).sleep.length,0),sessions.length);
|
||||
});
|
||||
|
||||
test('point heart rates and hourly summaries export separately with original timestamps', async () => {
|
||||
const h = setup();
|
||||
const point = { ...sample, spanMs: 1, startMs: sample.startMs + 123456, min: 72, max: 72, avg: 72, total: 72 };
|
||||
h.store.ingestSamples([sample, point]);
|
||||
await h.exporter.syncHealthExport();
|
||||
const body = JSON.parse(h.requests[0].options.body);
|
||||
assert.deepEqual(body.samples, [sample, point]);
|
||||
h.store.resetCache();
|
||||
assert.equal(h.store.ingestSamples([point]), 0);
|
||||
await h.exporter.syncHealthExport();
|
||||
assert.equal(h.requests.length, 1);
|
||||
});
|
||||
|
||||
test('a corrected point replaces its old value without changing hourly summaries', async () => {
|
||||
const h = setup();
|
||||
const point = { ...sample, spanMs: 1, min: 72, max: 72, avg: 72, total: 72 };
|
||||
h.store.ingestSamples([sample, point]);
|
||||
await h.exporter.syncHealthExport();
|
||||
const correction = { ...point, min: 74, max: 74, avg: 74, total: 74 };
|
||||
h.store.ingestSamples([correction]);
|
||||
await h.exporter.syncHealthExport();
|
||||
assert.deepEqual(JSON.parse(h.requests[1].options.body).samples, [correction]);
|
||||
assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1, true).length, 2);
|
||||
assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1)[0].avg, sample.avg);
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert/strict');
|
||||
const { loader } = require('./helpers/load-typescript.cjs');
|
||||
const { convertRecords } = loader()('app/health/health-ingest.ts');
|
||||
const receivedAtMs = Date.UTC(2026, 10, 8, 12);
|
||||
const page = {
|
||||
kind: 'hourly', metric: 'heartRate', anchorMs: receivedAtMs,
|
||||
groups: [{ hourIndex: 0, min: 50, max: 90, avg: 65 }],
|
||||
latest: { timestampSeconds: (receivedAtMs - 60000) / 1000, value: 72, receivedAtMs, clockOffsetMs: 0 },
|
||||
};
|
||||
|
||||
test('latest observation remains a point, independent of hourly averages and page anchor', () => {
|
||||
const result = convertRecords([{ ...page, anchorMs: null }]);
|
||||
assert.equal(result.samples.length, 1);
|
||||
assert.equal(result.samples[0].spanMs, 1);
|
||||
assert.equal(result.samples[0].startMs, receivedAtMs - 60000);
|
||||
assert.equal(result.samples[0].min, 72);
|
||||
assert.equal(result.samples[0].max, 72);
|
||||
assert.equal(result.samples[0].avg, 72);
|
||||
});
|
||||
|
||||
test('legacy handshake timezone offset is inverted once, preserving exact timestamps', () => {
|
||||
for (const clockOffsetMs of [-3600000, -7200000, 7 * 3600000, 0, -19800000]) {
|
||||
const latest = { ...page.latest, clockOffsetMs,
|
||||
timestampSeconds: (receivedAtMs - 60000 + clockOffsetMs) / 1000 };
|
||||
const samples = convertRecords([{ ...page, latest }]).samples;
|
||||
assert.equal(samples[0].startMs, receivedAtMs - 60000);
|
||||
assert.equal(samples[1].spanMs, 3600000);
|
||||
}
|
||||
});
|
||||
|
||||
test('invalid, stale and future latest values never corrupt hourly history', () => {
|
||||
for (const patch of [
|
||||
{ value: 0 }, { value: -1 }, { value: 301 }, { value: NaN }, { value: 72.5 },
|
||||
{ timestampSeconds: -1 }, { timestampSeconds: 0 }, { timestampSeconds: NaN },
|
||||
{ timestampSeconds: receivedAtMs / 1000 + 121 },
|
||||
{ timestampSeconds: receivedAtMs / 1000 - 1201 },
|
||||
{ receivedAtMs: NaN }, { clockOffsetMs: NaN },
|
||||
]) {
|
||||
const result = convertRecords([{ ...page, latest: { ...page.latest, ...patch } }]);
|
||||
assert.equal(result.samples.length, 1, JSON.stringify(patch));
|
||||
assert.equal(result.samples[0].spanMs, 3600000);
|
||||
assert.equal(result.skipped.length, 1);
|
||||
}
|
||||
for (const metric of ['spo2', 'hrv']) {
|
||||
assert.equal(convertRecords([{ ...page, metric }]).samples.length, 1);
|
||||
}
|
||||
});
|
||||
@@ -210,3 +210,56 @@ test("a failed journal replay is retried on the next sync and never blocks the l
|
||||
live.syncLiveRecords();
|
||||
assert.equal(failures, -1, "replay was not retried");
|
||||
});
|
||||
|
||||
test("live latest heart rate retains measurement time and survives restart without duplicates", () => {
|
||||
const measuredAt = Math.floor(Date.now() / 1000) * 1000 - 60_000;
|
||||
function sync(previous, time) {
|
||||
const env = liveEnvironment();
|
||||
if (previous) for (const [name, text] of previous.files) env.files.set(name, text);
|
||||
Object.assign(env.record, {
|
||||
receivedAtMs: time + 60_000,
|
||||
tagRaw: (time + new Date(time + 60_000).getTimezoneOffset() * 60_000) / 1000,
|
||||
current: 72,
|
||||
});
|
||||
env.load("health-live").syncLiveRecords();
|
||||
return env;
|
||||
}
|
||||
const first = sync(null, measuredAt);
|
||||
const repeated = sync(first, measuredAt);
|
||||
const next = sync(repeated, measuredAt + 600_000);
|
||||
const store = next.load("health-store-files").healthStore();
|
||||
const batch = store.exportBatch({});
|
||||
const points = Array.from(batch.samples).filter(s => s.spanMs === 1);
|
||||
assert.deepEqual(points.map(s => [s.startMs, s.avg]), [[measuredAt, 72], [measuredAt + 600_000, 72]]);
|
||||
assert.equal(batch.samples.filter(s => s.spanMs === 3600000).length, 1);
|
||||
assert.equal(store.samplesInRange(measuredAt - 86400000, measuredAt + 86400000).length, 1,
|
||||
"existing health charts only see bucket samples");
|
||||
const counts = () => Array.from(store.dailyRollups('heartRate', measuredAt - 86400000, measuredAt + 86400000).values()).reduce((n, r) => n + r.count, 0);
|
||||
assert.equal(counts(), 1);
|
||||
store.rebuildRollups();
|
||||
assert.equal(counts(), 1, "rebuilding must not mix point readings with hourly aggregates");
|
||||
});
|
||||
|
||||
test("aligned health collection requests all six ten-minute slots without timer drift", () => {
|
||||
const { msUntilNextAlignedPull, ALIGNED_PULL_MINUTES } = liveEnvironment().load('health-live').__alignedPullInternals;
|
||||
assert.deepEqual(Array.from(ALIGNED_PULL_MINUTES), [1, 11, 21, 31, 41, 51]);
|
||||
for (const minute of ALIGNED_PULL_MINUTES) {
|
||||
assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute, 0)), 600000);
|
||||
assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute - 1, 30)), 30000);
|
||||
}
|
||||
assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, 59, 0)), 120000);
|
||||
});
|
||||
|
||||
test("journal replay restores hourly history without assigning a clock to old point readings", () => {
|
||||
const receivedAtMs = Math.floor(Date.now() / 1000) * 1000;
|
||||
let replay;
|
||||
const ringProtocol = { decodeFrameJournal: () => ({ size: () => 1, get: () => replay }) };
|
||||
const env = liveEnvironment({ ringProtocol });
|
||||
replay = { ...env.record, current: 80, receivedAtMs,
|
||||
tagRaw: (receivedAtMs + new Date(receivedAtMs).getTimezoneOffset() * 60000) / 1000 };
|
||||
env.files.set('/health/ring-frames.jsonl', 'synthetic journal');
|
||||
env.load('health-live').syncLiveRecords();
|
||||
const samples = env.load('health-store-files').healthStore().exportBatch({}).samples;
|
||||
assert.equal(samples.length, 1);
|
||||
assert.equal(samples[0].spanMs, 3600000);
|
||||
});
|
||||
Reference in new issue
Block a user