diff --git a/app/health/health-ingest.ts b/app/health/health-ingest.ts index 70afcfc0..aca95ce9 100644 --- a/app/health/health-ingest.ts +++ b/app/health/health-ingest.ts @@ -48,6 +48,7 @@ import { HOUR_MS, + HEART_RATE_MEASUREMENT_SPAN_MS, TEN_MINUTES_MS, type HealthSample, type SleepSession, @@ -65,6 +66,8 @@ export type WireHourlyRecord = { * zero bytes instead. */ anchorMs: number | null; + /** Latest reading, only from live pages; raw journal replay does not guess its clock. */ + latest?: { timestampSeconds: number; value: number; receivedAtMs: number; clockOffsetMs: number }; groups: readonly { hourIndex: number; avg: number; max: number; min: number }[]; }; @@ -146,6 +149,22 @@ export function convertRecords(records: readonly WireRecord[]): ConversionResult * guesses. Dropping loses a page; guessing corrupts the history silently. */ function convertHourly(record: WireHourlyRecord, result: ConversionResult): void { + if (record.metric === "heartRate" && record.latest) { + const { timestampSeconds, value, receivedAtMs, clockOffsetMs } = record.latest; + const startMs = timestampSeconds * 1000 - clockOffsetMs; + // Invert the existing handshake clock shift, never replace measurement time + // with reception time or snap it onto an invented ten-minute bucket. + // Older pages can mix clock domains: only accept a recent live reading. + if (Number.isSafeInteger(timestampSeconds) && Number.isSafeInteger(receivedAtMs) + && Number.isFinite(clockOffsetMs) && Number.isInteger(value) && value > 0 && value <= 300 + && startMs >= 946684800000 && startMs <= receivedAtMs + 120_000 + && receivedAtMs - startMs <= 20 * 60_000) { + result.samples.push({ metric: "heartRate", startMs, spanMs: HEART_RATE_MEASUREMENT_SPAN_MS, + min: value, max: value, avg: value, total: value }); + } else { + result.skipped.push({ what: "latest heart rate", why: "invalid, stale or unresolved measurement timestamp/value" }); + } + } if (record.anchorMs === null) { result.skipped.push({ what: `${record.metric} page, ${record.groups.length} groups`, diff --git a/app/health/health-live.ts b/app/health/health-live.ts index 7dc86c04..0c1b74d0 100644 --- a/app/health/health-live.ts +++ b/app/health/health-live.ts @@ -240,7 +240,7 @@ function accumulateStepBuckets( return merged; } -function toWire(record: any): WireRecord | null { +function toWire(record: any, includeLatest: boolean): WireRecord | null { const cmdHi = Number(record.cmdHi); const metric = HOURLY_METRIC[cmdHi]; @@ -256,7 +256,12 @@ function toWire(record: any): WireRecord | null { min: Number(group.min), }); } - return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups }; + const receivedAtMs = Number(record.receivedAtMs); + const latest = includeLatest && metric === "heartRate" && Number(record.tagRaw) > 0 ? { + timestampSeconds: Number(record.tagRaw), value: Number(record.current), receivedAtMs, + clockOffsetMs: ringClockOffsetMs(receivedAtMs), + } : undefined; + return { kind: "hourly", metric, anchorMs: anchorToMs(Number(record.anchorUnixSeconds), true), groups, latest }; } if (cmdHi === CMD_STEPS) { @@ -391,12 +396,12 @@ type StoreResult = Omit; * fails, so the caller keeps whatever it would otherwise mark as consumed. * Shared by the live batch and the journal replay. */ -function storeRecords(records: JavaRecordList): StoreResult { +function storeRecords(records: JavaRecordList, includeLatest = true): StoreResult { const wire: WireRecord[] = []; const size = records.size(); for (let i = 0; i < size; i++) { try { - const converted = toWire(records.get(i)); + const converted = toWire(records.get(i), includeLatest); if (converted) wire.push(converted); } catch (error) { console.warn("health live: skipped an undecodable record", error); @@ -480,7 +485,7 @@ function replayFrameJournalOnce(): void { const records: JavaRecordList = ringProtocol.decodeFrameJournal(File.fromPath(journalPath).readTextSync(), after); const size = records.size(); if (size > 0) { - const result = storeRecords(records); + const result = storeRecords(records, false); let through = after; for (let i = 0; i < size; i++) through = Math.max(through, Number(records.get(i).receivedAtMs)); const next: ReplayCursor = { version: JOURNAL_DECODER_VERSION, throughReceivedAtMs: through }; @@ -543,7 +548,7 @@ export function syncLiveRecords(): LiveSyncResult { /** * Ask the ring for a fresh pull now, rather than waiting out the automatic - * 30-minute cycle. Returns immediately; the pull itself takes ~15s on the + * ten-minute cycle. Returns immediately; the pull itself takes ~15s on the * communicator's worker thread and lands through the next `syncLiveRecords()`. * * Throttled communicator-side to one a minute — see @@ -584,19 +589,17 @@ export function startLiveHealthSync(): void { } /** - * Pull the ring on a wall-clock-aligned cadence: :01 and :31, every hour. + * Pull the ring every ten minutes, one minute after each bucket boundary. * - * Chris, 2026-09-12: *"I think we should be polling the ring on the 30 minute - * cycle (that was my intent, not a floor honestly)."* The 30-minute value in - * `RING_HEALTH_MIN_PULL_INTERVAL_MS` was always meant as a collection rhythm - * and got built as a throttle, so a connected, stable ring pulled never — - * nothing in the system drives a pull on a timer at all. + * This collects the latest heart-rate observation at roughly the cadence + * the stock app documents, as well as the existing historical buckets. + * It does not force the sensor to take a new measurement. * * Two reasons this is aligned to the wall clock rather than a plain interval: * * 1. **The ring's own step buckets close on 10-minute boundaries.** Pulling on * the boundary collects a just-closed bucket instead of a half-formed one. - * 2. **An elapsed timer drifts.** A pull at :07 sets the next at :37, then :07, + * 2. **An elapsed timer drifts.** A pull at :07 sets the next at :17, then :27, * wandering away from the boundary it is supposed to track. Re-arming * against the clock each time cannot drift. * @@ -606,7 +609,7 @@ export function startLiveHealthSync(): void { * This only *requests* a pull. The communicator still applies its own anti-spam * floor, so an extra call here can never hammer the ring. */ -const ALIGNED_PULL_MINUTES = [1, 31]; +const ALIGNED_PULL_MINUTES = [1, 11, 21, 31, 41, 51]; let alignedPullTimer: ReturnType | null = null; function msUntilNextAlignedPull(now: Date): number { diff --git a/app/health/health-store.ts b/app/health/health-store.ts index 6c07d041..5eeaab20 100644 --- a/app/health/health-store.ts +++ b/app/health/health-store.ts @@ -59,6 +59,7 @@ import { type HealthSample, + isHeartRateMeasurement, type Rollup, type SampleMetric, type SleepSession, @@ -265,12 +266,14 @@ export class HealthStore { return { cursor: next, samples, sleep, scanned }; } - /** Every stored sample overlapping [startMs, endMs). Exact - reads shards. */ - samplesInRange(startMs: number, endMs: number): HealthSample[] { + /** Bucket samples in [startMs, endMs). Point observations are opt-in so + * existing charts and daily averages never combine two overlapping series. */ + samplesInRange(startMs: number, endMs: number, includeMeasurements = false): HealthSample[] { const out: HealthSample[] = []; for (const name of this.shardNamesInRange(startMs, endMs)) { for (const sample of this.loadShard(name).values()) { - if (sample.startMs >= startMs && sample.startMs < endMs) out.push(sample); + if (sample.startMs >= startMs && sample.startMs < endMs + && (includeMeasurements || !isHeartRateMeasurement(sample))) out.push(sample); } } out.sort((a, b) => a.startMs - b.startMs); @@ -334,6 +337,7 @@ export class HealthStore { for (const name of this.backend.list()) { if (!name.startsWith(SAMPLE_PREFIX) || !name.endsWith(SAMPLE_SUFFIX)) continue; for (const sample of this.loadShard(name).values()) { + if (isHeartRateMeasurement(sample)) continue; const key = dayKey(sample.startMs); let byMetric = perDay.get(key); if (!byMetric) { @@ -360,7 +364,9 @@ export class HealthStore { private updateRollupsFor(samples: readonly HealthSample[]): void { const file = this.loadRollups(); const touched = new Set(); - for (const sample of samples) touched.add(dayKey(sample.startMs)); + for (const sample of samples) { + if (!isHeartRateMeasurement(sample)) touched.add(dayKey(sample.startMs)); + } for (const key of touched) { const parts = key.split("-").map(Number); const dayStart = new Date(parts[0]!, parts[1]! - 1, parts[2]!).getTime(); diff --git a/app/health/health-types.ts b/app/health/health-types.ts index f9aeb698..e8713cc8 100644 --- a/app/health/health-types.ts +++ b/app/health/health-types.ts @@ -36,6 +36,12 @@ export function isCumulative(metric: SampleMetric): boolean { export const HOUR_MS = 3_600_000; export const TEN_MINUTES_MS = 600_000; +/** Point observation, not a ten-minute average. Distinct from hourly buckets. */ +export const HEART_RATE_MEASUREMENT_SPAN_MS = 1; + +export function isHeartRateMeasurement(sample: HealthSample): boolean { + return sample.metric === "heartRate" && sample.spanMs === HEART_RATE_MEASUREMENT_SPAN_MS; +} export const DAY_MS = 86_400_000; /** diff --git a/docs/validation-health-export.md b/docs/validation-health-export.md index 2e298cdf..e22c16c4 100644 --- a/docs/validation-health-export.md +++ b/docs/validation-health-export.md @@ -30,3 +30,42 @@ the deployed URL/key, then compare measured timestamps/values and sleep duration with Grafana. Test offline/reconnect, app restart, disable/re-enable and rotated keys. Check key masking, log redaction and preservation of other preferences. No firmware modification is needed. + +## Latest heart-rate observations + +Live HR pages also carry a last-measured timestamp and value. These are now +stored as `heartRate` with `spanMs: 1`, at the exact measurement timestamp: +this denotes a point observation, **not** a ten-minute average. Existing hourly +records retain `spanMs: 3600000`. The existing local and PostgreSQL keys include +span, so readings and summaries coexist without collisions. Equal BPM at a new +timestamp is a new measurement; repeated timestamps are deduplicated across +restarts and corrected values update the same observation. Local bucket charts +and daily rollups exclude points, including when the cache is rebuilt. + +The existing aligned ring pull changes from :01/:31 to :01/:11/:21/:31/:41/:51. +This is a collection cadence, not a sensor setting or a guarantee of six readings +per hour. It uses the same ring worker, connection windows and anti-spam guards; +it makes more collection attempts than before. Offline gaps remain gaps and +there is no new background service. The exporter still sends at most one batch +per minute while the Android process runs. + +Timestamp handling deliberately retains the current handshake's legacy clock +convention, inverting its existing phone-timezone offset once. Only a live point +within 20 minutes of page reception and no more than two minutes in the future +is accepted; invalid/missing timestamps and BPM outside 1–300 are skipped. +Reception time is never substituted for measurement time. Old frame-journal +replay continues restoring hourly history but does not reconstruct point readings: +old pages may mix clock domains. No firmware/time-sync migration is attempted. +A timezone change around measurement or an older value can therefore leave a gap; +correct placement across such transitions still requires hardware validation. + +The deployed v1 API already accepts span 1 and upserts on device/metric/time/span. +No API image, schema migration, Infrastructure change or key rotation is needed. +The Grafana overview shows separate hourly-average and latest-reading series, +with point markers for readings, not interpolated ten-minute averages. + +Software checks cover latest-page conversion, invalid/stale/future readings, +timezone offsets, restart/deduplication, unchanged hourly rollups, export and +all six scheduling slots. Hardware checks remain pending: record raw/current +measurement times across several pulls, compare BPM/time with the stock app, +and check reconnect, timezone/DST transitions and collection battery impact. diff --git a/tests/health-export.test.cjs b/tests/health-export.test.cjs index ed948469..44416356 100644 --- a/tests/health-export.test.cjs +++ b/tests/health-export.test.cjs @@ -94,3 +94,29 @@ test('large sleep histories split into bounded bodies without losing sessions', assert.ok(h.requests.every(r=>r.options.body.length<=240000)); assert.equal(h.requests.reduce((n,r)=>n+JSON.parse(r.options.body).sleep.length,0),sessions.length); }); + +test('point heart rates and hourly summaries export separately with original timestamps', async () => { + const h = setup(); + const point = { ...sample, spanMs: 1, startMs: sample.startMs + 123456, min: 72, max: 72, avg: 72, total: 72 }; + h.store.ingestSamples([sample, point]); + await h.exporter.syncHealthExport(); + const body = JSON.parse(h.requests[0].options.body); + assert.deepEqual(body.samples, [sample, point]); + h.store.resetCache(); + assert.equal(h.store.ingestSamples([point]), 0); + await h.exporter.syncHealthExport(); + assert.equal(h.requests.length, 1); +}); + +test('a corrected point replaces its old value without changing hourly summaries', async () => { + const h = setup(); + const point = { ...sample, spanMs: 1, min: 72, max: 72, avg: 72, total: 72 }; + h.store.ingestSamples([sample, point]); + await h.exporter.syncHealthExport(); + const correction = { ...point, min: 74, max: 74, avg: 74, total: 74 }; + h.store.ingestSamples([correction]); + await h.exporter.syncHealthExport(); + assert.deepEqual(JSON.parse(h.requests[1].options.body).samples, [correction]); + assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1, true).length, 2); + assert.equal(h.store.samplesInRange(sample.startMs, sample.startMs + 1)[0].avg, sample.avg); +}); diff --git a/tests/health-heart-rate-measurements.test.cjs b/tests/health-heart-rate-measurements.test.cjs new file mode 100644 index 00000000..bcc1456d --- /dev/null +++ b/tests/health-heart-rate-measurements.test.cjs @@ -0,0 +1,48 @@ +const test = require('node:test'); +const assert = require('node:assert/strict'); +const { loader } = require('./helpers/load-typescript.cjs'); +const { convertRecords } = loader()('app/health/health-ingest.ts'); +const receivedAtMs = Date.UTC(2026, 10, 8, 12); +const page = { + kind: 'hourly', metric: 'heartRate', anchorMs: receivedAtMs, + groups: [{ hourIndex: 0, min: 50, max: 90, avg: 65 }], + latest: { timestampSeconds: (receivedAtMs - 60000) / 1000, value: 72, receivedAtMs, clockOffsetMs: 0 }, +}; + +test('latest observation remains a point, independent of hourly averages and page anchor', () => { + const result = convertRecords([{ ...page, anchorMs: null }]); + assert.equal(result.samples.length, 1); + assert.equal(result.samples[0].spanMs, 1); + assert.equal(result.samples[0].startMs, receivedAtMs - 60000); + assert.equal(result.samples[0].min, 72); + assert.equal(result.samples[0].max, 72); + assert.equal(result.samples[0].avg, 72); +}); + +test('legacy handshake timezone offset is inverted once, preserving exact timestamps', () => { + for (const clockOffsetMs of [-3600000, -7200000, 7 * 3600000, 0, -19800000]) { + const latest = { ...page.latest, clockOffsetMs, + timestampSeconds: (receivedAtMs - 60000 + clockOffsetMs) / 1000 }; + const samples = convertRecords([{ ...page, latest }]).samples; + assert.equal(samples[0].startMs, receivedAtMs - 60000); + assert.equal(samples[1].spanMs, 3600000); + } +}); + +test('invalid, stale and future latest values never corrupt hourly history', () => { + for (const patch of [ + { value: 0 }, { value: -1 }, { value: 301 }, { value: NaN }, { value: 72.5 }, + { timestampSeconds: -1 }, { timestampSeconds: 0 }, { timestampSeconds: NaN }, + { timestampSeconds: receivedAtMs / 1000 + 121 }, + { timestampSeconds: receivedAtMs / 1000 - 1201 }, + { receivedAtMs: NaN }, { clockOffsetMs: NaN }, + ]) { + const result = convertRecords([{ ...page, latest: { ...page.latest, ...patch } }]); + assert.equal(result.samples.length, 1, JSON.stringify(patch)); + assert.equal(result.samples[0].spanMs, 3600000); + assert.equal(result.skipped.length, 1); + } + for (const metric of ['spo2', 'hrv']) { + assert.equal(convertRecords([{ ...page, metric }]).samples.length, 1); + } +}); diff --git a/tests/health-live.test.cjs b/tests/health-live.test.cjs index 9b2b2e25..bbac1b6e 100644 --- a/tests/health-live.test.cjs +++ b/tests/health-live.test.cjs @@ -210,3 +210,56 @@ test("a failed journal replay is retried on the next sync and never blocks the l live.syncLiveRecords(); assert.equal(failures, -1, "replay was not retried"); }); + +test("live latest heart rate retains measurement time and survives restart without duplicates", () => { + const measuredAt = Math.floor(Date.now() / 1000) * 1000 - 60_000; + function sync(previous, time) { + const env = liveEnvironment(); + if (previous) for (const [name, text] of previous.files) env.files.set(name, text); + Object.assign(env.record, { + receivedAtMs: time + 60_000, + tagRaw: (time + new Date(time + 60_000).getTimezoneOffset() * 60_000) / 1000, + current: 72, + }); + env.load("health-live").syncLiveRecords(); + return env; + } + const first = sync(null, measuredAt); + const repeated = sync(first, measuredAt); + const next = sync(repeated, measuredAt + 600_000); + const store = next.load("health-store-files").healthStore(); + const batch = store.exportBatch({}); + const points = Array.from(batch.samples).filter(s => s.spanMs === 1); + assert.deepEqual(points.map(s => [s.startMs, s.avg]), [[measuredAt, 72], [measuredAt + 600_000, 72]]); + assert.equal(batch.samples.filter(s => s.spanMs === 3600000).length, 1); + assert.equal(store.samplesInRange(measuredAt - 86400000, measuredAt + 86400000).length, 1, + "existing health charts only see bucket samples"); + const counts = () => Array.from(store.dailyRollups('heartRate', measuredAt - 86400000, measuredAt + 86400000).values()).reduce((n, r) => n + r.count, 0); + assert.equal(counts(), 1); + store.rebuildRollups(); + assert.equal(counts(), 1, "rebuilding must not mix point readings with hourly aggregates"); +}); + +test("aligned health collection requests all six ten-minute slots without timer drift", () => { + const { msUntilNextAlignedPull, ALIGNED_PULL_MINUTES } = liveEnvironment().load('health-live').__alignedPullInternals; + assert.deepEqual(Array.from(ALIGNED_PULL_MINUTES), [1, 11, 21, 31, 41, 51]); + for (const minute of ALIGNED_PULL_MINUTES) { + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute, 0)), 600000); + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, minute - 1, 30)), 30000); + } + assert.equal(msUntilNextAlignedPull(new Date(2026, 10, 8, 12, 59, 0)), 120000); +}); + +test("journal replay restores hourly history without assigning a clock to old point readings", () => { + const receivedAtMs = Math.floor(Date.now() / 1000) * 1000; + let replay; + const ringProtocol = { decodeFrameJournal: () => ({ size: () => 1, get: () => replay }) }; + const env = liveEnvironment({ ringProtocol }); + replay = { ...env.record, current: 80, receivedAtMs, + tagRaw: (receivedAtMs + new Date(receivedAtMs).getTimezoneOffset() * 60000) / 1000 }; + env.files.set('/health/ring-frames.jsonl', 'synthetic journal'); + env.load('health-live').syncLiveRecords(); + const samples = env.load('health-store-files').healthStore().exportBatch({}).samples; + assert.equal(samples.length, 1); + assert.equal(samples[0].spanMs, 3600000); +});