From 20026c918a1460854cff2e81b6136cff725a29b0 Mon Sep 17 00:00:00 2001 From: Kalani Helekunihi Date: Tue, 25 Aug 2026 00:41:43 -0500 Subject: [PATCH] Replace Unicode property escapes, which crash the glasses' V8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The NativeScript Android runtime embeds a V8 built without ICU, so a regex literal containing `\p{...}` cannot be parsed there. Because the app ships as a single ES module, the three such literals added with the calculator and microphone apps made the whole of bundle.mjs fail to compile: com.tns.NativeScriptException: Cannot compile ES module .../app/bundle.mjs SyntaxError: Invalid regular expression: /\p{L}/: Invalid property name That throws inside Application.onCreate, so the app died on launch before any of its own code ran — every screen, not just the two new ones. Neither tsc nor the test suite caught it: the syntax is valid ES2018 and node has ICU. app/util/unicode-class.ts replaces them with explicit ranges. The two uses pull in opposite directions, so they are stated differently: - isLetter (maths lexer) is a letter table. The lexer's alphabet is small and known, and an unrecognised letter just becomes an identifier matching no known function, which is already a handled error. - NON_WORD_RUN (conversation search, sentiment) is a *separator* table rather than the complement of that letter table. Complementing it would make every letter the table omits — minority scripts, later Unicode additions — act as a separator and silently shred that text; listing separators instead keeps an unlisted character as word content. It also holds decomposed graphemes together, which `\p{L}` did not. Verified against the real properties under node's ICU: identical output on a prose corpus except where the new split is better (a Lao vowel sign and a combining acute now stay attached). The new test's last case scans app/ for `\p{` outside comments — the check that would have caught this before it reached a device. --- app/apps/calculator/math/lexer.ts | 5 +- app/apps/microphones/sentiment.ts | 4 +- app/phone-ui/conversation-search.ts | 3 +- app/util/unicode-class.ts | 100 ++++++++++++++++++++++++++++ tests/unicode-class.test.cjs | 92 +++++++++++++++++++++++++ 5 files changed, 198 insertions(+), 6 deletions(-) create mode 100644 app/util/unicode-class.ts create mode 100644 tests/unicode-class.test.cjs diff --git a/app/apps/calculator/math/lexer.ts b/app/apps/calculator/math/lexer.ts index 895bc1c1..a751dcee 100644 --- a/app/apps/calculator/math/lexer.ts +++ b/app/apps/calculator/math/lexer.ts @@ -2,6 +2,7 @@ import { type MathNumber, fromDecimal, integer, real } from "./math-number"; import { type MathRelation } from "./expression"; +import { isLetter } from "../../../util/unicode-class"; export type MathToken = | { type: "number"; value: MathNumber } @@ -71,10 +72,6 @@ function isDigit(character: string): boolean { return character >= "0" && character <= "9"; } -function isLetter(character: string): boolean { - return /\p{L}/u.test(character); -} - export function tokenize(input: string): MathToken[] { const tokens: MathToken[] = []; const characters = Array.from(input); diff --git a/app/apps/microphones/sentiment.ts b/app/apps/microphones/sentiment.ts index 0cd4f1b6..157129e7 100644 --- a/app/apps/microphones/sentiment.ts +++ b/app/apps/microphones/sentiment.ts @@ -3,6 +3,8 @@ // scoring. No platform NLP dependency — the deterministic tiers carry the // whole signal. +import { NON_WORD_RUN } from "../../util/unicode-class"; + export type EmotionLabel = | "happy" | "sad" @@ -143,7 +145,7 @@ function emotionTokens(text: string): string[] { .replace(/can't/g, "cannot") .replace(/i'm/g, "i am") .replace(/n't/g, " not"); - return normalized.split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0); + return normalized.split(NON_WORD_RUN).filter((token) => token.length > 0); } function emotionCueIsNegated(start: number, tokens: string[]): boolean { diff --git a/app/phone-ui/conversation-search.ts b/app/phone-ui/conversation-search.ts index dd590ca2..97fa5a0e 100644 --- a/app/phone-ui/conversation-search.ts +++ b/app/phone-ui/conversation-search.ts @@ -1,4 +1,5 @@ import { inferredEmotionFilter, type EmotionLabel } from "../apps/microphones/sentiment"; +import { NON_WORD_RUN } from "../util/unicode-class"; /** * Natural-language search parsing for the conversations list. The grammar is @@ -90,7 +91,7 @@ function startOfDay(nowMs: number): number { } function tokenize(text: string): string[] { - return text.split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0); + return text.split(NON_WORD_RUN).filter((token) => token.length > 0); } export function parseConversationQuery(raw: string, nowMs: number = Date.now()): ParsedConversationQuery { diff --git a/app/util/unicode-class.ts b/app/util/unicode-class.ts new file mode 100644 index 00000000..cfe59657 --- /dev/null +++ b/app/util/unicode-class.ts @@ -0,0 +1,100 @@ +// Letter and digit character classes that avoid Unicode property escapes. +// +// The NativeScript Android runtime embeds a V8 built without ICU, so a regex +// literal containing `\p{...}` cannot be *parsed* there. Since the app ships as +// one ES module, a single such literal anywhere makes the whole of bundle.mjs +// fail to compile with +// SyntaxError: Invalid regular expression: /\p{L}/: Invalid property name +// which kills the process inside Application.onCreate, before any of our code +// runs. The ranges below are plain ECMAScript and parse on every engine. +// +// Coverage is the BMP only: a letter encoded as a surrogate pair (CJK ext. B, +// Deseret, Old Italic, ...) is not classified as a letter. `\p{L}` did cover +// those; nothing in this app has a use for them today, and the alternative is +// surrogate bookkeeping at every call site. + +/** Class body — no enclosing brackets — approximating `\p{L}` over the BMP. */ +export const LETTER_CLASS = + "A-Za-z" + + "\\u00AA\\u00B5\\u00BA" + // ª µ º + "\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02AF" + // Latin-1 sup. + ext. A/B + IPA, minus × and ÷ + "\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03FF" + // Greek (incl. π), minus its punctuation + "\\u0400-\\u052F" + // Cyrillic + supplement + "\\u0531-\\u0556\\u0561-\\u0587" + // Armenian + "\\u05D0-\\u05EA\\u05EF-\\u05F2" + // Hebrew + "\\u0620-\\u064A\\u066E-\\u06D3\\u06D5" + // Arabic + "\\u02B0-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4" + // Modifier letters + "\\u0710-\\u072F\\u0780-\\u07B1" + // Syriac, Thaana + "\\u0900-\\u097F\\u0980-\\u09FF\\u0A00-\\u0A7F\\u0A80-\\u0AFF" + // Devanagari, Bengali, Gurmukhi, Gujarati + "\\u0B00-\\u0B7F\\u0B80-\\u0BFF\\u0C00-\\u0C7F\\u0C80-\\u0CFF" + // Oriya, Tamil, Telugu, Kannada + "\\u0D00-\\u0D7F\\u0D80-\\u0DFF" + // Malayalam, Sinhala + "\\u0E01-\\u0E3A\\u0E40-\\u0E4E\\u0E81-\\u0EDF" + // Thai, Lao + "\\u0F40-\\u0FBC" + // Tibetan + "\\u1000-\\u109F" + // Myanmar + "\\u10A0-\\u10FF\\u2D00-\\u2D2F" + // Georgian + supplement + "\\u1200-\\u137F\\u1380-\\u139F" + // Ethiopic + "\\u13A0-\\u13FD\\u1400-\\u167F" + // Cherokee, Canadian Aboriginal syllabics + "\\u1780-\\u17DC" + // Khmer + "\\u1820-\\u18AA" + // Mongolian + "\\u1E00-\\u1EFF" + // Latin extended additional (Vietnamese) + "\\u1F00-\\u1FFF" + // Greek extended + "\\u2C60-\\u2C7F\\u2C80-\\u2CFF\\u2D30-\\u2D67" + // Latin ext. C/D, Coptic, Tifinagh + "\\u3041-\\u3096\\u30A1-\\u30FA\\u30FC" + // Hiragana, Katakana + "\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF" + // CJK + "\\uA500-\\uA62B\\uA640-\\uA69F" + // Vai, Cyrillic extended-B + "\\u1100-\\u11FF\\u3130-\\u318F\\uA960-\\uA97C\\uAC00-\\uD7A3"; // Hangul + +/** Class body — no enclosing brackets — approximating `\p{N}` over the BMP. */ +export const NUMBER_CLASS = + "0-9" + + "\\u00B2\\u00B3\\u00B9\\u00BC-\\u00BE" + // ² ³ ¹ ¼ ½ ¾ + "\\u0660-\\u0669\\u06F0-\\u06F9" + // Arabic-Indic + "\\u0966-\\u096F\\u09E6-\\u09EF" + // Devanagari, Bengali + "\\u0E50-\\u0E59" + // Thai + "\\uFF10-\\uFF19"; // Fullwidth + +const LETTER_RE = new RegExp(`[${LETTER_CLASS}]`); + +/** + * True when `character` contains a letter. Stands in for `/\p{L}/u.test`. + * + * Used by the maths lexer, whose alphabet is small and known: ASCII identifiers + * plus the operator glyphs it enumerates by hand. Over-inclusiveness is cheap + * there — an unrecognised letter becomes an identifier that resolves to no + * known function, which is already a handled error. + */ +export function isLetter(character: string): boolean { + return LETTER_RE.test(character); +} + +/** + * Separator runs for word-splitting prose — whitespace, punctuation, and + * symbols. Stands in for `/[^\p{L}\p{N}]+/u`. + * + * Stated as what to *split on* rather than as the complement of LETTER_CLASS, + * because the two directions fail differently. Complementing the table would + * make every letter it omits (minority scripts, later Unicode additions) act as + * a separator, silently shredding that text into fragments. Listing separators + * instead means an unlisted character is kept as word content: the tokenizer + * degrades to "this stayed joined" rather than "this disappeared". + * + * Combining marks are deliberately absent, so a decomposed "e" + U+0301 stays + * one token — `\p{L}` alone would have split it. + * + * Deliberately not global: `String.prototype.split` ignores `lastIndex`, so one + * shared instance is safe to reuse across call sites. + */ +export const NON_WORD_RUN = new RegExp( + "[\\s" + + "\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E" + // ASCII punctuation + "\\u00A0-\\u00A9\\u00AB-\\u00B1\\u00B4\\u00B6-\\u00B8\\u00BB\\u00BF\\u00D7\\u00F7" + // Latin-1 punctuation/symbols, minus superscripts + "\\u02C2-\\u02C5\\u02D2-\\u02DF\\u02E5-\\u02EB\\u02ED\\u02EF-\\u02FF" + // Spacing modifier symbols, minus modifier letters + "\\u2000-\\u206F" + // General punctuation (dashes, quotes, bullets) + "\\u20A0-\\u20BF" + // Currency + "\\u2190-\\u245F\\u2500-\\u2BFF" + // Arrows/maths/geometric/misc symbols, skipping + // letterlike (U+2100-214F), number forms (U+2150-218F), enclosed alphanumerics (U+2460-24FF) + "\\u3000-\\u303F" + // CJK punctuation + "\\uFE30-\\uFE4F\\uFE50-\\uFE6B" + // CJK compatibility + small form punctuation + "\\uFF00-\\uFF0F\\uFF1A-\\uFF20\\uFF3B-\\uFF40\\uFF5B-\\uFF65" + // Fullwidth punctuation + "]+", +); diff --git a/tests/unicode-class.test.cjs b/tests/unicode-class.test.cjs new file mode 100644 index 00000000..00591570 --- /dev/null +++ b/tests/unicode-class.test.cjs @@ -0,0 +1,92 @@ +// The NativeScript Android runtime embeds a V8 built without ICU, so a regex +// literal containing a Unicode property escape (`\p{L}`) cannot be parsed +// there. One anywhere in the app makes bundle.mjs fail to compile and takes the +// process down inside Application.onCreate, before any of our code runs — a +// crash no typecheck catches, because the syntax is valid ES2018. +// +// These pin the replacement classes and guard against `\p{` coming back. +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); + +const { isLetter, NON_WORD_RUN, LETTER_CLASS, NUMBER_CLASS } = require("../.test-build/app/util/unicode-class.js"); + +const split = (text) => text.split(NON_WORD_RUN).filter((token) => token.length > 0); + +test("the maths lexer's operator glyphs are not letters", () => { + // If any of these read as a letter, tokenize() swallows them into an + // identifier instead of reaching the operator switch. + for (const glyph of ["×", "÷", "−", "≤", "≥", "√", "_", " ", "1"]) { + assert.equal(isLetter(glyph), false, `expected ${JSON.stringify(glyph)} not to be a letter`); + } +}); + +test("identifier characters are letters", () => { + for (const glyph of ["x", "Q", "π", "µ", "é", "й"]) { + assert.equal(isLetter(glyph), true, `expected ${JSON.stringify(glyph)} to be a letter`); + } +}); + +test("prose splits on punctuation and whitespace", () => { + assert.deepEqual(split("i am not happy, but angry"), ["i", "am", "not", "happy", "but", "angry"]); + assert.deepEqual(split("cannot wait!! 100%"), ["cannot", "wait", "100"]); + assert.deepEqual(split("hello—world (test)"), ["hello", "world", "test"]); + assert.deepEqual(split("half-life (2004) — 9/10"), ["half", "life", "2004", "9", "10"]); +}); + +test("prose keeps accented and non-Latin words whole", () => { + assert.deepEqual(split("café naïve résumé"), ["café", "naïve", "résumé"]); + assert.deepEqual(split("привет мир 42"), ["привет", "мир", "42"]); + assert.deepEqual(split("东京 tokyo"), ["东京", "tokyo"]); +}); + +test("a decomposed accent stays attached to its base letter", () => { + // "e" + COMBINING ACUTE. `\p{L}` alone treated the mark as a separator and + // split this into ["e"]; keeping combining marks out of the separator set + // holds the grapheme together. + assert.deepEqual(split("é combining"), ["é", "combining"]); +}); + +test("an unlisted character is kept as word content, never dropped", () => { + // The separator set is stated positively so that anything it does not know + // about — a minority script, a later Unicode addition — degrades to "stayed + // joined" rather than "disappeared". + const unassigned = "ꟲ"; // Latin Extended-D, absent from LETTER_CLASS + assert.deepEqual(split(`alpha${unassigned}beta gamma`), [`alpha${unassigned}beta`, "gamma"]); +}); + +test("the class bodies build valid regexes", () => { + assert.doesNotThrow(() => new RegExp(`[${LETTER_CLASS}]`)); + assert.doesNotThrow(() => new RegExp(`[${NUMBER_CLASS}]`)); + assert.doesNotThrow(() => new RegExp(`[^${LETTER_CLASS}${NUMBER_CLASS}]`)); +}); + +test("no Unicode property escape survives anywhere in app/", () => { + const appRoot = path.join(__dirname, "..", "app"); + + const sources = []; + (function walk(dir) { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) walk(full); + else if (entry.name.endsWith(".ts") || entry.name.endsWith(".js")) sources.push(full); + } + })(appRoot); + + assert.ok(sources.length > 0, "found no sources to scan"); + + // Comments may name the escape (this file's own header does); code may not. + const stripComments = (text) => + text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:"'`\\])\/\/.*$/gm, "$1"); + + const offenders = sources.filter((file) => + /\\p\{/.test(stripComments(fs.readFileSync(file, "utf8"))), + ); + + assert.deepEqual( + offenders.map((file) => path.relative(appRoot, file)), + [], + "Unicode property escapes crash the ICU-less V8 on the glasses' Android host", + ); +});