mirror of
https://git.undefined-dev.fr/Blendman974/faceclaw.git
synced 2026-11-19 21:03:09 +00:00
Replace Unicode property escapes, which crash the glasses' V8
The NativeScript Android runtime embeds a V8 built without ICU, so a regex
literal containing `\p{...}` cannot be parsed there. Because the app ships as a
single ES module, the three such literals added with the calculator and
microphone apps made the whole of bundle.mjs fail to compile:
com.tns.NativeScriptException: Cannot compile ES module .../app/bundle.mjs
SyntaxError: Invalid regular expression: /\p{L}/: Invalid property name
That throws inside Application.onCreate, so the app died on launch before any
of its own code ran — every screen, not just the two new ones. Neither tsc nor
the test suite caught it: the syntax is valid ES2018 and node has ICU.
app/util/unicode-class.ts replaces them with explicit ranges. The two uses pull
in opposite directions, so they are stated differently:
- isLetter (maths lexer) is a letter table. The lexer's alphabet is small and
known, and an unrecognised letter just becomes an identifier matching no
known function, which is already a handled error.
- NON_WORD_RUN (conversation search, sentiment) is a *separator* table rather
than the complement of that letter table. Complementing it would make every
letter the table omits — minority scripts, later Unicode additions — act as
a separator and silently shred that text; listing separators instead keeps
an unlisted character as word content. It also holds decomposed graphemes
together, which `\p{L}` did not.
Verified against the real properties under node's ICU: identical output on a
prose corpus except where the new split is better (a Lao vowel sign and a
combining acute now stay attached).
The new test's last case scans app/ for `\p{` outside comments — the check that
would have caught this before it reached a device.
This commit is contained in:
5 files changed
+198
-6
No files matched your search
@@ -2,6 +2,7 @@
|
||||
|
||||
import { type MathNumber, fromDecimal, integer, real } from "./math-number";
|
||||
import { type MathRelation } from "./expression";
|
||||
import { isLetter } from "../../../util/unicode-class";
|
||||
|
||||
export type MathToken =
|
||||
| { type: "number"; value: MathNumber }
|
||||
@@ -71,10 +72,6 @@ function isDigit(character: string): boolean {
|
||||
return character >= "0" && character <= "9";
|
||||
}
|
||||
|
||||
function isLetter(character: string): boolean {
|
||||
return /\p{L}/u.test(character);
|
||||
}
|
||||
|
||||
export function tokenize(input: string): MathToken[] {
|
||||
const tokens: MathToken[] = [];
|
||||
const characters = Array.from(input);
|
||||
|
||||
@@ -3,6 +3,8 @@
|
||||
// scoring. No platform NLP dependency — the deterministic tiers carry the
|
||||
// whole signal.
|
||||
|
||||
import { NON_WORD_RUN } from "../../util/unicode-class";
|
||||
|
||||
export type EmotionLabel =
|
||||
| "happy"
|
||||
| "sad"
|
||||
@@ -143,7 +145,7 @@ function emotionTokens(text: string): string[] {
|
||||
.replace(/can't/g, "cannot")
|
||||
.replace(/i'm/g, "i am")
|
||||
.replace(/n't/g, " not");
|
||||
return normalized.split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0);
|
||||
return normalized.split(NON_WORD_RUN).filter((token) => token.length > 0);
|
||||
}
|
||||
|
||||
function emotionCueIsNegated(start: number, tokens: string[]): boolean {
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { inferredEmotionFilter, type EmotionLabel } from "../apps/microphones/sentiment";
|
||||
import { NON_WORD_RUN } from "../util/unicode-class";
|
||||
|
||||
/**
|
||||
* Natural-language search parsing for the conversations list. The grammar is
|
||||
@@ -90,7 +91,7 @@ function startOfDay(nowMs: number): number {
|
||||
}
|
||||
|
||||
function tokenize(text: string): string[] {
|
||||
return text.split(/[^\p{L}\p{N}]+/u).filter((token) => token.length > 0);
|
||||
return text.split(NON_WORD_RUN).filter((token) => token.length > 0);
|
||||
}
|
||||
|
||||
export function parseConversationQuery(raw: string, nowMs: number = Date.now()): ParsedConversationQuery {
|
||||
|
||||
@@ -0,0 +1,100 @@
|
||||
// Letter and digit character classes that avoid Unicode property escapes.
|
||||
//
|
||||
// The NativeScript Android runtime embeds a V8 built without ICU, so a regex
|
||||
// literal containing `\p{...}` cannot be *parsed* there. Since the app ships as
|
||||
// one ES module, a single such literal anywhere makes the whole of bundle.mjs
|
||||
// fail to compile with
|
||||
// SyntaxError: Invalid regular expression: /\p{L}/: Invalid property name
|
||||
// which kills the process inside Application.onCreate, before any of our code
|
||||
// runs. The ranges below are plain ECMAScript and parse on every engine.
|
||||
//
|
||||
// Coverage is the BMP only: a letter encoded as a surrogate pair (CJK ext. B,
|
||||
// Deseret, Old Italic, ...) is not classified as a letter. `\p{L}` did cover
|
||||
// those; nothing in this app has a use for them today, and the alternative is
|
||||
// surrogate bookkeeping at every call site.
|
||||
|
||||
/** Class body — no enclosing brackets — approximating `\p{L}` over the BMP. */
|
||||
export const LETTER_CLASS =
|
||||
"A-Za-z" +
|
||||
"\\u00AA\\u00B5\\u00BA" + // ª µ º
|
||||
"\\u00C0-\\u00D6\\u00D8-\\u00F6\\u00F8-\\u02AF" + // Latin-1 sup. + ext. A/B + IPA, minus × and ÷
|
||||
"\\u0386\\u0388-\\u038A\\u038C\\u038E-\\u03A1\\u03A3-\\u03FF" + // Greek (incl. π), minus its punctuation
|
||||
"\\u0400-\\u052F" + // Cyrillic + supplement
|
||||
"\\u0531-\\u0556\\u0561-\\u0587" + // Armenian
|
||||
"\\u05D0-\\u05EA\\u05EF-\\u05F2" + // Hebrew
|
||||
"\\u0620-\\u064A\\u066E-\\u06D3\\u06D5" + // Arabic
|
||||
"\\u02B0-\\u02C1\\u02C6-\\u02D1\\u02E0-\\u02E4" + // Modifier letters
|
||||
"\\u0710-\\u072F\\u0780-\\u07B1" + // Syriac, Thaana
|
||||
"\\u0900-\\u097F\\u0980-\\u09FF\\u0A00-\\u0A7F\\u0A80-\\u0AFF" + // Devanagari, Bengali, Gurmukhi, Gujarati
|
||||
"\\u0B00-\\u0B7F\\u0B80-\\u0BFF\\u0C00-\\u0C7F\\u0C80-\\u0CFF" + // Oriya, Tamil, Telugu, Kannada
|
||||
"\\u0D00-\\u0D7F\\u0D80-\\u0DFF" + // Malayalam, Sinhala
|
||||
"\\u0E01-\\u0E3A\\u0E40-\\u0E4E\\u0E81-\\u0EDF" + // Thai, Lao
|
||||
"\\u0F40-\\u0FBC" + // Tibetan
|
||||
"\\u1000-\\u109F" + // Myanmar
|
||||
"\\u10A0-\\u10FF\\u2D00-\\u2D2F" + // Georgian + supplement
|
||||
"\\u1200-\\u137F\\u1380-\\u139F" + // Ethiopic
|
||||
"\\u13A0-\\u13FD\\u1400-\\u167F" + // Cherokee, Canadian Aboriginal syllabics
|
||||
"\\u1780-\\u17DC" + // Khmer
|
||||
"\\u1820-\\u18AA" + // Mongolian
|
||||
"\\u1E00-\\u1EFF" + // Latin extended additional (Vietnamese)
|
||||
"\\u1F00-\\u1FFF" + // Greek extended
|
||||
"\\u2C60-\\u2C7F\\u2C80-\\u2CFF\\u2D30-\\u2D67" + // Latin ext. C/D, Coptic, Tifinagh
|
||||
"\\u3041-\\u3096\\u30A1-\\u30FA\\u30FC" + // Hiragana, Katakana
|
||||
"\\u3400-\\u4DBF\\u4E00-\\u9FFF\\uF900-\\uFAFF" + // CJK
|
||||
"\\uA500-\\uA62B\\uA640-\\uA69F" + // Vai, Cyrillic extended-B
|
||||
"\\u1100-\\u11FF\\u3130-\\u318F\\uA960-\\uA97C\\uAC00-\\uD7A3"; // Hangul
|
||||
|
||||
/** Class body — no enclosing brackets — approximating `\p{N}` over the BMP. */
|
||||
export const NUMBER_CLASS =
|
||||
"0-9" +
|
||||
"\\u00B2\\u00B3\\u00B9\\u00BC-\\u00BE" + // ² ³ ¹ ¼ ½ ¾
|
||||
"\\u0660-\\u0669\\u06F0-\\u06F9" + // Arabic-Indic
|
||||
"\\u0966-\\u096F\\u09E6-\\u09EF" + // Devanagari, Bengali
|
||||
"\\u0E50-\\u0E59" + // Thai
|
||||
"\\uFF10-\\uFF19"; // Fullwidth
|
||||
|
||||
const LETTER_RE = new RegExp(`[${LETTER_CLASS}]`);
|
||||
|
||||
/**
|
||||
* True when `character` contains a letter. Stands in for `/\p{L}/u.test`.
|
||||
*
|
||||
* Used by the maths lexer, whose alphabet is small and known: ASCII identifiers
|
||||
* plus the operator glyphs it enumerates by hand. Over-inclusiveness is cheap
|
||||
* there — an unrecognised letter becomes an identifier that resolves to no
|
||||
* known function, which is already a handled error.
|
||||
*/
|
||||
export function isLetter(character: string): boolean {
|
||||
return LETTER_RE.test(character);
|
||||
}
|
||||
|
||||
/**
|
||||
* Separator runs for word-splitting prose — whitespace, punctuation, and
|
||||
* symbols. Stands in for `/[^\p{L}\p{N}]+/u`.
|
||||
*
|
||||
* Stated as what to *split on* rather than as the complement of LETTER_CLASS,
|
||||
* because the two directions fail differently. Complementing the table would
|
||||
* make every letter it omits (minority scripts, later Unicode additions) act as
|
||||
* a separator, silently shredding that text into fragments. Listing separators
|
||||
* instead means an unlisted character is kept as word content: the tokenizer
|
||||
* degrades to "this stayed joined" rather than "this disappeared".
|
||||
*
|
||||
* Combining marks are deliberately absent, so a decomposed "e" + U+0301 stays
|
||||
* one token — `\p{L}` alone would have split it.
|
||||
*
|
||||
* Deliberately not global: `String.prototype.split` ignores `lastIndex`, so one
|
||||
* shared instance is safe to reuse across call sites.
|
||||
*/
|
||||
export const NON_WORD_RUN = new RegExp(
|
||||
"[\\s" +
|
||||
"\\u0021-\\u002F\\u003A-\\u0040\\u005B-\\u0060\\u007B-\\u007E" + // ASCII punctuation
|
||||
"\\u00A0-\\u00A9\\u00AB-\\u00B1\\u00B4\\u00B6-\\u00B8\\u00BB\\u00BF\\u00D7\\u00F7" + // Latin-1 punctuation/symbols, minus superscripts
|
||||
"\\u02C2-\\u02C5\\u02D2-\\u02DF\\u02E5-\\u02EB\\u02ED\\u02EF-\\u02FF" + // Spacing modifier symbols, minus modifier letters
|
||||
"\\u2000-\\u206F" + // General punctuation (dashes, quotes, bullets)
|
||||
"\\u20A0-\\u20BF" + // Currency
|
||||
"\\u2190-\\u245F\\u2500-\\u2BFF" + // Arrows/maths/geometric/misc symbols, skipping
|
||||
// letterlike (U+2100-214F), number forms (U+2150-218F), enclosed alphanumerics (U+2460-24FF)
|
||||
"\\u3000-\\u303F" + // CJK punctuation
|
||||
"\\uFE30-\\uFE4F\\uFE50-\\uFE6B" + // CJK compatibility + small form punctuation
|
||||
"\\uFF00-\\uFF0F\\uFF1A-\\uFF20\\uFF3B-\\uFF40\\uFF5B-\\uFF65" + // Fullwidth punctuation
|
||||
"]+",
|
||||
);
|
||||
@@ -0,0 +1,92 @@
|
||||
// The NativeScript Android runtime embeds a V8 built without ICU, so a regex
|
||||
// literal containing a Unicode property escape (`\p{L}`) cannot be parsed
|
||||
// there. One anywhere in the app makes bundle.mjs fail to compile and takes the
|
||||
// process down inside Application.onCreate, before any of our code runs — a
|
||||
// crash no typecheck catches, because the syntax is valid ES2018.
|
||||
//
|
||||
// These pin the replacement classes and guard against `\p{` coming back.
|
||||
const test = require("node:test");
|
||||
const assert = require("node:assert/strict");
|
||||
const fs = require("node:fs");
|
||||
const path = require("node:path");
|
||||
|
||||
const { isLetter, NON_WORD_RUN, LETTER_CLASS, NUMBER_CLASS } = require("../.test-build/app/util/unicode-class.js");
|
||||
|
||||
const split = (text) => text.split(NON_WORD_RUN).filter((token) => token.length > 0);
|
||||
|
||||
test("the maths lexer's operator glyphs are not letters", () => {
|
||||
// If any of these read as a letter, tokenize() swallows them into an
|
||||
// identifier instead of reaching the operator switch.
|
||||
for (const glyph of ["×", "÷", "−", "≤", "≥", "√", "_", " ", "1"]) {
|
||||
assert.equal(isLetter(glyph), false, `expected ${JSON.stringify(glyph)} not to be a letter`);
|
||||
}
|
||||
});
|
||||
|
||||
test("identifier characters are letters", () => {
|
||||
for (const glyph of ["x", "Q", "π", "µ", "é", "й"]) {
|
||||
assert.equal(isLetter(glyph), true, `expected ${JSON.stringify(glyph)} to be a letter`);
|
||||
}
|
||||
});
|
||||
|
||||
test("prose splits on punctuation and whitespace", () => {
|
||||
assert.deepEqual(split("i am not happy, but angry"), ["i", "am", "not", "happy", "but", "angry"]);
|
||||
assert.deepEqual(split("cannot wait!! 100%"), ["cannot", "wait", "100"]);
|
||||
assert.deepEqual(split("hello—world (test)"), ["hello", "world", "test"]);
|
||||
assert.deepEqual(split("half-life (2004) — 9/10"), ["half", "life", "2004", "9", "10"]);
|
||||
});
|
||||
|
||||
test("prose keeps accented and non-Latin words whole", () => {
|
||||
assert.deepEqual(split("café naïve résumé"), ["café", "naïve", "résumé"]);
|
||||
assert.deepEqual(split("привет мир 42"), ["привет", "мир", "42"]);
|
||||
assert.deepEqual(split("东京 tokyo"), ["东京", "tokyo"]);
|
||||
});
|
||||
|
||||
test("a decomposed accent stays attached to its base letter", () => {
|
||||
// "e" + COMBINING ACUTE. `\p{L}` alone treated the mark as a separator and
|
||||
// split this into ["e"]; keeping combining marks out of the separator set
|
||||
// holds the grapheme together.
|
||||
assert.deepEqual(split("é combining"), ["é", "combining"]);
|
||||
});
|
||||
|
||||
test("an unlisted character is kept as word content, never dropped", () => {
|
||||
// The separator set is stated positively so that anything it does not know
|
||||
// about — a minority script, a later Unicode addition — degrades to "stayed
|
||||
// joined" rather than "disappeared".
|
||||
const unassigned = "ꟲ"; // Latin Extended-D, absent from LETTER_CLASS
|
||||
assert.deepEqual(split(`alpha${unassigned}beta gamma`), [`alpha${unassigned}beta`, "gamma"]);
|
||||
});
|
||||
|
||||
test("the class bodies build valid regexes", () => {
|
||||
assert.doesNotThrow(() => new RegExp(`[${LETTER_CLASS}]`));
|
||||
assert.doesNotThrow(() => new RegExp(`[${NUMBER_CLASS}]`));
|
||||
assert.doesNotThrow(() => new RegExp(`[^${LETTER_CLASS}${NUMBER_CLASS}]`));
|
||||
});
|
||||
|
||||
test("no Unicode property escape survives anywhere in app/", () => {
|
||||
const appRoot = path.join(__dirname, "..", "app");
|
||||
|
||||
const sources = [];
|
||||
(function walk(dir) {
|
||||
for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
|
||||
const full = path.join(dir, entry.name);
|
||||
if (entry.isDirectory()) walk(full);
|
||||
else if (entry.name.endsWith(".ts") || entry.name.endsWith(".js")) sources.push(full);
|
||||
}
|
||||
})(appRoot);
|
||||
|
||||
assert.ok(sources.length > 0, "found no sources to scan");
|
||||
|
||||
// Comments may name the escape (this file's own header does); code may not.
|
||||
const stripComments = (text) =>
|
||||
text.replace(/\/\*[\s\S]*?\*\//g, "").replace(/(^|[^:"'`\\])\/\/.*$/gm, "$1");
|
||||
|
||||
const offenders = sources.filter((file) =>
|
||||
/\\p\{/.test(stripComments(fs.readFileSync(file, "utf8"))),
|
||||
);
|
||||
|
||||
assert.deepEqual(
|
||||
offenders.map((file) => path.relative(appRoot, file)),
|
||||
[],
|
||||
"Unicode property escapes crash the ICU-less V8 on the glasses' Android host",
|
||||
);
|
||||
});
|
||||
Reference in new issue
Block a user