From 123a5d1366454a708640df036da121259a4428dc Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Thu, 9 Jul 2026 14:55:25 -0400 Subject: [PATCH] feat(tn): split interior apostrophe so possessives normalize (+2) Add the apostrophe to the interior-split set: "80's" -> "eighty's" (the number normalizes, the possessive stays attached). Contractions are unaffected because the glued reconstruction still matches them as whole tokens with no tagger, so "don't"/"they're"/"I've" stay literal. en TN punctuation 41->42; punctuation_match_input 5->6. No regressions. --- src/lib.rs | 2 +- tests/parity_baseline.tsv | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 19f57f8..34965cf 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1062,7 +1062,7 @@ fn is_split_punct(c: char) -> bool { /// never occur inside glued numeric/semiotic forms (unlike `.`/`:`/`,`), so /// splitting them cannot break decimals, times, or IPs. fn is_interior_hard(c: char) -> bool { - matches!(c, '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}') + matches!(c, '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'') } /// Fold NeMo's double-backtick quotes to a straight double quote so the diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index 9a17626..bd1c38f 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -58,8 +58,8 @@ en tn measure 14 21 en tn money 71 71 en tn normalize_with_audio 40 58 en tn ordinal 27 27 -en tn punctuation 41 63 -en tn punctuation_match_input 5 13 +en tn punctuation 42 63 +en tn punctuation_match_input 6 13 en tn range 20 20 en tn roman 4 4 en tn serial 29 32