From 026544e882a3e566e19f40796bc80234b37f0546 Mon Sep 17 00:00:00 2001
From: Alex-Wengg <hanweng9@gmail.com>
Date: Thu, 9 Jul 2026 15:00:16 -0400
Subject: [PATCH] =?UTF-8?q?feat(tn):=20en-dash=20kept=20literal=20+=20deto?=
 =?UTF-8?q?kenized=20(punctuation=2042=E2=86=9243)?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

An en-dash between values ("1961–1965") is kept literal and detokenized with
a trailing space ("nineteen sixty one– nineteen sixty five"), matching NeMo's
punctuation output. The en-dash is removed from the range tagger (which only
tests hyphens) and added to the interior-split + detok close set.

en TN punctuation 42->43; range stays 20/20. No regressions.
---
 src/lib.rs                | 10 ++++++++--
 src/tn/en/range.rs        | 11 +++++------
 tests/parity_baseline.tsv |  2 +-
 3 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/lib.rs b/src/lib.rs
index 34965cf..f43e983 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -1062,7 +1062,10 @@ fn is_split_punct(c: char) -> bool {
 /// never occur inside glued numeric/semiotic forms (unlike `.`/`:`/`,`), so
 /// splitting them cannot break decimals, times, or IPs.
 fn is_interior_hard(c: char) -> bool {
-    matches!(c, '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'')
+    matches!(
+        c,
+        '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'' | '\u{2013}'
+    )
 }
 
 /// Fold NeMo's double-backtick quotes to a straight double quote so the
@@ -1242,7 +1245,10 @@ where
 /// "hello" → "one! hello"), matching NeMo's punctuation post-processing.
 fn push_detokenized(out: &mut String, sep: &str, text: &str) {
     let glue_after_close = sep.is_empty()
-        && matches!(out.chars().last(), Some('!' | '?' | ')' | ']' | '}'))
+        && matches!(
+            out.chars().last(),
+            Some('!' | '?' | ')' | ']' | '}' | '\u{2013}')
+        )
         && text
             .chars()
             .next()
diff --git a/src/tn/en/range.rs b/src/tn/en/range.rs
index 71b09a9..20e3c55 100644
--- a/src/tn/en/range.rs
+++ b/src/tn/en/range.rs
@@ -32,12 +32,11 @@ pub fn parse(input: &str) -> Option<String> {
         }
     }
 
-    // Hyphen or en-dash range.
-    for sep in ['-', '\u{2013}'] {
-        if let Some((a, b)) = split_binary(t, sep) {
-            if let Some(result) = range_words(a, b) {
-                return Some(result);
-            }
+    // Hyphen range ("1980-1986" → "… to …"). An en-dash is kept literal by the
+    // tokenizer/detokenizer, so it is not treated as a range here.
+    if let Some((a, b)) = split_binary(t, '-') {
+        if let Some(result) = range_words(a, b) {
+            return Some(result);
         }
     }
 
diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv
index bd1c38f..46ff06d 100644
--- a/tests/parity_baseline.tsv
+++ b/tests/parity_baseline.tsv
@@ -58,7 +58,7 @@ en	tn	measure	14	21
 en	tn	money	71	71
 en	tn	normalize_with_audio	40	58
 en	tn	ordinal	27	27
-en	tn	punctuation	42	63
+en	tn	punctuation	43	63
 en	tn	punctuation_match_input	6	13
 en	tn	range	20	20
 en	tn	roman	4	4