diff --git a/src/lib.rs b/src/lib.rs index 34965cf..f43e983 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1062,7 +1062,10 @@ fn is_split_punct(c: char) -> bool { /// never occur inside glued numeric/semiotic forms (unlike `.`/`:`/`,`), so /// splitting them cannot break decimals, times, or IPs. fn is_interior_hard(c: char) -> bool { - matches!(c, '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'') + matches!( + c, + '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'' | '\u{2013}' + ) } /// Fold NeMo's double-backtick quotes to a straight double quote so the @@ -1242,7 +1245,10 @@ where /// "hello" → "one! hello"), matching NeMo's punctuation post-processing. fn push_detokenized(out: &mut String, sep: &str, text: &str) { let glue_after_close = sep.is_empty() - && matches!(out.chars().last(), Some('!' | '?' | ')' | ']' | '}')) + && matches!( + out.chars().last(), + Some('!' | '?' | ')' | ']' | '}' | '\u{2013}') + ) && text .chars() .next() diff --git a/src/tn/en/range.rs b/src/tn/en/range.rs index 71b09a9..20e3c55 100644 --- a/src/tn/en/range.rs +++ b/src/tn/en/range.rs @@ -32,12 +32,11 @@ pub fn parse(input: &str) -> Option { } } - // Hyphen or en-dash range. - for sep in ['-', '\u{2013}'] { - if let Some((a, b)) = split_binary(t, sep) { - if let Some(result) = range_words(a, b) { - return Some(result); - } + // Hyphen range ("1980-1986" → "… to …"). An en-dash is kept literal by the + // tokenizer/detokenizer, so it is not treated as a range here. + if let Some((a, b)) = split_binary(t, '-') { + if let Some(result) = range_words(a, b) { + return Some(result); } } diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index bd1c38f..46ff06d 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -58,7 +58,7 @@ en tn measure 14 21 en tn money 71 71 en tn normalize_with_audio 40 58 en tn ordinal 27 27 -en tn punctuation 42 63 +en tn punctuation 43 63 en tn punctuation_match_input 6 13 en tn range 20 20 en tn roman 4 4