Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -1062,7 +1062,10 @@ fn is_split_punct(c: char) -> bool {
/// never occur inside glued numeric/semiotic forms (unlike `.`/`:`/`,`), so
/// splitting them cannot break decimals, times, or IPs.
fn is_interior_hard(c: char) -> bool {
matches!(c, '!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'')
matches!(
c,
'!' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '\'' | '\u{2013}'
)
}

/// Fold NeMo's double-backtick quotes to a straight double quote so the
Expand Down Expand Up @@ -1242,7 +1245,10 @@ where
/// "hello" → "one! hello"), matching NeMo's punctuation post-processing.
fn push_detokenized(out: &mut String, sep: &str, text: &str) {
let glue_after_close = sep.is_empty()
&& matches!(out.chars().last(), Some('!' | '?' | ')' | ']' | '}'))
&& matches!(
out.chars().last(),
Some('!' | '?' | ')' | ']' | '}' | '\u{2013}')
)
&& text
.chars()
.next()
Expand Down
11 changes: 5 additions & 6 deletions src/tn/en/range.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,12 +32,11 @@ pub fn parse(input: &str) -> Option<String> {
}
}

// Hyphen or en-dash range.
for sep in ['-', '\u{2013}'] {
if let Some((a, b)) = split_binary(t, sep) {
if let Some(result) = range_words(a, b) {
return Some(result);
}
// Hyphen range ("1980-1986" → "… to …"). An en-dash is kept literal by the
// tokenizer/detokenizer, so it is not treated as a range here.
if let Some((a, b)) = split_binary(t, '-') {
if let Some(result) = range_words(a, b) {
return Some(result);
}
}

Expand Down
2 changes: 1 addition & 1 deletion tests/parity_baseline.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ en tn measure 14 21
en tn money 71 71
en tn normalize_with_audio 40 58
en tn ordinal 27 27
en tn punctuation 42 63
en tn punctuation 43 63
en tn punctuation_match_input 6 13
en tn range 20 20
en tn roman 4 4
Expand Down
Loading