From 6001c67b47066b87ebbbbd4f16285946078640bf Mon Sep 17 00:00:00 2001 From: Alex-Wengg Date: Thu, 9 Jul 2026 12:52:05 -0400 Subject: [PATCH] =?UTF-8?q?feat(tn/en/serial):=20alphanumeric=20serial=20c?= =?UTF-8?q?odes=20(8=E2=86=9227);=20tighten=20vanity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a `serial` tagger (sentence-mode priority 65, below cardinal, above word) for mixed letter/digit/symbol codes: - "B2A23C" → "B two A twenty three C" - "133-ABC" → "one hundred thirty three-ABC" - "$12@12%" → "dollar twelve at twelve percent" Letter runs are kept; digit runs read as cardinals (digit-by-digit with a leading zero or beyond four digits); "-" and "/" are kept literal and glue, other symbols spell out. Requires a digit (pure-symbol tokens stay with the `word` tagger) and skips ARPABET phonemes ("AH0"). Also tighten telephone vanity detection to 3+ groups with an upper-case mnemonic so short serials ("133-ABC") no longer take the phone path. en TN serial 8→27; normalize_with_audio 31→39, punctuation 34→35 as side wins. The five unreproduced cases are NeMo's irregular forms (hyphen spaced/dropped in "1-8090"/"7-eleven", "1/f"→"one per F", 4-digit "9453" digit-by-digit). No regressions. --- src/lib.rs | 4 + src/tn/en/mod.rs | 1 + src/tn/en/serial.rs | 162 ++++++++++++++++++++++++++++++++++++++ src/tn/en/telephone.rs | 5 +- tests/parity_baseline.tsv | 6 +- 5 files changed, 174 insertions(+), 4 deletions(-) create mode 100644 src/tn/en/serial.rs diff --git a/src/lib.rs b/src/lib.rs index 8d7cd8a..a970bb8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1343,6 +1343,10 @@ fn tn_parse_span(span: &str) -> Option<(String, u8)> { if let Some(result) = tn::en::cardinal::parse(span) { return Some((result, 70)); } + // Serial codes (mixed letter/digit/symbol) read cardinally. + if let Some(result) = tn::en::serial::parse(span) { + return Some((result, 65)); + } // Last-resort spell-out for leftover symbol / alphanumeric tokens. if let Some(result) = tn::en::word::parse(span) { return Some((result, 60)); diff --git a/src/tn/en/mod.rs b/src/tn/en/mod.rs index 9b80a92..6d0c533 100644 --- a/src/tn/en/mod.rs +++ b/src/tn/en/mod.rs @@ -16,6 +16,7 @@ pub mod money; pub mod ordinal; pub mod range; pub mod roman; +pub mod serial; pub mod telephone; pub mod time; pub mod whitelist; diff --git a/src/tn/en/serial.rs b/src/tn/en/serial.rs new file mode 100644 index 0000000..f704c44 --- /dev/null +++ b/src/tn/en/serial.rs @@ -0,0 +1,162 @@ +//! Serial TN tagger — reads alphanumeric serial codes NeMo's `serial` class +//! handles, e.g. part numbers and mixed letter/digit/symbol tokens: +//! - "B2A23C" → "B two A twenty three C" +//! - "133-ABC" → "one hundred thirty three-ABC" +//! - "$12@12%" → "dollar twelve at twelve percent" +//! +//! Letter runs are kept verbatim; digit runs read as cardinals (digit-by-digit +//! when they carry a leading zero or exceed four digits); `-` and `/` are kept +//! literally and glue their neighbours, while other symbols spell out as words. +//! +//! NeMo's serial grammar is internally inconsistent in places (a hyphen is kept +//! in "133-ABC" but spaced in "1-8090" and dropped in "7-eleven"); those +//! irregular forms are not reproduced. + +use super::{number_to_words, spell_digits}; + +/// Spoken name for a spell-out symbol (glue symbols `-` and `/` are handled +/// separately and kept literal). +fn symbol_word(c: char) -> Option<&'static str> { + Some(match c { + '$' => "dollar", + '€' => "euro", + '£' => "pound", + '¥' => "yen", + '₩' => "won", + '#' => "hash", + '%' => "percent", + '@' => "at", + '*' => "asterisk", + '+' => "plus", + '&' => "and", + '=' => "equals", + _ => return None, + }) +} + +/// True for characters a serial code may contain. +fn is_serial_char(c: char) -> bool { + c.is_ascii_alphanumeric() || c == '-' || c == '/' || symbol_word(c).is_some() +} + +/// Parse a serial code to spoken form. +pub fn parse(input: &str) -> Option { + let token = input.trim(); + if token.is_empty() || !token.chars().all(is_serial_char) { + return None; + } + // Require a digit so plain words, hyphenated words ("well-known"), and + // pure-symbol tokens (left to the `word` tagger, which reads "/" as "slash") + // are not swallowed. + if !token.chars().any(|c| c.is_ascii_digit()) { + return None; + } + // Leave ARPABET phonemes ("AH0", "OW1") to be kept verbatim. + if is_arpabet(token) { + return None; + } + + let mut out = String::new(); + let mut chars = token.chars().peekable(); + while let Some(&c) = chars.peek() { + if c == '-' || c == '/' { + // Glue: keep literal, no surrounding spaces. + out.push(c); + chars.next(); + continue; + } + let piece = if c.is_ascii_alphabetic() { + let mut run = String::new(); + while matches!(chars.peek(), Some(d) if d.is_ascii_alphabetic()) { + run.push(chars.next().unwrap()); + } + run + } else if c.is_ascii_digit() { + let mut run = String::new(); + while matches!(chars.peek(), Some(d) if d.is_ascii_digit()) { + run.push(chars.next().unwrap()); + } + read_digits(&run)? + } else { + let word = symbol_word(c)?.to_string(); + chars.next(); + word + }; + if !out.is_empty() && !out.ends_with(['-', '/']) { + out.push(' '); + } + out.push_str(&piece); + } + + if out == token { + return None; + } + Some(out) +} + +/// An ARPABET phoneme token: upper-case letters followed by a single stress +/// digit (0/1/2), e.g. "AH0", "OW1". +fn is_arpabet(token: &str) -> bool { + let bytes = token.as_bytes(); + if bytes.len() < 2 { + return false; + } + let (letters, last) = bytes.split_at(bytes.len() - 1); + letters.iter().all(|b| b.is_ascii_uppercase()) && matches!(last[0], b'0'..=b'2') +} + +/// Read a digit run: digit-by-digit with a leading zero or beyond four digits, +/// otherwise a cardinal ("25" → "twenty five", "2000" → "two thousand"). +fn read_digits(run: &str) -> Option { + if run.starts_with('0') || run.len() > 4 { + Some(spell_digits(run)) + } else { + Some(number_to_words(run.parse().ok()?)) + } +} + +#[cfg(test)] +mod tests { + use super::parse; + + #[test] + fn test_alphanumeric() { + assert_eq!(parse("B2A23C"), Some("B two A twenty three C".to_string())); + assert_eq!(parse("C24"), Some("C twenty four".to_string())); + assert_eq!( + parse("25d08A"), + Some("twenty five d zero eight A".to_string()) + ); + } + + #[test] + fn test_hyphen_kept() { + assert_eq!( + parse("133-ABC"), + Some("one hundred thirty three-ABC".to_string()) + ); + assert_eq!(parse("covid-19"), Some("covid-nineteen".to_string())); + assert_eq!( + parse("t-0t25d12-f"), + Some("t-zero t twenty five d twelve-f".to_string()) + ); + } + + #[test] + fn test_symbols() { + assert_eq!( + parse("$12@12%"), + Some("dollar twelve at twelve percent".to_string()) + ); + assert_eq!(parse("2*8"), Some("two asterisk eight".to_string())); + // Pure-symbol tokens (no digit) are left to the `word` tagger. + assert_eq!(parse("#mytext#"), None); + } + + #[test] + fn test_leaves_plain() { + assert_eq!(parse("hello"), None); + assert_eq!(parse("well-known"), None); + assert_eq!(parse("1.2.3"), None); // contains a dot + } +} diff --git a/src/tn/en/telephone.rs b/src/tn/en/telephone.rs index bcf1a84..d1d1e62 100644 --- a/src/tn/en/telephone.rs +++ b/src/tn/en/telephone.rs @@ -142,7 +142,10 @@ fn parse_vanity(input: &str) -> Option { return None; } let groups: Vec<&str> = input.split('-').collect(); - if groups.len() < 2 { + // Real vanity numbers have three or more groups and an upper-case mnemonic + // ("1-800-GO-U-HAUL"); this keeps short mixed serials ("133-ABC") and + // lower-case codes ("1-413-te-b") out of the phone path. + if groups.len() < 3 || input.chars().any(|c| c.is_ascii_lowercase()) { return None; } let mut has_digit = false; diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index 4fa2b44..9dc9be3 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -56,13 +56,13 @@ en tn fraction 16 16 en tn math 4 4 en tn measure 10 21 en tn money 71 71 -en tn normalize_with_audio 31 58 +en tn normalize_with_audio 39 58 en tn ordinal 27 27 -en tn punctuation 34 63 +en tn punctuation 35 63 en tn punctuation_match_input 4 13 en tn range 19 20 en tn roman 4 4 -en tn serial 8 32 +en tn serial 27 32 en tn special_text 9 10 en tn telephone 20 20 en tn time 21 21