feat(tn/en): abbreviation collapse + date comma preservation (punctuation 35→39)

examples/probe.rs

@@ -0,0 +1,18 @@
+use std::fs;
+use text_processing_rs::tn_normalize_sentence_lang;
+fn main() {
+    let path="/tmp/nemo-parity/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt";
+    for line in fs::read_to_string(path).unwrap().lines() {
+        let l = line.trim();
+        if l.is_empty() || l.starts_with('#') {
+            continue;
+        }
+        let Some((i, e)) = l.split_once('~') else {
+            continue;
+        };
+        let g = tn_normalize_sentence_lang(i, "en");
+        if g != e {
+            println!("[{}]\n  got  [{}]\n  want [{}]", i, g, e);
+        }
+    }
+}

src/lib.rs

@@ -1541,7 +1541,7 @@ mod tests {
     fn test_tn_date() {
         assert_eq!(
             tn_normalize("January 5, 2025"),
-            "january fifth twenty twenty five"
+            "january fifth, twenty twenty five"
         );
         assert_eq!(tn_normalize("1980s"), "nineteen eighties");
     }

src/tn/en/date.rs

@@ -178,11 +178,18 @@ fn parse_word_date(input: &str) -> Option<String> {
     } else {
         None
     };
+    // NeMo keeps a comma written before the year ("august 23, 2002" → "august
+    // twenty third, two thousand two").
+    let comma = year.is_some() && input.contains(',');
 
     // US order: Month Day [Year].
     if let Some(month) = parse_month(tokens[0]) {
         let day = parse_day(tokens[1])?;
-        return Some(with_year(format!("{} {}", month, ordinal_word(day)), year));
+        return Some(with_year(
+            format!("{} {}", month, ordinal_word(day)),
+            year,
+            comma,
+        ));
     }
 
     // British order: Day Month [Year].
@@ -191,14 +198,16 @@ fn parse_word_date(input: &str) -> Option<String> {
         return Some(with_year(
             format!("the {} of {}", ordinal_word(day), month),
             year,
+            comma,
         ));
     }
 
     None
 }
 
-fn with_year(base: String, year: Option<String>) -> String {
+fn with_year(base: String, year: Option<String>, comma: bool) -> String {
     match year {
+        Some(y) if comma => format!("{}, {}", base, y),
         Some(y) => format!("{} {}", base, y),
         None => base,
     }
@@ -526,13 +535,18 @@ mod tests {
 
     #[test]
     fn test_month_day_year() {
+        // A written comma before the year is kept (NeMo).
         assert_eq!(
             parse("January 5, 2025"),
-            Some("january fifth twenty twenty five".to_string())
+            Some("january fifth, twenty twenty five".to_string())
         );
         assert_eq!(
             parse("July 4, 1776"),
-            Some("july fourth seventeen seventy six".to_string())
+            Some("july fourth, seventeen seventy six".to_string())
+        );
+        assert_eq!(
+            parse("January 5 2025"),
+            Some("january fifth twenty twenty five".to_string())
         );
     }
 

src/tn/en/whitelist.rs

@@ -84,17 +84,19 @@ pub fn parse(input: &str) -> Option<String> {
     None
 }
 
-/// Collapse a run of dotted single upper-case initials ("C. S.", "U. S. A.")
-/// into a bare acronym ("CS", "USA").
+/// Collapse a run of dotted single upper-case initials into a bare acronym,
+/// whether space- or dot-separated ("C. S." → "CS", "U.S.A." → "USA").
 fn merge_initials(s: &str) -> Option<String> {
-    let parts: Vec<&str> = s.split_whitespace().collect();
+    if !s.contains('.') {
+        return None;
+    }
+    let parts: Vec<&str> = s.split(['.', ' ']).filter(|p| !p.is_empty()).collect();
     if parts.len() < 2 {
         return None;
     }
     let mut letters = String::new();
     for part in &parts {
-        let letter = part.strip_suffix('.')?;
-        let mut chars = letter.chars();
+        let mut chars = part.chars();
         let c = chars.next()?;
         if chars.next().is_some() || !c.is_ascii_uppercase() {
             return None;

swift-test/Sources/NemoTest/NemoTest.swift

@@ -273,7 +273,7 @@ let tnTimeTests: [TC] = [
 let tnDateTests: [TC] = [
     ("January 5", "january fifth"),
     ("December 25", "december twenty fifth"),
-    ("January 5, 2025", "january fifth twenty twenty five"),
+    ("January 5, 2025", "january fifth, twenty twenty five"),
     ("1980s", "nineteen eighties"),
     ("1990s", "nineteen nineties"),
 ]

tests/extensive_tests.rs

@@ -338,21 +338,22 @@ fn test_tn_date_month_day() {
 
 #[test]
 fn test_tn_date_month_day_year() {
+    // A written comma before the year is kept (NeMo).
     assert_eq!(
         tn_normalize("January 5, 2025"),
-        "january fifth twenty twenty five"
+        "january fifth, twenty twenty five"
     );
     assert_eq!(
         tn_normalize("July 4, 1776"),
-        "july fourth seventeen seventy six"
+        "july fourth, seventeen seventy six"
     );
     assert_eq!(
         tn_normalize("December 31, 1999"),
-        "december thirty first nineteen ninety nine"
+        "december thirty first, nineteen ninety nine"
     );
     assert_eq!(
         tn_normalize("January 1, 2000"),
-        "january first two thousand"
+        "january first, two thousand"
     );
 }
 
@@ -361,12 +362,15 @@ fn test_tn_date_month_day_year_2001_to_2009() {
     // Years 2001-2009 should use "two thousand X" form
     assert_eq!(
         tn_normalize("March 15, 2001"),
-        "march fifteenth two thousand one"
+        "march fifteenth, two thousand one"
+    );
+    assert_eq!(
+        tn_normalize("June 1, 2005"),
+        "june first, two thousand five"
     );
-    assert_eq!(tn_normalize("June 1, 2005"), "june first two thousand five");
     assert_eq!(
         tn_normalize("August 20, 2009"),
-        "august twentieth two thousand nine"
+        "august twentieth, two thousand nine"
     );
 }
 
@@ -504,23 +508,23 @@ fn test_tn_date_numeric_invalid_day() {
 fn test_tn_date_year_verbalization() {
     assert_eq!(
         tn_normalize("January 1, 2025"),
-        "january first twenty twenty five"
+        "january first, twenty twenty five"
     );
     assert_eq!(
         tn_normalize("January 1, 2000"),
-        "january first two thousand"
+        "january first, two thousand"
     );
     assert_eq!(
         tn_normalize("January 1, 2001"),
-        "january first two thousand one"
+        "january first, two thousand one"
     );
     assert_eq!(
         tn_normalize("January 1, 1900"),
-        "january first nineteen hundred"
+        "january first, nineteen hundred"
     );
     assert_eq!(
         tn_normalize("January 1, 1776"),
-        "january first seventeen seventy six"
+        "january first, seventeen seventy six"
     );
 }
 
@@ -535,10 +539,10 @@ fn test_tn_date_with_ordinal_suffix_in_day() {
 
 #[test]
 fn test_tn_date_trailing_punctuation() {
-    // Date with trailing period (common in sentences)
+    // Date with trailing period (common in sentences); the comma is kept.
     assert_eq!(
         tn_normalize("March 8, 2026."),
-        "march eighth twenty twenty six"
+        "march eighth, twenty twenty six"
     );
 }
 
@@ -1396,24 +1400,30 @@ fn test_tn_date_year_oh_pattern() {
     // e.g. 1901 → "nineteen oh one", not "nineteen one"
     assert_eq!(
         tn_normalize("January 1, 1901"),
-        "january first nineteen oh one"
+        "january first, nineteen oh one"
+    );
+    assert_eq!(
+        tn_normalize("July 4, 1805"),
+        "july fourth, eighteen oh five"
     );
-    assert_eq!(tn_normalize("July 4, 1805"), "july fourth eighteen oh five");
     assert_eq!(
         tn_normalize("March 15, 1709"),
-        "march fifteenth seventeen oh nine"
+        "march fifteenth, seventeen oh nine"
     );
     // 2001-2009 should still use "two thousand X" form (special case)
-    assert_eq!(tn_normalize("June 1, 2001"), "june first two thousand one");
-    assert_eq!(tn_normalize("June 1, 2009"), "june first two thousand nine");
+    assert_eq!(tn_normalize("June 1, 2001"), "june first, two thousand one");
+    assert_eq!(
+        tn_normalize("June 1, 2009"),
+        "june first, two thousand nine"
+    );
     // Years with remainder >= 10 should NOT have "oh"
     assert_eq!(
         tn_normalize("January 1, 1910"),
-        "january first nineteen ten"
+        "january first, nineteen ten"
     );
     assert_eq!(
         tn_normalize("January 1, 1776"),
-        "january first seventeen seventy six"
+        "january first, seventeen seventy six"
     );
 }
 

tests/parity_baseline.tsv

@@ -58,7 +58,7 @@ en	tn	measure	10	21
 en	tn	money	71	71
 en	tn	normalize_with_audio	39	58
 en	tn	ordinal	27	27
-en	tn	punctuation	35	63
+en	tn	punctuation	39	63
 en	tn	punctuation_match_input	4	13
 en	tn	range	19	20
 en	tn	roman	4	4