diff --git a/examples/probe.rs b/examples/probe.rs new file mode 100644 index 0000000..0bc6f6b --- /dev/null +++ b/examples/probe.rs @@ -0,0 +1,18 @@ +use std::fs; +use text_processing_rs::tn_normalize_sentence_lang; +fn main() { + let path="/tmp/nemo-parity/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt"; + for line in fs::read_to_string(path).unwrap().lines() { + let l = line.trim(); + if l.is_empty() || l.starts_with('#') { + continue; + } + let Some((i, e)) = l.split_once('~') else { + continue; + }; + let g = tn_normalize_sentence_lang(i, "en"); + if g != e { + println!("[{}]\n got [{}]\n want [{}]", i, g, e); + } + } +} diff --git a/src/lib.rs b/src/lib.rs index a970bb8..9070f9d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1541,7 +1541,7 @@ mod tests { fn test_tn_date() { assert_eq!( tn_normalize("January 5, 2025"), - "january fifth twenty twenty five" + "january fifth, twenty twenty five" ); assert_eq!(tn_normalize("1980s"), "nineteen eighties"); } diff --git a/src/tn/en/date.rs b/src/tn/en/date.rs index 8f8b42c..70c99fb 100644 --- a/src/tn/en/date.rs +++ b/src/tn/en/date.rs @@ -178,11 +178,18 @@ fn parse_word_date(input: &str) -> Option { } else { None }; + // NeMo keeps a comma written before the year ("august 23, 2002" → "august + // twenty third, two thousand two"). + let comma = year.is_some() && input.contains(','); // US order: Month Day [Year]. if let Some(month) = parse_month(tokens[0]) { let day = parse_day(tokens[1])?; - return Some(with_year(format!("{} {}", month, ordinal_word(day)), year)); + return Some(with_year( + format!("{} {}", month, ordinal_word(day)), + year, + comma, + )); } // British order: Day Month [Year]. @@ -191,14 +198,16 @@ fn parse_word_date(input: &str) -> Option { return Some(with_year( format!("the {} of {}", ordinal_word(day), month), year, + comma, )); } None } -fn with_year(base: String, year: Option) -> String { +fn with_year(base: String, year: Option, comma: bool) -> String { match year { + Some(y) if comma => format!("{}, {}", base, y), Some(y) => format!("{} {}", base, y), None => base, } @@ -526,13 +535,18 @@ mod tests { #[test] fn test_month_day_year() { + // A written comma before the year is kept (NeMo). assert_eq!( parse("January 5, 2025"), - Some("january fifth twenty twenty five".to_string()) + Some("january fifth, twenty twenty five".to_string()) ); assert_eq!( parse("July 4, 1776"), - Some("july fourth seventeen seventy six".to_string()) + Some("july fourth, seventeen seventy six".to_string()) + ); + assert_eq!( + parse("January 5 2025"), + Some("january fifth twenty twenty five".to_string()) ); } diff --git a/src/tn/en/whitelist.rs b/src/tn/en/whitelist.rs index 54b2a14..d412675 100644 --- a/src/tn/en/whitelist.rs +++ b/src/tn/en/whitelist.rs @@ -84,17 +84,19 @@ pub fn parse(input: &str) -> Option { None } -/// Collapse a run of dotted single upper-case initials ("C. S.", "U. S. A.") -/// into a bare acronym ("CS", "USA"). +/// Collapse a run of dotted single upper-case initials into a bare acronym, +/// whether space- or dot-separated ("C. S." → "CS", "U.S.A." → "USA"). fn merge_initials(s: &str) -> Option { - let parts: Vec<&str> = s.split_whitespace().collect(); + if !s.contains('.') { + return None; + } + let parts: Vec<&str> = s.split(['.', ' ']).filter(|p| !p.is_empty()).collect(); if parts.len() < 2 { return None; } let mut letters = String::new(); for part in &parts { - let letter = part.strip_suffix('.')?; - let mut chars = letter.chars(); + let mut chars = part.chars(); let c = chars.next()?; if chars.next().is_some() || !c.is_ascii_uppercase() { return None; diff --git a/swift-test/Sources/NemoTest/NemoTest.swift b/swift-test/Sources/NemoTest/NemoTest.swift index 16d9113..e5670cd 100644 --- a/swift-test/Sources/NemoTest/NemoTest.swift +++ b/swift-test/Sources/NemoTest/NemoTest.swift @@ -273,7 +273,7 @@ let tnTimeTests: [TC] = [ let tnDateTests: [TC] = [ ("January 5", "january fifth"), ("December 25", "december twenty fifth"), - ("January 5, 2025", "january fifth twenty twenty five"), + ("January 5, 2025", "january fifth, twenty twenty five"), ("1980s", "nineteen eighties"), ("1990s", "nineteen nineties"), ] diff --git a/tests/extensive_tests.rs b/tests/extensive_tests.rs index 57f788c..9cb24e8 100644 --- a/tests/extensive_tests.rs +++ b/tests/extensive_tests.rs @@ -338,21 +338,22 @@ fn test_tn_date_month_day() { #[test] fn test_tn_date_month_day_year() { + // A written comma before the year is kept (NeMo). assert_eq!( tn_normalize("January 5, 2025"), - "january fifth twenty twenty five" + "january fifth, twenty twenty five" ); assert_eq!( tn_normalize("July 4, 1776"), - "july fourth seventeen seventy six" + "july fourth, seventeen seventy six" ); assert_eq!( tn_normalize("December 31, 1999"), - "december thirty first nineteen ninety nine" + "december thirty first, nineteen ninety nine" ); assert_eq!( tn_normalize("January 1, 2000"), - "january first two thousand" + "january first, two thousand" ); } @@ -361,12 +362,15 @@ fn test_tn_date_month_day_year_2001_to_2009() { // Years 2001-2009 should use "two thousand X" form assert_eq!( tn_normalize("March 15, 2001"), - "march fifteenth two thousand one" + "march fifteenth, two thousand one" + ); + assert_eq!( + tn_normalize("June 1, 2005"), + "june first, two thousand five" ); - assert_eq!(tn_normalize("June 1, 2005"), "june first two thousand five"); assert_eq!( tn_normalize("August 20, 2009"), - "august twentieth two thousand nine" + "august twentieth, two thousand nine" ); } @@ -504,23 +508,23 @@ fn test_tn_date_numeric_invalid_day() { fn test_tn_date_year_verbalization() { assert_eq!( tn_normalize("January 1, 2025"), - "january first twenty twenty five" + "january first, twenty twenty five" ); assert_eq!( tn_normalize("January 1, 2000"), - "january first two thousand" + "january first, two thousand" ); assert_eq!( tn_normalize("January 1, 2001"), - "january first two thousand one" + "january first, two thousand one" ); assert_eq!( tn_normalize("January 1, 1900"), - "january first nineteen hundred" + "january first, nineteen hundred" ); assert_eq!( tn_normalize("January 1, 1776"), - "january first seventeen seventy six" + "january first, seventeen seventy six" ); } @@ -535,10 +539,10 @@ fn test_tn_date_with_ordinal_suffix_in_day() { #[test] fn test_tn_date_trailing_punctuation() { - // Date with trailing period (common in sentences) + // Date with trailing period (common in sentences); the comma is kept. assert_eq!( tn_normalize("March 8, 2026."), - "march eighth twenty twenty six" + "march eighth, twenty twenty six" ); } @@ -1396,24 +1400,30 @@ fn test_tn_date_year_oh_pattern() { // e.g. 1901 → "nineteen oh one", not "nineteen one" assert_eq!( tn_normalize("January 1, 1901"), - "january first nineteen oh one" + "january first, nineteen oh one" + ); + assert_eq!( + tn_normalize("July 4, 1805"), + "july fourth, eighteen oh five" ); - assert_eq!(tn_normalize("July 4, 1805"), "july fourth eighteen oh five"); assert_eq!( tn_normalize("March 15, 1709"), - "march fifteenth seventeen oh nine" + "march fifteenth, seventeen oh nine" ); // 2001-2009 should still use "two thousand X" form (special case) - assert_eq!(tn_normalize("June 1, 2001"), "june first two thousand one"); - assert_eq!(tn_normalize("June 1, 2009"), "june first two thousand nine"); + assert_eq!(tn_normalize("June 1, 2001"), "june first, two thousand one"); + assert_eq!( + tn_normalize("June 1, 2009"), + "june first, two thousand nine" + ); // Years with remainder >= 10 should NOT have "oh" assert_eq!( tn_normalize("January 1, 1910"), - "january first nineteen ten" + "january first, nineteen ten" ); assert_eq!( tn_normalize("January 1, 1776"), - "january first seventeen seventy six" + "january first, seventeen seventy six" ); } diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index 9dc9be3..3fa75a6 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -58,7 +58,7 @@ en tn measure 10 21 en tn money 71 71 en tn normalize_with_audio 39 58 en tn ordinal 27 27 -en tn punctuation 35 63 +en tn punctuation 39 63 en tn punctuation_match_input 4 13 en tn range 19 20 en tn roman 4 4