diff --git a/examples/probe.rs b/examples/probe.rs index 0bc6f6b..1fdb1a0 100644 --- a/examples/probe.rs +++ b/examples/probe.rs @@ -1,7 +1,8 @@ use std::fs; use text_processing_rs::tn_normalize_sentence_lang; fn main() { - let path="/tmp/nemo-parity/tests/nemo_text_processing/en/data_text_normalization/test_cases_punctuation.txt"; + let path="/tmp/nemo-parity/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt"; + let (mut p, mut f) = (0, 0); for line in fs::read_to_string(path).unwrap().lines() { let l = line.trim(); if l.is_empty() || l.starts_with('#') { @@ -10,9 +11,17 @@ fn main() { let Some((i, e)) = l.split_once('~') else { continue; }; - let g = tn_normalize_sentence_lang(i, "en"); - if g != e { - println!("[{}]\n got [{}]\n want [{}]", i, g, e); + if tn_normalize_sentence_lang(i, "en") == e { + p += 1; + } else { + f += 1; + println!( + "F [{}]\n got [{}]\n want [{}]", + i, + tn_normalize_sentence_lang(i, "en"), + e + ); } } + eprintln!("address: {}/{}", p, p + f); } diff --git a/src/lib.rs b/src/lib.rs index 9070f9d..ac7e833 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1315,6 +1315,11 @@ fn tn_parse_span(span: &str) -> Option<(String, u8)> { if let Some(result) = tn::en::measure::parse(span) { return Some((result, 90)); } + // Street addresses (multi-token; before date so the house number reads + // address-style). + if let Some(result) = tn::en::address::parse(span) { + return Some((result, 89)); + } if let Some(result) = tn::en::date::parse(span) { return Some((result, 88)); } diff --git a/src/tn/en/address.rs b/src/tn/en/address.rs new file mode 100644 index 0000000..83d04d0 --- /dev/null +++ b/src/tn/en/address.rs @@ -0,0 +1,200 @@ +//! Address TN tagger — reads US street addresses NeMo's `address` class handles, +//! e.g. "1428 Elm St" → "fourteen twenty eight Elm Street" and +//! "708 N 1st St, San City" → "seven zero eight North first Street, San City". +//! +//! An address is a leading house number (read year-style, with "zero" for a +//! mid "oh"), an optional directional, street words, and a recognized street +//! suffix, followed optionally by comma-separated city, state, and ZIP. + +use super::date::verbalize_year; +use super::{ordinal, spell_digits}; +use lazy_static::lazy_static; +use std::collections::HashMap; + +lazy_static! { + /// Street-type suffixes (matched case-insensitively). + static ref SUFFIX: HashMap<&'static str, &'static str> = [ + ("st", "Street"), ("ave", "Avenue"), ("blvd", "Boulevard"), ("rd", "Road"), + ("dr", "Drive"), ("ln", "Lane"), ("ct", "Court"), ("pl", "Place"), + ("ter", "Terrace"), ("cir", "Circle"), ("way", "Way"), ("pkwy", "Parkway"), + ("hwy", "Highway"), ("expy", "Expressway"), ("sq", "Square"), ("trl", "Trail"), + ] + .into_iter() + .collect(); + + /// Directional prefixes. + static ref DIRECTIONAL: HashMap<&'static str, &'static str> = [ + ("N", "North"), ("S", "South"), ("E", "East"), ("W", "West"), + ("NE", "Northeast"), ("NW", "Northwest"), ("SE", "Southeast"), ("SW", "Southwest"), + ] + .into_iter() + .collect(); + + /// US state postal codes. + static ref STATE: HashMap<&'static str, &'static str> = [ + ("AL", "Alabama"), ("AK", "Alaska"), ("AZ", "Arizona"), ("AR", "Arkansas"), + ("CA", "California"), ("CO", "Colorado"), ("CT", "Connecticut"), ("DE", "Delaware"), + ("FL", "Florida"), ("GA", "Georgia"), ("HI", "Hawaii"), ("ID", "Idaho"), + ("IL", "Illinois"), ("IN", "Indiana"), ("IA", "Iowa"), ("KS", "Kansas"), + ("KY", "Kentucky"), ("LA", "Louisiana"), ("ME", "Maine"), ("MD", "Maryland"), + ("MA", "Massachusetts"), ("MI", "Michigan"), ("MN", "Minnesota"), ("MS", "Mississippi"), + ("MO", "Missouri"), ("MT", "Montana"), ("NE", "Nebraska"), ("NV", "Nevada"), + ("NH", "New Hampshire"), ("NJ", "New Jersey"), ("NM", "New Mexico"), ("NY", "New York"), + ("NC", "North Carolina"), ("ND", "North Dakota"), ("OH", "Ohio"), ("OK", "Oklahoma"), + ("OR", "Oregon"), ("PA", "Pennsylvania"), ("RI", "Rhode Island"), ("SC", "South Carolina"), + ("SD", "South Dakota"), ("TN", "Tennessee"), ("TX", "Texas"), ("UT", "Utah"), + ("VT", "Vermont"), ("VA", "Virginia"), ("WA", "Washington"), ("WV", "West Virginia"), + ("WI", "Wisconsin"), ("WY", "Wyoming"), + ] + .into_iter() + .collect(); +} + +/// Parse a street address to spoken form. +pub fn parse(input: &str) -> Option { + let span = input.trim(); + // Reject spans padded with trailing punctuation so a following "." is not + // swallowed (the shorter, address-only span wins instead). + if span + .chars() + .last() + .is_some_and(|c| !c.is_ascii_alphanumeric()) + { + return None; + } + + // Separate commas into their own tokens. + let spaced = span.replace(',', " , "); + let tokens: Vec<&str> = spaced.split_whitespace().collect(); + if tokens.len() < 2 || !tokens[0].chars().all(|c| c.is_ascii_digit()) { + return None; + } + + let mut pieces: Vec = vec![read_house_number(tokens[0])?]; + let mut i = 1; + + // Optional directional right after the house number. + if let Some(&dir) = DIRECTIONAL.get(tokens[i]) { + pieces.push(dir.to_string()); + i += 1; + } + + // Street words up to and including a recognized suffix. + let mut found_suffix = false; + while i < tokens.len() { + let token = tokens[i]; + if token == "," { + break; + } + if let Some(&suffix) = SUFFIX.get(token.to_ascii_lowercase().as_str()) { + pieces.push(suffix.to_string()); + i += 1; + found_suffix = true; + break; + } + if let Some(ord) = ordinal::parse(token) { + pieces.push(ord); + } else { + pieces.push(token.to_string()); + } + i += 1; + } + if !found_suffix { + return None; + } + + // After the suffix only comma-introduced city / state / ZIP may follow. + if i < tokens.len() { + if tokens[i] != "," { + return None; + } + while i < tokens.len() { + let token = tokens[i]; + if token == "," { + pieces.push(",".to_string()); + } else if let Some(&state) = STATE.get(token) { + pieces.push(state.to_string()); + } else if token.len() == 5 && token.chars().all(|c| c.is_ascii_digit()) { + pieces.push(spell_digits(token)); + } else { + pieces.push(token.to_string()); + } + i += 1; + } + } + + Some(assemble(&pieces)) +} + +/// A house number reads year-style, but a middle "oh" is spoken "zero" +/// ("708" → "seven zero eight", "2788" → "twenty seven eighty eight"). +fn read_house_number(s: &str) -> Option { + let n: u32 = s.parse().ok()?; + let words = verbalize_year(n)?; + Some( + words + .split(' ') + .map(|w| if w == "oh" { "zero" } else { w }) + .collect::>() + .join(" "), + ) +} + +/// Join pieces with spaces, attaching commas to the preceding word. +fn assemble(pieces: &[String]) -> String { + let mut out = String::new(); + for piece in pieces { + if piece == "," { + out.push(','); + } else { + if !out.is_empty() { + out.push(' '); + } + out.push_str(piece); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::parse; + + #[test] + fn test_basic() { + assert_eq!( + parse("1428 Elm St"), + Some("fourteen twenty eight Elm Street".to_string()) + ); + assert_eq!( + parse("1211 E Arques Ave"), + Some("twelve eleven East Arques Avenue".to_string()) + ); + assert_eq!( + parse("12 S 1st st"), + Some("twelve South first Street".to_string()) + ); + } + + #[test] + fn test_with_city_state_zip() { + assert_eq!( + parse("2788 San Tomas Expy, Santa Clara, CA 95051"), + Some( + "twenty seven eighty eight San Tomas Expressway, Santa Clara, California nine five zero five one" + .to_string() + ) + ); + assert_eq!( + parse("123 Smth St, City, NY"), + Some("one twenty three Smth Street, City, New York".to_string()) + ); + } + + #[test] + fn test_not_address() { + assert_eq!(parse("Main St"), None); // no house number + assert_eq!(parse("1428 Elm St."), None); // trailing period + assert_eq!(parse("hello world"), None); + } +} diff --git a/src/tn/en/mod.rs b/src/tn/en/mod.rs index 6d0c533..40b7d4a 100644 --- a/src/tn/en/mod.rs +++ b/src/tn/en/mod.rs @@ -5,6 +5,7 @@ //! - "$5.50" → "five dollars and fifty cents" //! - "January 5, 2025" → "january fifth twenty twenty five" +pub mod address; pub mod cardinal; pub mod date; pub mod decimal; diff --git a/tests/extensive_tests.rs b/tests/extensive_tests.rs index 9cb24e8..a50610e 100644 --- a/tests/extensive_tests.rs +++ b/tests/extensive_tests.rs @@ -1284,11 +1284,10 @@ fn test_itn_decimal_basic() { #[test] fn test_tts_scenario_address() { - let result = tn_normalize_sentence("123 Main St"); - assert!( - result.contains("one hundred and twenty three"), - "Address number should be spoken: {}", - result + // House numbers read address year-style, and "St" expands to "Street". + assert_eq!( + tn_normalize_sentence("123 Main St"), + "one twenty three Main Street" ); } diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index 3fa75a6..c49e7fd 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -47,14 +47,14 @@ en itn whitelist 12 12 en itn whitelist_cased 9 9 en itn word 55 55 en itn word_cased 49 49 -en tn address 1 11 +en tn address 8 11 en tn cardinal 18 18 en tn date 52 54 en tn decimal 12 12 en tn electronic 40 45 en tn fraction 16 16 en tn math 4 4 -en tn measure 10 21 +en tn measure 11 21 en tn money 71 71 en tn normalize_with_audio 39 58 en tn ordinal 27 27