diff --git a/examples/probe.rs b/examples/probe.rs deleted file mode 100644 index 1fdb1a0..0000000 --- a/examples/probe.rs +++ /dev/null @@ -1,27 +0,0 @@ -use std::fs; -use text_processing_rs::tn_normalize_sentence_lang; -fn main() { - let path="/tmp/nemo-parity/tests/nemo_text_processing/en/data_text_normalization/test_cases_address.txt"; - let (mut p, mut f) = (0, 0); - for line in fs::read_to_string(path).unwrap().lines() { - let l = line.trim(); - if l.is_empty() || l.starts_with('#') { - continue; - } - let Some((i, e)) = l.split_once('~') else { - continue; - }; - if tn_normalize_sentence_lang(i, "en") == e { - p += 1; - } else { - f += 1; - println!( - "F [{}]\n got [{}]\n want [{}]", - i, - tn_normalize_sentence_lang(i, "en"), - e - ); - } - } - eprintln!("address: {}/{}", p, p + f); -} diff --git a/src/tn/en/electronic.rs b/src/tn/en/electronic.rs index 8fcd5e4..563bca7 100644 --- a/src/tn/en/electronic.rs +++ b/src/tn/en/electronic.rs @@ -37,6 +37,10 @@ lazy_static! { /// Product acronyms upper-cased in email/URL context. static ref BRAND: HashSet<&'static str> = ["nvidia", "cuda", "dgx", "rtx", "basepod"].into_iter().collect(); + + /// File-extension acronyms upper-cased when they are the final ".ext". + static ref EXT_UPPER: HashSet<&'static str> = + ["html", "htm", "xml", "css", "json", "php", "asp", "sql"].into_iter().collect(); } /// Parse an email or URL to spoken form. @@ -221,6 +225,18 @@ fn render_remainder(s: &str, context: bool) -> String { } } flush(&mut buf, &mut out); + + // Upper-case a known acronym file extension in the final ".ext" label + // ("intro.html" → "… dot HTML"), leaving an identically-named directory + // ("…/html/…") lower case. + let n = out.len(); + if n >= 2 && out[n - 2] == "dot" { + let lower = out[n - 1].to_ascii_lowercase(); + if EXT_UPPER.contains(lower.as_str()) { + out[n - 1] = lower.to_ascii_uppercase(); + } + } + out.join(" ") } diff --git a/src/tn/en/measure.rs b/src/tn/en/measure.rs index a9a57bd..fbc7d51 100644 --- a/src/tn/en/measure.rs +++ b/src/tn/en/measure.rs @@ -145,6 +145,12 @@ pub fn parse(input: &str) -> Option { continue; } + // A spaced 4-digit decade ("1980 s") is not "1980 seconds" — leave it + // to the date tagger, which reads it as "nineteen eighties". + if unit_str == "s" && is_spaced_decade(num_part) { + continue; + } + // Scale word between the number and unit ("100 million kg"). if let Some(scaled) = parse_scaled(num_part) { return Some(format!("{} {}", scaled, unit_info.plural)); @@ -213,6 +219,13 @@ pub fn parse(input: &str) -> Option { parse_per(trimmed) } +/// A 4-digit multiple of ten ("1980") — a decade written before a spaced "s". +fn is_spaced_decade(num: &str) -> bool { + num.len() == 4 + && num.chars().all(|c| c.is_ascii_digit()) + && num.parse::().map(|n| n % 10 == 0).unwrap_or(false) +} + /// Read a "value/unit" form where the right side is a bare unit. fn parse_per(input: &str) -> Option { let (left, right) = input.split_once('/')?; diff --git a/src/tn/en/serial.rs b/src/tn/en/serial.rs index f704c44..b2b4c9c 100644 --- a/src/tn/en/serial.rs +++ b/src/tn/en/serial.rs @@ -56,6 +56,21 @@ pub fn parse(input: &str) -> Option { return None; } + // Rate notation: "/" → " per ". + if let Some((num, unit)) = token.split_once('/') { + if !num.is_empty() + && num.chars().all(|c| c.is_ascii_digit()) + && unit.len() == 1 + && unit.chars().all(|c| c.is_ascii_alphabetic()) + { + return Some(format!( + "{} per {}", + number_to_words(num.parse().ok()?), + unit.to_ascii_uppercase() + )); + } + } + let mut out = String::new(); let mut chars = token.chars().peekable(); while let Some(&c) = chars.peek() { diff --git a/tests/parity_baseline.tsv b/tests/parity_baseline.tsv index c49e7fd..e2742d1 100644 --- a/tests/parity_baseline.tsv +++ b/tests/parity_baseline.tsv @@ -49,9 +49,9 @@ en itn word 55 55 en itn word_cased 49 49 en tn address 8 11 en tn cardinal 18 18 -en tn date 52 54 +en tn date 53 54 en tn decimal 12 12 -en tn electronic 40 45 +en tn electronic 41 45 en tn fraction 16 16 en tn math 4 4 en tn measure 11 21 @@ -62,7 +62,7 @@ en tn punctuation 39 63 en tn punctuation_match_input 4 13 en tn range 19 20 en tn roman 4 4 -en tn serial 27 32 +en tn serial 28 32 en tn special_text 9 10 en tn telephone 20 20 en tn time 21 21