From 72688f94f7b5a4f7b649571bc0c090854fe86678 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 12:44:33 +0300 Subject: [PATCH 01/13] fix(markdown): escape a paired dollar so document text is not read as math A dollar pair delimits math for every renderer that supports it, so a document saying "costs $100 and $80" was serialized unescaped and read back as an equation spanning the two amounts. Escape a dollar that has a later dollar in the same run, on the same pairing rule the other inert-when- lone delimiters already use; a single dollar opens nothing and stays literal, which leaves the common currency case untouched. --- src/render/markdown/escape.rs | 7 ++++++- src/render/markdown/tests.rs | 11 +++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/render/markdown/escape.rs b/src/render/markdown/escape.rs index f2654376..281c8b66 100644 --- a/src/render/markdown/escape.rs +++ b/src/render/markdown/escape.rs @@ -33,7 +33,7 @@ pub(crate) fn escape_text(text: &str, ctx: InlineContext, opts: EscapeOpts) -> S let EscapeOpts { at_line_start, styled, trailing_active, in_label } = opts; let chars: Vec = text.chars().collect(); // Last position of each pairable delimiter; a lone one is inert. - let mut last: [Option; 5] = [None; 5]; // * _ ~ ` ] + let mut last: [Option; 6] = [None; 6]; // * _ ~ ` ] $ for (j, &c) in chars.iter().enumerate() { match c { '*' => last[0] = Some(j), @@ -41,6 +41,7 @@ pub(crate) fn escape_text(text: &str, ctx: InlineContext, opts: EscapeOpts) -> S '~' => last[2] = Some(j), '`' => last[3] = Some(j), ']' => last[4] = Some(j), + '$' => last[5] = Some(j), _ => {} } } @@ -76,6 +77,10 @@ pub(crate) fn escape_text(text: &str, ctx: InlineContext, opts: EscapeOpts) -> S styled || (next_nonspace && !(prev_alnum && next_alnum) && paired(1)) } '~' => styled || (next_nonspace && paired(2)), + // A dollar pair delimits math for every renderer that supports it, so document + // text like `$5 and $10` is otherwise read as an equation. A lone `$` opens + // nothing and stays literal, which keeps the common currency case unescaped. + '$' => paired(5), '[' => in_label || paired(4), '<' => next.is_some_and(|n| n.is_ascii_alphabetic() || matches!(n, '/' | '!' | '?')), '!' => next.is_none() && trailing_active, diff --git a/src/render/markdown/tests.rs b/src/render/markdown/tests.rs index a4aa8067..3fbced98 100644 --- a/src/render/markdown/tests.rs +++ b/src/render/markdown/tests.rs @@ -45,6 +45,17 @@ fn lone_syntax_chars_left_alone() { assert_eq!(md, "x < 5, ~10%, file_name, a[1\n"); } +#[test] +fn paired_dollars_escaped_lone_dollar_kept() { + // A pair delimits math for renderers that support it, so document text that + // happens to hold two dollars would be read as an equation. + let md = doc(vec![Block::Paragraph(vec![Inline::plain("costs $100 and $80 total")])]); + assert_eq!(md, "costs \\$100 and $80 total\n"); + // One dollar opens nothing; the common currency case stays literal. + let md = doc(vec![Block::Paragraph(vec![Inline::plain("costs $100 total")])]); + assert_eq!(md, "costs $100 total\n"); +} + #[test] fn intraword_underscores_unescaped() { let md = doc(vec![Block::Paragraph(vec![Inline::plain("snake_case_name vs _lead_")])]); From 5bb32e516a3a373ce3cb5f2b019027f5f51edd7c Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 13:09:16 +0300 Subject: [PATCH 02/13] feat(docx): superscript, subscript and Office Math equations Word carries three things this converter dropped. A `w:vertAlign` run is super- or subscript, and flattening it changes a value rather than a presentation: `10^-3` became `10-3`, `H_2 O` became `H2O`. An `m:oMath` equation vanished entirely, and an `m:oMathPara` took its whole paragraph with it, because all three namespace filters in the docx frontend admit only `w:*` and OMML lives in its own namespace. `Style` gains `vert_align`. It is a value, not a toggle: ECMA-376 s17.7.3 closes the toggle set and `w:vertAlign` is not in it, so it cannot ride the XOR parity `Toggles` uses, and `on_off` cannot even tell `superscript` from `subscript` since both are outside the false-set. It resolves through the ordinary nearest-specification path instead, and the other frontends fill it too: DrawingML `a:rPr/@baseline` and ODF `style:text-position`. `Inline::Math` carries LaTeX, rendered between dollars. It is the only inline whose payload reaches the output unescaped, since `escape_text` escapes every backslash and would destroy each command, so the producer owns making the body safe. `formats::docx::omml` translates the OMML element set. It parses to a small tree before emitting, because bracing is a question about a node's shape -- `y^{2}` used as another script's base needs braces or LaTeX reads a double superscript, while `\frac{a}{b}` does not -- and that cannot be decided by inspecting the emitted string. Property bags are read only through named lookups and never enumerated: each is optional in the schema, Word omits them whenever every property takes its default, and an absent bag must resolve exactly as an empty one does. Defaults follow the spec, including the n-ary operator defaulting to the integral rather than the sum. Hostile input is bounded and neutralised. Nesting past 64 levels degrades to the subtree's text rather than recursing, output is capped per equation because wrappers multiply through nesting, and text runs are LaTeX-escaped so a dollar cannot close the span and hand the rest of the document to the Markdown parser, nor a backslash smuggle in a command. A delimiter separator is emitted as `\mid` rather than a bare pipe, which would split the table row the equation sits in. Emitted commands stay inside the subset KaTeX implements, and a body holding a dollar takes the longer fence: Markdown math parsers do not honour a backslash-escaped dollar when scanning for the closing delimiter. --- node/index.d.ts | 9 +- node/src/document.rs | 22 +- python/anydoc/_anydoc.pyi | 2 +- python/src/document.rs | 21 +- src/formats/docx/content.rs | 28 + src/formats/docx/mod.rs | 1 + src/formats/docx/omml.rs | 758 ++++++++++++++++++ src/formats/docx/styles.rs | 17 +- src/formats/odf/styles.rs | 22 + src/formats/odf/table.rs | 1 + src/formats/ppt/mod.rs | 3 +- src/formats/pptx/cascade.rs | 13 + src/model/inline.rs | 14 +- src/model/mod.rs | 2 +- src/model/style.rs | 40 +- src/package/xml.rs | 1 + src/render/markdown/escape.rs | 4 +- src/render/markdown/inline.rs | 32 +- src/render/markdown/tests.rs | 26 +- src/shared/delta.rs | 5 +- src/shared/html.rs | 2 +- tests/fixtures/docx/handmade-math.docx | Bin 0 -> 1502 bytes tests/gen_fixtures.py | 61 ++ .../snapshots__docx__handmade-math.docx.snap | 9 + wasm/src/document.rs | 24 +- wasm/src/typescript.rs | 7 + 26 files changed, 1098 insertions(+), 26 deletions(-) create mode 100644 src/formats/docx/omml.rs create mode 100644 tests/fixtures/docx/handmade-math.docx create mode 100644 tests/snapshots/snapshots__docx__handmade-math.docx.snap diff --git a/node/index.d.ts b/node/index.d.ts index 27ceba0a..cd0ecfe2 100644 --- a/node/index.d.ts +++ b/node/index.d.ts @@ -168,6 +168,10 @@ export interface Inline { anchor?: string /** noteRef: the id of the note in `Document.notes`. */ noteId?: string + /** math: the expression as LaTeX, without delimiters. */ + latex?: string + /** math: true for an equation that stands on its own line. */ + display?: boolean } export declare const enum InlineKind { @@ -177,7 +181,8 @@ export declare const enum InlineKind { /** Zero-width marker for an internal link target at this position. */ anchor = 'anchor', noteRef = 'noteRef', - lineBreak = 'lineBreak' + lineBreak = 'lineBreak', + math = 'math' } export interface LinkTarget { @@ -241,6 +246,8 @@ export interface Style { italic: boolean strike: boolean code: boolean + /** `baseline`, `superscript` or `subscript`. */ + vertAlign: string } /** diff --git a/node/src/document.rs b/node/src/document.rs index 1cd66171..8763f62e 100644 --- a/node/src/document.rs +++ b/node/src/document.rs @@ -100,6 +100,7 @@ pub enum InlineKind { anchor, noteRef, lineBreak, + math, } #[napi(object)] @@ -121,6 +122,10 @@ pub struct Inline { pub anchor: Option, /// noteRef: the id of the note in `Document.notes`. pub note_id: Option, + /// math: the expression as LaTeX, without delimiters. + pub latex: Option, + /// math: true for an equation that stands on its own line. + pub display: Option, } impl Inline { @@ -135,6 +140,8 @@ impl Inline { source: None, anchor: None, note_id: None, + latex: None, + display: None, } } } @@ -163,6 +170,11 @@ impl From for Inline { model::Inline::NoteRef(id) => { Inline { note_id: Some(id), ..Inline::of(InlineKind::noteRef) } } + model::Inline::Math { latex, display } => Inline { + latex: Some(latex), + display: Some(display), + ..Inline::of(InlineKind::math) + }, model::Inline::LineBreak => Inline::of(InlineKind::lineBreak), } } @@ -175,11 +187,19 @@ pub struct Style { pub italic: bool, pub strike: bool, pub code: bool, + /// `baseline`, `superscript` or `subscript`. + pub vert_align: String, } impl From for Style { fn from(style: model::Style) -> Self { - Style { bold: style.bold, italic: style.italic, strike: style.strike, code: style.code } + Style { + bold: style.bold, + italic: style.italic, + strike: style.strike, + code: style.code, + vert_align: style.vert_align.as_str().into(), + } } } diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index 19dadbad..cc4ab7fd 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -99,7 +99,7 @@ class Block: @final class Inline: - kind: Literal["text", "link", "image", "anchor", "note_ref", "line_break"] + kind: Literal["text", "link", "image", "anchor", "note_ref", "math", "line_break"] """`anchor` is a zero-width marker for an internal link target at this position.""" text: str | None diff --git a/python/src/document.rs b/python/src/document.rs index e24cb666..7a8be099 100644 --- a/python/src/document.rs +++ b/python/src/document.rs @@ -87,7 +87,7 @@ fn block(py: Python<'_>, block: model::Block) -> PyResult { #[pyclass(frozen, get_all, module = "anydoc")] pub struct Inline { /// text, link, image, anchor (a zero-width marker for an internal link - /// target at this position), note_ref, or line_break. + /// target at this position), note_ref, math, or line_break. kind: &'static str, /// text. text: Option, @@ -105,6 +105,10 @@ pub struct Inline { anchor: Option, /// note_ref: the id of the note in `Document.notes`. note_id: Option, + /// math: the expression as LaTeX, without delimiters. + latex: Option, + /// math: True for an equation that stands on its own line. + display: Option, } impl Inline { @@ -119,6 +123,8 @@ impl Inline { source: None, anchor: None, note_id: None, + latex: None, + display: None, } } } @@ -142,6 +148,9 @@ fn inline(py: Python<'_>, inline: model::Inline) -> PyResult { }, model::Inline::Anchor(id) => Inline { anchor: Some(id), ..Inline::of("anchor") }, model::Inline::NoteRef(id) => Inline { note_id: Some(id), ..Inline::of("note_ref") }, + model::Inline::Math { latex, display } => { + Inline { latex: Some(latex), display: Some(display), ..Inline::of("math") } + } model::Inline::LineBreak => Inline::of("line_break"), }) } @@ -153,11 +162,19 @@ pub struct Style { italic: bool, strike: bool, code: bool, + /// `baseline`, `superscript` or `subscript`. + vert_align: &'static str, } impl From for Style { fn from(style: model::Style) -> Self { - Style { bold: style.bold, italic: style.italic, strike: style.strike, code: style.code } + Style { + bold: style.bold, + italic: style.italic, + strike: style.strike, + code: style.code, + vert_align: style.vert_align.as_str(), + } } } diff --git a/src/formats/docx/content.rs b/src/formats/docx/content.rs index c53c1ecf..6c7a035b 100644 --- a/src/formats/docx/content.rs +++ b/src/formats/docx/content.rs @@ -2,6 +2,7 @@ use crate::error::ConvertError; use crate::formats::docx::numbering::{Counters, Numbering}; +use crate::formats::docx::omml; use crate::formats::docx::styles::{Styles, on_off, rpr_delta}; use crate::model::{ Block, Cell, GridBuilder, ImageSource, Inline, LinkTarget, Style, TableKind, inlines_are_empty, @@ -22,6 +23,7 @@ use std::collections::HashMap; /// requiring anything else fall back to `mc:Fallback`. const SUPPORTED_NS: &[&str] = &[ ns::W, + ns::M, ns::A, ns::PIC, ns::WP, @@ -148,6 +150,13 @@ fn collect_blocks( } continue; } + if child.is(ns::M, "oMathPara") { + runs.flush(blocks); + if let Some(math) = omml::to_inline(child, true) { + blocks.push(Block::Paragraph(vec![math])); + } + continue; + } if child.ns.as_deref().is_none_or(|n| n != ns::W) { continue; } @@ -380,6 +389,19 @@ impl<'a, 'b, 'e> InlineWalker<'a, 'b, 'e> { } continue; } + if child.is(ns::M, "oMath") { + if let Some(math) = omml::to_inline(child, false) { + self.push(math); + } + continue; + } + // A display equation inside the paragraph splits the run, as a text box does. + if child.is(ns::M, "oMathPara") { + if let Some(math) = omml::to_inline(child, true) { + self.push_blocks(vec![Block::Paragraph(vec![math])]); + } + continue; + } if child.ns.as_deref().is_none_or(|n| n != ns::W) { continue; } @@ -469,6 +491,12 @@ impl<'a, 'b, 'e> InlineWalker<'a, 'b, 'e> { } continue; } + if child.is(ns::M, "oMath") { + if let Some(math) = omml::to_inline(child, false) { + self.push(math); + } + continue; + } let in_w = child.ns.as_deref().is_some_and(|n| n == ns::W); if !in_w { continue; diff --git a/src/formats/docx/mod.rs b/src/formats/docx/mod.rs index 21dfef57..d1a219e1 100644 --- a/src/formats/docx/mod.rs +++ b/src/formats/docx/mod.rs @@ -5,6 +5,7 @@ mod content; mod numbering; +mod omml; mod styles; use crate::error::ConvertError; diff --git a/src/formats/docx/omml.rs b/src/formats/docx/omml.rs new file mode 100644 index 00000000..787ed7fa --- /dev/null +++ b/src/formats/docx/omml.rs @@ -0,0 +1,758 @@ +//! Office Math (OMML) to LaTeX. +//! +//! Parsing builds a tree before emitting: bracing depends on a node's shape -- +//! `y^{2}` as another script's base needs braces, `\frac{a}{b}` does not -- and +//! the emitted string cannot answer that. Property elements are read through +//! named lookups only, never enumerated, so an absent bag and an empty one +//! resolve alike. Emitted commands stay inside the subset KaTeX implements. + +use crate::model::Inline; +use crate::package::xml::{Element, ns}; + +/// Deepest OMML nesting translated. Real mathematics stays under a dozen +/// levels; past this the subtree degrades to its text, which keeps a hostile +/// document from driving recursion on a bounded stack. +const MAX_DEPTH: usize = 64; + +/// Largest LaTeX one equation may produce. Wrappers multiply through nesting +/// (`\left(\right)` is twelve characters per level), so a small document can +/// otherwise amplify without bound. A dense page of mathematics is under 4 KiB. +const MAX_LATEX_BYTES: usize = 64 * 1024; + +/// Most cells one matrix may contribute. +const MAX_MATRIX_CELLS: usize = 10_000; + +/// The default n-ary operator when `m:naryPr/m:chr` is absent (ECMA-376). +const DEFAULT_NARY: char = '∫'; +/// The default accent when `m:accPr/m:chr` is absent: combining circumflex. +const DEFAULT_ACCENT: char = '\u{0302}'; +/// The default group character when `m:groupChrPr/m:chr` is absent. +const DEFAULT_GROUP: char = '\u{23DF}'; + +/// A parsed equation, shaped for LaTeX emission. +enum Node { + /// Literal text from `m:t`, still unescaped. + Run(String), + Seq(Vec), + Frac { + num: Box, + den: Box, + kind: FracKind, + }, + Rad { + deg: Option>, + base: Box, + }, + Script { + base: Box, + sub: Option>, + sup: Option>, + pre: bool, + }, + Nary { + op: char, + sub: Option>, + sup: Option>, + base: Box, + limits: bool, + }, + Delim { + open: Option, + close: Option, + sep: char, + parts: Vec, + }, + Accent { + chr: char, + base: Box, + }, + Bar { + top: bool, + base: Box, + }, + Group { + chr: Option, + top: bool, + base: Box, + }, + Func { + name: Box, + arg: Box, + }, + Limit { + upper: bool, + base: Box, + lim: Box, + }, + Matrix(Vec>), + EqArr(Vec), + Boxed { + strike: Strike, + base: Box, + }, + Phantom { + base: Box, + }, +} + +enum FracKind { + Bar, + Skewed, + NoBar, + Linear, +} + +enum Strike { + None, + Forward, + Back, + Cross, +} + +/// Translate an `m:oMath` or `m:oMathPara` element. +/// +/// Returns `None` only when the element carries nothing renderable, so a +/// caller can drop an empty equation rather than emit empty delimiters. +pub fn to_inline(elem: &Element, display: bool) -> Option { + let node = parse_seq(elem, 0); + let mut latex = String::new(); + emit(&node, &mut latex, false); + let latex = latex.trim().to_string(); + if latex.is_empty() { + return None; + } + Some(Inline::Math { latex, display }) +} + +/// The `m:val` of a named property inside a named property bag, if every +/// level is present. Property bags are never enumerated: a `m:ctrlPr` holds +/// the formatting Word uses to draw its own glyphs and has no content of ours. +fn pr_val<'a>(parent: &'a Element, bag: &str, prop: &str) -> Option<&'a str> { + parent.find(ns::M, bag)?.find(ns::M, prop)?.attr(ns::M, "val") +} + +/// True when a property is present and explicitly on. +fn pr_on(parent: &Element, bag: &str, prop: &str) -> bool { + matches!(pr_val(parent, bag, prop), Some("1" | "true" | "on")) +} + +fn first_char(value: &str) -> Option { + value.chars().next() +} + +/// Children of an argument container (`m:e`, `m:num`, `m:den`, ...) as one node. +fn parse_arg(parent: &Element, name: &str, depth: usize) -> Option { + Some(parse_seq(parent.find(ns::M, name)?, depth)) +} + +fn boxed_arg(parent: &Element, name: &str, depth: usize) -> Box { + Box::new(parse_arg(parent, name, depth).unwrap_or(Node::Seq(Vec::new()))) +} + +/// Every child of `elem`, flattened into one sequence. +fn parse_seq(elem: &Element, depth: usize) -> Node { + if depth >= MAX_DEPTH { + return Node::Run(elem.text()); + } + let mut parts: Vec = Vec::new(); + for child in elem.child_elems() { + if child.ns.as_deref() != Some(ns::M) { + // Revision marks and bookmarks wrap runs that are part of the maths. + if child.ns.as_deref() == Some(ns::W) + && let Node::Seq(inner) = parse_seq(child, depth + 1) + { + parts.extend(inner); + } + continue; + } + if let Some(node) = parse_elem(child, depth) { + parts.push(node); + } + } + if parts.len() == 1 { parts.pop().unwrap() } else { Node::Seq(parts) } +} + +fn parse_elem(elem: &Element, depth: usize) -> Option { + let d = depth + 1; + match elem.local.as_str() { + name if name.ends_with("Pr") => None, + "t" => { + let text = elem.text(); + if text.is_empty() { None } else { Some(Node::Run(text)) } + } + "r" => Some(parse_seq(elem, d)), + "oMath" | "oMathPara" | "box" | "e" | "num" | "den" | "lim" | "deg" | "fName" | "sub" + | "sup" => Some(parse_seq(elem, d)), + "f" => Some(Node::Frac { + num: boxed_arg(elem, "num", d), + den: boxed_arg(elem, "den", d), + kind: match pr_val(elem, "fPr", "type") { + Some("skw") => FracKind::Skewed, + Some("noBar") => FracKind::NoBar, + Some("lin") => FracKind::Linear, + _ => FracKind::Bar, + }, + }), + "rad" => { + // An empty `m:deg` is a square root; `m:degHide` says the same. + let deg = elem + .find(ns::M, "deg") + .filter(|d| !d.text().trim().is_empty()) + .filter(|_| !pr_on(elem, "radPr", "degHide")) + .map(|deg| Box::new(parse_seq(deg, d))); + Some(Node::Rad { deg, base: boxed_arg(elem, "e", d) }) + } + "sSup" => Some(Node::Script { + base: boxed_arg(elem, "e", d), + sub: None, + sup: parse_arg(elem, "sup", d).map(Box::new), + pre: false, + }), + "sSub" => Some(Node::Script { + base: boxed_arg(elem, "e", d), + sub: parse_arg(elem, "sub", d).map(Box::new), + sup: None, + pre: false, + }), + "sSubSup" => Some(Node::Script { + base: boxed_arg(elem, "e", d), + sub: parse_arg(elem, "sub", d).map(Box::new), + sup: parse_arg(elem, "sup", d).map(Box::new), + pre: false, + }), + "sPre" => Some(Node::Script { + base: boxed_arg(elem, "e", d), + sub: parse_arg(elem, "sub", d).map(Box::new), + sup: parse_arg(elem, "sup", d).map(Box::new), + pre: true, + }), + "nary" => Some(Node::Nary { + op: pr_val(elem, "naryPr", "chr").and_then(first_char).unwrap_or(DEFAULT_NARY), + sub: parse_arg(elem, "sub", d) + .filter(|_| !pr_on(elem, "naryPr", "subHide")) + .map(Box::new), + sup: parse_arg(elem, "sup", d) + .filter(|_| !pr_on(elem, "naryPr", "supHide")) + .map(Box::new), + base: boxed_arg(elem, "e", d), + limits: pr_val(elem, "naryPr", "limLoc") == Some("undOvr"), + }), + "d" => { + // An explicitly empty delimiter means "no glyph", which is not the + // same as an absent property taking its default. + let bag = elem.find(ns::M, "dPr"); + let chr = |name: &str, default: char| match bag.and_then(|b| b.find(ns::M, name)) { + None => Some(default), + Some(e) => e.attr(ns::M, "val").and_then(first_char), + }; + Some(Node::Delim { + open: chr("begChr", '('), + close: chr("endChr", ')'), + sep: pr_val(elem, "dPr", "sepChr").and_then(first_char).unwrap_or('|'), + parts: elem.find_all(ns::M, "e").map(|e| parse_seq(e, d)).collect(), + }) + } + "func" => { + Some(Node::Func { name: boxed_arg(elem, "fName", d), arg: boxed_arg(elem, "e", d) }) + } + "limLow" => Some(Node::Limit { + upper: false, + base: boxed_arg(elem, "e", d), + lim: boxed_arg(elem, "lim", d), + }), + "limUpp" => Some(Node::Limit { + upper: true, + base: boxed_arg(elem, "e", d), + lim: boxed_arg(elem, "lim", d), + }), + "m" => { + let mut rows = Vec::new(); + let mut cells = 0usize; + for mr in elem.find_all(ns::M, "mr") { + let row: Vec = mr.find_all(ns::M, "e").map(|e| parse_seq(e, d)).collect(); + cells += row.len(); + if cells > MAX_MATRIX_CELLS { + break; + } + rows.push(row); + } + Some(Node::Matrix(rows)) + } + "eqArr" => Some(Node::EqArr(elem.find_all(ns::M, "e").map(|e| parse_seq(e, d)).collect())), + "acc" => Some(Node::Accent { + chr: pr_val(elem, "accPr", "chr").and_then(first_char).unwrap_or(DEFAULT_ACCENT), + base: boxed_arg(elem, "e", d), + }), + "bar" => Some(Node::Bar { + top: pr_val(elem, "barPr", "pos") == Some("top"), + base: boxed_arg(elem, "e", d), + }), + "groupChr" => { + let bag = elem.find(ns::M, "groupChrPr"); + let chr = match bag.and_then(|b| b.find(ns::M, "chr")) { + None => Some(DEFAULT_GROUP), + Some(e) => e.attr(ns::M, "val").and_then(first_char), + }; + Some(Node::Group { + chr, + top: pr_val(elem, "groupChrPr", "pos") == Some("top"), + base: boxed_arg(elem, "e", d), + }) + } + "borderBox" => Some(Node::Boxed { + strike: match ( + pr_on(elem, "borderBoxPr", "strikeBLTR"), + pr_on(elem, "borderBoxPr", "strikeTLBR"), + ) { + (true, true) => Strike::Cross, + (true, false) => Strike::Forward, + (false, true) => Strike::Back, + (false, false) => Strike::None, + }, + base: boxed_arg(elem, "e", d), + }), + "phant" => Some(Node::Phantom { base: boxed_arg(elem, "e", d) }), + "br" => Some(Node::Run(" ".into())), + // Unmodelled elements still hold content; descend rather than drop it. + _ => Some(parse_seq(elem, d)), + } +} + +/// LaTeX-escape literal text. A `$` would close the surrounding math span and +/// hand the rest of the document to the Markdown parser, so it is escaped +/// here rather than trusted; newlines cannot appear inside inline math. +fn push_text(text: &str, out: &mut String) { + // A command runs until a non-letter, so `\int` + `f` would lex as `\intf`. + if text.starts_with(|c: char| c.is_ascii_alphabetic()) && ends_with_control_word(out) { + out.push(' '); + } + for c in text.chars() { + match c { + '\\' => out.push_str("\\backslash "), + '{' => out.push_str("\\{"), + '}' => out.push_str("\\}"), + '$' => out.push_str("\\$"), + '&' => out.push_str("\\&"), + '#' => out.push_str("\\#"), + '%' => out.push_str("\\%"), + '_' => out.push_str("\\_"), + '^' => out.push_str("\\^{}"), + '~' => out.push_str("\\~{}"), + '\n' | '\r' => out.push(' '), + c => out.push(c), + } + } +} + +/// True when `out` ends in a LaTeX control word, whose name would absorb a +/// following letter. +fn ends_with_control_word(out: &str) -> bool { + let trailing_letters = + out.len() - out.trim_end_matches(|c: char| c.is_ascii_alphabetic()).len(); + trailing_letters > 0 + && out[..out.len() - trailing_letters].ends_with('\\') + && !out[..out.len() - trailing_letters].ends_with("\\\\") +} + +/// A script's base needs braces when it already ends in a script (`y^{2}^{3}` +/// is an error) or when it is a sequence the script would otherwise bind only +/// the last atom of. A fraction, radical or delimiter is a single atom already. +fn base_needs_braces(node: &Node) -> bool { + match node { + Node::Seq(parts) => parts.len() != 1, + Node::Script { .. } | Node::Nary { .. } | Node::Limit { .. } => true, + _ => false, + } +} + +/// Emit `node` as a braced group unless it is a single atom. +fn emit_group(node: &Node, out: &mut String) { + out.push('{'); + emit(node, out, false); + out.push('}'); +} + +fn emit(node: &Node, out: &mut String, _in_group: bool) { + if out.len() >= MAX_LATEX_BYTES { + return; + } + match node { + Node::Run(text) => push_text(text, out), + Node::Seq(parts) => { + for part in parts { + emit(part, out, false); + } + } + Node::Frac { num, den, kind } => match kind { + FracKind::Bar => { + out.push_str("\\frac"); + emit_group(num, out); + emit_group(den, out); + } + FracKind::NoBar => { + out.push_str("\\binom"); + emit_group(num, out); + emit_group(den, out); + } + FracKind::Skewed | FracKind::Linear => { + emit_group(num, out); + out.push('/'); + emit_group(den, out); + } + }, + Node::Rad { deg, base } => { + out.push_str("\\sqrt"); + if let Some(deg) = deg { + out.push('['); + emit(deg, out, false); + out.push(']'); + } + emit_group(base, out); + } + Node::Script { base, sub, sup, pre } => { + if *pre { + // Scripts hang off an empty base; the real base follows. + out.push_str("{}"); + if let Some(sub) = sub { + out.push('_'); + emit_group(sub, out); + } + if let Some(sup) = sup { + out.push('^'); + emit_group(sup, out); + } + emit(base, out, false); + return; + } + if base_needs_braces(base) { + emit_group(base, out); + } else { + emit(base, out, false); + } + if let Some(sub) = sub { + out.push('_'); + emit_group(sub, out); + } + if let Some(sup) = sup { + out.push('^'); + emit_group(sup, out); + } + } + Node::Nary { op, sub, sup, base, limits } => { + out.push_str(nary_command(*op)); + if *limits { + out.push_str("\\limits"); + } + if let Some(sub) = sub { + out.push('_'); + emit_group(sub, out); + } + if let Some(sup) = sup { + out.push('^'); + emit_group(sup, out); + } + emit(base, out, false); + } + Node::Delim { open, close, sep, parts } => { + out.push_str("\\left"); + out.push_str(&delim_glyph(*open)); + for (i, part) in parts.iter().enumerate() { + if i > 0 { + out.push_str(&delim_sep(*sep)); + } + emit(part, out, false); + } + out.push_str("\\right"); + out.push_str(&delim_glyph(*close)); + } + Node::Accent { chr, base } => { + out.push_str(accent_command(*chr)); + emit_group(base, out); + } + Node::Bar { top, base } => { + out.push_str(if *top { "\\overline" } else { "\\underline" }); + emit_group(base, out); + } + Node::Group { chr, top, base } => match chr.map(group_command) { + Some(cmd) => { + out.push_str(cmd); + emit_group(base, out); + } + // An explicitly blank group character draws nothing. + None => { + let _ = top; + emit(base, out, false); + } + }, + Node::Func { name, arg } => { + let mut rendered = String::new(); + emit(name, &mut rendered, false); + let trimmed = rendered.trim(); + if KATEX_FUNCTIONS.contains(&trimmed) { + out.push('\\'); + out.push_str(trimmed); + } else { + out.push_str("\\operatorname"); + out.push('{'); + out.push_str(trimmed); + out.push('}'); + } + emit_group(arg, out); + } + Node::Limit { upper, base, lim } => { + let mut rendered = String::new(); + emit(base, &mut rendered, false); + let trimmed = rendered.trim(); + // An operator name takes a real script; anything else stacks. + if KATEX_FUNCTIONS.contains(&trimmed) { + out.push('\\'); + out.push_str(trimmed); + out.push(if *upper { '^' } else { '_' }); + emit_group(lim, out); + return; + } + out.push_str(if *upper { "\\overset" } else { "\\underset" }); + emit_group(lim, out); + out.push('{'); + out.push_str(trimmed); + out.push('}'); + } + Node::Matrix(rows) => { + out.push_str("\\begin{matrix}"); + for (i, row) in rows.iter().enumerate() { + if i > 0 { + out.push_str(" \\\\ "); + } + for (j, cell) in row.iter().enumerate() { + if j > 0 { + out.push_str(" & "); + } + emit(cell, out, false); + } + } + out.push_str("\\end{matrix}"); + } + Node::EqArr(rows) => { + out.push_str("\\begin{aligned}"); + for (i, row) in rows.iter().enumerate() { + if i > 0 { + out.push_str(" \\\\ "); + } + emit(row, out, false); + } + out.push_str("\\end{aligned}"); + } + Node::Boxed { strike, base } => { + out.push_str(match strike { + Strike::None => "\\boxed", + Strike::Forward => "\\cancel", + Strike::Back => "\\bcancel", + Strike::Cross => "\\xcancel", + }); + emit_group(base, out); + } + Node::Phantom { base } => { + out.push_str("\\phantom"); + emit_group(base, out); + } + } +} + +/// Operator names KaTeX spells with a leading backslash. +const KATEX_FUNCTIONS: &[&str] = &[ + "arccos", "arcsin", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc", "deg", "det", "dim", + "exp", "gcd", "hom", "inf", "ker", "lg", "lim", "liminf", "limsup", "ln", "log", "max", "min", + "sec", "sin", "sinh", "sup", "tan", "tanh", +]; + +fn nary_command(chr: char) -> &'static str { + match chr { + '∑' => "\\sum", + '∏' => "\\prod", + '∐' => "\\coprod", + '∫' => "\\int", + '∬' => "\\iint", + '∭' => "\\iiint", + '∮' => "\\oint", + '∯' => "\\oiint", + '∰' => "\\oiiint", + '⋀' => "\\bigwedge", + '⋁' => "\\bigvee", + '⋂' => "\\bigcap", + '⋃' => "\\bigcup", + '⨀' => "\\bigodot", + '⨁' => "\\bigoplus", + '⨂' => "\\bigotimes", + '⨄' => "\\biguplus", + '⨆' => "\\bigsqcup", + _ => "\\int", + } +} + +/// OMML gives a combining codepoint; KaTeX wants the accent command. +fn accent_command(chr: char) -> &'static str { + match chr { + '\u{0300}' => "\\grave", + '\u{0301}' => "\\acute", + '\u{0303}' | '~' => "\\widetilde", + '\u{0304}' => "\\bar", + '\u{0305}' => "\\overline", + '\u{0306}' => "\\breve", + '\u{0307}' => "\\dot", + '\u{0308}' => "\\ddot", + '\u{030A}' => "\\mathring", + '\u{030C}' => "\\check", + '\u{0332}' => "\\underline", + '\u{20D6}' => "\\overleftarrow", + '\u{20D7}' | '→' => "\\vec", + '\u{20DB}' => "\\dddot", + '\u{20E1}' => "\\overleftrightarrow", + _ => "\\widehat", + } +} + +fn group_command(chr: char) -> &'static str { + match chr { + '\u{23DE}' | '\u{FE37}' => "\\overbrace", + '\u{23B4}' => "\\overbracket", + '\u{23B5}' => "\\underbracket", + '\u{23DC}' => "\\overgroup", + '\u{23DD}' => "\\undergroup", + '←' => "\\overleftarrow", + '→' => "\\overrightarrow", + _ => "\\underbrace", + } +} + +/// A delimiter glyph in `\left`/`\right` position. An absent delimiter is a +/// bare `.`, which is how LaTeX spells "no glyph but keep the pair balanced". +fn delim_glyph(chr: Option) -> String { + match chr { + None => ".".into(), + Some('{') => "\\{".into(), + Some('}') => "\\}".into(), + Some('|') => "\\vert".into(), + Some('‖') => "\\Vert".into(), + Some('⌈') => "\\lceil".into(), + Some('⌉') => "\\rceil".into(), + Some('⌊') => "\\lfloor".into(), + Some('⌋') => "\\rfloor".into(), + Some('⟨') => "\\langle".into(), + Some('⟩') => "\\rangle".into(), + Some(c) => c.to_string(), + } +} + +/// A separator between delimiter parts. A bare `|` would split a Markdown +/// table row, so it is always spelled as a command. +fn delim_sep(chr: char) -> String { + match chr { + '|' => "\\mid ".into(), + '‖' => "\\Vert ".into(), + c => c.to_string(), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::package::xml::parse_xml; + + /// Convert one `m:oMath` written as a source fragment. + fn latex(body: &str) -> String { + let doc = format!( + r#"{body}"# + ); + let root = parse_xml(doc.as_bytes()).unwrap(); + let math = root.find(ns::M, "oMath").unwrap(); + match to_inline(math, false) { + Some(Inline::Math { latex, .. }) => latex, + _ => String::new(), + } + } + + fn run(text: &str) -> String { + format!("{text}") + } + + #[test] + fn absent_property_bags_take_their_spec_defaults() { + assert_eq!(latex(&format!("{}", run("a"))), r"\left(a\right)"); + assert_eq!(latex(&format!("{}", run("y"))), r"\widehat{y}"); + assert_eq!(latex(&format!("{}", run("x"))), r"\underline{x}"); + assert_eq!( + latex(&format!("{}", run("x"))), + r"\underbrace{x}" + ); + assert_eq!( + latex(&format!("{}{}", run("a"), run("b"))), + r"\frac{a}{b}" + ); + } + + #[test] + fn absent_nary_operator_is_the_integral_not_the_sum() { + let xml = format!("{}", run("f")); + assert_eq!(latex(&xml), r"\int f"); + } + + #[test] + fn an_explicitly_empty_delimiter_draws_no_glyph() { + let xml = format!( + r#"{}"#, + run("v") + ); + assert_eq!(latex(&xml), r"\left.v\right."); + } + + #[test] + fn a_script_base_that_is_itself_a_script_is_braced() { + // `y^{2}^{3}` is a double superscript, which is an error. + let inner = format!("{}{}", run("y"), run("2")); + let outer = format!("{inner}{}", run("3")); + assert_eq!(latex(&outer), "{y^{2}}^{3}"); + } + + #[test] + fn a_fraction_base_needs_no_braces() { + let frac = format!("{}{}", run("a"), run("b")); + let xml = format!("{frac}{}", run("2")); + assert_eq!(latex(&xml), r"\frac{a}{b}^{2}"); + } + + #[test] + fn text_cannot_escape_the_math_span() { + let out = latex(&run(r"a$b \href{x}{y} 50%")); + assert!(!out.contains('$') || out.contains(r"\$"), "{out}"); + assert!(!out.contains(r"\href"), "{out}"); + assert_eq!(out, r"a\$b \backslash href\{x\}\{y\} 50\%"); + } + + #[test] + fn a_delimiter_separator_never_emits_a_bare_pipe() { + // A raw pipe would split the row the equation sits in. + let xml = format!("{}{}", run("x"), run("y")); + assert!(!latex(&xml).contains('|')); + } + + #[test] + fn an_unmodelled_element_keeps_its_content() { + let xml = format!("{}", run("keep")); + assert_eq!(latex(&xml), "keep"); + } + + #[test] + fn nesting_past_the_bound_degrades_to_text() { + let mut xml = run("deep"); + for _ in 0..MAX_DEPTH + 8 { + xml = format!("{xml}"); + } + assert!(latex(&xml).contains("deep")); + } + + #[test] + fn an_empty_equation_produces_nothing() { + assert_eq!(latex(""), ""); + } +} diff --git a/src/formats/docx/styles.rs b/src/formats/docx/styles.rs index d2e82e34..940e9cca 100644 --- a/src/formats/docx/styles.rs +++ b/src/formats/docx/styles.rs @@ -7,7 +7,7 @@ //! formatting is absolute on/off. use crate::error::ConvertError; -use crate::model::Style; +use crate::model::{Style, VertAlign}; use crate::package::xml::{Element, ns}; use crate::shared::blockstyle::{self, BlockStyle}; use crate::shared::chain::StyleChains; @@ -36,6 +36,9 @@ impl Toggles { italic: base.italic ^ self.italic, strike: base.strike ^ self.strike, code: base.code, + // Not a toggle: ECMA-376 s17.7.3 closes the toggle set, and vertical + // alignment is an ordinary property whose nearest specification wins. + vert_align: base.vert_align, } } } @@ -184,6 +187,18 @@ pub fn rpr_delta(rpr: &Element) -> StyleDelta { None }, code: None, + vert_align: vert_align(rpr), + } +} + +/// ST_VerticalAlignRun. Not `on_off`: that reads any value outside the +/// false-set as `true`, which cannot tell `superscript` from `subscript`. +pub fn vert_align(rpr: &Element) -> Option { + match rpr.find(ns::W, "vertAlign")?.attr(ns::W, "val")? { + "superscript" => Some(VertAlign::Superscript), + "subscript" => Some(VertAlign::Subscript), + "baseline" => Some(VertAlign::Baseline), + _ => None, } } diff --git a/src/formats/odf/styles.rs b/src/formats/odf/styles.rs index 2ad6db37..fa06c7b3 100644 --- a/src/formats/odf/styles.rs +++ b/src/formats/odf/styles.rs @@ -6,6 +6,7 @@ //! from two separately parsed trees (`styles.xml` and `content.xml`). use crate::error::ConvertError; +use crate::model::VertAlign; use crate::package::xml::{Element, ns}; use crate::shared::blockstyle::{self, BlockStyle}; use crate::shared::delta::StyleDelta; @@ -222,6 +223,26 @@ fn parse_list_style(style: &Element) -> [ListLevel; LIST_LEVELS] { levels } +/// `style:text-position` is a raise followed by an optional font size, where +/// the raise is `super`, `sub`, or a signed percentage. +fn text_position(value: &str) -> Option { + let raise = value.split_whitespace().next()?; + match raise { + "super" => Some(VertAlign::Superscript), + "sub" => Some(VertAlign::Subscript), + _ => { + let percent: f32 = raise.trim_end_matches('%').parse().ok()?; + Some(if percent > 0.0 { + VertAlign::Superscript + } else if percent < 0.0 { + VertAlign::Subscript + } else { + VertAlign::Baseline + }) + } + } +} + /// Delta carried by a style's `style:text-properties`. pub fn text_properties_delta(elem: &Element) -> StyleDelta { let Some(props) = elem.find(ns::STYLE, "text-properties") else { @@ -234,5 +255,6 @@ pub fn text_properties_delta(elem: &Element) -> StyleDelta { italic: props.attr(ns::FO, "font-style").map(|s| s == "italic" || s == "oblique"), strike: props.attr(ns::STYLE, "text-line-through-style").map(|lt| lt != "none"), code: None, + vert_align: props.attr(ns::STYLE, "text-position").and_then(text_position), } } diff --git a/src/formats/odf/table.rs b/src/formats/odf/table.rs index 48e42fb4..d9e63dd0 100644 --- a/src/formats/odf/table.rs +++ b/src/formats/odf/table.rs @@ -94,6 +94,7 @@ fn block_bytes(blocks: &[Block]) -> u64 { } Inline::Image { alt, .. } => alt.len() as u64, Inline::Anchor(id) | Inline::NoteRef(id) => id.len() as u64, + Inline::Math { latex, .. } => latex.len() as u64, Inline::LineBreak => 1, }) .sum() diff --git a/src/formats/ppt/mod.rs b/src/formats/ppt/mod.rs index d0f38de2..e54151f9 100644 --- a/src/formats/ppt/mod.rs +++ b/src/formats/ppt/mod.rs @@ -9,7 +9,7 @@ mod styletext; use crate::error::ConvertError; -use crate::model::{Block, Document, Inline, Style, inlines_are_empty}; +use crate::model::{Block, Document, Inline, Style, VertAlign, inlines_are_empty}; use crate::package::limits; use crate::shared::binary::{get_u32, read_ole_stream}; use crate::shared::delta::{StyleDelta, rebase_emphasis}; @@ -552,6 +552,7 @@ impl Extractor { italic: char_run.and_then(|r| r.italic).or(d.italic).unwrap_or(false), strike: false, code: false, + vert_align: VertAlign::Baseline, }; if c == '\r' { if !run_text.is_empty() { diff --git a/src/formats/pptx/cascade.rs b/src/formats/pptx/cascade.rs index 76b59309..a0346c2c 100644 --- a/src/formats/pptx/cascade.rs +++ b/src/formats/pptx/cascade.rs @@ -3,6 +3,7 @@ //! placeholder / `txStyles` -> presentation `defaultTextStyle`, with //! explicit-off states honored at every layer. +use crate::model::VertAlign; use crate::package::xml::{Element, ns}; use crate::shared::delta::StyleDelta; use crate::shared::list::MarkerKind; @@ -129,6 +130,16 @@ pub fn paragraph_props(ppr: &Element) -> TextProps { TextProps { delta, bullet } } +/// A DrawingML `baseline` percentage as a vertical alignment: positive raises, +/// negative lowers, zero sits on the baseline. +fn baseline(percent: i32) -> VertAlign { + match percent.signum() { + 1 => VertAlign::Superscript, + -1 => VertAlign::Subscript, + _ => VertAlign::Baseline, + } +} + /// Delta from an `a:rPr`/`a:defRPr` element's attributes. pub fn rpr_delta(rpr: &Element) -> StyleDelta { let on_off = |name: &str| rpr.attr(ns::A, name).map(|v| matches!(v, "1" | "true" | "on")); @@ -137,6 +148,8 @@ pub fn rpr_delta(rpr: &Element) -> StyleDelta { italic: on_off("i"), strike: rpr.attr(ns::A, "strike").map(|v| matches!(v, "sngStrike" | "dblStrike")), code: None, + // A signed percentage of the raise, not an enum. + vert_align: rpr.attr(ns::A, "baseline").and_then(|v| v.parse::().ok()).map(baseline), } } diff --git a/src/model/inline.rs b/src/model/inline.rs index db6383a4..b2113f05 100644 --- a/src/model/inline.rs +++ b/src/model/inline.rs @@ -32,6 +32,17 @@ pub enum Inline { NoteRef(String), /// A line break inside a block, not a new block. LineBreak, + /// A mathematical expression, already translated to LaTeX. + /// + /// The only inline whose payload reaches the output unescaped, so the + /// producer owns making it safe: no bare `$`, which would close the span, + /// and no newline, which would end the construct. + Math { + /// LaTeX body, without delimiters. + latex: String, + /// Set for an equation that stands on its own line. + display: bool, + }, } impl Inline { @@ -57,6 +68,7 @@ fn collect_plain_text(inlines: &[Inline], out: &mut String) { Inline::Link { content, .. } => collect_plain_text(content, out), Inline::Image { alt, .. } => out.push_str(alt), Inline::Anchor(_) | Inline::NoteRef(_) => {} + Inline::Math { latex, .. } => out.push_str(latex), Inline::LineBreak => out.push('\n'), } } @@ -69,7 +81,7 @@ pub fn inlines_are_empty(inlines: &[Inline]) -> bool { inlines.iter().all(|i| match i { Inline::Text { text, .. } => text.trim().is_empty(), Inline::Link { content, target } => target.is_empty() && inlines_are_empty(content), - Inline::Image { .. } | Inline::NoteRef(_) => false, + Inline::Image { .. } | Inline::NoteRef(_) | Inline::Math { .. } => false, Inline::Anchor(_) | Inline::LineBreak => true, }) } diff --git a/src/model/mod.rs b/src/model/mod.rs index aa209215..d37940fb 100644 --- a/src/model/mod.rs +++ b/src/model/mod.rs @@ -18,7 +18,7 @@ pub use block::Block; pub use inline::{Inline, inlines_are_empty, inlines_to_plain_text}; pub use link::{AnchorId, ImageSource, LinkTarget}; pub use list::{List, ListItem, MarkerKind}; -pub use style::Style; +pub use style::{Style, VertAlign}; pub use table::{Cell, CellSlot, Table, TableKind}; /// Frontends build grids; consumers read them off [`Table::grid`]. diff --git a/src/model/style.rs b/src/model/style.rs index 438ab872..53006c6b 100644 --- a/src/model/style.rs +++ b/src/model/style.rs @@ -1,3 +1,31 @@ +/// Vertical position of a run relative to the baseline. +/// +/// Unlike the emphasis fields this is a value, not a toggle: OOXML models it +/// as `ST_VerticalAlignRun` and ODF as a percentage-plus-size pair, and in +/// both the nearest specification along the style chain wins outright rather +/// than flipping an inherited state. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum VertAlign { + /// On the baseline. + #[default] + Baseline, + /// Raised, as in `10⁻³` or an ordinal suffix. + Superscript, + /// Lowered, as in the 2 of `H₂O`. + Subscript, +} + +impl VertAlign { + /// The OOXML spelling, and the value the language bindings publish. + pub fn as_str(self) -> &'static str { + match self { + VertAlign::Baseline => "baseline", + VertAlign::Superscript => "superscript", + VertAlign::Subscript => "subscript", + } + } +} + /// Fully resolved character style. Tri-state deltas exist only during /// frontend resolution (`shared::delta`); by the time content reaches the /// model every toggle has a definite value. @@ -11,9 +39,17 @@ pub struct Style { pub strike: bool, /// Monospace, from a code or teletype character style. pub code: bool, + /// Position relative to the baseline. + pub vert_align: VertAlign, } impl Style { - /// No toggle set. - pub const PLAIN: Style = Style { bold: false, italic: false, strike: false, code: false }; + /// No toggle set, on the baseline. + pub const PLAIN: Style = Style { + bold: false, + italic: false, + strike: false, + code: false, + vert_align: VertAlign::Baseline, + }; } diff --git a/src/package/xml.rs b/src/package/xml.rs index 42af1d8c..57ce456e 100644 --- a/src/package/xml.rs +++ b/src/package/xml.rs @@ -22,6 +22,7 @@ pub mod ns { pub const PIC: &str = "http://schemas.openxmlformats.org/drawingml/2006/picture"; pub const WP: &str = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"; pub const MC: &str = "http://schemas.openxmlformats.org/markup-compatibility/2006"; + pub const M: &str = "http://schemas.openxmlformats.org/officeDocument/2006/math"; pub const CHART: &str = "http://schemas.openxmlformats.org/drawingml/2006/chart"; pub const DGM: &str = "http://schemas.openxmlformats.org/drawingml/2006/diagram"; pub const P: &str = "http://schemas.openxmlformats.org/presentationml/2006/main"; diff --git a/src/render/markdown/escape.rs b/src/render/markdown/escape.rs index 281c8b66..ace2b7fb 100644 --- a/src/render/markdown/escape.rs +++ b/src/render/markdown/escape.rs @@ -77,9 +77,7 @@ pub(crate) fn escape_text(text: &str, ctx: InlineContext, opts: EscapeOpts) -> S styled || (next_nonspace && !(prev_alnum && next_alnum) && paired(1)) } '~' => styled || (next_nonspace && paired(2)), - // A dollar pair delimits math for every renderer that supports it, so document - // text like `$5 and $10` is otherwise read as an equation. A lone `$` opens - // nothing and stays literal, which keeps the common currency case unescaped. + // A pair delimits math; a lone `$` opens nothing, so currency stays literal. '$' => paired(5), '[' => in_label || paired(4), '<' => next.is_some_and(|n| n.is_ascii_alphabetic() || matches!(n, '/' | '!' | '?')), diff --git a/src/render/markdown/inline.rs b/src/render/markdown/inline.rs index 1461e338..56ee9ea2 100644 --- a/src/render/markdown/inline.rs +++ b/src/render/markdown/inline.rs @@ -1,6 +1,6 @@ //! Inline run normalization and rendering. -use crate::model::{ImageSource, Inline, LinkTarget, Style, inlines_are_empty}; +use crate::model::{ImageSource, Inline, LinkTarget, Style, VertAlign, inlines_are_empty}; use crate::render::markdown::Ctx; use crate::render::markdown::escape::{ EscapeOpts, InlineContext, backtick_fence, escape_text, escape_url_as_text, format_url, @@ -14,6 +14,7 @@ pub(crate) enum Norm<'a> { Image { alt: &'a str, source: &'a ImageSource }, Anchor(&'a str), NoteRef(&'a str), + Math { latex: &'a str, display: bool }, LineBreak, } @@ -72,6 +73,7 @@ pub(crate) fn normalize<'a>(inlines: &'a [Inline], rc: &Ctx) -> Vec> { Inline::Anchor(id) if rc.anchors.html_id(id).is_none() => continue, Inline::Anchor(id) => out.push(Norm::Anchor(id)), Inline::NoteRef(id) => out.push(Norm::NoteRef(id)), + Inline::Math { latex, display } => out.push(Norm::Math { latex, display: *display }), Inline::LineBreak => out.push(Norm::LineBreak), } } @@ -90,7 +92,12 @@ fn render_inlines_mode(inlines: &[Inline], ctx: InlineContext, in_label: bool, r Norm::Text { text, style } => { let next_active = matches!( runs.get(idx + 1), - Some(Norm::Link { .. } | Norm::Image { .. } | Norm::NoteRef(_)) + Some( + Norm::Link { .. } + | Norm::Image { .. } + | Norm::NoteRef(_) + | Norm::Math { .. } + ) ) || matches!( runs.get(idx + 1), Some(Norm::Text { style, .. }) if *style != Style::PLAIN @@ -109,6 +116,14 @@ fn render_inlines_mode(inlines: &[Inline], ctx: InlineContext, in_label: bool, r let _ = write!(out, ""); } } + // The only payload that reaches the output unescaped: `escape_text` + // would destroy every command. The producer keeps the body newline-free. + Norm::Math { latex, display } => { + // Markdown math parsers ignore `\$` when scanning for the close, + // so a body holding a dollar needs the longer fence. + let fence = if *display || latex.contains('$') { "$$" } else { "$" }; + let _ = write!(out, "{fence}{latex}{fence}"); + } Norm::LineBreak => match ctx { InlineContext::Block => out.push_str("\\\n"), InlineContext::Heading => out.push(' '), @@ -204,6 +219,14 @@ fn render_text_run( out.push_str(lead); } if !core.is_empty() { + // Markdown has no super/subscript syntax; GFM inline HTML is the only + // way to say it, and dropping it changes a value (`10-3` for `10⁻³`). + let (raise_open, raise_close) = match style.vert_align { + VertAlign::Superscript => ("", ""), + VertAlign::Subscript => ("", ""), + VertAlign::Baseline => ("", ""), + }; + out.push_str(raise_open); if style.code { push_code_span(core, out); } else { @@ -222,10 +245,13 @@ fn render_text_run( out.push_str(&escape_text( core, ctx, - EscapeOpts { styled: true, in_label, ..Default::default() }, + // `styled` means "inside emphasis delimiters", which a raised + // but otherwise plain run has none of. + EscapeOpts { styled: !open.is_empty(), in_label, ..Default::default() }, )); out.push_str(&close); } + out.push_str(raise_close); } if !trail.is_empty() { out.push_str(trail); diff --git a/src/render/markdown/tests.rs b/src/render/markdown/tests.rs index 3fbced98..96ecafec 100644 --- a/src/render/markdown/tests.rs +++ b/src/render/markdown/tests.rs @@ -20,8 +20,8 @@ fn table_from(rows: Vec>, header_rows: usize) -> Block { Block::Table(Table::from_rows(rows, header_rows, TableKind::Data)) } -const BOLD: Style = Style { bold: true, italic: false, strike: false, code: false }; -const ITALIC: Style = Style { bold: false, italic: true, strike: false, code: false }; +const BOLD: Style = Style { bold: true, ..Style::PLAIN }; +const ITALIC: Style = Style { italic: true, ..Style::PLAIN }; #[test] fn heading_and_paragraph() { @@ -47,15 +47,29 @@ fn lone_syntax_chars_left_alone() { #[test] fn paired_dollars_escaped_lone_dollar_kept() { - // A pair delimits math for renderers that support it, so document text that - // happens to hold two dollars would be read as an equation. let md = doc(vec![Block::Paragraph(vec![Inline::plain("costs $100 and $80 total")])]); assert_eq!(md, "costs \\$100 and $80 total\n"); - // One dollar opens nothing; the common currency case stays literal. let md = doc(vec![Block::Paragraph(vec![Inline::plain("costs $100 total")])]); assert_eq!(md, "costs $100 total\n"); } +#[test] +fn math_renders_between_dollars_without_escaping() { + let math = |latex: &str, display| Inline::Math { latex: latex.into(), display }; + let md = doc(vec![Block::Paragraph(vec![Inline::plain("see "), math(r"\frac{a}{b}", false)])]); + assert_eq!(md, "see $\\frac{a}{b}$\n"); + let md = doc(vec![Block::Paragraph(vec![math(r"\sum_{i=1}^{n}", true)])]); + assert_eq!(md, "$$\\sum_{i=1}^{n}$$\n"); +} + +#[test] +fn math_holding_a_dollar_takes_the_longer_fence() { + // Markdown math parsers end the span at a backslash-escaped dollar. + let md = + doc(vec![Block::Paragraph(vec![Inline::Math { latex: r"a\$b".into(), display: false }])]); + assert_eq!(md, "$$a\\$b$$\n"); +} + #[test] fn intraword_underscores_unescaped() { let md = doc(vec![Block::Paragraph(vec![Inline::plain("snake_case_name vs _lead_")])]); @@ -120,7 +134,7 @@ fn adjacent_same_style_runs_merged() { fn bold_italic_combo() { let md = doc(vec![Block::Paragraph(vec![styled( "both", - Style { bold: true, italic: true, strike: false, code: false }, + Style { bold: true, italic: true, ..Style::PLAIN }, )])]); assert_eq!(md, "***both***\n"); } diff --git a/src/shared/delta.rs b/src/shared/delta.rs index e26d16d0..e293145c 100644 --- a/src/shared/delta.rs +++ b/src/shared/delta.rs @@ -2,7 +2,7 @@ //! either explicitly on, explicitly off, or unset (inherit); only after the //! full cascade is a delta collapsed into the model's resolved [`Style`]. -use crate::model::{Inline, Style}; +use crate::model::{Inline, Style, VertAlign}; #[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] pub struct StyleDelta { @@ -10,6 +10,7 @@ pub struct StyleDelta { pub italic: Option, pub strike: Option, pub code: Option, + pub vert_align: Option, } impl StyleDelta { @@ -21,6 +22,7 @@ impl StyleDelta { italic: child.italic.or(self.italic), strike: child.strike.or(self.strike), code: child.code.or(self.code), + vert_align: child.vert_align.or(self.vert_align), } } @@ -30,6 +32,7 @@ impl StyleDelta { italic: self.italic.unwrap_or(base.italic), strike: self.strike.unwrap_or(base.strike), code: self.code.unwrap_or(base.code), + vert_align: self.vert_align.unwrap_or(base.vert_align), } } diff --git a/src/shared/html.rs b/src/shared/html.rs index a55d8078..f8e137ca 100644 --- a/src/shared/html.rs +++ b/src/shared/html.rs @@ -243,7 +243,7 @@ fn at_space_boundary(inlines: &[Inline], start: bool) -> bool { } return at_space_boundary(content, false); } - Inline::Image { .. } | Inline::NoteRef(_) => return false, + Inline::Image { .. } | Inline::NoteRef(_) | Inline::Math { .. } => return false, } } start diff --git a/tests/fixtures/docx/handmade-math.docx b/tests/fixtures/docx/handmade-math.docx new file mode 100644 index 0000000000000000000000000000000000000000..986286c27ee484c2494559656adceb28b967db5f GIT binary patch literal 1502 zcmWIWW@Zs#U|`??V#S!Qpz5@rK$Z~`1A{P-j&{z^D@n~Oi4UnPNG*=ltH{k+J8`$) zVFQu2@_C0oMXB=fa4cLVyQAkofZT%N9RL zSA3th`0IPy^l8V|ZI+t;=gPFAD;`JIzl+_gA(s>J*Rti*iwXbgkKNh*;p)rN=JEk3 zA!Wbz->>z+5Yk~}VBiMQ@kOaQ#rk?6viH0--ys7Yh6{6pUG8N@v9$y^i?>`6oyzRe zIXlcm{^-NYwKFqOPy^J^XrlKvojw*&a4nCU%$sCdRd90 zYtP}#20`JFwAP5DKiSZ5ejUwMH^0~yDen{c8LdllykGE!M zFPx1t$ z->JzPaz-+IkND5JtKirXMlIPzH4;+C*uLy#e>Cym-j#RG^qKh{syh*-oSo1-lh@FW zNB3dHrK`t1!~O-FdVO?;M8<@!l&$L85^DdX{Xa`yh_e5=Bu8U&{i7}6rUh5$Z8;h( z1S6x zd1{^dR@Sb{WJ}B3{;#&y%DbnUKk@c|o zZntLNvU3Uzl*~_SuX(=a^lImC{;SWWa_*mS=HmIwE=$!tX3bsnf;DhuuiNuP0fpO& z7k}DF{MB8U>6Le}?CcX>)TAKn^(0^hFbP}-Vt!asC@!hYNi7B?gxEe%G8b@JFSJu# zT0@}g(FKJpE{lfLe7i||r7L+Aw;!MA#Q1dfMQPvSI`;TI2B%Iexw>ynNMp*)-YeN( zcf^;ghk6)2=}C85D<)E!H+$!_EfrIgUaNdvb9KU{5ate90}0Jar&F=V4QEU-{AN&G z5}eYlRB2mb5pq?qNz!v|bKhH5>(5_2!+N&wc<6S)?1S{-`7HOt-?9gIGct)V<1W*H zJ^_OUU@FI2!l4_0p4B1R85kNEwSYz-Wqfpv=y?gDkqKD7V&p7z?dai;(9X*O)sBdb V0B=?{kOCGU{05{i0hKT?001}XTu1-_ literal 0 HcmV?d00001 diff --git a/tests/gen_fixtures.py b/tests/gen_fixtures.py index 21552bf7..0d87b584 100644 --- a/tests/gen_fixtures.py +++ b/tests/gen_fixtures.py @@ -173,6 +173,7 @@ def write_zip(path, entries, mimetype_first=None): W = 'xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"' R = 'xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"' +M = 'xmlns:m="http://schemas.openxmlformats.org/officeDocument/2006/math"' CONTENT_TYPES_BASE = """ @@ -1034,6 +1035,65 @@ def defaults_odf(): # --------------------------------------------------------------------------- # R15: DOCX gridBefore/gridAfter, legacy hMerge, ST_OnOff tblHeader +# --------------------------------------------------------------------------- +# Office Math: the OMML constructs Word's equation editor emits, plus the +# w:vertAlign runs that carry super/subscript outside an equation. Property +# elements are omitted wherever the spec makes them optional, because that is +# what Word writes when every property takes its default and it is the shape a +# converter is most likely to mishandle. + +def math_docx(): + def r(text): + return f'{text}' + + def arg(name, text): + return f'{r(text)}' + + inline = ( + # No m:fPr: the fraction takes its default bar form. + '' + r("a") + '' + r("b") + '' + + r("+") + # A script whose base is itself a script must brace, or LaTeX reads a + # double superscript. + + '' + r("y") + '' + arg("sup", "2") + + '' + arg("sup", "3") + '' + # No m:dPr: the delimiter takes its default parentheses. + + '' + r("x,y") + '' + + '' + r("z") + '' + ) + # No m:naryPr: the n-ary operator defaults to the integral, not the sum. + display = ( + '' + '' + + arg("sub", "i=1") + arg("sup", "n") + + '' + r("x") + '' + arg("sub", "i") + '' + '' + ) + + def raised(text, val): + return (f'' + f'{text}') + + document = f""" + +Inline {inline} + done. +{display} +Concentration 10{raised("-3", "superscript")} + mol/L of H{raised("2", "subscript")} +O costs $100 and $80. +""" + styles = (f'' + f'' + '') + write_zip(OUT / "docx" / "handmade-math.docx", [ + ("[Content_Types].xml", CONTENT_TYPES_BASE.format(extra="")), + ("_rels/.rels", ROOT_RELS), + ("word/document.xml", document), + ("word/styles.xml", styles), + ]) + + def tables_docx(): def tc(text, extra=""): return (f'{extra}' @@ -1915,6 +1975,7 @@ def main(): multimaster_ppt() sparsenotes_ppt() tables_docx() + math_docx() outline_docx() blockstyle_docx() blockstyle_odt() diff --git a/tests/snapshots/snapshots__docx__handmade-math.docx.snap b/tests/snapshots/snapshots__docx__handmade-math.docx.snap new file mode 100644 index 00000000..366d0c93 --- /dev/null +++ b/tests/snapshots/snapshots__docx__handmade-math.docx.snap @@ -0,0 +1,9 @@ +--- +source: tests/snapshots.rs +expression: output +--- +Inline $\frac{a}{b}+{y^{2}}^{3}\left(x,y\right)\sqrt{z}$ done. + +$$\sum_{i=1}^{n}x_{i}$$ + +Concentration 10-3 mol/L of H2O costs \$100 and $80. diff --git a/wasm/src/document.rs b/wasm/src/document.rs index 8b5d971d..0b6cbaa0 100644 --- a/wasm/src/document.rs +++ b/wasm/src/document.rs @@ -108,6 +108,7 @@ pub enum InlineKind { Anchor, NoteRef, LineBreak, + Math, } #[derive(Serialize)] @@ -138,6 +139,12 @@ pub struct Inline { /// noteRef: the id of the note in `Document.notes`. #[serde(skip_serializing_if = "Option::is_none")] pub note_id: Option, + /// math: the expression as LaTeX, without delimiters. + #[serde(skip_serializing_if = "Option::is_none")] + pub latex: Option, + /// math: true for an equation that stands on its own line. + #[serde(skip_serializing_if = "Option::is_none")] + pub display: Option, } impl Inline { @@ -152,6 +159,8 @@ impl Inline { source: None, anchor: None, note_id: None, + latex: None, + display: None, } } } @@ -180,6 +189,11 @@ impl From for Inline { model::Inline::NoteRef(id) => { Inline { note_id: Some(id), ..Inline::of(InlineKind::NoteRef) } } + model::Inline::Math { latex, display } => Inline { + latex: Some(latex), + display: Some(display), + ..Inline::of(InlineKind::Math) + }, model::Inline::LineBreak => Inline::of(InlineKind::LineBreak), } } @@ -192,11 +206,19 @@ pub struct Style { pub italic: bool, pub strike: bool, pub code: bool, + /// `baseline`, `superscript` or `subscript`. + pub vert_align: String, } impl From for Style { fn from(style: model::Style) -> Self { - Style { bold: style.bold, italic: style.italic, strike: style.strike, code: style.code } + Style { + bold: style.bold, + italic: style.italic, + strike: style.strike, + code: style.code, + vert_align: style.vert_align.as_str().into(), + } } } diff --git a/wasm/src/typescript.rs b/wasm/src/typescript.rs index 9a5cba68..91578375 100644 --- a/wasm/src/typescript.rs +++ b/wasm/src/typescript.rs @@ -72,6 +72,7 @@ export type InlineKind = | 'anchor' | 'noteRef' | 'lineBreak' + | 'math' export interface Inline { kind: InlineKind @@ -91,6 +92,10 @@ export interface Inline { anchor?: string /** noteRef: the id of the note in `Document.notes`. */ noteId?: string + /** math: the expression as LaTeX, without delimiters. */ + latex?: string + /** math: true for an equation that stands on its own line. */ + display?: boolean } /** Fully resolved character style. */ @@ -99,6 +104,8 @@ export interface Style { italic: boolean strike: boolean code: boolean + /** `baseline`, `superscript` or `subscript`. */ + vertAlign: string } export type LinkTargetKind = From e0f0c7d6041c68eecceefd1826d5d35dbea399d1 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 14:27:10 +0300 Subject: [PATCH 03/13] fix(math): four defects the first pass left, and the EPUB path Four bugs, all silent, found by adversarial review of the previous commit and each reproduced before it was fixed. A `w:ins` wrapping exactly one element dropped it. `parse_seq` collapses a one-element sequence to that element, so the `if let Node::Seq(..)` guard on the revision-mark branch never matched and control fell through to `continue`. Tracked changes around a single fraction lost the whole equation. An unmapped n-ary operator became an integral and an unmapped accent became a hat. Well-formed, KaTeX-valid and a different expression than the document's. Both now pass the glyph through, which says what the document said; an unmapped group character draws nothing rather than an underbrace. `m:begChr`, `m:endChr` and `m:sepChr` are author-supplied and reached the LaTeX body without escaping, so a delimiter of `$` put a bare dollar in a payload whose contract says there is none, and one of `\` could open a command. They take the same escaping as any other text now. Degradation was silent everywhere: the crate routes recovery through `log` and this module logged nothing. EPUB and any HTML input carried the same losses and one more. `` and `` were flattened, so the same content read `10-3` from DOCX and `10-3` from EPUB. MathML was worse than dropped: `` is not a container tag, so the walker descended into it and emitted the presentation tree as symbol soup *and* the `` beside it as visible text, backslashes doubled by the escaper. The annotation is exact where re-deriving LaTeX from the presentation tree is not, so it is taken when present; without one the characters are kept and the shape is lost. --- src/formats/docx/omml.rs | 106 ++++++++++++++---- src/shared/html.rs | 34 +++++- tests/fixtures/epub/handmade-math.epub | Bin 0 -> 1280 bytes tests/gen_fixtures.py | 40 +++++++ .../snapshots__epub__handmade-math.epub.snap | 15 +++ 5 files changed, 174 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/epub/handmade-math.epub create mode 100644 tests/snapshots/snapshots__epub__handmade-math.epub.snap diff --git a/src/formats/docx/omml.rs b/src/formats/docx/omml.rs index 787ed7fa..e39d2182 100644 --- a/src/formats/docx/omml.rs +++ b/src/formats/docx/omml.rs @@ -152,16 +152,18 @@ fn boxed_arg(parent: &Element, name: &str, depth: usize) -> Box { /// Every child of `elem`, flattened into one sequence. fn parse_seq(elem: &Element, depth: usize) -> Node { if depth >= MAX_DEPTH { + log::warn!("OMML nesting deeper than {MAX_DEPTH} levels; keeping its text only"); return Node::Run(elem.text()); } let mut parts: Vec = Vec::new(); for child in elem.child_elems() { if child.ns.as_deref() != Some(ns::M) { // Revision marks and bookmarks wrap runs that are part of the maths. - if child.ns.as_deref() == Some(ns::W) - && let Node::Seq(inner) = parse_seq(child, depth + 1) - { - parts.extend(inner); + if child.ns.as_deref() == Some(ns::W) { + match parse_seq(child, depth + 1) { + Node::Seq(inner) => parts.extend(inner), + node => parts.push(node), + } } continue; } @@ -439,7 +441,7 @@ fn emit(node: &Node, out: &mut String, _in_group: bool) { } } Node::Nary { op, sub, sup, base, limits } => { - out.push_str(nary_command(*op)); + out.push_str(&nary_command(*op)); if *limits { out.push_str("\\limits"); } @@ -465,20 +467,29 @@ fn emit(node: &Node, out: &mut String, _in_group: bool) { out.push_str("\\right"); out.push_str(&delim_glyph(*close)); } - Node::Accent { chr, base } => { - out.push_str(accent_command(*chr)); - emit_group(base, out); - } + Node::Accent { chr, base } => match accent_command(*chr) { + Some(cmd) => { + out.push_str(cmd); + emit_group(base, out); + } + None => { + out.push_str("\\overset"); + out.push('{'); + out.push_str(&escaped(*chr)); + out.push('}'); + emit_group(base, out); + } + }, Node::Bar { top, base } => { out.push_str(if *top { "\\overline" } else { "\\underline" }); emit_group(base, out); } - Node::Group { chr, top, base } => match chr.map(group_command) { + // An explicitly blank, or unmapped, group character draws nothing. + Node::Group { chr, top, base } => match chr.and_then(group_command) { Some(cmd) => { out.push_str(cmd); emit_group(base, out); } - // An explicitly blank group character draws nothing. None => { let _ = top; emit(base, out, false); @@ -565,7 +576,7 @@ const KATEX_FUNCTIONS: &[&str] = &[ "sec", "sin", "sinh", "sup", "tan", "tanh", ]; -fn nary_command(chr: char) -> &'static str { +fn nary_command(chr: char) -> String { match chr { '∑' => "\\sum", '∏' => "\\prod", @@ -585,15 +596,20 @@ fn nary_command(chr: char) -> &'static str { '⨂' => "\\bigotimes", '⨄' => "\\biguplus", '⨆' => "\\bigsqcup", - _ => "\\int", + // Passing the glyph through says what the document said; guessing a + // command would emit a different operator, well-formed and wrong. + _ => return escaped(chr), } + .into() } -/// OMML gives a combining codepoint; KaTeX wants the accent command. -fn accent_command(chr: char) -> &'static str { +/// OMML gives a combining codepoint; KaTeX wants the accent command. An +/// unmapped mark is stacked over the base rather than replaced by a guess. +fn accent_command(chr: char) -> Option<&'static str> { match chr { '\u{0300}' => "\\grave", '\u{0301}' => "\\acute", + '\u{0302}' | '^' => "\\widehat", '\u{0303}' | '~' => "\\widetilde", '\u{0304}' => "\\bar", '\u{0305}' => "\\overline", @@ -607,11 +623,12 @@ fn accent_command(chr: char) -> &'static str { '\u{20D7}' | '→' => "\\vec", '\u{20DB}' => "\\dddot", '\u{20E1}' => "\\overleftrightarrow", - _ => "\\widehat", + _ => return None, } + .into() } -fn group_command(chr: char) -> &'static str { +fn group_command(chr: char) -> Option<&'static str> { match chr { '\u{23DE}' | '\u{FE37}' => "\\overbrace", '\u{23B4}' => "\\overbracket", @@ -620,8 +637,10 @@ fn group_command(chr: char) -> &'static str { '\u{23DD}' => "\\undergroup", '←' => "\\overleftarrow", '→' => "\\overrightarrow", - _ => "\\underbrace", + '\u{23DF}' | '\u{FE38}' => "\\underbrace", + _ => return None, } + .into() } /// A delimiter glyph in `\left`/`\right` position. An absent delimiter is a @@ -639,17 +658,25 @@ fn delim_glyph(chr: Option) -> String { Some('⌋') => "\\rfloor".into(), Some('⟨') => "\\langle".into(), Some('⟩') => "\\rangle".into(), - Some(c) => c.to_string(), + Some(c) => escaped(c), } } +/// One author-supplied character, LaTeX-escaped. Delimiters and operators come +/// from the document and reach the body outside `push_text`. +fn escaped(chr: char) -> String { + let mut out = String::new(); + push_text(&chr.to_string(), &mut out); + out +} + /// A separator between delimiter parts. A bare `|` would split a Markdown /// table row, so it is always spelled as a command. fn delim_sep(chr: char) -> String { match chr { '|' => "\\mid ".into(), '‖' => "\\Vert ".into(), - c => c.to_string(), + c => escaped(c), } } @@ -736,6 +763,45 @@ mod tests { assert!(!latex(&xml).contains('|')); } + #[test] + fn a_revision_wrapper_with_one_child_keeps_it() { + // `` around a single element is the ordinary tracked-changes + // shape, and the content is the whole equation. + let xml = format!( + "{}{}", + run("a"), + run("b") + ); + assert_eq!(latex(&xml), r"\frac{a}{b}"); + } + + #[test] + fn author_supplied_delimiters_are_escaped() { + // These reach the body outside `push_text`, so they need escaping of + // their own or the payload's no-bare-dollar contract is a lie. + let xml = + format!(r#"{}"#, run("v")); + assert!(!latex(&xml).contains(r"left$"), "{}", latex(&xml)); + let xml = format!( + r#"{}{}"#, + run("x"), + run("y") + ); + assert!(latex(&xml).contains(r"\$"), "{}", latex(&xml)); + } + + #[test] + fn an_unmapped_operator_is_passed_through_not_guessed() { + // Emitting `\int` for an unknown n-ary glyph is well-formed and wrong. + let xml = format!( + r#"{}"#, + run("f") + ); + let out = latex(&xml); + assert!(out.contains('⨌'), "{out}"); + assert!(!out.contains(r"\int"), "{out}"); + } + #[test] fn an_unmodelled_element_keeps_its_content() { let xml = format!("{}", run("keep")); diff --git a/src/shared/html.rs b/src/shared/html.rs index f8e137ca..8ef05c28 100644 --- a/src/shared/html.rs +++ b/src/shared/html.rs @@ -9,7 +9,7 @@ use crate::error::ConvertError; use crate::model::{ AnchorId, Block, Cell, GridBuilder, ImageSource, Inline, LinkTarget, List, ListItem, - MarkerKind, TableKind, inlines_are_empty, inlines_to_plain_text, + MarkerKind, TableKind, VertAlign, inlines_are_empty, inlines_to_plain_text, }; use crate::package::xml::{Element, Node}; use crate::shared::delta::{StyleDelta, rebase_emphasis}; @@ -442,15 +442,32 @@ impl Builder<'_> { } } "script" | "style" | "head" | "template" | "noscript" => {} + // MathML metadata, not content: an annotation holds a second + // encoding of the same expression. + "annotation" | "annotation-xml" => {} _ => self.walk_inline(elem, delta)?, } Ok(()) } + /// MathML. A `<semantics>` wrapper usually carries the source LaTeX in an + /// `annotation`, which is exact where re-deriving it from the presentation + /// tree is not; without one the characters are kept but the shape is lost. + fn walk_math(&mut self, elem: &Element, delta: StyleDelta) -> Result<(), ConvertError> { + let display = elem.attr_any("display") == Some("block"); + if let Some(latex) = tex_annotation(elem) { + self.inlines.push(Inline::Math { latex, display }); + return Ok(()); + } + log::debug!("MathML without a TeX annotation; keeping its characters only"); + self.walk_children(elem, delta) + } + fn walk_inline(&mut self, elem: &Element, delta: StyleDelta) -> Result<(), ConvertError> { self.push_anchor(elem); match elem.local.as_str() { "br" => self.inlines.push(Inline::LineBreak), + "math" => self.walk_math(elem, delta)?, "img" | "image" => { let alt = clean_text(elem.attr_any("alt").unwrap_or("")); let src = elem.attr_any("src").or_else(|| elem.attr_any("href")).unwrap_or(""); @@ -682,12 +699,27 @@ impl Builder<'_> { } } +/// The LaTeX an `annotation` carries, if the MathML supplies one. +fn tex_annotation(math: &Element) -> Option { + let text = math + .descendant_elems() + .filter(|e| e.local == "annotation") + .find(|e| { + matches!(e.attr_any("encoding"), Some("application/x-tex" | "application/x-latex")) + })? + .text(); + let text = text.trim(); + if text.is_empty() { None } else { Some(text.replace(['\n', '\r'], " ")) } +} + fn merge_inline_tag(elem: &Element, mut delta: StyleDelta) -> StyleDelta { match elem.local.as_str() { "b" | "strong" => delta.bold = Some(true), "i" | "em" | "cite" | "dfn" | "var" => delta.italic = Some(true), "s" | "del" | "strike" => delta.strike = Some(true), "code" | "kbd" | "samp" | "tt" => delta.code = Some(true), + "sup" => delta.vert_align = Some(VertAlign::Superscript), + "sub" => delta.vert_align = Some(VertAlign::Subscript), _ => {} } delta diff --git a/tests/fixtures/epub/handmade-math.epub b/tests/fixtures/epub/handmade-math.epub new file mode 100644 index 0000000000000000000000000000000000000000..a521dbc00d42849eca96d1f5298a5d9e7e1adda5 GIT binary patch literal 1280 zcmWIWW@Zs#fB?mq{KVBdARY*F0C8?+ZfZ$oL26<_K~83JVo7Fxo_=aUX_9tTW|Lxio zp1IhZ)0fq8J=ZJtBPG$zC-X9bF77w4TJa#4@AEyM&6QQ*b_G4tJP)plbLvdlvDrjp z$DHq1rOa|KN-SHV`|!v0vnH;rmpvwxZV8<9O(NB35_8b;v`-$Mb=gJJ9L)O8tlZ9* zY<+yw{5h+e{LER~?|!Qgl2`g{+AinSKlf^A6>G=>=Ynf5rp)I!mfay~TyFb7$-gl} z`r~Ku-Wk(xo1^;f*8G1WDvS&aZ-7`3=sSN`r+{Fv*HZIJ^zsYRawqQ0J8Zzy_T8p+ z!PX6_+nu=%mo@O+-L;ip!DEZBOL)pk&-?qf%w=X><+z}t#@=&ld$;Ku=|?^HO84k# zpF5zr{$j7}|6euwaX%N{-|Dz(u}ov&w%OKpOGTs<&pj{hwBlLXzqn%Es=Xf)AMoEf z-E^AIUy|#}10g<+jgKWw)mOBtZ1pKxY!-F*prZR`r>#N>2ZCPeyxJt@Ju~bq^Ra;V zO3Phpldpzc5B^hpabuDF)<^s)&1ybuzxZdWeS7w|@a)YR?<*eWX;F^+dA)b;?+&}e z7MV7Qk9|h){UoN{Wy`b9E#qL0433RkmcpAZW^zjDFUyM^JO>v{>MT>c_Th4>C+9`> z4$elgyp`IYI_Dk9+7uTtSfiFzW;SUYLGVlzdtnw7@TLB7#R2w zL7HKxSCLVYn=>`&wBKO^fjyr^r>$R6=&YanWJlAg4rgt>0%woD$7X_cnr14YSL=`W zi`{Zo-I4O>=FPcxuRi*z!209S+}y2NiSv3DA9TIk6?$7}-#(kZWd(kD1&{Zf@t&%i z_;`+{z^mVOuhnP1+om!9Y+43~)C`x_Jxa3|nU?zgSrcv`@@$LRjjetKLJ>We?A619kw<{i(GH%dWGF_m5n)j-h(`&9*lwA%K%eX1j6?o*! zDnI>_AXZyn)4xvV!W?@Ab-Q(>-1_(@>}l<>wSBXic?av^-?18>XTQl__&IEmXNEw| z7J*&cCEe8cJ~~+Q`5e~`*Zi6!k!gQ+^UByhk>Di(F(}Kp8wI!K+jhQ pGbDg^W8^Y)?dWL;p* + +Math +

Math Chapter

+

Annotated: +ab +\\frac{a}{b} + x^2 + follows.

+

Bare presentation: x2 follows.

+

Block: E +E = mc^2 +

+

Concentration 10-3 mol/L of H2O.

+""" + opf = """ + + +urn:uuid:00000000-0000-0000-0000-0000000ma7h +Math Booken + + + +""" + container = """ + + +""" + write_zip(OUT / "epub" / "handmade-math.epub", [ + ("META-INF/container.xml", container), + ("OEBPS/content.opf", opf), + ("OEBPS/ch1.xhtml", ch1), + ], mimetype_first="application/epub+zip") + + def features_epub(): css = "p.hidden { display: none; }\n.crossed { text-decoration: line-through; }\n" ch1 = """ @@ -1987,6 +2026,7 @@ def main(): defaults_odf() merged_xlsx() features_epub() + math_epub() bin_rtf() csvs() malformed() diff --git a/tests/snapshots/snapshots__epub__handmade-math.epub.snap b/tests/snapshots/snapshots__epub__handmade-math.epub.snap new file mode 100644 index 00000000..97b5e5fd --- /dev/null +++ b/tests/snapshots/snapshots__epub__handmade-math.epub.snap @@ -0,0 +1,15 @@ +--- +source: tests/snapshots.rs +expression: output +--- +# Math Book + +# Math Chapter + +Annotated: $\frac{a}{b} + x^2$ follows. + +Bare presentation: x2 follows. + +Block: $$E = mc^2$$ + +Concentration 10-3 mol/L of H2O. From 191b99c846061822800f2587ed3ad85090792d91 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 16:04:59 +0300 Subject: [PATCH 04/13] feat(xml): resolve every HTML5 named character reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit resolve_entity knew about forty names. Everything else fell through to the literal `&name;`, which the Markdown writer then escaped again, so `β` reached the reader as `&beta;` — the reference destroyed rather than merely unresolved. MathML depends on these names more than prose does: `α`, `∑`, `⁢` are how most producers spell their operators. The table is generated from the WHATWG entities.json, restricted to the names that carry a trailing semicolon, which is every name XML admits. Sorted for binary search; the format, separator and combining characters are written as escapes so nothing invisible sits in the source. --- src/package/entities.rs | 2163 +++++++++++++++++++++++++++++++++++++++ src/package/mod.rs | 1 + src/package/xml.rs | 49 +- 3 files changed, 2165 insertions(+), 48 deletions(-) create mode 100644 src/package/entities.rs diff --git a/src/package/entities.rs b/src/package/entities.rs new file mode 100644 index 00000000..6af8094e --- /dev/null +++ b/src/package/entities.rs @@ -0,0 +1,2163 @@ +//! HTML5 named character references. +//! +//! Generated from : the names that +//! require a trailing semicolon, which is every name XML admits. MathML leans +//! on these heavily (`α`, `∑`, `⁢`), and an unresolved +//! reference reaches the writer as literal text. + +/// Name, without `&` and `;`, to its replacement. Sorted, for binary search. +static NAMED: &[(&str, &str)] = &[ + ("AElig", "Æ"), + ("AMP", "&"), + ("Aacute", "Á"), + ("Abreve", "Ă"), + ("Acirc", "Â"), + ("Acy", "А"), + ("Afr", "𝔄"), + ("Agrave", "À"), + ("Alpha", "Α"), + ("Amacr", "Ā"), + ("And", "⩓"), + ("Aogon", "Ą"), + ("Aopf", "𝔸"), + ("ApplyFunction", "\u{2061}"), + ("Aring", "Å"), + ("Ascr", "𝒜"), + ("Assign", "≔"), + ("Atilde", "Ã"), + ("Auml", "Ä"), + ("Backslash", "∖"), + ("Barv", "⫧"), + ("Barwed", "⌆"), + ("Bcy", "Б"), + ("Because", "∵"), + ("Bernoullis", "ℬ"), + ("Beta", "Β"), + ("Bfr", "𝔅"), + ("Bopf", "𝔹"), + ("Breve", "˘"), + ("Bscr", "ℬ"), + ("Bumpeq", "≎"), + ("CHcy", "Ч"), + ("COPY", "©"), + ("Cacute", "Ć"), + ("Cap", "⋒"), + ("CapitalDifferentialD", "ⅅ"), + ("Cayleys", "ℭ"), + ("Ccaron", "Č"), + ("Ccedil", "Ç"), + ("Ccirc", "Ĉ"), + ("Cconint", "∰"), + ("Cdot", "Ċ"), + ("Cedilla", "¸"), + ("CenterDot", "·"), + ("Cfr", "ℭ"), + ("Chi", "Χ"), + ("CircleDot", "⊙"), + ("CircleMinus", "⊖"), + ("CirclePlus", "⊕"), + ("CircleTimes", "⊗"), + ("ClockwiseContourIntegral", "∲"), + ("CloseCurlyDoubleQuote", "”"), + ("CloseCurlyQuote", "’"), + ("Colon", "∷"), + ("Colone", "⩴"), + ("Congruent", "≡"), + ("Conint", "∯"), + ("ContourIntegral", "∮"), + ("Copf", "ℂ"), + ("Coproduct", "∐"), + ("CounterClockwiseContourIntegral", "∳"), + ("Cross", "⨯"), + ("Cscr", "𝒞"), + ("Cup", "⋓"), + ("CupCap", "≍"), + ("DD", "ⅅ"), + ("DDotrahd", "⤑"), + ("DJcy", "Ђ"), + ("DScy", "Ѕ"), + ("DZcy", "Џ"), + ("Dagger", "‡"), + ("Darr", "↡"), + ("Dashv", "⫤"), + ("Dcaron", "Ď"), + ("Dcy", "Д"), + ("Del", "∇"), + ("Delta", "Δ"), + ("Dfr", "𝔇"), + ("DiacriticalAcute", "´"), + ("DiacriticalDot", "˙"), + ("DiacriticalDoubleAcute", "˝"), + ("DiacriticalGrave", "`"), + ("DiacriticalTilde", "˜"), + ("Diamond", "⋄"), + ("DifferentialD", "ⅆ"), + ("Dopf", "𝔻"), + ("Dot", "¨"), + ("DotDot", "\u{20dc}"), + ("DotEqual", "≐"), + ("DoubleContourIntegral", "∯"), + ("DoubleDot", "¨"), + ("DoubleDownArrow", "⇓"), + ("DoubleLeftArrow", "⇐"), + ("DoubleLeftRightArrow", "⇔"), + ("DoubleLeftTee", "⫤"), + ("DoubleLongLeftArrow", "⟸"), + ("DoubleLongLeftRightArrow", "⟺"), + ("DoubleLongRightArrow", "⟹"), + ("DoubleRightArrow", "⇒"), + ("DoubleRightTee", "⊨"), + ("DoubleUpArrow", "⇑"), + ("DoubleUpDownArrow", "⇕"), + ("DoubleVerticalBar", "∥"), + ("DownArrow", "↓"), + ("DownArrowBar", "⤓"), + ("DownArrowUpArrow", "⇵"), + ("DownBreve", "\u{311}"), + ("DownLeftRightVector", "⥐"), + ("DownLeftTeeVector", "⥞"), + ("DownLeftVector", "↽"), + ("DownLeftVectorBar", "⥖"), + ("DownRightTeeVector", "⥟"), + ("DownRightVector", "⇁"), + ("DownRightVectorBar", "⥗"), + ("DownTee", "⊤"), + ("DownTeeArrow", "↧"), + ("Downarrow", "⇓"), + ("Dscr", "𝒟"), + ("Dstrok", "Đ"), + ("ENG", "Ŋ"), + ("ETH", "Ð"), + ("Eacute", "É"), + ("Ecaron", "Ě"), + ("Ecirc", "Ê"), + ("Ecy", "Э"), + ("Edot", "Ė"), + ("Efr", "𝔈"), + ("Egrave", "È"), + ("Element", "∈"), + ("Emacr", "Ē"), + ("EmptySmallSquare", "◻"), + ("EmptyVerySmallSquare", "▫"), + ("Eogon", "Ę"), + ("Eopf", "𝔼"), + ("Epsilon", "Ε"), + ("Equal", "⩵"), + ("EqualTilde", "≂"), + ("Equilibrium", "⇌"), + ("Escr", "ℰ"), + ("Esim", "⩳"), + ("Eta", "Η"), + ("Euml", "Ë"), + ("Exists", "∃"), + ("ExponentialE", "ⅇ"), + ("Fcy", "Ф"), + ("Ffr", "𝔉"), + ("FilledSmallSquare", "◼"), + ("FilledVerySmallSquare", "▪"), + ("Fopf", "𝔽"), + ("ForAll", "∀"), + ("Fouriertrf", "ℱ"), + ("Fscr", "ℱ"), + ("GJcy", "Ѓ"), + ("GT", ">"), + ("Gamma", "Γ"), + ("Gammad", "Ϝ"), + ("Gbreve", "Ğ"), + ("Gcedil", "Ģ"), + ("Gcirc", "Ĝ"), + ("Gcy", "Г"), + ("Gdot", "Ġ"), + ("Gfr", "𝔊"), + ("Gg", "⋙"), + ("Gopf", "𝔾"), + ("GreaterEqual", "≥"), + ("GreaterEqualLess", "⋛"), + ("GreaterFullEqual", "≧"), + ("GreaterGreater", "⪢"), + ("GreaterLess", "≷"), + ("GreaterSlantEqual", "⩾"), + ("GreaterTilde", "≳"), + ("Gscr", "𝒢"), + ("Gt", "≫"), + ("HARDcy", "Ъ"), + ("Hacek", "ˇ"), + ("Hat", "^"), + ("Hcirc", "Ĥ"), + ("Hfr", "ℌ"), + ("HilbertSpace", "ℋ"), + ("Hopf", "ℍ"), + ("HorizontalLine", "─"), + ("Hscr", "ℋ"), + ("Hstrok", "Ħ"), + ("HumpDownHump", "≎"), + ("HumpEqual", "≏"), + ("IEcy", "Е"), + ("IJlig", "IJ"), + ("IOcy", "Ё"), + ("Iacute", "Í"), + ("Icirc", "Î"), + ("Icy", "И"), + ("Idot", "İ"), + ("Ifr", "ℑ"), + ("Igrave", "Ì"), + ("Im", "ℑ"), + ("Imacr", "Ī"), + ("ImaginaryI", "ⅈ"), + ("Implies", "⇒"), + ("Int", "∬"), + ("Integral", "∫"), + ("Intersection", "⋂"), + ("InvisibleComma", "\u{2063}"), + ("InvisibleTimes", "\u{2062}"), + ("Iogon", "Į"), + ("Iopf", "𝕀"), + ("Iota", "Ι"), + ("Iscr", "ℐ"), + ("Itilde", "Ĩ"), + ("Iukcy", "І"), + ("Iuml", "Ï"), + ("Jcirc", "Ĵ"), + ("Jcy", "Й"), + ("Jfr", "𝔍"), + ("Jopf", "𝕁"), + ("Jscr", "𝒥"), + ("Jsercy", "Ј"), + ("Jukcy", "Є"), + ("KHcy", "Х"), + ("KJcy", "Ќ"), + ("Kappa", "Κ"), + ("Kcedil", "Ķ"), + ("Kcy", "К"), + ("Kfr", "𝔎"), + ("Kopf", "𝕂"), + ("Kscr", "𝒦"), + ("LJcy", "Љ"), + ("LT", "<"), + ("Lacute", "Ĺ"), + ("Lambda", "Λ"), + ("Lang", "⟪"), + ("Laplacetrf", "ℒ"), + ("Larr", "↞"), + ("Lcaron", "Ľ"), + ("Lcedil", "Ļ"), + ("Lcy", "Л"), + ("LeftAngleBracket", "⟨"), + ("LeftArrow", "←"), + ("LeftArrowBar", "⇤"), + ("LeftArrowRightArrow", "⇆"), + ("LeftCeiling", "⌈"), + ("LeftDoubleBracket", "⟦"), + ("LeftDownTeeVector", "⥡"), + ("LeftDownVector", "⇃"), + ("LeftDownVectorBar", "⥙"), + ("LeftFloor", "⌊"), + ("LeftRightArrow", "↔"), + ("LeftRightVector", "⥎"), + ("LeftTee", "⊣"), + ("LeftTeeArrow", "↤"), + ("LeftTeeVector", "⥚"), + ("LeftTriangle", "⊲"), + ("LeftTriangleBar", "⧏"), + ("LeftTriangleEqual", "⊴"), + ("LeftUpDownVector", "⥑"), + ("LeftUpTeeVector", "⥠"), + ("LeftUpVector", "↿"), + ("LeftUpVectorBar", "⥘"), + ("LeftVector", "↼"), + ("LeftVectorBar", "⥒"), + ("Leftarrow", "⇐"), + ("Leftrightarrow", "⇔"), + ("LessEqualGreater", "⋚"), + ("LessFullEqual", "≦"), + ("LessGreater", "≶"), + ("LessLess", "⪡"), + ("LessSlantEqual", "⩽"), + ("LessTilde", "≲"), + ("Lfr", "𝔏"), + ("Ll", "⋘"), + ("Lleftarrow", "⇚"), + ("Lmidot", "Ŀ"), + ("LongLeftArrow", "⟵"), + ("LongLeftRightArrow", "⟷"), + ("LongRightArrow", "⟶"), + ("Longleftarrow", "⟸"), + ("Longleftrightarrow", "⟺"), + ("Longrightarrow", "⟹"), + ("Lopf", "𝕃"), + ("LowerLeftArrow", "↙"), + ("LowerRightArrow", "↘"), + ("Lscr", "ℒ"), + ("Lsh", "↰"), + ("Lstrok", "Ł"), + ("Lt", "≪"), + ("Map", "⤅"), + ("Mcy", "М"), + ("MediumSpace", "\u{205f}"), + ("Mellintrf", "ℳ"), + ("Mfr", "𝔐"), + ("MinusPlus", "∓"), + ("Mopf", "𝕄"), + ("Mscr", "ℳ"), + ("Mu", "Μ"), + ("NJcy", "Њ"), + ("Nacute", "Ń"), + ("Ncaron", "Ň"), + ("Ncedil", "Ņ"), + ("Ncy", "Н"), + ("NegativeMediumSpace", "\u{200b}"), + ("NegativeThickSpace", "\u{200b}"), + ("NegativeThinSpace", "\u{200b}"), + ("NegativeVeryThinSpace", "\u{200b}"), + ("NestedGreaterGreater", "≫"), + ("NestedLessLess", "≪"), + ("NewLine", "\u{a}"), + ("Nfr", "𝔑"), + ("NoBreak", "\u{2060}"), + ("NonBreakingSpace", "\u{a0}"), + ("Nopf", "ℕ"), + ("Not", "⫬"), + ("NotCongruent", "≢"), + ("NotCupCap", "≭"), + ("NotDoubleVerticalBar", "∦"), + ("NotElement", "∉"), + ("NotEqual", "≠"), + ("NotEqualTilde", "≂\u{338}"), + ("NotExists", "∄"), + ("NotGreater", "≯"), + ("NotGreaterEqual", "≱"), + ("NotGreaterFullEqual", "≧\u{338}"), + ("NotGreaterGreater", "≫\u{338}"), + ("NotGreaterLess", "≹"), + ("NotGreaterSlantEqual", "⩾\u{338}"), + ("NotGreaterTilde", "≵"), + ("NotHumpDownHump", "≎\u{338}"), + ("NotHumpEqual", "≏\u{338}"), + ("NotLeftTriangle", "⋪"), + ("NotLeftTriangleBar", "⧏\u{338}"), + ("NotLeftTriangleEqual", "⋬"), + ("NotLess", "≮"), + ("NotLessEqual", "≰"), + ("NotLessGreater", "≸"), + ("NotLessLess", "≪\u{338}"), + ("NotLessSlantEqual", "⩽\u{338}"), + ("NotLessTilde", "≴"), + ("NotNestedGreaterGreater", "⪢\u{338}"), + ("NotNestedLessLess", "⪡\u{338}"), + ("NotPrecedes", "⊀"), + ("NotPrecedesEqual", "⪯\u{338}"), + ("NotPrecedesSlantEqual", "⋠"), + ("NotReverseElement", "∌"), + ("NotRightTriangle", "⋫"), + ("NotRightTriangleBar", "⧐\u{338}"), + ("NotRightTriangleEqual", "⋭"), + ("NotSquareSubset", "⊏\u{338}"), + ("NotSquareSubsetEqual", "⋢"), + ("NotSquareSuperset", "⊐\u{338}"), + ("NotSquareSupersetEqual", "⋣"), + ("NotSubset", "⊂\u{20d2}"), + ("NotSubsetEqual", "⊈"), + ("NotSucceeds", "⊁"), + ("NotSucceedsEqual", "⪰\u{338}"), + ("NotSucceedsSlantEqual", "⋡"), + ("NotSucceedsTilde", "≿\u{338}"), + ("NotSuperset", "⊃\u{20d2}"), + ("NotSupersetEqual", "⊉"), + ("NotTilde", "≁"), + ("NotTildeEqual", "≄"), + ("NotTildeFullEqual", "≇"), + ("NotTildeTilde", "≉"), + ("NotVerticalBar", "∤"), + ("Nscr", "𝒩"), + ("Ntilde", "Ñ"), + ("Nu", "Ν"), + ("OElig", "Œ"), + ("Oacute", "Ó"), + ("Ocirc", "Ô"), + ("Ocy", "О"), + ("Odblac", "Ő"), + ("Ofr", "𝔒"), + ("Ograve", "Ò"), + ("Omacr", "Ō"), + ("Omega", "Ω"), + ("Omicron", "Ο"), + ("Oopf", "𝕆"), + ("OpenCurlyDoubleQuote", "“"), + ("OpenCurlyQuote", "‘"), + ("Or", "⩔"), + ("Oscr", "𝒪"), + ("Oslash", "Ø"), + ("Otilde", "Õ"), + ("Otimes", "⨷"), + ("Ouml", "Ö"), + ("OverBar", "‾"), + ("OverBrace", "⏞"), + ("OverBracket", "⎴"), + ("OverParenthesis", "⏜"), + ("PartialD", "∂"), + ("Pcy", "П"), + ("Pfr", "𝔓"), + ("Phi", "Φ"), + ("Pi", "Π"), + ("PlusMinus", "±"), + ("Poincareplane", "ℌ"), + ("Popf", "ℙ"), + ("Pr", "⪻"), + ("Precedes", "≺"), + ("PrecedesEqual", "⪯"), + ("PrecedesSlantEqual", "≼"), + ("PrecedesTilde", "≾"), + ("Prime", "″"), + ("Product", "∏"), + ("Proportion", "∷"), + ("Proportional", "∝"), + ("Pscr", "𝒫"), + ("Psi", "Ψ"), + ("QUOT", "\""), + ("Qfr", "𝔔"), + ("Qopf", "ℚ"), + ("Qscr", "𝒬"), + ("RBarr", "⤐"), + ("REG", "®"), + ("Racute", "Ŕ"), + ("Rang", "⟫"), + ("Rarr", "↠"), + ("Rarrtl", "⤖"), + ("Rcaron", "Ř"), + ("Rcedil", "Ŗ"), + ("Rcy", "Р"), + ("Re", "ℜ"), + ("ReverseElement", "∋"), + ("ReverseEquilibrium", "⇋"), + ("ReverseUpEquilibrium", "⥯"), + ("Rfr", "ℜ"), + ("Rho", "Ρ"), + ("RightAngleBracket", "⟩"), + ("RightArrow", "→"), + ("RightArrowBar", "⇥"), + ("RightArrowLeftArrow", "⇄"), + ("RightCeiling", "⌉"), + ("RightDoubleBracket", "⟧"), + ("RightDownTeeVector", "⥝"), + ("RightDownVector", "⇂"), + ("RightDownVectorBar", "⥕"), + ("RightFloor", "⌋"), + ("RightTee", "⊢"), + ("RightTeeArrow", "↦"), + ("RightTeeVector", "⥛"), + ("RightTriangle", "⊳"), + ("RightTriangleBar", "⧐"), + ("RightTriangleEqual", "⊵"), + ("RightUpDownVector", "⥏"), + ("RightUpTeeVector", "⥜"), + ("RightUpVector", "↾"), + ("RightUpVectorBar", "⥔"), + ("RightVector", "⇀"), + ("RightVectorBar", "⥓"), + ("Rightarrow", "⇒"), + ("Ropf", "ℝ"), + ("RoundImplies", "⥰"), + ("Rrightarrow", "⇛"), + ("Rscr", "ℛ"), + ("Rsh", "↱"), + ("RuleDelayed", "⧴"), + ("SHCHcy", "Щ"), + ("SHcy", "Ш"), + ("SOFTcy", "Ь"), + ("Sacute", "Ś"), + ("Sc", "⪼"), + ("Scaron", "Š"), + ("Scedil", "Ş"), + ("Scirc", "Ŝ"), + ("Scy", "С"), + ("Sfr", "𝔖"), + ("ShortDownArrow", "↓"), + ("ShortLeftArrow", "←"), + ("ShortRightArrow", "→"), + ("ShortUpArrow", "↑"), + ("Sigma", "Σ"), + ("SmallCircle", "∘"), + ("Sopf", "𝕊"), + ("Sqrt", "√"), + ("Square", "□"), + ("SquareIntersection", "⊓"), + ("SquareSubset", "⊏"), + ("SquareSubsetEqual", "⊑"), + ("SquareSuperset", "⊐"), + ("SquareSupersetEqual", "⊒"), + ("SquareUnion", "⊔"), + ("Sscr", "𝒮"), + ("Star", "⋆"), + ("Sub", "⋐"), + ("Subset", "⋐"), + ("SubsetEqual", "⊆"), + ("Succeeds", "≻"), + ("SucceedsEqual", "⪰"), + ("SucceedsSlantEqual", "≽"), + ("SucceedsTilde", "≿"), + ("SuchThat", "∋"), + ("Sum", "∑"), + ("Sup", "⋑"), + ("Superset", "⊃"), + ("SupersetEqual", "⊇"), + ("Supset", "⋑"), + ("THORN", "Þ"), + ("TRADE", "™"), + ("TSHcy", "Ћ"), + ("TScy", "Ц"), + ("Tab", "\u{9}"), + ("Tau", "Τ"), + ("Tcaron", "Ť"), + ("Tcedil", "Ţ"), + ("Tcy", "Т"), + ("Tfr", "𝔗"), + ("Therefore", "∴"), + ("Theta", "Θ"), + ("ThickSpace", "\u{205f}\u{200a}"), + ("ThinSpace", "\u{2009}"), + ("Tilde", "∼"), + ("TildeEqual", "≃"), + ("TildeFullEqual", "≅"), + ("TildeTilde", "≈"), + ("Topf", "𝕋"), + ("TripleDot", "\u{20db}"), + ("Tscr", "𝒯"), + ("Tstrok", "Ŧ"), + ("Uacute", "Ú"), + ("Uarr", "↟"), + ("Uarrocir", "⥉"), + ("Ubrcy", "Ў"), + ("Ubreve", "Ŭ"), + ("Ucirc", "Û"), + ("Ucy", "У"), + ("Udblac", "Ű"), + ("Ufr", "𝔘"), + ("Ugrave", "Ù"), + ("Umacr", "Ū"), + ("UnderBar", "_"), + ("UnderBrace", "⏟"), + ("UnderBracket", "⎵"), + ("UnderParenthesis", "⏝"), + ("Union", "⋃"), + ("UnionPlus", "⊎"), + ("Uogon", "Ų"), + ("Uopf", "𝕌"), + ("UpArrow", "↑"), + ("UpArrowBar", "⤒"), + ("UpArrowDownArrow", "⇅"), + ("UpDownArrow", "↕"), + ("UpEquilibrium", "⥮"), + ("UpTee", "⊥"), + ("UpTeeArrow", "↥"), + ("Uparrow", "⇑"), + ("Updownarrow", "⇕"), + ("UpperLeftArrow", "↖"), + ("UpperRightArrow", "↗"), + ("Upsi", "ϒ"), + ("Upsilon", "Υ"), + ("Uring", "Ů"), + ("Uscr", "𝒰"), + ("Utilde", "Ũ"), + ("Uuml", "Ü"), + ("VDash", "⊫"), + ("Vbar", "⫫"), + ("Vcy", "В"), + ("Vdash", "⊩"), + ("Vdashl", "⫦"), + ("Vee", "⋁"), + ("Verbar", "‖"), + ("Vert", "‖"), + ("VerticalBar", "∣"), + ("VerticalLine", "|"), + ("VerticalSeparator", "❘"), + ("VerticalTilde", "≀"), + ("VeryThinSpace", "\u{200a}"), + ("Vfr", "𝔙"), + ("Vopf", "𝕍"), + ("Vscr", "𝒱"), + ("Vvdash", "⊪"), + ("Wcirc", "Ŵ"), + ("Wedge", "⋀"), + ("Wfr", "𝔚"), + ("Wopf", "𝕎"), + ("Wscr", "𝒲"), + ("Xfr", "𝔛"), + ("Xi", "Ξ"), + ("Xopf", "𝕏"), + ("Xscr", "𝒳"), + ("YAcy", "Я"), + ("YIcy", "Ї"), + ("YUcy", "Ю"), + ("Yacute", "Ý"), + ("Ycirc", "Ŷ"), + ("Ycy", "Ы"), + ("Yfr", "𝔜"), + ("Yopf", "𝕐"), + ("Yscr", "𝒴"), + ("Yuml", "Ÿ"), + ("ZHcy", "Ж"), + ("Zacute", "Ź"), + ("Zcaron", "Ž"), + ("Zcy", "З"), + ("Zdot", "Ż"), + ("ZeroWidthSpace", "\u{200b}"), + ("Zeta", "Ζ"), + ("Zfr", "ℨ"), + ("Zopf", "ℤ"), + ("Zscr", "𝒵"), + ("aacute", "á"), + ("abreve", "ă"), + ("ac", "∾"), + ("acE", "∾\u{333}"), + ("acd", "∿"), + ("acirc", "â"), + ("acute", "´"), + ("acy", "а"), + ("aelig", "æ"), + ("af", "\u{2061}"), + ("afr", "𝔞"), + ("agrave", "à"), + ("alefsym", "ℵ"), + ("aleph", "ℵ"), + ("alpha", "α"), + ("amacr", "ā"), + ("amalg", "⨿"), + ("amp", "&"), + ("and", "∧"), + ("andand", "⩕"), + ("andd", "⩜"), + ("andslope", "⩘"), + ("andv", "⩚"), + ("ang", "∠"), + ("ange", "⦤"), + ("angle", "∠"), + ("angmsd", "∡"), + ("angmsdaa", "⦨"), + ("angmsdab", "⦩"), + ("angmsdac", "⦪"), + ("angmsdad", "⦫"), + ("angmsdae", "⦬"), + ("angmsdaf", "⦭"), + ("angmsdag", "⦮"), + ("angmsdah", "⦯"), + ("angrt", "∟"), + ("angrtvb", "⊾"), + ("angrtvbd", "⦝"), + ("angsph", "∢"), + ("angst", "Å"), + ("angzarr", "⍼"), + ("aogon", "ą"), + ("aopf", "𝕒"), + ("ap", "≈"), + ("apE", "⩰"), + ("apacir", "⩯"), + ("ape", "≊"), + ("apid", "≋"), + ("apos", "'"), + ("approx", "≈"), + ("approxeq", "≊"), + ("aring", "å"), + ("ascr", "𝒶"), + ("ast", "*"), + ("asymp", "≈"), + ("asympeq", "≍"), + ("atilde", "ã"), + ("auml", "ä"), + ("awconint", "∳"), + ("awint", "⨑"), + ("bNot", "⫭"), + ("backcong", "≌"), + ("backepsilon", "϶"), + ("backprime", "‵"), + ("backsim", "∽"), + ("backsimeq", "⋍"), + ("barvee", "⊽"), + ("barwed", "⌅"), + ("barwedge", "⌅"), + ("bbrk", "⎵"), + ("bbrktbrk", "⎶"), + ("bcong", "≌"), + ("bcy", "б"), + ("bdquo", "„"), + ("becaus", "∵"), + ("because", "∵"), + ("bemptyv", "⦰"), + ("bepsi", "϶"), + ("bernou", "ℬ"), + ("beta", "β"), + ("beth", "ℶ"), + ("between", "≬"), + ("bfr", "𝔟"), + ("bigcap", "⋂"), + ("bigcirc", "◯"), + ("bigcup", "⋃"), + ("bigodot", "⨀"), + ("bigoplus", "⨁"), + ("bigotimes", "⨂"), + ("bigsqcup", "⨆"), + ("bigstar", "★"), + ("bigtriangledown", "▽"), + ("bigtriangleup", "△"), + ("biguplus", "⨄"), + ("bigvee", "⋁"), + ("bigwedge", "⋀"), + ("bkarow", "⤍"), + ("blacklozenge", "⧫"), + ("blacksquare", "▪"), + ("blacktriangle", "▴"), + ("blacktriangledown", "▾"), + ("blacktriangleleft", "◂"), + ("blacktriangleright", "▸"), + ("blank", "␣"), + ("blk12", "▒"), + ("blk14", "░"), + ("blk34", "▓"), + ("block", "█"), + ("bne", "=\u{20e5}"), + ("bnequiv", "≡\u{20e5}"), + ("bnot", "⌐"), + ("bopf", "𝕓"), + ("bot", "⊥"), + ("bottom", "⊥"), + ("bowtie", "⋈"), + ("boxDL", "╗"), + ("boxDR", "╔"), + ("boxDl", "╖"), + ("boxDr", "╓"), + ("boxH", "═"), + ("boxHD", "╦"), + ("boxHU", "╩"), + ("boxHd", "╤"), + ("boxHu", "╧"), + ("boxUL", "╝"), + ("boxUR", "╚"), + ("boxUl", "╜"), + ("boxUr", "╙"), + ("boxV", "║"), + ("boxVH", "╬"), + ("boxVL", "╣"), + ("boxVR", "╠"), + ("boxVh", "╫"), + ("boxVl", "╢"), + ("boxVr", "╟"), + ("boxbox", "⧉"), + ("boxdL", "╕"), + ("boxdR", "╒"), + ("boxdl", "┐"), + ("boxdr", "┌"), + ("boxh", "─"), + ("boxhD", "╥"), + ("boxhU", "╨"), + ("boxhd", "┬"), + ("boxhu", "┴"), + ("boxminus", "⊟"), + ("boxplus", "⊞"), + ("boxtimes", "⊠"), + ("boxuL", "╛"), + ("boxuR", "╘"), + ("boxul", "┘"), + ("boxur", "└"), + ("boxv", "│"), + ("boxvH", "╪"), + ("boxvL", "╡"), + ("boxvR", "╞"), + ("boxvh", "┼"), + ("boxvl", "┤"), + ("boxvr", "├"), + ("bprime", "‵"), + ("breve", "˘"), + ("brvbar", "¦"), + ("bscr", "𝒷"), + ("bsemi", "⁏"), + ("bsim", "∽"), + ("bsime", "⋍"), + ("bsol", "\\"), + ("bsolb", "⧅"), + ("bsolhsub", "⟈"), + ("bull", "•"), + ("bullet", "•"), + ("bump", "≎"), + ("bumpE", "⪮"), + ("bumpe", "≏"), + ("bumpeq", "≏"), + ("cacute", "ć"), + ("cap", "∩"), + ("capand", "⩄"), + ("capbrcup", "⩉"), + ("capcap", "⩋"), + ("capcup", "⩇"), + ("capdot", "⩀"), + ("caps", "∩\u{fe00}"), + ("caret", "⁁"), + ("caron", "ˇ"), + ("ccaps", "⩍"), + ("ccaron", "č"), + ("ccedil", "ç"), + ("ccirc", "ĉ"), + ("ccups", "⩌"), + ("ccupssm", "⩐"), + ("cdot", "ċ"), + ("cedil", "¸"), + ("cemptyv", "⦲"), + ("cent", "¢"), + ("centerdot", "·"), + ("cfr", "𝔠"), + ("chcy", "ч"), + ("check", "✓"), + ("checkmark", "✓"), + ("chi", "χ"), + ("cir", "○"), + ("cirE", "⧃"), + ("circ", "ˆ"), + ("circeq", "≗"), + ("circlearrowleft", "↺"), + ("circlearrowright", "↻"), + ("circledR", "®"), + ("circledS", "Ⓢ"), + ("circledast", "⊛"), + ("circledcirc", "⊚"), + ("circleddash", "⊝"), + ("cire", "≗"), + ("cirfnint", "⨐"), + ("cirmid", "⫯"), + ("cirscir", "⧂"), + ("clubs", "♣"), + ("clubsuit", "♣"), + ("colon", ":"), + ("colone", "≔"), + ("coloneq", "≔"), + ("comma", ","), + ("commat", "@"), + ("comp", "∁"), + ("compfn", "∘"), + ("complement", "∁"), + ("complexes", "ℂ"), + ("cong", "≅"), + ("congdot", "⩭"), + ("conint", "∮"), + ("copf", "𝕔"), + ("coprod", "∐"), + ("copy", "©"), + ("copysr", "℗"), + ("crarr", "↵"), + ("cross", "✗"), + ("cscr", "𝒸"), + ("csub", "⫏"), + ("csube", "⫑"), + ("csup", "⫐"), + ("csupe", "⫒"), + ("ctdot", "⋯"), + ("cudarrl", "⤸"), + ("cudarrr", "⤵"), + ("cuepr", "⋞"), + ("cuesc", "⋟"), + ("cularr", "↶"), + ("cularrp", "⤽"), + ("cup", "∪"), + ("cupbrcap", "⩈"), + ("cupcap", "⩆"), + ("cupcup", "⩊"), + ("cupdot", "⊍"), + ("cupor", "⩅"), + ("cups", "∪\u{fe00}"), + ("curarr", "↷"), + ("curarrm", "⤼"), + ("curlyeqprec", "⋞"), + ("curlyeqsucc", "⋟"), + ("curlyvee", "⋎"), + ("curlywedge", "⋏"), + ("curren", "¤"), + ("curvearrowleft", "↶"), + ("curvearrowright", "↷"), + ("cuvee", "⋎"), + ("cuwed", "⋏"), + ("cwconint", "∲"), + ("cwint", "∱"), + ("cylcty", "⌭"), + ("dArr", "⇓"), + ("dHar", "⥥"), + ("dagger", "†"), + ("daleth", "ℸ"), + ("darr", "↓"), + ("dash", "‐"), + ("dashv", "⊣"), + ("dbkarow", "⤏"), + ("dblac", "˝"), + ("dcaron", "ď"), + ("dcy", "д"), + ("dd", "ⅆ"), + ("ddagger", "‡"), + ("ddarr", "⇊"), + ("ddotseq", "⩷"), + ("deg", "°"), + ("delta", "δ"), + ("demptyv", "⦱"), + ("dfisht", "⥿"), + ("dfr", "𝔡"), + ("dharl", "⇃"), + ("dharr", "⇂"), + ("diam", "⋄"), + ("diamond", "⋄"), + ("diamondsuit", "♦"), + ("diams", "♦"), + ("die", "¨"), + ("digamma", "ϝ"), + ("disin", "⋲"), + ("div", "÷"), + ("divide", "÷"), + ("divideontimes", "⋇"), + ("divonx", "⋇"), + ("djcy", "ђ"), + ("dlcorn", "⌞"), + ("dlcrop", "⌍"), + ("dollar", "$"), + ("dopf", "𝕕"), + ("dot", "˙"), + ("doteq", "≐"), + ("doteqdot", "≑"), + ("dotminus", "∸"), + ("dotplus", "∔"), + ("dotsquare", "⊡"), + ("doublebarwedge", "⌆"), + ("downarrow", "↓"), + ("downdownarrows", "⇊"), + ("downharpoonleft", "⇃"), + ("downharpoonright", "⇂"), + ("drbkarow", "⤐"), + ("drcorn", "⌟"), + ("drcrop", "⌌"), + ("dscr", "𝒹"), + ("dscy", "ѕ"), + ("dsol", "⧶"), + ("dstrok", "đ"), + ("dtdot", "⋱"), + ("dtri", "▿"), + ("dtrif", "▾"), + ("duarr", "⇵"), + ("duhar", "⥯"), + ("dwangle", "⦦"), + ("dzcy", "џ"), + ("dzigrarr", "⟿"), + ("eDDot", "⩷"), + ("eDot", "≑"), + ("eacute", "é"), + ("easter", "⩮"), + ("ecaron", "ě"), + ("ecir", "≖"), + ("ecirc", "ê"), + ("ecolon", "≕"), + ("ecy", "э"), + ("edot", "ė"), + ("ee", "ⅇ"), + ("efDot", "≒"), + ("efr", "𝔢"), + ("eg", "⪚"), + ("egrave", "è"), + ("egs", "⪖"), + ("egsdot", "⪘"), + ("el", "⪙"), + ("elinters", "⏧"), + ("ell", "ℓ"), + ("els", "⪕"), + ("elsdot", "⪗"), + ("emacr", "ē"), + ("empty", "∅"), + ("emptyset", "∅"), + ("emptyv", "∅"), + ("emsp", "\u{2003}"), + ("emsp13", "\u{2004}"), + ("emsp14", "\u{2005}"), + ("eng", "ŋ"), + ("ensp", "\u{2002}"), + ("eogon", "ę"), + ("eopf", "𝕖"), + ("epar", "⋕"), + ("eparsl", "⧣"), + ("eplus", "⩱"), + ("epsi", "ε"), + ("epsilon", "ε"), + ("epsiv", "ϵ"), + ("eqcirc", "≖"), + ("eqcolon", "≕"), + ("eqsim", "≂"), + ("eqslantgtr", "⪖"), + ("eqslantless", "⪕"), + ("equals", "="), + ("equest", "≟"), + ("equiv", "≡"), + ("equivDD", "⩸"), + ("eqvparsl", "⧥"), + ("erDot", "≓"), + ("erarr", "⥱"), + ("escr", "ℯ"), + ("esdot", "≐"), + ("esim", "≂"), + ("eta", "η"), + ("eth", "ð"), + ("euml", "ë"), + ("euro", "€"), + ("excl", "!"), + ("exist", "∃"), + ("expectation", "ℰ"), + ("exponentiale", "ⅇ"), + ("fallingdotseq", "≒"), + ("fcy", "ф"), + ("female", "♀"), + ("ffilig", "ffi"), + ("fflig", "ff"), + ("ffllig", "ffl"), + ("ffr", "𝔣"), + ("filig", "fi"), + ("fjlig", "fj"), + ("flat", "♭"), + ("fllig", "fl"), + ("fltns", "▱"), + ("fnof", "ƒ"), + ("fopf", "𝕗"), + ("forall", "∀"), + ("fork", "⋔"), + ("forkv", "⫙"), + ("fpartint", "⨍"), + ("frac12", "½"), + ("frac13", "⅓"), + ("frac14", "¼"), + ("frac15", "⅕"), + ("frac16", "⅙"), + ("frac18", "⅛"), + ("frac23", "⅔"), + ("frac25", "⅖"), + ("frac34", "¾"), + ("frac35", "⅗"), + ("frac38", "⅜"), + ("frac45", "⅘"), + ("frac56", "⅚"), + ("frac58", "⅝"), + ("frac78", "⅞"), + ("frasl", "⁄"), + ("frown", "⌢"), + ("fscr", "𝒻"), + ("gE", "≧"), + ("gEl", "⪌"), + ("gacute", "ǵ"), + ("gamma", "γ"), + ("gammad", "ϝ"), + ("gap", "⪆"), + ("gbreve", "ğ"), + ("gcirc", "ĝ"), + ("gcy", "г"), + ("gdot", "ġ"), + ("ge", "≥"), + ("gel", "⋛"), + ("geq", "≥"), + ("geqq", "≧"), + ("geqslant", "⩾"), + ("ges", "⩾"), + ("gescc", "⪩"), + ("gesdot", "⪀"), + ("gesdoto", "⪂"), + ("gesdotol", "⪄"), + ("gesl", "⋛\u{fe00}"), + ("gesles", "⪔"), + ("gfr", "𝔤"), + ("gg", "≫"), + ("ggg", "⋙"), + ("gimel", "ℷ"), + ("gjcy", "ѓ"), + ("gl", "≷"), + ("glE", "⪒"), + ("gla", "⪥"), + ("glj", "⪤"), + ("gnE", "≩"), + ("gnap", "⪊"), + ("gnapprox", "⪊"), + ("gne", "⪈"), + ("gneq", "⪈"), + ("gneqq", "≩"), + ("gnsim", "⋧"), + ("gopf", "𝕘"), + ("grave", "`"), + ("gscr", "ℊ"), + ("gsim", "≳"), + ("gsime", "⪎"), + ("gsiml", "⪐"), + ("gt", ">"), + ("gtcc", "⪧"), + ("gtcir", "⩺"), + ("gtdot", "⋗"), + ("gtlPar", "⦕"), + ("gtquest", "⩼"), + ("gtrapprox", "⪆"), + ("gtrarr", "⥸"), + ("gtrdot", "⋗"), + ("gtreqless", "⋛"), + ("gtreqqless", "⪌"), + ("gtrless", "≷"), + ("gtrsim", "≳"), + ("gvertneqq", "≩\u{fe00}"), + ("gvnE", "≩\u{fe00}"), + ("hArr", "⇔"), + ("hairsp", "\u{200a}"), + ("half", "½"), + ("hamilt", "ℋ"), + ("hardcy", "ъ"), + ("harr", "↔"), + ("harrcir", "⥈"), + ("harrw", "↭"), + ("hbar", "ℏ"), + ("hcirc", "ĥ"), + ("hearts", "♥"), + ("heartsuit", "♥"), + ("hellip", "…"), + ("hercon", "⊹"), + ("hfr", "𝔥"), + ("hksearow", "⤥"), + ("hkswarow", "⤦"), + ("hoarr", "⇿"), + ("homtht", "∻"), + ("hookleftarrow", "↩"), + ("hookrightarrow", "↪"), + ("hopf", "𝕙"), + ("horbar", "―"), + ("hscr", "𝒽"), + ("hslash", "ℏ"), + ("hstrok", "ħ"), + ("hybull", "⁃"), + ("hyphen", "‐"), + ("iacute", "í"), + ("ic", "\u{2063}"), + ("icirc", "î"), + ("icy", "и"), + ("iecy", "е"), + ("iexcl", "¡"), + ("iff", "⇔"), + ("ifr", "𝔦"), + ("igrave", "ì"), + ("ii", "ⅈ"), + ("iiiint", "⨌"), + ("iiint", "∭"), + ("iinfin", "⧜"), + ("iiota", "℩"), + ("ijlig", "ij"), + ("imacr", "ī"), + ("image", "ℑ"), + ("imagline", "ℐ"), + ("imagpart", "ℑ"), + ("imath", "ı"), + ("imof", "⊷"), + ("imped", "Ƶ"), + ("in", "∈"), + ("incare", "℅"), + ("infin", "∞"), + ("infintie", "⧝"), + ("inodot", "ı"), + ("int", "∫"), + ("intcal", "⊺"), + ("integers", "ℤ"), + ("intercal", "⊺"), + ("intlarhk", "⨗"), + ("intprod", "⨼"), + ("iocy", "ё"), + ("iogon", "į"), + ("iopf", "𝕚"), + ("iota", "ι"), + ("iprod", "⨼"), + ("iquest", "¿"), + ("iscr", "𝒾"), + ("isin", "∈"), + ("isinE", "⋹"), + ("isindot", "⋵"), + ("isins", "⋴"), + ("isinsv", "⋳"), + ("isinv", "∈"), + ("it", "\u{2062}"), + ("itilde", "ĩ"), + ("iukcy", "і"), + ("iuml", "ï"), + ("jcirc", "ĵ"), + ("jcy", "й"), + ("jfr", "𝔧"), + ("jmath", "ȷ"), + ("jopf", "𝕛"), + ("jscr", "𝒿"), + ("jsercy", "ј"), + ("jukcy", "є"), + ("kappa", "κ"), + ("kappav", "ϰ"), + ("kcedil", "ķ"), + ("kcy", "к"), + ("kfr", "𝔨"), + ("kgreen", "ĸ"), + ("khcy", "х"), + ("kjcy", "ќ"), + ("kopf", "𝕜"), + ("kscr", "𝓀"), + ("lAarr", "⇚"), + ("lArr", "⇐"), + ("lAtail", "⤛"), + ("lBarr", "⤎"), + ("lE", "≦"), + ("lEg", "⪋"), + ("lHar", "⥢"), + ("lacute", "ĺ"), + ("laemptyv", "⦴"), + ("lagran", "ℒ"), + ("lambda", "λ"), + ("lang", "⟨"), + ("langd", "⦑"), + ("langle", "⟨"), + ("lap", "⪅"), + ("laquo", "«"), + ("larr", "←"), + ("larrb", "⇤"), + ("larrbfs", "⤟"), + ("larrfs", "⤝"), + ("larrhk", "↩"), + ("larrlp", "↫"), + ("larrpl", "⤹"), + ("larrsim", "⥳"), + ("larrtl", "↢"), + ("lat", "⪫"), + ("latail", "⤙"), + ("late", "⪭"), + ("lates", "⪭\u{fe00}"), + ("lbarr", "⤌"), + ("lbbrk", "❲"), + ("lbrace", "{"), + ("lbrack", "["), + ("lbrke", "⦋"), + ("lbrksld", "⦏"), + ("lbrkslu", "⦍"), + ("lcaron", "ľ"), + ("lcedil", "ļ"), + ("lceil", "⌈"), + ("lcub", "{"), + ("lcy", "л"), + ("ldca", "⤶"), + ("ldquo", "“"), + ("ldquor", "„"), + ("ldrdhar", "⥧"), + ("ldrushar", "⥋"), + ("ldsh", "↲"), + ("le", "≤"), + ("leftarrow", "←"), + ("leftarrowtail", "↢"), + ("leftharpoondown", "↽"), + ("leftharpoonup", "↼"), + ("leftleftarrows", "⇇"), + ("leftrightarrow", "↔"), + ("leftrightarrows", "⇆"), + ("leftrightharpoons", "⇋"), + ("leftrightsquigarrow", "↭"), + ("leftthreetimes", "⋋"), + ("leg", "⋚"), + ("leq", "≤"), + ("leqq", "≦"), + ("leqslant", "⩽"), + ("les", "⩽"), + ("lescc", "⪨"), + ("lesdot", "⩿"), + ("lesdoto", "⪁"), + ("lesdotor", "⪃"), + ("lesg", "⋚\u{fe00}"), + ("lesges", "⪓"), + ("lessapprox", "⪅"), + ("lessdot", "⋖"), + ("lesseqgtr", "⋚"), + ("lesseqqgtr", "⪋"), + ("lessgtr", "≶"), + ("lesssim", "≲"), + ("lfisht", "⥼"), + ("lfloor", "⌊"), + ("lfr", "𝔩"), + ("lg", "≶"), + ("lgE", "⪑"), + ("lhard", "↽"), + ("lharu", "↼"), + ("lharul", "⥪"), + ("lhblk", "▄"), + ("ljcy", "љ"), + ("ll", "≪"), + ("llarr", "⇇"), + ("llcorner", "⌞"), + ("llhard", "⥫"), + ("lltri", "◺"), + ("lmidot", "ŀ"), + ("lmoust", "⎰"), + ("lmoustache", "⎰"), + ("lnE", "≨"), + ("lnap", "⪉"), + ("lnapprox", "⪉"), + ("lne", "⪇"), + ("lneq", "⪇"), + ("lneqq", "≨"), + ("lnsim", "⋦"), + ("loang", "⟬"), + ("loarr", "⇽"), + ("lobrk", "⟦"), + ("longleftarrow", "⟵"), + ("longleftrightarrow", "⟷"), + ("longmapsto", "⟼"), + ("longrightarrow", "⟶"), + ("looparrowleft", "↫"), + ("looparrowright", "↬"), + ("lopar", "⦅"), + ("lopf", "𝕝"), + ("loplus", "⨭"), + ("lotimes", "⨴"), + ("lowast", "∗"), + ("lowbar", "_"), + ("loz", "◊"), + ("lozenge", "◊"), + ("lozf", "⧫"), + ("lpar", "("), + ("lparlt", "⦓"), + ("lrarr", "⇆"), + ("lrcorner", "⌟"), + ("lrhar", "⇋"), + ("lrhard", "⥭"), + ("lrm", "\u{200e}"), + ("lrtri", "⊿"), + ("lsaquo", "‹"), + ("lscr", "𝓁"), + ("lsh", "↰"), + ("lsim", "≲"), + ("lsime", "⪍"), + ("lsimg", "⪏"), + ("lsqb", "["), + ("lsquo", "‘"), + ("lsquor", "‚"), + ("lstrok", "ł"), + ("lt", "<"), + ("ltcc", "⪦"), + ("ltcir", "⩹"), + ("ltdot", "⋖"), + ("lthree", "⋋"), + ("ltimes", "⋉"), + ("ltlarr", "⥶"), + ("ltquest", "⩻"), + ("ltrPar", "⦖"), + ("ltri", "◃"), + ("ltrie", "⊴"), + ("ltrif", "◂"), + ("lurdshar", "⥊"), + ("luruhar", "⥦"), + ("lvertneqq", "≨\u{fe00}"), + ("lvnE", "≨\u{fe00}"), + ("mDDot", "∺"), + ("macr", "¯"), + ("male", "♂"), + ("malt", "✠"), + ("maltese", "✠"), + ("map", "↦"), + ("mapsto", "↦"), + ("mapstodown", "↧"), + ("mapstoleft", "↤"), + ("mapstoup", "↥"), + ("marker", "▮"), + ("mcomma", "⨩"), + ("mcy", "м"), + ("mdash", "—"), + ("measuredangle", "∡"), + ("mfr", "𝔪"), + ("mho", "℧"), + ("micro", "µ"), + ("mid", "∣"), + ("midast", "*"), + ("midcir", "⫰"), + ("middot", "·"), + ("minus", "−"), + ("minusb", "⊟"), + ("minusd", "∸"), + ("minusdu", "⨪"), + ("mlcp", "⫛"), + ("mldr", "…"), + ("mnplus", "∓"), + ("models", "⊧"), + ("mopf", "𝕞"), + ("mp", "∓"), + ("mscr", "𝓂"), + ("mstpos", "∾"), + ("mu", "μ"), + ("multimap", "⊸"), + ("mumap", "⊸"), + ("nGg", "⋙\u{338}"), + ("nGt", "≫\u{20d2}"), + ("nGtv", "≫\u{338}"), + ("nLeftarrow", "⇍"), + ("nLeftrightarrow", "⇎"), + ("nLl", "⋘\u{338}"), + ("nLt", "≪\u{20d2}"), + ("nLtv", "≪\u{338}"), + ("nRightarrow", "⇏"), + ("nVDash", "⊯"), + ("nVdash", "⊮"), + ("nabla", "∇"), + ("nacute", "ń"), + ("nang", "∠\u{20d2}"), + ("nap", "≉"), + ("napE", "⩰\u{338}"), + ("napid", "≋\u{338}"), + ("napos", "ʼn"), + ("napprox", "≉"), + ("natur", "♮"), + ("natural", "♮"), + ("naturals", "ℕ"), + ("nbsp", "\u{a0}"), + ("nbump", "≎\u{338}"), + ("nbumpe", "≏\u{338}"), + ("ncap", "⩃"), + ("ncaron", "ň"), + ("ncedil", "ņ"), + ("ncong", "≇"), + ("ncongdot", "⩭\u{338}"), + ("ncup", "⩂"), + ("ncy", "н"), + ("ndash", "–"), + ("ne", "≠"), + ("neArr", "⇗"), + ("nearhk", "⤤"), + ("nearr", "↗"), + ("nearrow", "↗"), + ("nedot", "≐\u{338}"), + ("nequiv", "≢"), + ("nesear", "⤨"), + ("nesim", "≂\u{338}"), + ("nexist", "∄"), + ("nexists", "∄"), + ("nfr", "𝔫"), + ("ngE", "≧\u{338}"), + ("nge", "≱"), + ("ngeq", "≱"), + ("ngeqq", "≧\u{338}"), + ("ngeqslant", "⩾\u{338}"), + ("nges", "⩾\u{338}"), + ("ngsim", "≵"), + ("ngt", "≯"), + ("ngtr", "≯"), + ("nhArr", "⇎"), + ("nharr", "↮"), + ("nhpar", "⫲"), + ("ni", "∋"), + ("nis", "⋼"), + ("nisd", "⋺"), + ("niv", "∋"), + ("njcy", "њ"), + ("nlArr", "⇍"), + ("nlE", "≦\u{338}"), + ("nlarr", "↚"), + ("nldr", "‥"), + ("nle", "≰"), + ("nleftarrow", "↚"), + ("nleftrightarrow", "↮"), + ("nleq", "≰"), + ("nleqq", "≦\u{338}"), + ("nleqslant", "⩽\u{338}"), + ("nles", "⩽\u{338}"), + ("nless", "≮"), + ("nlsim", "≴"), + ("nlt", "≮"), + ("nltri", "⋪"), + ("nltrie", "⋬"), + ("nmid", "∤"), + ("nopf", "𝕟"), + ("not", "¬"), + ("notin", "∉"), + ("notinE", "⋹\u{338}"), + ("notindot", "⋵\u{338}"), + ("notinva", "∉"), + ("notinvb", "⋷"), + ("notinvc", "⋶"), + ("notni", "∌"), + ("notniva", "∌"), + ("notnivb", "⋾"), + ("notnivc", "⋽"), + ("npar", "∦"), + ("nparallel", "∦"), + ("nparsl", "⫽\u{20e5}"), + ("npart", "∂\u{338}"), + ("npolint", "⨔"), + ("npr", "⊀"), + ("nprcue", "⋠"), + ("npre", "⪯\u{338}"), + ("nprec", "⊀"), + ("npreceq", "⪯\u{338}"), + ("nrArr", "⇏"), + ("nrarr", "↛"), + ("nrarrc", "⤳\u{338}"), + ("nrarrw", "↝\u{338}"), + ("nrightarrow", "↛"), + ("nrtri", "⋫"), + ("nrtrie", "⋭"), + ("nsc", "⊁"), + ("nsccue", "⋡"), + ("nsce", "⪰\u{338}"), + ("nscr", "𝓃"), + ("nshortmid", "∤"), + ("nshortparallel", "∦"), + ("nsim", "≁"), + ("nsime", "≄"), + ("nsimeq", "≄"), + ("nsmid", "∤"), + ("nspar", "∦"), + ("nsqsube", "⋢"), + ("nsqsupe", "⋣"), + ("nsub", "⊄"), + ("nsubE", "⫅\u{338}"), + ("nsube", "⊈"), + ("nsubset", "⊂\u{20d2}"), + ("nsubseteq", "⊈"), + ("nsubseteqq", "⫅\u{338}"), + ("nsucc", "⊁"), + ("nsucceq", "⪰\u{338}"), + ("nsup", "⊅"), + ("nsupE", "⫆\u{338}"), + ("nsupe", "⊉"), + ("nsupset", "⊃\u{20d2}"), + ("nsupseteq", "⊉"), + ("nsupseteqq", "⫆\u{338}"), + ("ntgl", "≹"), + ("ntilde", "ñ"), + ("ntlg", "≸"), + ("ntriangleleft", "⋪"), + ("ntrianglelefteq", "⋬"), + ("ntriangleright", "⋫"), + ("ntrianglerighteq", "⋭"), + ("nu", "ν"), + ("num", "#"), + ("numero", "№"), + ("numsp", "\u{2007}"), + ("nvDash", "⊭"), + ("nvHarr", "⤄"), + ("nvap", "≍\u{20d2}"), + ("nvdash", "⊬"), + ("nvge", "≥\u{20d2}"), + ("nvgt", ">\u{20d2}"), + ("nvinfin", "⧞"), + ("nvlArr", "⤂"), + ("nvle", "≤\u{20d2}"), + ("nvlt", "<\u{20d2}"), + ("nvltrie", "⊴\u{20d2}"), + ("nvrArr", "⤃"), + ("nvrtrie", "⊵\u{20d2}"), + ("nvsim", "∼\u{20d2}"), + ("nwArr", "⇖"), + ("nwarhk", "⤣"), + ("nwarr", "↖"), + ("nwarrow", "↖"), + ("nwnear", "⤧"), + ("oS", "Ⓢ"), + ("oacute", "ó"), + ("oast", "⊛"), + ("ocir", "⊚"), + ("ocirc", "ô"), + ("ocy", "о"), + ("odash", "⊝"), + ("odblac", "ő"), + ("odiv", "⨸"), + ("odot", "⊙"), + ("odsold", "⦼"), + ("oelig", "œ"), + ("ofcir", "⦿"), + ("ofr", "𝔬"), + ("ogon", "˛"), + ("ograve", "ò"), + ("ogt", "⧁"), + ("ohbar", "⦵"), + ("ohm", "Ω"), + ("oint", "∮"), + ("olarr", "↺"), + ("olcir", "⦾"), + ("olcross", "⦻"), + ("oline", "‾"), + ("olt", "⧀"), + ("omacr", "ō"), + ("omega", "ω"), + ("omicron", "ο"), + ("omid", "⦶"), + ("ominus", "⊖"), + ("oopf", "𝕠"), + ("opar", "⦷"), + ("operp", "⦹"), + ("oplus", "⊕"), + ("or", "∨"), + ("orarr", "↻"), + ("ord", "⩝"), + ("order", "ℴ"), + ("orderof", "ℴ"), + ("ordf", "ª"), + ("ordm", "º"), + ("origof", "⊶"), + ("oror", "⩖"), + ("orslope", "⩗"), + ("orv", "⩛"), + ("oscr", "ℴ"), + ("oslash", "ø"), + ("osol", "⊘"), + ("otilde", "õ"), + ("otimes", "⊗"), + ("otimesas", "⨶"), + ("ouml", "ö"), + ("ovbar", "⌽"), + ("par", "∥"), + ("para", "¶"), + ("parallel", "∥"), + ("parsim", "⫳"), + ("parsl", "⫽"), + ("part", "∂"), + ("pcy", "п"), + ("percnt", "%"), + ("period", "."), + ("permil", "‰"), + ("perp", "⊥"), + ("pertenk", "‱"), + ("pfr", "𝔭"), + ("phi", "φ"), + ("phiv", "ϕ"), + ("phmmat", "ℳ"), + ("phone", "☎"), + ("pi", "π"), + ("pitchfork", "⋔"), + ("piv", "ϖ"), + ("planck", "ℏ"), + ("planckh", "ℎ"), + ("plankv", "ℏ"), + ("plus", "+"), + ("plusacir", "⨣"), + ("plusb", "⊞"), + ("pluscir", "⨢"), + ("plusdo", "∔"), + ("plusdu", "⨥"), + ("pluse", "⩲"), + ("plusmn", "±"), + ("plussim", "⨦"), + ("plustwo", "⨧"), + ("pm", "±"), + ("pointint", "⨕"), + ("popf", "𝕡"), + ("pound", "£"), + ("pr", "≺"), + ("prE", "⪳"), + ("prap", "⪷"), + ("prcue", "≼"), + ("pre", "⪯"), + ("prec", "≺"), + ("precapprox", "⪷"), + ("preccurlyeq", "≼"), + ("preceq", "⪯"), + ("precnapprox", "⪹"), + ("precneqq", "⪵"), + ("precnsim", "⋨"), + ("precsim", "≾"), + ("prime", "′"), + ("primes", "ℙ"), + ("prnE", "⪵"), + ("prnap", "⪹"), + ("prnsim", "⋨"), + ("prod", "∏"), + ("profalar", "⌮"), + ("profline", "⌒"), + ("profsurf", "⌓"), + ("prop", "∝"), + ("propto", "∝"), + ("prsim", "≾"), + ("prurel", "⊰"), + ("pscr", "𝓅"), + ("psi", "ψ"), + ("puncsp", "\u{2008}"), + ("qfr", "𝔮"), + ("qint", "⨌"), + ("qopf", "𝕢"), + ("qprime", "⁗"), + ("qscr", "𝓆"), + ("quaternions", "ℍ"), + ("quatint", "⨖"), + ("quest", "?"), + ("questeq", "≟"), + ("quot", "\""), + ("rAarr", "⇛"), + ("rArr", "⇒"), + ("rAtail", "⤜"), + ("rBarr", "⤏"), + ("rHar", "⥤"), + ("race", "∽\u{331}"), + ("racute", "ŕ"), + ("radic", "√"), + ("raemptyv", "⦳"), + ("rang", "⟩"), + ("rangd", "⦒"), + ("range", "⦥"), + ("rangle", "⟩"), + ("raquo", "»"), + ("rarr", "→"), + ("rarrap", "⥵"), + ("rarrb", "⇥"), + ("rarrbfs", "⤠"), + ("rarrc", "⤳"), + ("rarrfs", "⤞"), + ("rarrhk", "↪"), + ("rarrlp", "↬"), + ("rarrpl", "⥅"), + ("rarrsim", "⥴"), + ("rarrtl", "↣"), + ("rarrw", "↝"), + ("ratail", "⤚"), + ("ratio", "∶"), + ("rationals", "ℚ"), + ("rbarr", "⤍"), + ("rbbrk", "❳"), + ("rbrace", "}"), + ("rbrack", "]"), + ("rbrke", "⦌"), + ("rbrksld", "⦎"), + ("rbrkslu", "⦐"), + ("rcaron", "ř"), + ("rcedil", "ŗ"), + ("rceil", "⌉"), + ("rcub", "}"), + ("rcy", "р"), + ("rdca", "⤷"), + ("rdldhar", "⥩"), + ("rdquo", "”"), + ("rdquor", "”"), + ("rdsh", "↳"), + ("real", "ℜ"), + ("realine", "ℛ"), + ("realpart", "ℜ"), + ("reals", "ℝ"), + ("rect", "▭"), + ("reg", "®"), + ("rfisht", "⥽"), + ("rfloor", "⌋"), + ("rfr", "𝔯"), + ("rhard", "⇁"), + ("rharu", "⇀"), + ("rharul", "⥬"), + ("rho", "ρ"), + ("rhov", "ϱ"), + ("rightarrow", "→"), + ("rightarrowtail", "↣"), + ("rightharpoondown", "⇁"), + ("rightharpoonup", "⇀"), + ("rightleftarrows", "⇄"), + ("rightleftharpoons", "⇌"), + ("rightrightarrows", "⇉"), + ("rightsquigarrow", "↝"), + ("rightthreetimes", "⋌"), + ("ring", "˚"), + ("risingdotseq", "≓"), + ("rlarr", "⇄"), + ("rlhar", "⇌"), + ("rlm", "\u{200f}"), + ("rmoust", "⎱"), + ("rmoustache", "⎱"), + ("rnmid", "⫮"), + ("roang", "⟭"), + ("roarr", "⇾"), + ("robrk", "⟧"), + ("ropar", "⦆"), + ("ropf", "𝕣"), + ("roplus", "⨮"), + ("rotimes", "⨵"), + ("rpar", ")"), + ("rpargt", "⦔"), + ("rppolint", "⨒"), + ("rrarr", "⇉"), + ("rsaquo", "›"), + ("rscr", "𝓇"), + ("rsh", "↱"), + ("rsqb", "]"), + ("rsquo", "’"), + ("rsquor", "’"), + ("rthree", "⋌"), + ("rtimes", "⋊"), + ("rtri", "▹"), + ("rtrie", "⊵"), + ("rtrif", "▸"), + ("rtriltri", "⧎"), + ("ruluhar", "⥨"), + ("rx", "℞"), + ("sacute", "ś"), + ("sbquo", "‚"), + ("sc", "≻"), + ("scE", "⪴"), + ("scap", "⪸"), + ("scaron", "š"), + ("sccue", "≽"), + ("sce", "⪰"), + ("scedil", "ş"), + ("scirc", "ŝ"), + ("scnE", "⪶"), + ("scnap", "⪺"), + ("scnsim", "⋩"), + ("scpolint", "⨓"), + ("scsim", "≿"), + ("scy", "с"), + ("sdot", "⋅"), + ("sdotb", "⊡"), + ("sdote", "⩦"), + ("seArr", "⇘"), + ("searhk", "⤥"), + ("searr", "↘"), + ("searrow", "↘"), + ("sect", "§"), + ("semi", ";"), + ("seswar", "⤩"), + ("setminus", "∖"), + ("setmn", "∖"), + ("sext", "✶"), + ("sfr", "𝔰"), + ("sfrown", "⌢"), + ("sharp", "♯"), + ("shchcy", "щ"), + ("shcy", "ш"), + ("shortmid", "∣"), + ("shortparallel", "∥"), + ("shy", "\u{ad}"), + ("sigma", "σ"), + ("sigmaf", "ς"), + ("sigmav", "ς"), + ("sim", "∼"), + ("simdot", "⩪"), + ("sime", "≃"), + ("simeq", "≃"), + ("simg", "⪞"), + ("simgE", "⪠"), + ("siml", "⪝"), + ("simlE", "⪟"), + ("simne", "≆"), + ("simplus", "⨤"), + ("simrarr", "⥲"), + ("slarr", "←"), + ("smallsetminus", "∖"), + ("smashp", "⨳"), + ("smeparsl", "⧤"), + ("smid", "∣"), + ("smile", "⌣"), + ("smt", "⪪"), + ("smte", "⪬"), + ("smtes", "⪬\u{fe00}"), + ("softcy", "ь"), + ("sol", "/"), + ("solb", "⧄"), + ("solbar", "⌿"), + ("sopf", "𝕤"), + ("spades", "♠"), + ("spadesuit", "♠"), + ("spar", "∥"), + ("sqcap", "⊓"), + ("sqcaps", "⊓\u{fe00}"), + ("sqcup", "⊔"), + ("sqcups", "⊔\u{fe00}"), + ("sqsub", "⊏"), + ("sqsube", "⊑"), + ("sqsubset", "⊏"), + ("sqsubseteq", "⊑"), + ("sqsup", "⊐"), + ("sqsupe", "⊒"), + ("sqsupset", "⊐"), + ("sqsupseteq", "⊒"), + ("squ", "□"), + ("square", "□"), + ("squarf", "▪"), + ("squf", "▪"), + ("srarr", "→"), + ("sscr", "𝓈"), + ("ssetmn", "∖"), + ("ssmile", "⌣"), + ("sstarf", "⋆"), + ("star", "☆"), + ("starf", "★"), + ("straightepsilon", "ϵ"), + ("straightphi", "ϕ"), + ("strns", "¯"), + ("sub", "⊂"), + ("subE", "⫅"), + ("subdot", "⪽"), + ("sube", "⊆"), + ("subedot", "⫃"), + ("submult", "⫁"), + ("subnE", "⫋"), + ("subne", "⊊"), + ("subplus", "⪿"), + ("subrarr", "⥹"), + ("subset", "⊂"), + ("subseteq", "⊆"), + ("subseteqq", "⫅"), + ("subsetneq", "⊊"), + ("subsetneqq", "⫋"), + ("subsim", "⫇"), + ("subsub", "⫕"), + ("subsup", "⫓"), + ("succ", "≻"), + ("succapprox", "⪸"), + ("succcurlyeq", "≽"), + ("succeq", "⪰"), + ("succnapprox", "⪺"), + ("succneqq", "⪶"), + ("succnsim", "⋩"), + ("succsim", "≿"), + ("sum", "∑"), + ("sung", "♪"), + ("sup", "⊃"), + ("sup1", "¹"), + ("sup2", "²"), + ("sup3", "³"), + ("supE", "⫆"), + ("supdot", "⪾"), + ("supdsub", "⫘"), + ("supe", "⊇"), + ("supedot", "⫄"), + ("suphsol", "⟉"), + ("suphsub", "⫗"), + ("suplarr", "⥻"), + ("supmult", "⫂"), + ("supnE", "⫌"), + ("supne", "⊋"), + ("supplus", "⫀"), + ("supset", "⊃"), + ("supseteq", "⊇"), + ("supseteqq", "⫆"), + ("supsetneq", "⊋"), + ("supsetneqq", "⫌"), + ("supsim", "⫈"), + ("supsub", "⫔"), + ("supsup", "⫖"), + ("swArr", "⇙"), + ("swarhk", "⤦"), + ("swarr", "↙"), + ("swarrow", "↙"), + ("swnwar", "⤪"), + ("szlig", "ß"), + ("target", "⌖"), + ("tau", "τ"), + ("tbrk", "⎴"), + ("tcaron", "ť"), + ("tcedil", "ţ"), + ("tcy", "т"), + ("tdot", "\u{20db}"), + ("telrec", "⌕"), + ("tfr", "𝔱"), + ("there4", "∴"), + ("therefore", "∴"), + ("theta", "θ"), + ("thetasym", "ϑ"), + ("thetav", "ϑ"), + ("thickapprox", "≈"), + ("thicksim", "∼"), + ("thinsp", "\u{2009}"), + ("thkap", "≈"), + ("thksim", "∼"), + ("thorn", "þ"), + ("tilde", "˜"), + ("times", "×"), + ("timesb", "⊠"), + ("timesbar", "⨱"), + ("timesd", "⨰"), + ("tint", "∭"), + ("toea", "⤨"), + ("top", "⊤"), + ("topbot", "⌶"), + ("topcir", "⫱"), + ("topf", "𝕥"), + ("topfork", "⫚"), + ("tosa", "⤩"), + ("tprime", "‴"), + ("trade", "™"), + ("triangle", "▵"), + ("triangledown", "▿"), + ("triangleleft", "◃"), + ("trianglelefteq", "⊴"), + ("triangleq", "≜"), + ("triangleright", "▹"), + ("trianglerighteq", "⊵"), + ("tridot", "◬"), + ("trie", "≜"), + ("triminus", "⨺"), + ("triplus", "⨹"), + ("trisb", "⧍"), + ("tritime", "⨻"), + ("trpezium", "⏢"), + ("tscr", "𝓉"), + ("tscy", "ц"), + ("tshcy", "ћ"), + ("tstrok", "ŧ"), + ("twixt", "≬"), + ("twoheadleftarrow", "↞"), + ("twoheadrightarrow", "↠"), + ("uArr", "⇑"), + ("uHar", "⥣"), + ("uacute", "ú"), + ("uarr", "↑"), + ("ubrcy", "ў"), + ("ubreve", "ŭ"), + ("ucirc", "û"), + ("ucy", "у"), + ("udarr", "⇅"), + ("udblac", "ű"), + ("udhar", "⥮"), + ("ufisht", "⥾"), + ("ufr", "𝔲"), + ("ugrave", "ù"), + ("uharl", "↿"), + ("uharr", "↾"), + ("uhblk", "▀"), + ("ulcorn", "⌜"), + ("ulcorner", "⌜"), + ("ulcrop", "⌏"), + ("ultri", "◸"), + ("umacr", "ū"), + ("uml", "¨"), + ("uogon", "ų"), + ("uopf", "𝕦"), + ("uparrow", "↑"), + ("updownarrow", "↕"), + ("upharpoonleft", "↿"), + ("upharpoonright", "↾"), + ("uplus", "⊎"), + ("upsi", "υ"), + ("upsih", "ϒ"), + ("upsilon", "υ"), + ("upuparrows", "⇈"), + ("urcorn", "⌝"), + ("urcorner", "⌝"), + ("urcrop", "⌎"), + ("uring", "ů"), + ("urtri", "◹"), + ("uscr", "𝓊"), + ("utdot", "⋰"), + ("utilde", "ũ"), + ("utri", "▵"), + ("utrif", "▴"), + ("uuarr", "⇈"), + ("uuml", "ü"), + ("uwangle", "⦧"), + ("vArr", "⇕"), + ("vBar", "⫨"), + ("vBarv", "⫩"), + ("vDash", "⊨"), + ("vangrt", "⦜"), + ("varepsilon", "ϵ"), + ("varkappa", "ϰ"), + ("varnothing", "∅"), + ("varphi", "ϕ"), + ("varpi", "ϖ"), + ("varpropto", "∝"), + ("varr", "↕"), + ("varrho", "ϱ"), + ("varsigma", "ς"), + ("varsubsetneq", "⊊\u{fe00}"), + ("varsubsetneqq", "⫋\u{fe00}"), + ("varsupsetneq", "⊋\u{fe00}"), + ("varsupsetneqq", "⫌\u{fe00}"), + ("vartheta", "ϑ"), + ("vartriangleleft", "⊲"), + ("vartriangleright", "⊳"), + ("vcy", "в"), + ("vdash", "⊢"), + ("vee", "∨"), + ("veebar", "⊻"), + ("veeeq", "≚"), + ("vellip", "⋮"), + ("verbar", "|"), + ("vert", "|"), + ("vfr", "𝔳"), + ("vltri", "⊲"), + ("vnsub", "⊂\u{20d2}"), + ("vnsup", "⊃\u{20d2}"), + ("vopf", "𝕧"), + ("vprop", "∝"), + ("vrtri", "⊳"), + ("vscr", "𝓋"), + ("vsubnE", "⫋\u{fe00}"), + ("vsubne", "⊊\u{fe00}"), + ("vsupnE", "⫌\u{fe00}"), + ("vsupne", "⊋\u{fe00}"), + ("vzigzag", "⦚"), + ("wcirc", "ŵ"), + ("wedbar", "⩟"), + ("wedge", "∧"), + ("wedgeq", "≙"), + ("weierp", "℘"), + ("wfr", "𝔴"), + ("wopf", "𝕨"), + ("wp", "℘"), + ("wr", "≀"), + ("wreath", "≀"), + ("wscr", "𝓌"), + ("xcap", "⋂"), + ("xcirc", "◯"), + ("xcup", "⋃"), + ("xdtri", "▽"), + ("xfr", "𝔵"), + ("xhArr", "⟺"), + ("xharr", "⟷"), + ("xi", "ξ"), + ("xlArr", "⟸"), + ("xlarr", "⟵"), + ("xmap", "⟼"), + ("xnis", "⋻"), + ("xodot", "⨀"), + ("xopf", "𝕩"), + ("xoplus", "⨁"), + ("xotime", "⨂"), + ("xrArr", "⟹"), + ("xrarr", "⟶"), + ("xscr", "𝓍"), + ("xsqcup", "⨆"), + ("xuplus", "⨄"), + ("xutri", "△"), + ("xvee", "⋁"), + ("xwedge", "⋀"), + ("yacute", "ý"), + ("yacy", "я"), + ("ycirc", "ŷ"), + ("ycy", "ы"), + ("yen", "¥"), + ("yfr", "𝔶"), + ("yicy", "ї"), + ("yopf", "𝕪"), + ("yscr", "𝓎"), + ("yucy", "ю"), + ("yuml", "ÿ"), + ("zacute", "ź"), + ("zcaron", "ž"), + ("zcy", "з"), + ("zdot", "ż"), + ("zeetrf", "ℨ"), + ("zeta", "ζ"), + ("zfr", "𝔷"), + ("zhcy", "ж"), + ("zigrarr", "⇝"), + ("zopf", "𝕫"), + ("zscr", "𝓏"), + ("zwj", "\u{200d}"), + ("zwnj", "\u{200c}"), +]; + +pub(crate) fn named(name: &str) -> Option<&'static str> { + NAMED.binary_search_by_key(&name, |(n, _)| *n).ok().map(|i| NAMED[i].1) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn the_table_is_sorted_so_binary_search_holds() { + assert!(NAMED.windows(2).all(|w| w[0].0 < w[1].0)); + } + + #[test] + fn resolves_the_names_mathml_actually_uses() { + assert_eq!(named("alpha"), Some("\u{3b1}")); + assert_eq!(named("sum"), Some("\u{2211}")); + assert_eq!(named("InvisibleTimes"), Some("\u{2062}")); + assert_eq!(named("nleqq"), Some("\u{2266}\u{338}")); + assert_eq!(named("nope"), None); + } + + #[test] + fn names_are_case_sensitive() { + assert_ne!(named("Sigma"), named("sigma")); + } +} diff --git a/src/package/mod.rs b/src/package/mod.rs index a3c974f7..6ad67881 100644 --- a/src/package/mod.rs +++ b/src/package/mod.rs @@ -3,6 +3,7 @@ //! OPC/EPUB target resolution. pub mod archive; +mod entities; pub mod limits; pub mod path; pub mod relationships; diff --git a/src/package/xml.rs b/src/package/xml.rs index 57ce456e..baeb3115 100644 --- a/src/package/xml.rs +++ b/src/package/xml.rs @@ -440,54 +440,7 @@ fn resolve_entity(name: &str) -> Option { }; return char::from_u32(code).map(String::from); } - let ch = match name { - "amp" => '&', - "lt" => '<', - "gt" => '>', - "apos" => '\'', - "quot" => '"', - "nbsp" => '\u{a0}', - "shy" => '\u{ad}', - "mdash" => '\u{2014}', - "ndash" => '\u{2013}', - "lsquo" => '\u{2018}', - "rsquo" => '\u{2019}', - "ldquo" => '\u{201c}', - "rdquo" => '\u{201d}', - "hellip" => '\u{2026}', - "copy" => '\u{a9}', - "reg" => '\u{ae}', - "trade" => '\u{2122}', - "deg" => '\u{b0}', - "middot" => '\u{b7}', - "bull" => '\u{2022}', - "sect" => '\u{a7}', - "para" => '\u{b6}', - "laquo" => '\u{ab}', - "raquo" => '\u{bb}', - "times" => '\u{d7}', - "divide" => '\u{f7}', - "plusmn" => '\u{b1}', - "frac12" => '\u{bd}', - "frac14" => '\u{bc}', - "eacute" => '\u{e9}', - "egrave" => '\u{e8}', - "agrave" => '\u{e0}', - "ccedil" => '\u{e7}', - "uuml" => '\u{fc}', - "ouml" => '\u{f6}', - "auml" => '\u{e4}', - "szlig" => '\u{df}', - "aring" => '\u{e5}', - "oslash" => '\u{f8}', - "aelig" => '\u{e6}', - "euro" => '\u{20ac}', - "pound" => '\u{a3}', - "yen" => '\u{a5}', - "cent" => '\u{a2}', - _ => return None, - }; - Some(ch.to_string()) + super::entities::named(name).map(String::from) } #[cfg(test)] From af17a403568ba9f1f91c878d6168d6ace631d3d4 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 16:04:59 +0300 Subject: [PATCH 05/13] feat(math): translate MathML that carries no TeX annotation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The EPUB path only recovered an equation when the producer had embedded a TeX annotation. Most do not, and without one the markup reached the writer as its bare characters: a over 1 and 3 read as "13", which is not a degraded fraction but a different number. Structure is what gets translated. Glyphs are left alone — KaTeX takes Unicode operators and Greek directly and has metrics for them, so mapping them to commands would only be a chance to pick the wrong one. The invisible operators are the exception: they carry no glyph, so passing them through would put unreadable codepoints in front of a reader. Covers the presentation set: scripts (including mmultiscripts), fractions, radicals, under/over with limits and accents, tables, mfenced, semantics, and the token elements. Symbols are trimmed and prose is not, so the space in if survives as a word boundary. The escaping and Unicode-to-command tables move to shared/latex.rs, which both math frontends now resolve against, with the same rule as before: an unmapped glyph is passed through, never guessed at. --- src/formats/docx/omml.rs | 156 +---- src/shared/html.rs | 6 +- src/shared/latex.rs | 158 +++++ src/shared/mathml.rs | 548 ++++++++++++++++++ src/shared/mod.rs | 2 + .../snapshots__epub__handmade-math.epub.snap | 2 +- 6 files changed, 718 insertions(+), 154 deletions(-) create mode 100644 src/shared/latex.rs create mode 100644 src/shared/mathml.rs diff --git a/src/formats/docx/omml.rs b/src/formats/docx/omml.rs index e39d2182..b1e2257a 100644 --- a/src/formats/docx/omml.rs +++ b/src/formats/docx/omml.rs @@ -8,17 +8,16 @@ use crate::model::Inline; use crate::package::xml::{Element, ns}; +use crate::shared::latex::{ + KATEX_FUNCTIONS, MAX_LATEX_BYTES, accent_command, delim_glyph, delim_sep, escaped, + group_command, nary_command, push_text, +}; /// Deepest OMML nesting translated. Real mathematics stays under a dozen /// levels; past this the subtree degrades to its text, which keeps a hostile /// document from driving recursion on a bounded stack. const MAX_DEPTH: usize = 64; -/// Largest LaTeX one equation may produce. Wrappers multiply through nesting -/// (`\left(\right)` is twelve characters per level), so a small document can -/// otherwise amplify without bound. A dense page of mathematics is under 4 KiB. -const MAX_LATEX_BYTES: usize = 64 * 1024; - /// Most cells one matrix may contribute. const MAX_MATRIX_CELLS: usize = 10_000; @@ -320,42 +319,6 @@ fn parse_elem(elem: &Element, depth: usize) -> Option { } } -/// LaTeX-escape literal text. A `$` would close the surrounding math span and -/// hand the rest of the document to the Markdown parser, so it is escaped -/// here rather than trusted; newlines cannot appear inside inline math. -fn push_text(text: &str, out: &mut String) { - // A command runs until a non-letter, so `\int` + `f` would lex as `\intf`. - if text.starts_with(|c: char| c.is_ascii_alphabetic()) && ends_with_control_word(out) { - out.push(' '); - } - for c in text.chars() { - match c { - '\\' => out.push_str("\\backslash "), - '{' => out.push_str("\\{"), - '}' => out.push_str("\\}"), - '$' => out.push_str("\\$"), - '&' => out.push_str("\\&"), - '#' => out.push_str("\\#"), - '%' => out.push_str("\\%"), - '_' => out.push_str("\\_"), - '^' => out.push_str("\\^{}"), - '~' => out.push_str("\\~{}"), - '\n' | '\r' => out.push(' '), - c => out.push(c), - } - } -} - -/// True when `out` ends in a LaTeX control word, whose name would absorb a -/// following letter. -fn ends_with_control_word(out: &str) -> bool { - let trailing_letters = - out.len() - out.trim_end_matches(|c: char| c.is_ascii_alphabetic()).len(); - trailing_letters > 0 - && out[..out.len() - trailing_letters].ends_with('\\') - && !out[..out.len() - trailing_letters].ends_with("\\\\") -} - /// A script's base needs braces when it already ends in a script (`y^{2}^{3}` /// is an error) or when it is a sequence the script would otherwise bind only /// the last atom of. A fraction, radical or delimiter is a single atom already. @@ -569,117 +532,6 @@ fn emit(node: &Node, out: &mut String, _in_group: bool) { } } -/// Operator names KaTeX spells with a leading backslash. -const KATEX_FUNCTIONS: &[&str] = &[ - "arccos", "arcsin", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc", "deg", "det", "dim", - "exp", "gcd", "hom", "inf", "ker", "lg", "lim", "liminf", "limsup", "ln", "log", "max", "min", - "sec", "sin", "sinh", "sup", "tan", "tanh", -]; - -fn nary_command(chr: char) -> String { - match chr { - '∑' => "\\sum", - '∏' => "\\prod", - '∐' => "\\coprod", - '∫' => "\\int", - '∬' => "\\iint", - '∭' => "\\iiint", - '∮' => "\\oint", - '∯' => "\\oiint", - '∰' => "\\oiiint", - '⋀' => "\\bigwedge", - '⋁' => "\\bigvee", - '⋂' => "\\bigcap", - '⋃' => "\\bigcup", - '⨀' => "\\bigodot", - '⨁' => "\\bigoplus", - '⨂' => "\\bigotimes", - '⨄' => "\\biguplus", - '⨆' => "\\bigsqcup", - // Passing the glyph through says what the document said; guessing a - // command would emit a different operator, well-formed and wrong. - _ => return escaped(chr), - } - .into() -} - -/// OMML gives a combining codepoint; KaTeX wants the accent command. An -/// unmapped mark is stacked over the base rather than replaced by a guess. -fn accent_command(chr: char) -> Option<&'static str> { - match chr { - '\u{0300}' => "\\grave", - '\u{0301}' => "\\acute", - '\u{0302}' | '^' => "\\widehat", - '\u{0303}' | '~' => "\\widetilde", - '\u{0304}' => "\\bar", - '\u{0305}' => "\\overline", - '\u{0306}' => "\\breve", - '\u{0307}' => "\\dot", - '\u{0308}' => "\\ddot", - '\u{030A}' => "\\mathring", - '\u{030C}' => "\\check", - '\u{0332}' => "\\underline", - '\u{20D6}' => "\\overleftarrow", - '\u{20D7}' | '→' => "\\vec", - '\u{20DB}' => "\\dddot", - '\u{20E1}' => "\\overleftrightarrow", - _ => return None, - } - .into() -} - -fn group_command(chr: char) -> Option<&'static str> { - match chr { - '\u{23DE}' | '\u{FE37}' => "\\overbrace", - '\u{23B4}' => "\\overbracket", - '\u{23B5}' => "\\underbracket", - '\u{23DC}' => "\\overgroup", - '\u{23DD}' => "\\undergroup", - '←' => "\\overleftarrow", - '→' => "\\overrightarrow", - '\u{23DF}' | '\u{FE38}' => "\\underbrace", - _ => return None, - } - .into() -} - -/// A delimiter glyph in `\left`/`\right` position. An absent delimiter is a -/// bare `.`, which is how LaTeX spells "no glyph but keep the pair balanced". -fn delim_glyph(chr: Option) -> String { - match chr { - None => ".".into(), - Some('{') => "\\{".into(), - Some('}') => "\\}".into(), - Some('|') => "\\vert".into(), - Some('‖') => "\\Vert".into(), - Some('⌈') => "\\lceil".into(), - Some('⌉') => "\\rceil".into(), - Some('⌊') => "\\lfloor".into(), - Some('⌋') => "\\rfloor".into(), - Some('⟨') => "\\langle".into(), - Some('⟩') => "\\rangle".into(), - Some(c) => escaped(c), - } -} - -/// One author-supplied character, LaTeX-escaped. Delimiters and operators come -/// from the document and reach the body outside `push_text`. -fn escaped(chr: char) -> String { - let mut out = String::new(); - push_text(&chr.to_string(), &mut out); - out -} - -/// A separator between delimiter parts. A bare `|` would split a Markdown -/// table row, so it is always spelled as a command. -fn delim_sep(chr: char) -> String { - match chr { - '|' => "\\mid ".into(), - '‖' => "\\Vert ".into(), - c => escaped(c), - } -} - #[cfg(test)] mod tests { use super::*; diff --git a/src/shared/html.rs b/src/shared/html.rs index 8ef05c28..cb619042 100644 --- a/src/shared/html.rs +++ b/src/shared/html.rs @@ -14,6 +14,7 @@ use crate::model::{ use crate::package::xml::{Element, Node}; use crate::shared::delta::{StyleDelta, rebase_emphasis}; use crate::shared::header::resolve_header_rows; +use crate::shared::mathml; use crate::shared::text::{clean_text, collapse_ws}; use std::collections::HashMap; @@ -459,7 +460,10 @@ impl Builder<'_> { self.inlines.push(Inline::Math { latex, display }); return Ok(()); } - log::debug!("MathML without a TeX annotation; keeping its characters only"); + if let Some(math) = mathml::to_inline(elem, display) { + self.inlines.push(math); + return Ok(()); + } self.walk_children(elem, delta) } diff --git a/src/shared/latex.rs b/src/shared/latex.rs new file mode 100644 index 00000000..665ce6df --- /dev/null +++ b/src/shared/latex.rs @@ -0,0 +1,158 @@ +//! Shared LaTeX emission: escaping, and the Unicode-to-command tables the +//! math frontends resolve against. +//! +//! An unmapped glyph is passed through rather than guessed at. A guess emits a +//! different operator, well-formed and wrong, which is worse than a character +//! the reader can still see. + +/// Largest LaTeX one equation may produce. Wrappers multiply through nesting +/// (`\left(\right)` is twelve characters per level), so a small document can +/// otherwise amplify without bound. A dense page of mathematics is under 4 KiB. +pub(crate) const MAX_LATEX_BYTES: usize = 64 * 1024; + +/// LaTeX-escape literal text. A `$` would close the surrounding math span and +/// hand the rest of the document to the Markdown parser, so it is escaped +/// here rather than trusted; newlines cannot appear inside inline math. +pub(crate) fn push_text(text: &str, out: &mut String) { + // A command runs until a non-letter, so `\int` + `f` would lex as `\intf`. + if text.starts_with(|c: char| c.is_ascii_alphabetic()) && ends_with_control_word(out) { + out.push(' '); + } + for c in text.chars() { + match c { + '\\' => out.push_str("\\backslash "), + '{' => out.push_str("\\{"), + '}' => out.push_str("\\}"), + '$' => out.push_str("\\$"), + '&' => out.push_str("\\&"), + '#' => out.push_str("\\#"), + '%' => out.push_str("\\%"), + '_' => out.push_str("\\_"), + '^' => out.push_str("\\^{}"), + '~' => out.push_str("\\~{}"), + '\n' | '\r' => out.push(' '), + c => out.push(c), + } + } +} + +/// True when `out` ends in a LaTeX control word, whose name would absorb a +/// following letter. +pub(crate) fn ends_with_control_word(out: &str) -> bool { + let trailing_letters = + out.len() - out.trim_end_matches(|c: char| c.is_ascii_alphabetic()).len(); + trailing_letters > 0 + && out[..out.len() - trailing_letters].ends_with('\\') + && !out[..out.len() - trailing_letters].ends_with("\\\\") +} + +/// Operator names KaTeX spells with a leading backslash. +pub(crate) const KATEX_FUNCTIONS: &[&str] = &[ + "arccos", "arcsin", "arctan", "arg", "cos", "cosh", "cot", "coth", "csc", "deg", "det", "dim", + "exp", "gcd", "hom", "inf", "ker", "lg", "lim", "liminf", "limsup", "ln", "log", "max", "min", + "sec", "sin", "sinh", "sup", "tan", "tanh", +]; + +pub(crate) fn nary_command(chr: char) -> String { + match chr { + '∑' => "\\sum", + '∏' => "\\prod", + '∐' => "\\coprod", + '∫' => "\\int", + '∬' => "\\iint", + '∭' => "\\iiint", + '∮' => "\\oint", + '∯' => "\\oiint", + '∰' => "\\oiiint", + '⋀' => "\\bigwedge", + '⋁' => "\\bigvee", + '⋂' => "\\bigcap", + '⋃' => "\\bigcup", + '⨀' => "\\bigodot", + '⨁' => "\\bigoplus", + '⨂' => "\\bigotimes", + '⨄' => "\\biguplus", + '⨆' => "\\bigsqcup", + // Passing the glyph through says what the document said; guessing a + // command would emit a different operator, well-formed and wrong. + _ => return escaped(chr), + } + .into() +} + +/// OMML gives a combining codepoint; KaTeX wants the accent command. An +/// unmapped mark is stacked over the base rather than replaced by a guess. +pub(crate) fn accent_command(chr: char) -> Option<&'static str> { + match chr { + '\u{0300}' => "\\grave", + '\u{0301}' => "\\acute", + '\u{0302}' | '^' => "\\widehat", + '\u{0303}' | '~' => "\\widetilde", + '\u{0304}' => "\\bar", + '\u{0305}' => "\\overline", + '\u{0306}' => "\\breve", + '\u{0307}' => "\\dot", + '\u{0308}' => "\\ddot", + '\u{030A}' => "\\mathring", + '\u{030C}' => "\\check", + '\u{0332}' => "\\underline", + '\u{20D6}' => "\\overleftarrow", + '\u{20D7}' | '→' => "\\vec", + '\u{20DB}' => "\\dddot", + '\u{20E1}' => "\\overleftrightarrow", + _ => return None, + } + .into() +} + +pub(crate) fn group_command(chr: char) -> Option<&'static str> { + match chr { + '\u{23DE}' | '\u{FE37}' => "\\overbrace", + '\u{23B4}' => "\\overbracket", + '\u{23B5}' => "\\underbracket", + '\u{23DC}' => "\\overgroup", + '\u{23DD}' => "\\undergroup", + '←' => "\\overleftarrow", + '→' => "\\overrightarrow", + '\u{23DF}' | '\u{FE38}' => "\\underbrace", + _ => return None, + } + .into() +} + +/// A delimiter glyph in `\left`/`\right` position. An absent delimiter is a +/// bare `.`, which is how LaTeX spells "no glyph but keep the pair balanced". +pub(crate) fn delim_glyph(chr: Option) -> String { + match chr { + None => ".".into(), + Some('{') => "\\{".into(), + Some('}') => "\\}".into(), + Some('|') => "\\vert".into(), + Some('‖') => "\\Vert".into(), + Some('⌈') => "\\lceil".into(), + Some('⌉') => "\\rceil".into(), + Some('⌊') => "\\lfloor".into(), + Some('⌋') => "\\rfloor".into(), + Some('⟨') => "\\langle".into(), + Some('⟩') => "\\rangle".into(), + Some(c) => escaped(c), + } +} + +/// One author-supplied character, LaTeX-escaped. Delimiters and operators come +/// from the document and reach the body outside `push_text`. +pub(crate) fn escaped(chr: char) -> String { + let mut out = String::new(); + push_text(&chr.to_string(), &mut out); + out +} + +/// A separator between delimiter parts. A bare `|` would split a Markdown +/// table row, so it is always spelled as a command. +pub(crate) fn delim_sep(chr: char) -> String { + match chr { + '|' => "\\mid ".into(), + '‖' => "\\Vert ".into(), + c => escaped(c), + } +} diff --git a/src/shared/mathml.rs b/src/shared/mathml.rs new file mode 100644 index 00000000..6678c41f --- /dev/null +++ b/src/shared/mathml.rs @@ -0,0 +1,548 @@ +//! MathML presentation markup to LaTeX. +//! +//! Only part of the MathML in the wild carries a TeX annotation. Without one +//! the markup used to reach the writer as its bare characters, which keeps the +//! glyphs and drops every relation the layout expressed: a `` over 1 and +//! 3 read as `13`. +//! +//! Structure is translated; glyphs are not. KaTeX accepts Unicode operators and +//! Greek directly and has metrics for them, so `α` and `∑` are left as the +//! document wrote them and only the invisible operators, which would otherwise +//! reach the reader as nothing at all, are dropped. + +use crate::model::Inline; +use crate::package::xml::Element; +use crate::shared::latex::{ + KATEX_FUNCTIONS, MAX_LATEX_BYTES, accent_command, delim_glyph, delim_sep, push_text, +}; + +const MAX_DEPTH: usize = 64; +const MAX_TABLE_CELLS: usize = 10_000; + +/// Operators that take their scripts as limits rather than as corner scripts. +const BIG_OPERATORS: &str = "∑∏∐∫∬∭∮∯∰⋀⋁⋂⋃⨀⨁⨂⨄⨆"; + +pub(crate) fn to_inline(math: &Element, display: bool) -> Option { + let mut out = String::new(); + emit_children(math, &mut out, 0); + let latex = out.trim().to_string(); + if latex.is_empty() { + return None; + } + Some(Inline::Math { latex, display }) +} + +fn emit_children(parent: &Element, out: &mut String, depth: usize) { + for child in parent.child_elems() { + emit(child, out, depth); + } +} + +fn arg(parent: &Element, index: usize) -> Option<&Element> { + parent.child_elems().nth(index) +} + +/// Emit `elem` as a braced group. An absent argument still needs its braces: +/// `\frac{a}` is a syntax error where `\frac{a}{}` is not. +fn group(elem: Option<&Element>, out: &mut String, depth: usize) { + out.push('{'); + if let Some(elem) = elem { + emit(elem, out, depth); + } + out.push('}'); +} + +/// A base needs no braces when it is one character or one control word; +/// anything longer would let the script bind to its last atom only. Braces +/// around a large operator would also cost it its limits, so the two cases +/// that may go bare are exactly the two that are already single atoms. +fn emit_base(elem: Option<&Element>, out: &mut String, depth: usize) { + let mut base = String::new(); + if let Some(elem) = elem { + emit(elem, &mut base, depth); + } + let bare = base.chars().count() == 1 + || (base.starts_with('\\') && base[1..].chars().all(|c| c.is_ascii_alphabetic())); + if bare { + out.push_str(&base); + } else { + out.push('{'); + out.push_str(&base); + out.push('}'); + } +} + +fn emit(elem: &Element, out: &mut String, depth: usize) { + if out.len() >= MAX_LATEX_BYTES { + return; + } + if depth > MAX_DEPTH { + log::debug!("MathML nested past {MAX_DEPTH} levels; the subtree degrades to its text"); + push_text(elem.text().trim(), out); + return; + } + let depth = depth + 1; + match elem.local.as_str() { + "mi" => identifier(elem, out), + "mn" => push_text(elem.text().trim(), out), + "mo" => operator(elem, out), + "mtext" => text_run(&collapsed(&elem.text()), out), + "ms" => quoted(elem, out), + "mspace" => space(elem, out), + "mglyph" => push_text(elem.attr_any("alt").unwrap_or_default(), out), + "mphantom" => { + out.push_str("\\phantom"); + out.push('{'); + emit_children(elem, out, depth); + out.push('}'); + } + "msqrt" => { + out.push_str("\\sqrt{"); + emit_children(elem, out, depth); + out.push('}'); + } + "mroot" => { + out.push_str("\\sqrt["); + if let Some(index) = arg(elem, 1) { + emit(index, out, depth); + } + out.push_str("]{"); + if let Some(base) = arg(elem, 0) { + emit(base, out, depth); + } + out.push('}'); + } + "mfrac" => fraction(elem, out, depth), + "msub" => { + emit_base(arg(elem, 0), out, depth); + out.push('_'); + group(arg(elem, 1), out, depth); + } + "msup" => { + emit_base(arg(elem, 0), out, depth); + out.push('^'); + group(arg(elem, 1), out, depth); + } + "msubsup" => { + emit_base(arg(elem, 0), out, depth); + out.push('_'); + group(arg(elem, 1), out, depth); + out.push('^'); + group(arg(elem, 2), out, depth); + } + "munder" => under_over(elem, out, depth, true, false), + "mover" => under_over(elem, out, depth, false, true), + "munderover" => under_over(elem, out, depth, true, true), + "mmultiscripts" => multiscripts(elem, out, depth), + "mtable" => table(elem, out, depth), + "mfenced" => fenced(elem, out, depth), + "semantics" => { + let presentation = elem + .child_elems() + .find(|c| !matches!(c.local.as_str(), "annotation" | "annotation-xml")); + if let Some(presentation) = presentation { + emit(presentation, out, depth); + } + } + "maction" => { + if let Some(shown) = arg(elem, 0) { + emit(shown, out, depth); + } + } + "annotation" | "annotation-xml" | "none" | "mprescripts" => {} + _ => emit_children(elem, out, depth), + } +} + +/// A single character is a variable and italic already; anything longer is a +/// name, which MathML sets upright and LaTeX would otherwise set as a product +/// of its letters. +fn identifier(elem: &Element, out: &mut String) { + let text = elem.text(); + let text = text.trim(); + if text.is_empty() { + return; + } + if KATEX_FUNCTIONS.contains(&text) { + out.push('\\'); + out.push_str(text); + return; + } + if text.chars().count() == 1 && elem.attr_any("mathvariant") != Some("normal") { + push_text(text, out); + return; + } + out.push_str("\\mathrm{"); + push_text(text, out); + out.push('}'); +} + +/// The invisible operators carry grouping that the surrounding markup already +/// states, and no glyph. Passing them through would put unreadable codepoints +/// in front of a reader. +fn operator(elem: &Element, out: &mut String) { + let text = elem.text(); + let text = text.trim(); + if text.is_empty() || text.chars().all(|c| ('\u{2061}'..='\u{2064}').contains(&c)) { + return; + } + for chr in text.chars() { + out.push_str(&delim_sep(chr)); + } +} + +fn text_run(text: &str, out: &mut String) { + if text.is_empty() { + return; + } + out.push_str("\\text{"); + push_text(text, out); + out.push('}'); +} + +fn quoted(elem: &Element, out: &mut String) { + let open = elem.attr_any("lquote").unwrap_or("\""); + let close = elem.attr_any("rquote").unwrap_or("\""); + text_run(&format!("{open}{}{close}", collapsed(&elem.text())), out); +} + +/// Symbols are trimmed, prose is not: a space between `if ` and +/// what follows is a word boundary, and dropping it reads as one word. +fn collapsed(text: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut in_space = false; + for chr in text.chars() { + if chr.is_whitespace() { + in_space = true; + continue; + } + if in_space { + out.push(' '); + } + in_space = false; + out.push(chr); + } + if in_space && !out.is_empty() { + out.push(' '); + } + out +} + +/// Only a CSS length can be handed to `\hspace`; the named widths +/// (`thickmathspace`) have no LaTeX spelling and are dropped rather than +/// guessed at. +fn space(elem: &Element, out: &mut String) { + let Some(width) = elem.attr_any("width") else { + return; + }; + let unit_at = width.find(|c: char| c.is_ascii_alphabetic()).unwrap_or(0); + let (amount, unit) = width.split_at(unit_at); + if amount.parse::().is_ok() + && matches!(unit, "em" | "ex" | "px" | "pt" | "cm" | "mm" | "in" | "pc") + { + out.push_str("\\hspace{"); + out.push_str(width); + out.push('}'); + } +} + +fn fraction(elem: &Element, out: &mut String, depth: usize) { + // A zero rule is a choose-style stack, which `\frac` would draw a bar under. + if elem.attr_any("linethickness").is_some_and(is_zero_length) { + out.push('{'); + group(arg(elem, 0), out, depth); + out.push_str("\\atop "); + group(arg(elem, 1), out, depth); + out.push('}'); + return; + } + out.push_str("\\frac"); + group(arg(elem, 0), out, depth); + group(arg(elem, 1), out, depth); +} + +fn is_zero_length(value: &str) -> bool { + let digits = value.trim_end_matches(|c: char| c.is_ascii_alphabetic() || c == '%'); + digits.trim().parse::() == Ok(0.0) +} + +/// A script under or over a large operator is a limit and belongs in `_`/`^`; +/// over anything else it is either an accent or a genuine overset. +fn under_over(elem: &Element, out: &mut String, depth: usize, under: bool, over: bool) { + let base = arg(elem, 0); + let first = arg(elem, 1); + let second = if under && over { arg(elem, 2) } else { None }; + + if let (false, Some(mark)) = (under, first.and_then(single_char)) + && let Some(command) = accent_command(mark) + { + out.push_str(command); + group(base, out, depth); + return; + } + + if base.is_some_and(takes_limits) { + emit_base(base, out, depth); + if under { + out.push('_'); + group(first, out, depth); + } + if over { + out.push('^'); + group(if under { second } else { first }, out, depth); + } + return; + } + + // `\overset` and `\underset` take the script first, so carrying both means + // nesting: the outer command's base is the whole inner one. + let mut stacked = String::new(); + if over { + stacked.push_str("\\overset"); + group(if under { second } else { first }, &mut stacked, depth); + } + if under { + let inner = std::mem::take(&mut stacked); + stacked.push_str("\\underset"); + group(first, &mut stacked, depth); + stacked.push('{'); + stacked.push_str(&inner); + group(base, &mut stacked, depth); + stacked.push('}'); + out.push_str(&stacked); + return; + } + out.push_str(&stacked); + group(base, out, depth); +} + +fn single_char(elem: &Element) -> Option { + let text = elem.text(); + let mut chars = text.trim().chars(); + let first = chars.next()?; + chars.next().is_none().then_some(first) +} + +fn takes_limits(elem: &Element) -> bool { + let text = elem.text(); + let text = text.trim(); + text.chars().all(|c| BIG_OPERATORS.contains(c)) && !text.is_empty() + || matches!(text, "lim" | "max" | "min" | "sup" | "inf" | "limsup" | "liminf") +} + +/// `mmultiscripts` lists post-scripts first, then `mprescripts` and the +/// pre-scripts, each as a sub/sup pair. An empty slot is spelled ``. +fn multiscripts(elem: &Element, out: &mut String, depth: usize) { + let children: Vec<&Element> = elem.child_elems().collect(); + let Some((base, scripts)) = children.split_first() else { + return; + }; + let split = scripts.iter().position(|c| c.local == "mprescripts"); + let (post, pre) = match split { + Some(at) => (&scripts[..at], &scripts[at + 1..]), + None => (scripts, &[][..]), + }; + + for pair in pre.chunks(2) { + out.push_str("{}"); + emit_script('_', pair.first().copied(), out, depth); + emit_script('^', pair.get(1).copied(), out, depth); + } + emit_base(Some(base), out, depth); + for pair in post.chunks(2) { + emit_script('_', pair.first().copied(), out, depth); + emit_script('^', pair.get(1).copied(), out, depth); + } +} + +fn emit_script(marker: char, elem: Option<&Element>, out: &mut String, depth: usize) { + let Some(elem) = elem else { + return; + }; + if elem.local == "none" { + return; + } + out.push(marker); + group(Some(elem), out, depth); +} + +fn table(elem: &Element, out: &mut String, depth: usize) { + out.push_str("\\begin{matrix}"); + let mut cells = 0usize; + for (row_index, row) in elem.child_elems().filter(|c| c.local == "mtr").enumerate() { + if row_index > 0 { + out.push_str("\\\\"); + } + for (cell_index, cell) in row.child_elems().filter(|c| c.local == "mtd").enumerate() { + if cells >= MAX_TABLE_CELLS { + log::warn!("MathML table past {MAX_TABLE_CELLS} cells; the rest is dropped"); + out.push_str("\\end{matrix}"); + return; + } + cells += 1; + if cell_index > 0 { + out.push('&'); + } + emit_children(cell, out, depth); + } + } + out.push_str("\\end{matrix}"); +} + +/// `mfenced` was dropped in MathML 4 but is still what many producers emit. +/// Its delimiters are attributes, so the pair is balanced by construction and +/// can safely take `\left`/`\right`. +fn fenced(elem: &Element, out: &mut String, depth: usize) { + let open = elem.attr_any("open").unwrap_or("("); + let close = elem.attr_any("close").unwrap_or(")"); + let separators: Vec = + elem.attr_any("separators").unwrap_or(",").chars().filter(|c| !c.is_whitespace()).collect(); + + out.push_str("\\left"); + out.push_str(&delim_glyph(first_char(open))); + for (index, child) in elem.child_elems().enumerate() { + if index > 0 { + let separator = separators.get(index - 1).or_else(|| separators.last()).copied(); + if let Some(separator) = separator { + out.push_str(&delim_sep(separator)); + } + } + emit(child, out, depth); + } + out.push_str("\\right"); + out.push_str(&delim_glyph(first_char(close))); +} + +/// An explicitly empty delimiter draws no glyph, which `delim_glyph` spells as +/// the LaTeX null delimiter. +fn first_char(value: &str) -> Option { + value.chars().next() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::package::xml::parse_xml; + + fn latex(body: &str) -> String { + let doc = format!(r#"{body}"#); + let root = parse_xml(doc.as_bytes()).unwrap(); + let math = root.descendants_any("math").next().unwrap(); + match to_inline(math, false) { + Some(Inline::Math { latex, .. }) => latex, + _ => String::new(), + } + } + + #[test] + fn a_fraction_keeps_the_relation_its_glyphs_lost() { + assert_eq!(latex("13"), r"\frac{1}{3}"); + } + + #[test] + fn a_zero_rule_is_a_stack_not_a_fraction() { + assert_eq!( + latex(r#"nk"#), + r"{{n}\atop {k}}" + ); + } + + #[test] + fn invisible_operators_carry_no_glyph_and_leave_none() { + let xml = "x\u{2062}y\u{2061}z"; + assert_eq!(latex(xml), "xyz"); + } + + #[test] + fn a_multi_character_identifier_is_a_name_not_a_product() { + assert_eq!(latex("RankA"), r"\mathrm{Rank}A"); + assert_eq!(latex("detA"), r"\det A"); + assert_eq!(latex(r#"d"#), r"\mathrm{d}"); + } + + #[test] + fn a_script_under_a_large_operator_is_a_limit() { + let xml = "\u{2211}in"; + assert_eq!(latex(xml), "\u{2211}_{i}^{n}"); + } + + #[test] + fn a_script_over_an_ordinary_base_is_an_overset() { + assert_eq!( + latex("A12"), + r"\underset{1}{\overset{2}{A}}" + ); + } + + #[test] + fn a_combining_mark_over_a_base_is_an_accent() { + assert_eq!(latex("x\u{302}"), r"\widehat{x}"); + } + + #[test] + fn a_sequence_base_is_braced_so_the_script_binds_all_of_it() { + let xml = "a+b2"; + assert_eq!(latex(xml), "{a+b}^{2}"); + } + + #[test] + fn prescripts_precede_the_base_on_an_empty_atom() { + let xml = "X12\ + 34"; + assert_eq!(latex(xml), "{}_{3}^{4}X_{1}^{2}"); + } + + #[test] + fn an_empty_multiscript_slot_emits_no_script() { + let xml = "X2"; + assert_eq!(latex(xml), "X^{2}"); + } + + #[test] + fn prose_keeps_the_space_that_separates_it_from_the_next_symbol() { + assert_eq!(latex("if x"), r"\text{if }x"); + } + + #[test] + fn an_annotation_never_reaches_the_body() { + let xml = "1\ + 1"; + assert_eq!(latex(xml), "1"); + } + + #[test] + fn a_named_width_has_no_latex_spelling_and_is_dropped() { + assert_eq!(latex(r#"ab"#), "ab"); + assert_eq!(latex(r#"ab"#), r"a\hspace{1em}b"); + } + + #[test] + fn a_bare_pipe_would_split_a_markdown_row() { + assert_eq!(latex("|x|"), r"\mid x\mid"); + } + + #[test] + fn text_cannot_escape_the_math_span() { + assert_eq!(latex("$x$"), r"\text{\$x\$}"); + } + + #[test] + fn an_unmodelled_element_keeps_its_content() { + assert_eq!(latex("q"), "q"); + } + + #[test] + fn nesting_past_the_bound_degrades_to_text() { + let mut xml = "x".to_string(); + for _ in 0..MAX_DEPTH + 2 { + xml = format!("{xml}"); + } + assert_eq!(latex(&xml), "x"); + } + + #[test] + fn an_empty_equation_produces_nothing() { + assert_eq!(latex(""), ""); + assert_eq!(latex(""), ""); + } +} diff --git a/src/shared/mod.rs b/src/shared/mod.rs index 560f0c4a..5d3a4890 100644 --- a/src/shared/mod.rs +++ b/src/shared/mod.rs @@ -13,7 +13,9 @@ pub mod fields; pub mod grid; pub mod header; pub mod html; +pub mod latex; pub mod list; +pub mod mathml; pub mod mc; pub mod numbering; pub mod officeart; diff --git a/tests/snapshots/snapshots__epub__handmade-math.epub.snap b/tests/snapshots/snapshots__epub__handmade-math.epub.snap index 97b5e5fd..2d2acbe7 100644 --- a/tests/snapshots/snapshots__epub__handmade-math.epub.snap +++ b/tests/snapshots/snapshots__epub__handmade-math.epub.snap @@ -8,7 +8,7 @@ expression: output Annotated: $\frac{a}{b} + x^2$ follows. -Bare presentation: x2 follows. +Bare presentation: $x^{2}$ follows. Block: $$E = mc^2$$ From 2113b484119f5411824584e08bb1ef2d24916d94 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 16:07:46 +0300 Subject: [PATCH 06/13] feat(rtf): read the script control words \super, \sub and \nosupersub were not in the dispatcher, so RTF alone kept flattening what the other frontends now preserve: 10\super -3 came out as 10-3, the exact shape of the original report. \upN and \dnN carry an offset in half-points rather than a toggle, so only 0 means the baseline and an absent parameter takes the spec's default of 6. \plain already resets vert_align through Style::PLAIN. A paragraph style may carry the property too, so the stylesheet parser records it in the delta it already builds for bold and italic. --- src/formats/rtf/mod.rs | 45 ++++++++++++++++++++++++++++++++++++++- src/formats/rtf/tables.rs | 10 +++++++++ 2 files changed, 54 insertions(+), 1 deletion(-) diff --git a/src/formats/rtf/mod.rs b/src/formats/rtf/mod.rs index 5a018ee9..c9b2c836 100644 --- a/src/formats/rtf/mod.rs +++ b/src/formats/rtf/mod.rs @@ -8,7 +8,7 @@ mod table; mod tables; use crate::error::ConvertError; -use crate::model::{Block, Document, Inline, Note, NoteKind, Style, inlines_are_empty}; +use crate::model::{Block, Document, Inline, Note, NoteKind, Style, VertAlign, inlines_are_empty}; use crate::shared::blockstyle::{BlockStyle, StyledRun}; use crate::shared::delta::rebase_emphasis; use crate::shared::fields::field_result; @@ -549,6 +549,23 @@ impl<'a> Parser<'a> { "b" => self.set_style(|s| s.bold = on), "i" => self.set_style(|s| s.italic = on), "strike" | "striked" => self.set_style(|s| s.strike = on), + "super" => { + self.set_vert_align(if on { VertAlign::Superscript } else { VertAlign::Baseline }) + } + "sub" => { + self.set_vert_align(if on { VertAlign::Subscript } else { VertAlign::Baseline }) + } + "nosupersub" => self.set_vert_align(VertAlign::Baseline), + // \upN and \dnN carry an offset in half-points rather than a + // toggle, and the spec's default of 6 applies when it is absent. + "up" | "dn" => { + let raised = word == "up"; + self.set_vert_align(match param.unwrap_or(6) { + 0 => VertAlign::Baseline, + _ if raised => VertAlign::Superscript, + _ => VertAlign::Subscript, + }) + } "plain" => { self.flush_pending(); let font = self.state.font; @@ -819,6 +836,10 @@ impl<'a> Parser<'a> { } } + fn set_vert_align(&mut self, vert_align: VertAlign) { + self.set_style(|s| s.vert_align = vert_align); + } + fn set_style(&mut self, f: impl FnOnce(&mut Style)) { self.flush_pending(); f(&mut self.state.style); @@ -1049,6 +1070,28 @@ mod tests { assert_eq!(list.items[1].marker_label.as_deref(), Some("2.")); } + #[test] + fn scripts_survive_as_scripts() { + let src = r"{\rtf1 H\sub 2\nosupersub O and 10\super -3\nosupersub mol\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "H2O and 10-3 mol\n"); + } + + #[test] + fn a_half_point_offset_says_which_way_and_zero_says_neither() { + // \upN and \dnN carry an offset, not a toggle: only 0 is the baseline. + let src = r"{\rtf1 a\up6 b\up0 c\dn4 d\plain e\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "abcde\n"); + } + + #[test] + fn a_paragraph_style_can_carry_the_script() { + let src = r"{\rtf1{\stylesheet{\s15\super Raised;}}\pard\s15 note\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "note\n"); + } + #[test] fn mid_paragraph_page_and_column_breaks_keep_the_word_boundary() { // \page and \column carry no paragraph mark: without a break of diff --git a/src/formats/rtf/tables.rs b/src/formats/rtf/tables.rs index d01b8425..d3cefec4 100644 --- a/src/formats/rtf/tables.rs +++ b/src/formats/rtf/tables.rs @@ -2,6 +2,7 @@ //! per-font charsets), the style sheet, and the list/list-override tables. use crate::formats::rtf::lexer::{Lexer, Token, destination_groups}; +use crate::model::VertAlign; use crate::shared::blockstyle::{self, BlockStyle}; use crate::shared::delta::StyleDelta; use crate::shared::list::MarkerKind; @@ -214,6 +215,15 @@ fn parse_stylesheet( def.delta.italic = Some(param != Some(0)); } } + "super" | "sub" | "nosupersub" => { + if let Some((_, def, _)) = current.as_mut() { + def.delta.vert_align = Some(match (word, param) { + (_, Some(0)) | ("nosupersub", _) => VertAlign::Baseline, + ("super", _) => VertAlign::Superscript, + _ => VertAlign::Subscript, + }); + } + } _ => {} }, _ => {} From 0c36785099a465c38c107b8e76b97ac15f108678 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 16:15:09 +0300 Subject: [PATCH 07/13] feat(doc,ppt): read the script properties the binary formats carry MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Word 97-2003 never read sprmCIss, and the PPT frontend hardcoded the baseline, so both flattened what every other frontend now keeps. DOC gains sprmCIss (0x2A48: 1 superscript, 2 subscript) and sprmCHpsPos (0x4845), the signed half-point offset Word writes for "raised by" rather than for the checkbox — the same pair as RTF's \super and \up. PPT already walked past the TextCFException position field, a signed percentage of the font size, so reading it is a matter of not skipping it. Both are covered by handmade fixtures with the property in real record bytes, because a unit test over a synthetic grpprl proves the parser and not the path that reaches it. --- src/formats/doc/sprm.rs | 74 +++++++++++- src/formats/ppt/mod.rs | 2 +- src/formats/ppt/styletext.rs | 50 +++++++- tests/fixtures/doc/handmade-script.doc | Bin 0 -> 9728 bytes tests/fixtures/ppt/handmade-script.ppt | Bin 0 -> 9728 bytes tests/gen_fixtures.py | 109 ++++++++++++++++++ .../snapshots__doc__handmade-script.doc.snap | 5 + .../snapshots__ppt__handmade-script.ppt.snap | 5 + 8 files changed, 240 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/doc/handmade-script.doc create mode 100644 tests/fixtures/ppt/handmade-script.ppt create mode 100644 tests/snapshots/snapshots__doc__handmade-script.doc.snap create mode 100644 tests/snapshots/snapshots__ppt__handmade-script.ppt.snap diff --git a/src/formats/doc/sprm.rs b/src/formats/doc/sprm.rs index d175c8b7..fee65fc3 100644 --- a/src/formats/doc/sprm.rs +++ b/src/formats/doc/sprm.rs @@ -2,7 +2,7 @@ //! character toggles resolve against the style-chain base (0 = off, 1 = on, //! 0x80 = style's value, 0x81 = style's value inverted). -use crate::model::Style; +use crate::model::{Style, VertAlign}; use crate::shared::binary::{get_u16, get_u32}; fn sprm_operand_len(sprm: u16, operand: &[u8]) -> usize { @@ -35,6 +35,10 @@ pub fn walk_sprms(grpprl: &[u8], mut f: impl FnMut(u16, &[u8])) { } } +fn get_i16(operand: &[u8], at: usize) -> Option { + operand.get(at..at + 2).map(|b| i16::from_le_bytes([b[0], b[1]])) +} + /// Resolve a toggle operand against the style-chain base value. fn toggle(operand: &[u8], base: bool) -> Option { match operand.first() { @@ -90,6 +94,28 @@ pub fn apply_chpx(grpprl: &[u8], current: Style, style_base: Style) -> Style { style.strike = v; } } + // sprmCIss: 0 none, 1 superscript, 2 subscript. Not a toggle, so it + // resolves against nothing and simply sets. + 0x2A48 => { + if let Some(&iss) = operand.first() { + style.vert_align = match iss { + 1 => VertAlign::Superscript, + 2 => VertAlign::Subscript, + _ => VertAlign::Baseline, + }; + } + } + // sprmCHpsPos: a signed half-point offset from the baseline, which is + // how Word records "raised/lowered by" rather than the checkbox. + 0x4845 => { + if let Some(pos) = get_i16(operand, 0) { + style.vert_align = match pos { + 0 => VertAlign::Baseline, + pos if pos > 0 => VertAlign::Superscript, + _ => VertAlign::Subscript, + }; + } + } _ => {} }); style @@ -249,3 +275,49 @@ fn parse_tdef_table(operand: &[u8]) -> Option { pub fn apply_style_chpx(grpprl: &[u8], parent: Style) -> Style { apply_chpx(grpprl, parent, parent) } + +#[cfg(test)] +mod tests { + use super::*; + + fn chpx(sprm: u16, operand: &[u8]) -> Vec { + let mut out = sprm.to_le_bytes().to_vec(); + out.extend_from_slice(operand); + out + } + + #[test] + fn sprm_ciss_says_which_way_and_zero_says_neither() { + for (iss, expected) in + [(0u8, VertAlign::Baseline), (1, VertAlign::Superscript), (2, VertAlign::Subscript)] + { + let style = apply_chpx(&chpx(0x2A48, &[iss]), Style::PLAIN, Style::PLAIN); + assert_eq!(style.vert_align, expected, "iss {iss}"); + } + } + + #[test] + fn sprm_chpspos_reads_the_sign_of_the_offset() { + for (offset, expected) in + [(0i16, VertAlign::Baseline), (6, VertAlign::Superscript), (-6, VertAlign::Subscript)] + { + let style = + apply_chpx(&chpx(0x4845, &offset.to_le_bytes()), Style::PLAIN, Style::PLAIN); + assert_eq!(style.vert_align, expected, "offset {offset}"); + } + } + + #[test] + fn a_script_sprm_leaves_the_other_properties_alone() { + let base = Style { bold: true, ..Style::PLAIN }; + let style = apply_chpx(&chpx(0x2A48, &[1]), base, base); + assert!(style.bold); + assert_eq!(style.vert_align, VertAlign::Superscript); + } + + #[test] + fn a_truncated_operand_changes_nothing() { + let style = apply_chpx(&[0x45, 0x48, 0x06], Style::PLAIN, Style::PLAIN); + assert_eq!(style.vert_align, VertAlign::Baseline); + } +} diff --git a/src/formats/ppt/mod.rs b/src/formats/ppt/mod.rs index e54151f9..16c1896a 100644 --- a/src/formats/ppt/mod.rs +++ b/src/formats/ppt/mod.rs @@ -552,7 +552,7 @@ impl Extractor { italic: char_run.and_then(|r| r.italic).or(d.italic).unwrap_or(false), strike: false, code: false, - vert_align: VertAlign::Baseline, + vert_align: char_run.and_then(|r| r.vert_align).unwrap_or(VertAlign::Baseline), }; if c == '\r' { if !run_text.is_empty() { diff --git a/src/formats/ppt/styletext.rs b/src/formats/ppt/styletext.rs index 55efb50f..c47401e7 100644 --- a/src/formats/ppt/styletext.rs +++ b/src/formats/ppt/styletext.rs @@ -3,6 +3,7 @@ //! TextPFException / TextCFException layouts. Parsing is defensive - a //! malformed exception aborts styling for that atom (logged), never the text. +use crate::model::VertAlign; use crate::shared::binary::{get_u16, get_u32}; #[derive(Debug, Clone, Copy, Default)] @@ -20,6 +21,7 @@ pub struct CharProps { pub count: usize, pub bold: Option, pub italic: Option, + pub vert_align: Option, } /// One indent level's defaults from a `TxMasterStyleAtom`, tri-state. @@ -72,7 +74,12 @@ pub fn parse_style_text(body: &[u8], text_len: usize) -> StyleRuns { break; }; pos = next; - runs.chars.push(CharProps { count, bold: cf.bold, italic: cf.italic }); + runs.chars.push(CharProps { + count, + bold: cf.bold, + italic: cf.italic, + vert_align: cf.vert_align, + }); covered += count; if count == 0 { break; @@ -156,6 +163,7 @@ fn parse_pf_exception(body: &[u8], mut pos: usize) -> Option<(Option, usiz struct CfStyle { bold: Option, italic: Option, + vert_align: Option, } /// TextCFException: mask (+ optional style bitfield) + sized fields. Each @@ -194,13 +202,20 @@ fn parse_cf_exception(body: &[u8], mut pos: usize) -> Option<(CfStyle, usize)> { if mask & 0x0004_0000 != 0 { pos += 4; // color } + // position: the baseline offset as a signed percentage of the font size. + let mut vert_align = None; if mask & 0x0008_0000 != 0 { - pos += 2; // position + vert_align = Some(match get_u16(body, pos)? as i16 { + 0 => VertAlign::Baseline, + offset if offset > 0 => VertAlign::Superscript, + _ => VertAlign::Subscript, + }); + pos += 2; } if pos > body.len() { return None; } - Some((CfStyle { bold, italic }, pos)) + Some((CfStyle { bold, italic, vert_align }, pos)) } /// A `TxMasterStyleAtom`: per-indent-level tri-state defaults @@ -229,3 +244,32 @@ pub fn parse_master_style(body: &[u8], instance: u16) -> Vec { } out } + +#[cfg(test)] +mod tests { + use super::*; + + /// A TextCFException carrying only the position field. + fn exception(position: i16) -> Vec { + let mut out = 0x0008_0000u32.to_le_bytes().to_vec(); + out.extend_from_slice(&position.to_le_bytes()); + out + } + + #[test] + fn the_position_percentage_says_which_side_of_the_baseline() { + for (position, expected) in + [(0i16, VertAlign::Baseline), (30, VertAlign::Superscript), (-25, VertAlign::Subscript)] + { + let (style, pos) = parse_cf_exception(&exception(position), 0).unwrap(); + assert_eq!(style.vert_align, Some(expected), "position {position}"); + assert_eq!(pos, 6); + } + } + + #[test] + fn an_absent_position_leaves_the_run_where_the_master_put_it() { + let (style, _) = parse_cf_exception(&0u32.to_le_bytes(), 0).unwrap(); + assert_eq!(style.vert_align, None); + } +} diff --git a/tests/fixtures/doc/handmade-script.doc b/tests/fixtures/doc/handmade-script.doc new file mode 100644 index 0000000000000000000000000000000000000000..6ee0164eb6a5edaa23cffb93ed1fb37098afe305 GIT binary patch literal 9728 zcmeI2&q_i;6vn@M)k-r!2{(w@PpqM9|u@oqs?I>Tz2106juaP>)diUO{i) z5hp}RZb<%#Z{*|5pEJyv-++wcd3(*jKR%Z}Fjh{GKtD=jZKU60eaE#~=|@pyO=t70 zvrA=#2p|a*sf0UMSrHHc5fA|p5CIVo0TB=Z5fA|p5P^S3;PdH)drI>KV5|(^1;%=( zHj7h7mvJz=j6o?T5tti+TIDRPw;SP6`QSKgb((|5edTa#ZV_mqg%aTRyhr&|hRRX} zYL~Ki;h+%g1*HIq+P=xBykxaD`nSMB!`I?75m=vTwkC^!2#A0Ph=2&JB!P2u__;-h zD%!MNJp5`O1*ANe?qK;Laf-_qG2b#SP{$RTxQR1$!Y2oRu)fa`i(|s!r;cGF%O!W; zMFXd_*SJHAaTtfVr8HdMF^Ds7!<@boY2wWIJ0CkUV`upvyzV1K+2!6YhIUSA+6=Wx W*=vdYF3eHel#QEuZu7D$N8k&+i@YrW literal 0 HcmV?d00001 diff --git a/tests/fixtures/ppt/handmade-script.ppt b/tests/fixtures/ppt/handmade-script.ppt new file mode 100644 index 0000000000000000000000000000000000000000..0cc299da82de90c7dc7b4350fe42232e36e1452d GIT binary patch literal 9728 zcmeI2Jxc>Y5QZl?qZhx{h^PdSD=aJ&`~XWE73>tjT0|`blfbEnwZFh$An9zaZ0#-d zCny$LsUX^!^PSD!B?uu{N%Y-gcII~Wc6OevZrIzK++F!1e=pzKym(THveI5Ro`T&M zzr(0RQKYZ2&cVhITMH2o`7qr$w*kn;)#<}EG#d$!011!)36KB@kN^pg011!)36Q|w zB4Fgn8$(J)Ud@P2?ak9#K)OCnE&QMvdG=1A)AHgC+mS}ynQ4Qo`q{bBy3q2>O0Zen z4yLCjXM;j%uScY8=HTd0)^JhWXTCf}zVlT#c7?=A)VV!KQ9 zx~^9nqD2BEKmsH{0wh2JBtQZrKmsH{0{@S|nw0Py;5T|niqN4P{|LgMWK!{UhxiA+ z*#hx>5uLXnhlm-v_PUYn+!zn{B%I{oU+iJkS>T8Tu`@> RUWkFHU*$7Sp1S1-d;#1}%KiWV literal 0 HcmV?d00001 diff --git a/tests/gen_fixtures.py b/tests/gen_fixtures.py index 5680b741..a4a26445 100644 --- a/tests/gen_fixtures.py +++ b/tests/gen_fixtures.py @@ -1455,6 +1455,61 @@ def std(name, istd): [("0Table", stsh + plcf), ("WordDocument", bytes(word_doc))]) +def script_doc(): + """Binary .doc whose CHPX FKP page carries sprmCIss and sprmCHpsPos.""" + text = "H2O and 10-3 mol and x2.\r".encode("cp1252") + + def chpx(*sprms): + out = b"" + for sprm, operand in sprms: + out += struct.pack("{text}' @@ -1561,6 +1616,58 @@ def persist_atom(persist_ref, sid): # this deck has notes only on the SECOND slide, which order-based zipping # would misattribute to the first. +def script_ppt(): + """Deck whose StyleTextPropAtom carries the position field: the run's + baseline offset as a signed percentage of the font size.""" + text = "H2O and 10-3 mol\r" + + def para_run(count): + return struct.pack("2
O and 10-3 mol and x2. diff --git a/tests/snapshots/snapshots__ppt__handmade-script.ppt.snap b/tests/snapshots/snapshots__ppt__handmade-script.ppt.snap new file mode 100644 index 00000000..d88a6767 --- /dev/null +++ b/tests/snapshots/snapshots__ppt__handmade-script.ppt.snap @@ -0,0 +1,5 @@ +--- +source: tests/snapshots.rs +expression: output +--- +H2O and 10-3 mol From 95a8e17afd3cfcae0fcb9320ef59196571bf2918 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 17:19:19 +0300 Subject: [PATCH 08/13] feat(pptx,odf): recover the equations these formats keep out of line Neither frontend reached the equation at all, and the loss was total rather than partial: a slide read "Energy: done" and a document "Einstein said and stopped." PowerPoint writes an equation as a14:m inside an mc:AlternateContent whose fallback is a picture of it. The paragraph walker skipped every child outside the a namespace, so not even the fallback text survived. It now resolves the AlternateContent and hands a14:m to the OMML converter, which moves to shared/ because two frontends read it. ODF keeps a formula in a sub-document of its own, referenced by draw:object. Following the reference gives MathML, which the MathML converter already translates; some producers put it inline in the element instead, so both spellings are read. A reference that resolves to nothing still degrades to the frame's alternative text. The TeX-annotation preference moves into the MathML converter so every caller gets it: what the author wrote beats anything derived from the layout, while ODF's StarMath annotation is not LaTeX and is ignored. --- src/formats/docx/content.rs | 2 +- src/formats/docx/mod.rs | 1 - src/formats/odf/text.rs | 38 ++++++++++ src/formats/pptx/mod.rs | 25 ++++++- src/package/xml.rs | 3 + src/shared/html.rs | 17 ----- src/shared/mathml.rs | 33 +++++++-- src/shared/mod.rs | 1 + src/{formats/docx => shared}/omml.rs | 0 tests/fixtures/odt/handmade-math.odt | Bin 0 -> 883 bytes tests/fixtures/pptx/handmade-math.pptx | Bin 0 -> 1848 bytes tests/gen_fixtures.py | 67 ++++++++++++++++++ .../snapshots__odt__handmade-math.odt.snap | 9 +++ .../snapshots__pptx__handmade-math.pptx.snap | 7 ++ 14 files changed, 178 insertions(+), 25 deletions(-) rename src/{formats/docx => shared}/omml.rs (100%) create mode 100644 tests/fixtures/odt/handmade-math.odt create mode 100644 tests/fixtures/pptx/handmade-math.pptx create mode 100644 tests/snapshots/snapshots__odt__handmade-math.odt.snap create mode 100644 tests/snapshots/snapshots__pptx__handmade-math.pptx.snap diff --git a/src/formats/docx/content.rs b/src/formats/docx/content.rs index 6c7a035b..64d0d3a4 100644 --- a/src/formats/docx/content.rs +++ b/src/formats/docx/content.rs @@ -2,7 +2,6 @@ use crate::error::ConvertError; use crate::formats::docx::numbering::{Counters, Numbering}; -use crate::formats::docx::omml; use crate::formats::docx::styles::{Styles, on_off, rpr_delta}; use crate::model::{ Block, Cell, GridBuilder, ImageSource, Inline, LinkTarget, Style, TableKind, inlines_are_empty, @@ -15,6 +14,7 @@ use crate::shared::delta::rebase_emphasis; use crate::shared::fields::{FieldFrame, field_result}; use crate::shared::header::resolve_header_rows; use crate::shared::list::{ListEntry, ListKey, flush_list}; +use crate::shared::omml; use crate::shared::text::{clean_text, is_xml_space}; use std::cell::RefCell; use std::collections::HashMap; diff --git a/src/formats/docx/mod.rs b/src/formats/docx/mod.rs index d1a219e1..21dfef57 100644 --- a/src/formats/docx/mod.rs +++ b/src/formats/docx/mod.rs @@ -5,7 +5,6 @@ mod content; mod numbering; -mod omml; mod styles; use crate::error::ConvertError; diff --git a/src/formats/odf/text.rs b/src/formats/odf/text.rs index 96628565..dfe78125 100644 --- a/src/formats/odf/text.rs +++ b/src/formats/odf/text.rs @@ -428,6 +428,12 @@ pub(super) fn walk_frame( .or_else(|| frame.first_descendant(ns::SVG_COMPAT, "desc").map(|d| d.text())) .unwrap_or_default(); let alt = clean_text(alt.trim()); + if let Some(object) = frame.find(ns::DRAW, "object") + && let Some(math) = load_formula(ctx, object)? + { + out.push(math); + return Ok(()); + } if let Some(image) = frame.first_descendant(ns::DRAW, "image") { let href = image.attr(ns::XLINK, "href").unwrap_or(""); let source = load_image(ctx, href)?; @@ -442,6 +448,38 @@ pub(super) fn walk_frame( Ok(()) } +/// A `draw:object` is a whole sub-document. When it is a formula its content +/// is MathML, either inline in the element or in its own package directory. +fn load_formula(ctx: &Ctx, object: &Element) -> Result, ConvertError> { + if let Some(math) = object.first_descendant(ns::MATHML, "math") { + return Ok(crate::shared::mathml::to_inline(math, false)); + } + let Some(href) = object.attr(ns::XLINK, "href") else { + return Ok(None); + }; + if href.is_empty() || crate::shared::uri::is_absolute_uri(href) { + return Ok(None); + } + let inner = format!("{}/content.xml", href.trim_end_matches('/')); + let target = match crate::package::path::resolve("content.xml", &inner) { + Ok(t) => t, + Err(e) => { + log::warn!("skipping unresolvable object reference {href:?}: {e}"); + return Ok(None); + } + }; + let Some(root) = ctx.pkg.borrow_mut().optional_xml_part(&target.path)? else { + log::debug!("object part {} is missing or not XML", target.path); + return Ok(None); + }; + let math = if root.is(ns::MATHML, "math") { + Some(&root) + } else { + root.first_descendant(ns::MATHML, "math") + }; + Ok(math.and_then(|m| crate::shared::mathml::to_inline(m, false))) +} + /// Failures degrade (log + `None`) per the unified policy; resource-limit /// errors always propagate. fn load_image(ctx: &Ctx, href: &str) -> Result, ConvertError> { diff --git a/src/formats/pptx/mod.rs b/src/formats/pptx/mod.rs index 800c26ec..aa8569a7 100644 --- a/src/formats/pptx/mod.rs +++ b/src/formats/pptx/mod.rs @@ -20,6 +20,7 @@ use crate::shared::delta::rebase_emphasis; use crate::shared::fields::classify_rel_target; use crate::shared::header::resolve_header_rows; use crate::shared::list::{ListEntry, ListKey, MarkerKind, flush_list}; +use crate::shared::omml; use crate::shared::text::clean_text; use cascade::{Bullet, LevelStyle, Placeholder, TextProps, TitleClass}; use std::cell::{Cell as StdCell, RefCell}; @@ -35,7 +36,7 @@ const SLIDE_REL: &str = "http://schemas.openxmlformats.org/officeDocument/2006/r /// Namespaces whose markup this frontend understands; `mc:Choice` branches /// requiring anything else fall back to `mc:Fallback`. -const SUPPORTED_NS: &[&str] = &[ns::P, ns::A, ns::R, ns::MC]; +const SUPPORTED_NS: &[&str] = &[ns::P, ns::A, ns::R, ns::MC, ns::A14, ns::M]; struct LayoutInfo { placeholders: Vec, @@ -510,7 +511,28 @@ fn parse_text_body( fn parse_para_inlines(p: &Element, ctx: &SlideCtx, base: Style) -> Vec { let mut out: Vec = Vec::new(); + push_para_inlines(p, ctx, base, &mut out); + out +} + +fn push_para_inlines(p: &Element, ctx: &SlideCtx, base: Style, out: &mut Vec) { for child in p.child_elems() { + // An equation arrives as a14:m, which PowerPoint wraps in an + // AlternateContent whose fallback is a picture of it. + if child.is(ns::MC, "AlternateContent") { + if let Some(branch) = crate::shared::mc::alternate_branch(child, SUPPORTED_NS) { + push_para_inlines(branch, ctx, base, out); + } + continue; + } + if child.is(ns::A14, "m") || child.is(ns::M, "oMath") { + let math = + if child.is(ns::M, "oMath") { Some(child) } else { child.find(ns::M, "oMath") }; + if let Some(inline) = math.and_then(|m| omml::to_inline(m, false)) { + out.push(inline); + } + continue; + } if child.ns.as_deref().is_none_or(|n| n != ns::A) { continue; } @@ -541,7 +563,6 @@ fn parse_para_inlines(p: &Element, ctx: &SlideCtx, base: Style) -> Vec { _ => {} } } - out } fn parse_graphic_frame( diff --git a/src/package/xml.rs b/src/package/xml.rs index baeb3115..3573fde6 100644 --- a/src/package/xml.rs +++ b/src/package/xml.rs @@ -23,6 +23,9 @@ pub mod ns { pub const WP: &str = "http://schemas.openxmlformats.org/drawingml/2006/wordprocessingDrawing"; pub const MC: &str = "http://schemas.openxmlformats.org/markup-compatibility/2006"; pub const M: &str = "http://schemas.openxmlformats.org/officeDocument/2006/math"; + /// DrawingML 2010 extensions, whose `a14:m` carries OMML in a shape. + pub const A14: &str = "http://schemas.microsoft.com/office/drawing/2010/main"; + pub const MATHML: &str = "http://www.w3.org/1998/Math/MathML"; pub const CHART: &str = "http://schemas.openxmlformats.org/drawingml/2006/chart"; pub const DGM: &str = "http://schemas.openxmlformats.org/drawingml/2006/diagram"; pub const P: &str = "http://schemas.openxmlformats.org/presentationml/2006/main"; diff --git a/src/shared/html.rs b/src/shared/html.rs index cb619042..df961bc2 100644 --- a/src/shared/html.rs +++ b/src/shared/html.rs @@ -456,10 +456,6 @@ impl Builder<'_> { /// tree is not; without one the characters are kept but the shape is lost. fn walk_math(&mut self, elem: &Element, delta: StyleDelta) -> Result<(), ConvertError> { let display = elem.attr_any("display") == Some("block"); - if let Some(latex) = tex_annotation(elem) { - self.inlines.push(Inline::Math { latex, display }); - return Ok(()); - } if let Some(math) = mathml::to_inline(elem, display) { self.inlines.push(math); return Ok(()); @@ -703,19 +699,6 @@ impl Builder<'_> { } } -/// The LaTeX an `annotation` carries, if the MathML supplies one. -fn tex_annotation(math: &Element) -> Option { - let text = math - .descendant_elems() - .filter(|e| e.local == "annotation") - .find(|e| { - matches!(e.attr_any("encoding"), Some("application/x-tex" | "application/x-latex")) - })? - .text(); - let text = text.trim(); - if text.is_empty() { None } else { Some(text.replace(['\n', '\r'], " ")) } -} - fn merge_inline_tag(elem: &Element, mut delta: StyleDelta) -> StyleDelta { match elem.local.as_str() { "b" | "strong" => delta.bold = Some(true), diff --git a/src/shared/mathml.rs b/src/shared/mathml.rs index 6678c41f..85221eab 100644 --- a/src/shared/mathml.rs +++ b/src/shared/mathml.rs @@ -23,6 +23,9 @@ const MAX_TABLE_CELLS: usize = 10_000; const BIG_OPERATORS: &str = "∑∏∐∫∬∭∮∯∰⋀⋁⋂⋃⨀⨁⨂⨄⨆"; pub(crate) fn to_inline(math: &Element, display: bool) -> Option { + if let Some(latex) = tex_annotation(math) { + return Some(Inline::Math { latex, display }); + } let mut out = String::new(); emit_children(math, &mut out, 0); let latex = out.trim().to_string(); @@ -32,6 +35,20 @@ pub(crate) fn to_inline(math: &Element, display: bool) -> Option { Some(Inline::Math { latex, display }) } +/// The LaTeX a `` may carry alongside the presentation tree. It is +/// what the author wrote, so it wins over anything derived from the layout. +fn tex_annotation(math: &Element) -> Option { + let text = math + .descendant_elems() + .filter(|e| e.local == "annotation") + .find(|e| { + matches!(e.attr_any("encoding"), Some("application/x-tex" | "application/x-latex")) + })? + .text(); + let text = text.trim(); + if text.is_empty() { None } else { Some(text.replace(['\n', '\r'], " ")) } +} + fn emit_children(parent: &Element, out: &mut String, depth: usize) { for child in parent.child_elems() { emit(child, out, depth); @@ -504,10 +521,18 @@ mod tests { } #[test] - fn an_annotation_never_reaches_the_body() { - let xml = "1\ - 1"; - assert_eq!(latex(xml), "1"); + fn the_authors_own_latex_wins_over_the_presentation_tree() { + let xml = r#"12 + \tfrac12"#; + assert_eq!(latex(xml), r"\tfrac12"); + } + + #[test] + fn an_annotation_in_another_encoding_is_not_latex() { + // ODF writes StarMath here, which would be nonsense as LaTeX. + let xml = r#"12 + 1 over 2"#; + assert_eq!(latex(xml), r"\frac{1}{2}"); } #[test] diff --git a/src/shared/mod.rs b/src/shared/mod.rs index 5d3a4890..ab9d4c88 100644 --- a/src/shared/mod.rs +++ b/src/shared/mod.rs @@ -19,5 +19,6 @@ pub mod mathml; pub mod mc; pub mod numbering; pub mod officeart; +pub mod omml; pub mod text; pub mod uri; diff --git a/src/formats/docx/omml.rs b/src/shared/omml.rs similarity index 100% rename from src/formats/docx/omml.rs rename to src/shared/omml.rs diff --git a/tests/fixtures/odt/handmade-math.odt b/tests/fixtures/odt/handmade-math.odt new file mode 100644 index 0000000000000000000000000000000000000000..d784604cd886396025c4992cdffcc8b1723455c0 GIT binary patch literal 883 zcmWIWW@Zs#fB?mqxMM~<>Oc+%a{zH}W^QUpWkG6UK|xMta$-qlex80=UW#6RVsU1% zUVcGpUP^v)X>Mv>iC#%+MM(hEFpyTo7^BEJ0nv;M46)1%4BSAGbM}IXAi5Ys=Z##5Zb--{<>pW{o{%{qWSKC$q%$-b>{sPP68&5K6gy;CPOk zb^gxagDiQWi<#T!><+uF+#SED@=a;E218Xt^h-f&Me(?=+b1cDt@ZtpxP9l;zJM+F zws4)D`6wUrgS*gh-3WoYfk>TrqknfNJ&tK6)cOROl&1Ni{rCuJiUX5jX zugfVWjkr`|3-zx9#u|IaWu<1t7rRFkz=H><4UuOnibSql@$RmK$ z@q*#8mnR~;R(kwC^s{klY3xQmRmok(Q*FM4Iht5KSXa(@@x(M^m%Otum4v}^dy09lph#m%p21Z{X6DeHKHKK32` LHv;K3OduWrhTuyF literal 0 HcmV?d00001 diff --git a/tests/fixtures/pptx/handmade-math.pptx b/tests/fixtures/pptx/handmade-math.pptx new file mode 100644 index 0000000000000000000000000000000000000000..afdd833dce92d2b5640b15cced9b9e8e6275643e GIT binary patch literal 1848 zcmWIWW@Zs#U|`??V#S!}X2MNBfh;Q~1_ogu9qpW-SCX1n5+71okXjt8SCN~uHqhGt zuz|q6pQ6*QYn7d95XidKc$|ahDf^r`-bq*Wf8JU3_j`ZVoi$&&Wg9V9 zDToTQx$T-^DfP4E1b^X%^Qz{(AYAH$`K_AL1s`LBHL$*+r>=IyKd%N2kU zQsHMa?r#8wkUk>=12>S4FG|fR*4G1(z32CG9WoGLcyM<)$M)M>_>?Yf>E?)CYUkjw zE_35gMe(Ei_8#=?Zhri?wmf{hzx8Hzw^@6H14LB~N>05#Dam;K-yU0r#VT{BZ7$bH zaBSLk^NRSK)KkVLevJ`sGMo~@T<+&Qdoxr;?G~J#deB`vm~Zhz$5pdUPL!waeE#ur zgXk$MU4^n;OxN@o{~!Eo;KSt~cUO1Um*nHClusK;`7k+o*tjg2Q&TP;$u>{_QGC6b zKB~uB_hjiG0(vL~=rK^jC@3h=FDOba2Bw0~*=fi}l66%ilNu`e^=F64lQ>h3i<>1O2KD^s_wF&yWa3 z_c=6z4+0~YhwZ^`|G=X1yEYvscE!5f2z|yp#q4%hZ1gOxcW-qcIUa33Zm$?WY1iKK z)9xBf`sn8nI_1Cx&Go&~taE-pu43<*R1y{SKHxwTOYY`X{u!Vcp6Q^Y+{fah*=qZI ziP{`+49`7yJaU=L;}gvx;+9JHr#zi@yqtMoz_m=qjh96?)N%gcf7gCc!=|Ei?!~3@ z))tYA4V#VnX0X^iUL7SG$vkiU1Nr@?{HT6avkF{#0O(H}76t}Us9%e7GE-8E^}&=O zD1zrsKAm^lK*06;Pp+olGs*-+S`{=;6pJflTnfF&>f5WMR5Nu+)a&=wOH}qxYMscd zTJ%oltZd(@*=xg|=>JeyyYfZW#w9F08+uJx1E#N zx9i`5i$N9AInv@Y%tKAe*(ZmeR*wF)dYi$Gi#36dLQP9|8GL`ncx?NYV{hl?w(WXy zRBFEdyoIyG^<;l+Ir8gb!+joB&B$9zuP%1$eiY{Vq0N2AcC|<9uS+WKN3Lo5(_p{% z$mvxYyOwiA#fnR@w$>h-6jXQUb<6salO<~Y=?}M-zvTQHr`l(d!_^jOk-5&#)bdQ4 z!fg9Kv13^`*4?{!K{!0??9tziKjVWw^9tOabc4|`xYDam{+m*A*-G)s-y2P>l^S)$ z19`iX-Cq5?^Yd}sL+{yQmtQk){&VGxNZtXN71x%pl1ZES!_)ql(v3Q;8y0Fg_U1Wq zs?+NK{jv}6W@Hj!#$B!h;~fkdfC&L>DUWUddT|HQ&cM*Xs0B0vsqjPBh+do^G%^G0 z7;J?Lx(Vo+6Jf%4s0nzoD!O6lsUKn3V`hkRF!DlxH!B-R87mO_06lw+6~qGoc|Y89 literal 0 HcmV?d00001 diff --git a/tests/gen_fixtures.py b/tests/gen_fixtures.py index a4a26445..30d67c71 100644 --- a/tests/gen_fixtures.py +++ b/tests/gen_fixtures.py @@ -1081,6 +1081,71 @@ def defaults_odf(): # what Word writes when every property takes its default and it is the shape a # converter is most likely to mishandle. +def math_pptx(): + """A slide whose equation is an a14:m inside an mc:AlternateContent, which + is how PowerPoint writes one, alongside a bare a14:m.""" + a14 = 'xmlns:a14="http://schemas.microsoft.com/office/drawing/2010/main"' + mc = 'xmlns:mc="http://schemas.openxmlformats.org/markup-compatibility/2006"' + sup = ("c" + "2") + frac = ("a" + "b") + slide = f""" + + +Energy: + +E=m{sup} +[image of the equation] + exactly. +{frac} +""" + presentation = f""" +""" + pres_rels = """ + + +""" + ct = """ + + + + + +""" + write_zip(OUT / "pptx" / "handmade-math.pptx", [ + ("[Content_Types].xml", ct), + ("_rels/.rels", ROOT_RELS.replace("word/document.xml", "ppt/presentation.xml")), + ("ppt/presentation.xml", presentation), + ("ppt/_rels/presentation.xml.rels", pres_rels), + ("ppt/slides/slide1.xml", slide), + ]) + + +def math_odt(): + """ODF keeps a formula in a sub-document of its own: one frame references + it, one carries the MathML inline, and one points at a part that is not + there, so its alternative text has to carry the meaning.""" + mathml = ('' + '' + "E=mc2" + 'E = m c^2') + content = """ + + +Einstein said and stopped. +Inline x here. +Missing the quadratic formula there. +""" + write_zip(OUT / "odt" / "handmade-math.odt", + [("content.xml", content), ("Object 1/content.xml", mathml)], + mimetype_first="application/vnd.oasis.opendocument.text") + + def math_docx(): def r(text): return f'{text}' @@ -2123,6 +2188,8 @@ def main(): script_ppt() tables_docx() math_docx() + math_pptx() + math_odt() outline_docx() blockstyle_docx() blockstyle_odt() diff --git a/tests/snapshots/snapshots__odt__handmade-math.odt.snap b/tests/snapshots/snapshots__odt__handmade-math.odt.snap new file mode 100644 index 00000000..df43e7f0 --- /dev/null +++ b/tests/snapshots/snapshots__odt__handmade-math.odt.snap @@ -0,0 +1,9 @@ +--- +source: tests/snapshots.rs +expression: output +--- +Einstein said $E=mc^{2}$ and stopped. + +Inline $\sqrt{x}$ here. + +Missing the quadratic formula there. diff --git a/tests/snapshots/snapshots__pptx__handmade-math.pptx.snap b/tests/snapshots/snapshots__pptx__handmade-math.pptx.snap new file mode 100644 index 00000000..b584a29a --- /dev/null +++ b/tests/snapshots/snapshots__pptx__handmade-math.pptx.snap @@ -0,0 +1,7 @@ +--- +source: tests/snapshots.rs +expression: output +--- +Energy: $E=mc^{2}$ exactly. + +$\frac{a}{b}$ From fa0e48f79a547060ded56f7f1eacdd8144040e82 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 18:02:42 +0300 Subject: [PATCH 09/13] test(math): render every equation the corpus produces in KaTeX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The math frontends say they emit LaTeX inside the subset KaTeX implements. Nothing measured that, so the claim held only as far as the reviewer's eye. The gate walks the document model rather than the Markdown — the model carries latex and display directly, so no re-parsing stands between what the converter produced and what is checked — and renders each equation with throwOnError. Every fixture named handmade-math must contribute at least one, because a walk that quietly stops finding anything would otherwise pass while measuring nothing. Verified to fail: emitting an undefined control sequence from the MathML converter breaks it, as it should. --- node/katex.test.mjs | 83 ++++++++++++++++++++++++++++++++++++++++++ node/package-lock.json | 41 ++++++++++++++++++--- node/package.json | 3 +- 3 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 node/katex.test.mjs diff --git a/node/katex.test.mjs b/node/katex.test.mjs new file mode 100644 index 00000000..7f195867 --- /dev/null +++ b/node/katex.test.mjs @@ -0,0 +1,83 @@ +// The math frontends claim to emit LaTeX inside the subset KaTeX implements. +// Nothing measured that claim, so this renders every equation the fixture +// corpus produces and fails on the first one KaTeX will not accept. +import assert from 'node:assert/strict' +import { readFile, readdir } from 'node:fs/promises' +import { extname, join } from 'node:path' +import { fileURLToPath } from 'node:url' +import { test } from 'node:test' + +import katex from 'katex' + +import { toDocument } from './index.js' + +const FIXTURES = fileURLToPath(new URL('../tests/fixtures', import.meta.url)) + +// PDFs bypass the document model, CSV carries no styling, and the malformed +// and abuse corpora exist to fail rather than to convert. +const SKIP_DIRS = new Set(['pdf', 'csv', 'malformed', 'abuse']) + +function collect(inlines, out) { + for (const inline of inlines ?? []) { + if (inline.kind === 'math') out.push(inline) + collect(inline.content, out) + } +} + +function walk(blocks, out) { + for (const block of blocks ?? []) { + collect(block.content, out) + walk(block.blocks, out) + for (const item of block.items ?? []) walk(item.blocks, out) + for (const row of block.rows ?? []) { + for (const cell of row.cells ?? []) walk(cell.blocks, out) + } + } +} + +async function equationsIn(path) { + const document = await toDocument(await readFile(path)) + const found = [] + walk(document.blocks, found) + for (const note of document.notes ?? []) walk(note.blocks, found) + return found +} + +async function fixturePaths() { + const paths = [] + for (const dir of await readdir(FIXTURES, { withFileTypes: true })) { + if (!dir.isDirectory() || SKIP_DIRS.has(dir.name)) continue + for (const name of await readdir(join(FIXTURES, dir.name))) { + if (extname(name)) paths.push(join(FIXTURES, dir.name, name)) + } + } + return paths.sort() +} + +test('every equation the corpus produces renders in KaTeX', async () => { + const counts = new Map() + for (const path of await fixturePaths()) { + let equations + try { + equations = await equationsIn(path) + } catch { + continue // Unconvertible fixtures are another test's subject. + } + for (const { latex, display } of equations) { + assert.doesNotThrow( + () => katex.renderToString(latex, { throwOnError: true, displayMode: display }), + `${path}: ${latex}`, + ) + counts.set(path, (counts.get(path) ?? 0) + 1) + } + } + + // A walk that quietly stops finding anything would otherwise pass while + // measuring nothing, so each format that carries equations must contribute. + const withMath = [...counts.keys()] + for (const path of await fixturePaths()) { + if (!path.includes('handmade-math')) continue + assert.ok(counts.has(path), `no equation reached the document model from ${path}`) + } + assert.ok(withMath.length >= 4, `only ${withMath.length} fixtures carried equations`) +}) diff --git a/node/package-lock.json b/node/package-lock.json index 07542e40..8f38c8a1 100644 --- a/node/package-lock.json +++ b/node/package-lock.json @@ -8,11 +8,15 @@ "name": "@firecrawl/anydoc", "version": "0.1.7", "license": "MIT", + "bin": { + "anydoc": "cli.js" + }, "devDependencies": { - "@napi-rs/cli": "^3.8.2" + "@napi-rs/cli": "^3.8.2", + "katex": "^0.18.3" }, "engines": { - "node": ">= 18" + "node": ">= 20" } }, "node_modules/@emnapi/core": { @@ -29,9 +33,9 @@ } }, "node_modules/@emnapi/runtime": { - "version": "1.11.3", - "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.11.3.tgz", - "integrity": "sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==", + "version": "2.0.0-alpha.3", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-2.0.0-alpha.3.tgz", + "integrity": "sha512-hFPAhMUjJD9BSyCANEISPOogeXC9Zo9ZQl7L6vKnaVsMkCtzznaW/naYypeyl0Gv5rYfWYsZbpixTMpjDJzQeA==", "dev": true, "license": "MIT", "optional": true, @@ -1726,6 +1730,16 @@ "dev": true, "license": "MIT" }, + "node_modules/commander": { + "version": "8.3.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-8.3.0.tgz", + "integrity": "sha512-OkTL9umf+He2DZkUq8f8J9of7yL6RJKI24dVITBmNfZBmri9zYZQrKkuXiKhyfPSu8tUhnVBB1iKXevvnlR4Ww==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, "node_modules/content-type": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/content-type/-/content-type-2.0.0.tgz", @@ -1859,6 +1873,23 @@ "dev": true, "license": "MIT" }, + "node_modules/katex": { + "version": "0.18.3", + "resolved": "https://registry.npmjs.org/katex/-/katex-0.18.3.tgz", + "integrity": "sha512-6kbPr8KiYZpRfNNIw+8Td33njnFzK/ELcw6bI9FzIHysYPatAUdepQimfCX1vG2VCD/5Lv+Gu6P5YA58k8HHmw==", + "dev": true, + "funding": [ + "https://opencollective.com/katex", + "https://github.com/sponsors/katex" + ], + "license": "MIT", + "dependencies": { + "commander": "^8.3.0" + }, + "bin": { + "katex": "cli.js" + } + }, "node_modules/ms": { "version": "2.1.3", "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", diff --git a/node/package.json b/node/package.json index fb594a6d..11473fcd 100644 --- a/node/package.json +++ b/node/package.json @@ -59,6 +59,7 @@ "version": "napi version" }, "devDependencies": { - "@napi-rs/cli": "^3.8.2" + "@napi-rs/cli": "^3.8.2", + "katex": "^0.18.3" } } From a70e48309a60794bbee24924363b0cb4e6c2f863 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 18:05:10 +0300 Subject: [PATCH 10/13] feat(rtf): apply character styles MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit \cs was not in the dispatcher and the stylesheet parser recorded only paragraph styles, so every property a character style carried was lost — bold and italic as much as the script this branch went after. RTF numbers character styles in a space of their own, so \cs15 and \s15 are different styles and need separate maps; the \sbasedon resolution is the same either way and moves into a function both use. A character style applies over the run's own formatting rather than replacing it, which is what the tri-state delta already does. --- src/formats/rtf/mod.rs | 35 +++++++++++++++++++++++++++++++++++ src/formats/rtf/tables.rs | 38 +++++++++++++++++++++++++++++--------- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/src/formats/rtf/mod.rs b/src/formats/rtf/mod.rs index c9b2c836..5c91c8b1 100644 --- a/src/formats/rtf/mod.rs +++ b/src/formats/rtf/mod.rs @@ -572,6 +572,16 @@ impl<'a> Parser<'a> { self.state.style = Style::PLAIN; self.state.font = font; } + // A character style applies over the run's own formatting; unlike + // \s it carries no paragraph properties. + "cs" => { + let def = param.and_then(|id| self.prelude.char_styles.get(&id)); + if let Some(delta) = def.map(|d| d.delta) { + self.flush_pending(); + self.state.style = delta.apply(self.state.style); + self.state.style_base = self.state.style; + } + } "s" => { // Paragraph style: outline level for headings plus its // formatting delta as the new base. @@ -1085,6 +1095,31 @@ mod tests { assert_eq!(markdown, "abcde\n"); } + #[test] + fn a_character_style_applies_over_the_runs_own_formatting() { + let src = r"{\rtf1{\stylesheet{\*\cs15 \additive\super Sup;}\ + {\*\cs16 \additive\b\sbasedon15 BoldSup;}}\ + x{\cs15 2} and w{\cs16 3}.\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "x2 and w**3**.\n"); + } + + #[test] + fn character_and_paragraph_styles_are_numbered_apart() { + // \s15 and \cs15 are different styles, and the run keeps both. + let src = r"{\rtf1{\stylesheet{\s15\i Italic;}{\*\cs15 \additive\super Sup;}}\ + \pard\s15 para {\cs15 raised} back.\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "*para* *raised* *back.*\n"); + } + + #[test] + fn an_undefined_character_style_leaves_the_run_alone() { + let src = r"{\rtf1{\stylesheet{\*\cs15\super Sup;}}a{\cs99 b}c\par}"; + let markdown = crate::to_markdown_bytes(src.as_bytes(), crate::Format::Rtf).unwrap(); + assert_eq!(markdown, "abc\n"); + } + #[test] fn a_paragraph_style_can_carry_the_script() { let src = r"{\rtf1{\stylesheet{\s15\super Raised;}}\pard\s15 note\par}"; diff --git a/src/formats/rtf/tables.rs b/src/formats/rtf/tables.rs index d3cefec4..2411ef40 100644 --- a/src/formats/rtf/tables.rs +++ b/src/formats/rtf/tables.rs @@ -77,6 +77,9 @@ pub struct Prelude { pub fonts: HashMap, /// Paragraph style id (`\sN`) -> definition. pub styles: HashMap, + /// Character-style id -> definition. RTF numbers `\csN` in a space of its + /// own, so a `\cs15` and a `\s15` are different styles. + pub char_styles: HashMap, /// `\lsN` -> resolved list definition (through the override table). pub lists: HashMap, } @@ -88,7 +91,7 @@ pub fn parse_prelude(bytes: &[u8], default_encoding: &'static encoding_rs::Encod parse_fonttbl(group, &mut prelude.fonts, default_encoding); } for group in destination_groups(bytes, "stylesheet") { - parse_stylesheet(group, &mut prelude.styles, default_encoding); + parse_stylesheet(group, &mut prelude.styles, &mut prelude.char_styles, default_encoding); } let mut by_list_id: HashMap = HashMap::new(); for group in destination_groups(bytes, "listtable") { @@ -165,12 +168,15 @@ const NULL_STYLE: i32 = 222; fn parse_stylesheet( group: &[u8], styles: &mut HashMap, + char_styles: &mut HashMap, enc: &'static encoding_rs::Encoding, ) { let mut lexer = Lexer::new(group); let mut depth = 0usize; let mut current: Option<(i32, StyleDef, Option)> = None; + let mut character = false; let mut raw: HashMap)> = HashMap::new(); + let mut raw_chars: HashMap)> = HashMap::new(); // A style's name is the text at the end of its group, before the `;`. let mut name: Vec = Vec::new(); while let Some(token) = lexer.next_token() { @@ -180,16 +186,22 @@ fn parse_stylesheet( if depth == 1 && let Some((id, mut def, base)) = current.take() { - let (text, _, _) = enc.decode(&name); - def.block = blockstyle::from_style_name(text.trim_end_matches(';')); - raw.insert(id, (def, base)); + if character { + raw_chars.insert(id, (def, base)); + } else { + let (text, _, _) = enc.decode(&name); + def.block = blockstyle::from_style_name(text.trim_end_matches(';')); + raw.insert(id, (def, base)); + } } + character = false; name.clear(); depth = depth.saturating_sub(1); } Token::Hex(b) | Token::Byte(b) if depth == 1 && current.is_some() => name.push(b), Token::Word { name: word, param } => match word { - "s" => { + "s" | "cs" => { + character = word == "cs"; current = Some((param.unwrap_or(0), StyleDef::default(), None)); name.clear(); } @@ -229,9 +241,17 @@ fn parse_stylesheet( _ => {} } } - // Resolve every \sbasedon chain root-to-leaf: the child's own settings - // win over inherited ones. A cycle is bounded by the visited set and - // resolves from the acyclic prefix. + resolve_inheritance(&raw, styles); + resolve_inheritance(&raw_chars, char_styles); +} + +/// Resolve every `\sbasedon` chain root-to-leaf: the child's own settings win +/// over inherited ones. A cycle is bounded by the visited set and resolves from +/// the acyclic prefix. +fn resolve_inheritance( + raw: &HashMap)>, + out: &mut HashMap, +) { for &id in raw.keys() { let mut chain: Vec<&StyleDef> = Vec::new(); let mut seen: std::collections::HashSet = std::collections::HashSet::new(); @@ -251,7 +271,7 @@ fn parse_stylesheet( resolved.outline = def.outline.or(resolved.outline); resolved.block = def.block.or(resolved.block); } - styles.insert(id, resolved); + out.insert(id, resolved); } } From 7b8d1112cb18dfe86ea7fd80be7b1662122a5a58 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 9 Aug 2026 18:47:42 +0300 Subject: [PATCH 11/13] fix(math): stop emitting LaTeX that cannot render MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Found by fuzzing the math XML inside the fixtures and rendering everything that survived: 9 of 1631 equations came out unrenderable, all from the same place. A TeX annotation was passed through verbatim, so a broken one became broken output while a perfectly good presentation tree sat beside it. The annotation still wins when it can parse. Three ways it certainly cannot now yield to the presentation tree instead: unbalanced braces, a trailing script marker with no argument, and a bare `$` — that last one ended the math span early and handed the rest of the document to the Markdown parser, which is the same defect the document-text path was already guarded against and this path bypassed. A literal `^` or `~` was spelled `\^{}`, which is a text-mode accent LaTeX rejects in math mode; it is now written as the text it is. The KaTeX gate runs in strict mode, because a construct KaTeX renders while warning about it is still not LaTeX. What remains unrenderable is an annotation whose control sequence the author misspelled. Telling that from a macro would take a command table, and a table would reject legitimate LaTeX that is not in it. --- node/katex.test.mjs | 5 ++-- src/shared/latex.rs | 6 +++-- src/shared/mathml.rs | 62 ++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 67 insertions(+), 6 deletions(-) diff --git a/node/katex.test.mjs b/node/katex.test.mjs index 7f195867..139deb5a 100644 --- a/node/katex.test.mjs +++ b/node/katex.test.mjs @@ -1,6 +1,7 @@ // The math frontends claim to emit LaTeX inside the subset KaTeX implements. // Nothing measured that claim, so this renders every equation the fixture -// corpus produces and fails on the first one KaTeX will not accept. +// corpus produces and fails on the first one KaTeX will not accept. Strict +// mode is on: a construct KaTeX renders while warning is still not LaTeX. import assert from 'node:assert/strict' import { readFile, readdir } from 'node:fs/promises' import { extname, join } from 'node:path' @@ -65,7 +66,7 @@ test('every equation the corpus produces renders in KaTeX', async () => { } for (const { latex, display } of equations) { assert.doesNotThrow( - () => katex.renderToString(latex, { throwOnError: true, displayMode: display }), + () => katex.renderToString(latex, { throwOnError: true, strict: 'error', displayMode: display }), `${path}: ${latex}`, ) counts.set(path, (counts.get(path) ?? 0) + 1) diff --git a/src/shared/latex.rs b/src/shared/latex.rs index 665ce6df..522951b1 100644 --- a/src/shared/latex.rs +++ b/src/shared/latex.rs @@ -28,8 +28,10 @@ pub(crate) fn push_text(text: &str, out: &mut String) { '#' => out.push_str("\\#"), '%' => out.push_str("\\%"), '_' => out.push_str("\\_"), - '^' => out.push_str("\\^{}"), - '~' => out.push_str("\\~{}"), + // `\^` and `\~` are text-mode accents; in math mode LaTeX rejects + // them, so a literal one is spelled as the text it is. + '^' => out.push_str("\\text{\\^{}}"), + '~' => out.push_str("\\text{\\~{}}"), '\n' | '\r' => out.push(' '), c => out.push(c), } diff --git a/src/shared/mathml.rs b/src/shared/mathml.rs index 85221eab..ae3f2216 100644 --- a/src/shared/mathml.rs +++ b/src/shared/mathml.rs @@ -36,7 +36,9 @@ pub(crate) fn to_inline(math: &Element, display: bool) -> Option { } /// The LaTeX a `` may carry alongside the presentation tree. It is -/// what the author wrote, so it wins over anything derived from the layout. +/// what the author wrote, so it wins over anything derived from the layout -- +/// but only while it can still parse, because the presentation tree beside it +/// is a better answer than LaTeX that renders as an error message. fn tex_annotation(math: &Element) -> Option { let text = math .descendant_elems() @@ -46,7 +48,43 @@ fn tex_annotation(math: &Element) -> Option { })? .text(); let text = text.trim(); - if text.is_empty() { None } else { Some(text.replace(['\n', '\r'], " ")) } + if text.is_empty() { + return None; + } + if !is_well_formed(text) { + log::debug!("TeX annotation cannot parse; translating the presentation tree instead"); + return None; + } + Some(text.replace(['\n', '\r'], " ")) +} + +/// Not a TeX parser: the three ways an annotation can be certainly broken. +/// Unbalanced braces and a dangling command cannot render, and a bare `$` is a +/// delimiter inside a body that is already delimited, which ends the span +/// early and hands the rest of the document to the Markdown parser. +fn is_well_formed(text: &str) -> bool { + let mut depth = 0i32; + let mut escaped = false; + for chr in text.chars() { + if escaped { + escaped = false; + continue; + } + match chr { + '\\' => escaped = true, + '{' => depth += 1, + '}' => { + depth -= 1; + if depth < 0 { + return false; + } + } + '$' => return false, + _ => {} + } + } + // A trailing escape, script marker or open group has no argument to take. + depth == 0 && !escaped && !text.ends_with(['^', '_']) } fn emit_children(parent: &Element, out: &mut String, depth: usize) { @@ -527,6 +565,26 @@ mod tests { assert_eq!(latex(xml), r"\tfrac12"); } + #[test] + fn an_annotation_that_cannot_parse_yields_to_the_presentation_tree() { + // Unbalanced braces, a dangling script marker, and a bare `$` -- which + // would close the span and hand the rest of the document away. + for broken in [r"\\frac{a}{b", "x^", "a$$b", "a$b", "}{"] { + let xml = format!( + r#"12 + {broken}"# + ); + assert_eq!(latex(&xml), r"\frac{1}{2}", "annotation: {broken}"); + } + } + + #[test] + fn an_escaped_brace_or_dollar_does_not_count_against_the_annotation() { + let xml = r#"1 + \text{\$5 \{a\}}"#; + assert_eq!(latex(xml), r"\text{\$5 \{a\}}"); + } + #[test] fn an_annotation_in_another_encoding_is_not_latex() { // ODF writes StarMath here, which would be nonsense as LaTeX. From e93a72b6f33dc17edead041524afc965794d6ba8 Mon Sep 17 00:00:00 2001 From: harun Date: Sun, 16 Aug 2026 14:59:37 +0300 Subject: [PATCH 12/13] Version this fork by date, so a recipe id names which build made it `almila-document-converter-service` derives the storage key of every converted document from `{converter}-{version}-{fingerprint}` (`docs/adr/0010`), so two builds that answer with the same version mint the same key while producing different Markdown. After merging upstream v0.1.9 this branch did exactly that: it reports 0.1.9 and preserves superscripts, subscripts and Office Math that upstream 0.1.9 drops. CalVer because the three standard ways to mark a fork all fail a constraint that consumer enforces or that packaging requires: 0.1.9+almila.1 PEP 440 local version, but `+` is refused by that service's object-key component rule ([A-Za-z0-9._-]+), which guards a key against traversal and is the wrong thing to loosen 0.1.9-almila.1 valid semver, not valid PEP 440 -- maturin cannot ship it 0.1.9.post1 valid PEP 440, not valid semver -- Cargo cannot declare it 0.2.0 valid everywhere, and collides the day upstream releases it 2026.8.16 is valid semver, valid PEP 440, passes the key rule, and cannot collide with upstream's 0.x line. `scripts/check-versions.sh` passes on all five declarations. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01JL9pEBeAREYDRA62h6z1n6 --- Cargo.lock | 6 +++--- Cargo.toml | 2 +- node/index.js | 54 +++++++++++++++++++++++------------------------ node/package.json | 2 +- python/Cargo.toml | 2 +- wasm/Cargo.toml | 2 +- 6 files changed, 34 insertions(+), 34 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a01c9140..7665ea21 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "anydoc" -version = "0.1.9" +version = "2026.8.16" dependencies = [ "calamine", "cfb", @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "anydoc-python" -version = "0.1.9" +version = "2026.8.16" dependencies = [ "anydoc", "pyo3", @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "anydoc-wasm" -version = "0.1.9" +version = "2026.8.16" dependencies = [ "anydoc", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index 04ed82f8..bafa76db 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["fuzz"] [package] name = "anydoc" -version = "0.1.9" +version = "2026.8.16" edition = "2024" # Edition 2024 needs 1.85; zip and calamine both raise it to 1.88. rust-version = "1.88" diff --git a/node/index.js b/node/index.js index 4f592d69..1a8a5169 100644 --- a/node/index.js +++ b/node/index.js @@ -77,7 +77,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-android-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-android-arm64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -93,7 +93,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-android-arm-eabi') const bindingPackageVersion = require('@firecrawl/anydoc-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -114,7 +114,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-x64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -130,7 +130,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-x64-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -147,7 +147,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-ia32-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -163,7 +163,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-arm64-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -182,7 +182,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-universal') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-universal/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -198,7 +198,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-x64') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-x64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -214,7 +214,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-arm64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -234,7 +234,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-freebsd-x64') const bindingPackageVersion = require('@firecrawl/anydoc-freebsd-x64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -250,7 +250,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-freebsd-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -271,7 +271,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-x64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -287,7 +287,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-x64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -305,7 +305,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -321,7 +321,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -339,7 +339,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm-musleabihf') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -355,7 +355,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm-gnueabihf') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -373,7 +373,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-loong64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -389,7 +389,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-loong64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -407,7 +407,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-riscv64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -423,7 +423,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-riscv64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -440,7 +440,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-ppc64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -456,7 +456,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-s390x-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -476,7 +476,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -492,7 +492,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-x64') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-x64/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -508,7 +508,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-arm') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-arm/package.json').version - if (bindingPackageVersion !== '0.1.9' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -648,7 +648,7 @@ if (!nativeBinding || forceWasi) { if (!candidateFailed) { if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { const bindingPackageVersion = require('@firecrawl/anydoc-wasm32-wasi/package.json').version - if (bindingPackageVersion !== '0.1.9') { + if (bindingPackageVersion !== '2026.8.16') { throw new Error(`WASI binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } } diff --git a/node/package.json b/node/package.json index e49ed89d..a2726bfd 100644 --- a/node/package.json +++ b/node/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/anydoc", - "version": "0.1.9", + "version": "2026.8.16", "description": "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", "license": "MIT", "homepage": "https://github.com/firecrawl/anydoc#readme", diff --git a/python/Cargo.toml b/python/Cargo.toml index b600464b..362e661a 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -2,7 +2,7 @@ # dynamic version), so bump it together with the workspace release version. [package] name = "anydoc-python" -version = "0.1.9" +version = "2026.8.16" edition = "2024" description = "Python bindings for anydoc" license = "MIT" diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 4adc8a70..78c66a76 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -4,7 +4,7 @@ # only keeps it off crates.io. [package] name = "anydoc-wasm" -version = "0.1.9" +version = "2026.8.16" edition = "2024" description = "WebAssembly bindings for anydoc: convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown in the browser" license = "MIT" From dcb466014f26cf77c2d40ff9b808633aea03f4e2 Mon Sep 17 00:00:00 2001 From: harun Date: Tue, 18 Aug 2026 05:35:52 +0300 Subject: [PATCH 13/13] Return the per-page OCR verdict a PDF caller cannot get any other way `to_markdown_bytes` concatenates a PDF into one string and reports the pages it could not read through `log::warn!`, so a 90-page document with 3 scanned pages converts to 87 pages of confident, non-empty Markdown. Nothing downstream can tell that from a complete conversion. A caller that can OCR the remainder needs to know which pages to send; a caller that cannot needs to know the output is short. `pdf_pages` returns `PdfPage { index, markdown, needs_ocr, ocr_reason }` per page, over `pdf_inspector::extract_pages_markdown_mem` -- the API pdf-inspector documents for exactly this, "so callers can mix direct extraction (for simple text pages) with GPU OCR (for complex/scanned pages)". `to_markdown` is deliberately unchanged. The two verdicts DISAGREE, and the disagreement is measured on this crate's own `tests/fixtures/pdf/text.pdf`: `process_pdf_mem`, which `to_markdown` reads, flags page 2, whose text layer in fact yields "i Endnote body text." and which `extract_pages_markdown_mem` correctly passes. Routing on the document-level flag would send text pages to an OCR engine, so `to_markdown` keeps its own semantics and the module docstring records which API is for routing. `tests/pdf_pages.rs` pins the disagreement so a pdf-inspector upgrade that resolves it says so rather than leaving it to be rediscovered. Versioned 2026.8.18 because a consumer derives its storage key from the version (`almila-document-converter-service` `docs/adr/0010`): a build with a new API must not mint the previous build's key. All five declarations agree. cargo fmt --all --check, cargo clippy --workspace --all-targets --all-features -D warnings, cargo test --locked (273), and python -m unittest (11) pass. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_011XWEecjRapkLVUV99gwuE4 --- Cargo.lock | 6 ++-- Cargo.toml | 2 +- node/index.js | 54 ++++++++++++++++----------------- node/package.json | 2 +- python/Cargo.toml | 2 +- python/anydoc/__init__.py | 4 +++ python/anydoc/_anydoc.pyi | 28 ++++++++++++++++++ python/src/lib.rs | 44 +++++++++++++++++++++++++++ python/tests/test_anydoc.py | 14 +++++++++ src/formats/pdf.rs | 59 ++++++++++++++++++++++++++++++++++--- src/lib.rs | 13 ++++++++ tests/pdf_pages.rs | 50 +++++++++++++++++++++++++++++++ wasm/Cargo.toml | 2 +- 13 files changed, 242 insertions(+), 38 deletions(-) create mode 100644 tests/pdf_pages.rs diff --git a/Cargo.lock b/Cargo.lock index 7665ea21..60ca9bcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,7 +89,7 @@ dependencies = [ [[package]] name = "anydoc" -version = "2026.8.16" +version = "2026.8.18" dependencies = [ "calamine", "cfb", @@ -116,7 +116,7 @@ dependencies = [ [[package]] name = "anydoc-python" -version = "2026.8.16" +version = "2026.8.18" dependencies = [ "anydoc", "pyo3", @@ -124,7 +124,7 @@ dependencies = [ [[package]] name = "anydoc-wasm" -version = "2026.8.16" +version = "2026.8.18" dependencies = [ "anydoc", "js-sys", diff --git a/Cargo.toml b/Cargo.toml index bafa76db..0da2c1a7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = ["fuzz"] [package] name = "anydoc" -version = "2026.8.16" +version = "2026.8.18" edition = "2024" # Edition 2024 needs 1.85; zip and calamine both raise it to 1.88. rust-version = "1.88" diff --git a/node/index.js b/node/index.js index 1a8a5169..2f7b46e0 100644 --- a/node/index.js +++ b/node/index.js @@ -77,7 +77,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-android-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-android-arm64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -93,7 +93,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-android-arm-eabi') const bindingPackageVersion = require('@firecrawl/anydoc-android-arm-eabi/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -114,7 +114,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-x64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-win32-x64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -130,7 +130,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-x64-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-x64-msvc/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -147,7 +147,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-ia32-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-ia32-msvc/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -163,7 +163,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-win32-arm64-msvc') const bindingPackageVersion = require('@firecrawl/anydoc-win32-arm64-msvc/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -182,7 +182,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-universal') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-universal/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -198,7 +198,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-x64') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-x64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -214,7 +214,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-darwin-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-darwin-arm64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -234,7 +234,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-freebsd-x64') const bindingPackageVersion = require('@firecrawl/anydoc-freebsd-x64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -250,7 +250,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-freebsd-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-freebsd-arm64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -271,7 +271,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-x64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-x64-musl/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -287,7 +287,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-x64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-x64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -305,7 +305,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm64-musl/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -321,7 +321,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -339,7 +339,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm-musleabihf') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm-musleabihf/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -355,7 +355,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-arm-gnueabihf') const bindingPackageVersion = require('@firecrawl/anydoc-linux-arm-gnueabihf/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -373,7 +373,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-loong64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-loong64-musl/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -389,7 +389,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-loong64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-loong64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -407,7 +407,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-riscv64-musl') const bindingPackageVersion = require('@firecrawl/anydoc-linux-riscv64-musl/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -423,7 +423,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-riscv64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-riscv64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -440,7 +440,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-ppc64-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-ppc64-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -456,7 +456,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-linux-s390x-gnu') const bindingPackageVersion = require('@firecrawl/anydoc-linux-s390x-gnu/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -476,7 +476,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-arm64') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-arm64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -492,7 +492,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-x64') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-x64/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -508,7 +508,7 @@ function requireNative() { try { const binding = require('@firecrawl/anydoc-openharmony-arm') const bindingPackageVersion = require('@firecrawl/anydoc-openharmony-arm/package.json').version - if (bindingPackageVersion !== '2026.8.16' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { + if (bindingPackageVersion !== '2026.8.18' && process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { throw new Error(`Native binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } return binding @@ -648,7 +648,7 @@ if (!nativeBinding || forceWasi) { if (!candidateFailed) { if (process.env.NAPI_RS_ENFORCE_VERSION_CHECK && process.env.NAPI_RS_ENFORCE_VERSION_CHECK !== '0') { const bindingPackageVersion = require('@firecrawl/anydoc-wasm32-wasi/package.json').version - if (bindingPackageVersion !== '2026.8.16') { + if (bindingPackageVersion !== '2026.8.18') { throw new Error(`WASI binding package version mismatch, expected 0.1.9 but got ${bindingPackageVersion}. You can reinstall dependencies to fix this issue.`) } } diff --git a/node/package.json b/node/package.json index a2726bfd..900418b1 100644 --- a/node/package.json +++ b/node/package.json @@ -1,6 +1,6 @@ { "name": "@firecrawl/anydoc", - "version": "2026.8.16", + "version": "2026.8.18", "description": "Convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown", "license": "MIT", "homepage": "https://github.com/firecrawl/anydoc#readme", diff --git a/python/Cargo.toml b/python/Cargo.toml index 362e661a..de9a0707 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -2,7 +2,7 @@ # dynamic version), so bump it together with the workspace release version. [package] name = "anydoc-python" -version = "2026.8.16" +version = "2026.8.18" edition = "2024" description = "Python bindings for anydoc" license = "MIT" diff --git a/python/anydoc/__init__.py b/python/anydoc/__init__.py index f7c01d71..12eb7287 100644 --- a/python/anydoc/__init__.py +++ b/python/anydoc/__init__.py @@ -18,6 +18,7 @@ MalformedError, MissingPartError, Note, + PdfPage, ResourceLimitError, Style, Table, @@ -25,6 +26,7 @@ format_from_bytes, format_from_extension, format_from_path, + pdf_pages, to_document, to_markdown, to_markdown_bytes, @@ -54,6 +56,7 @@ "MalformedError", "MissingPartError", "Note", + "PdfPage", "ResourceLimitError", "Style", "Table", @@ -61,6 +64,7 @@ "format_from_bytes", "format_from_extension", "format_from_path", + "pdf_pages", "to_document", "to_markdown", "to_markdown_bytes", diff --git a/python/anydoc/_anydoc.pyi b/python/anydoc/_anydoc.pyi index cc4ab7fd..6ac5a068 100644 --- a/python/anydoc/_anydoc.pyi +++ b/python/anydoc/_anydoc.pyi @@ -71,6 +71,34 @@ def to_document(data: bytes | bytearray, format: Format | None = None) -> Docume Unsupported for `pdf`: PDF conversion produces Markdown directly and has no document-model form; use `to_markdown_bytes`.""" +def pdf_pages(data: bytes | bytearray) -> list[PdfPage]: + """Extract a PDF page by page, keeping the per-page verdict on whether + that page's text layer can be trusted. + + `to_markdown_bytes` returns one string and cannot say that some pages did + not extract; it logs and degrades. This returns the verdict, which is what + a caller able to OCR the remainder needs. Route OCR by `needs_ocr` here + and never by a document-level flag: the two disagree, and this one is the + API documented for routing. + + The per-page Markdown is flatter than `to_markdown_bytes`, which sees + structure across a page break that a page on its own cannot. + + PDFs only. Anything else raises `MalformedError`.""" + +@final +class PdfPage: + index: int + """0-indexed page number, in document order.""" + markdown: str + """Markdown extracted from this page's text layer, empty when the text + layer answered for nothing.""" + needs_ocr: bool + """True when the text layer cannot be trusted here: no text at all, + GID-encoded fonts, broken encodings, or garbage output.""" + ocr_reason: str | None + """Machine-readable reason for `needs_ocr`, where the cause is known.""" + @final class Document: blocks: list[Block] diff --git a/python/src/lib.rs b/python/src/lib.rs index abcbbd62..49541e26 100644 --- a/python/src/lib.rs +++ b/python/src/lib.rs @@ -182,6 +182,48 @@ fn to_document( document::document(py, parsed) } +/// One page of a PDF, as its own text layer answered for it. +#[pyclass(frozen, get_all, module = "anydoc")] +pub struct PdfPage { + /// 0-indexed page number, in document order. + index: u32, + /// Markdown extracted from this page's text layer, empty when the text + /// layer answered for nothing. + markdown: String, + /// True when the text layer cannot be trusted here: no text at all, + /// GID-encoded fonts, broken encodings, or garbage output. + needs_ocr: bool, + /// Machine-readable reason for `needs_ocr`, where the cause is known. + ocr_reason: Option, +} + +/// Extract a PDF page by page, keeping the per-page verdict on whether that +/// page's text layer can be trusted. +/// +/// `to_markdown_bytes` returns one string and cannot say that some pages did +/// not extract; it logs and degrades. This returns the verdict, which is what +/// a caller able to OCR the remainder needs. Route OCR by `needs_ocr` here and +/// never by a document-level flag: the two disagree, and this one is the API +/// documented for routing. +/// +/// The per-page Markdown is flatter than `to_markdown_bytes`, which sees +/// structure across a page break that a page on its own cannot. +/// +/// PDFs only. Anything else raises `MalformedError`. +#[pyfunction] +fn pdf_pages(py: Python<'_>, data: Vec) -> PyResult> { + let pages = py.detach(|| anydoc::pdf_pages(&data)).map_err(|e| convert_error(py, e))?; + Ok(pages + .into_iter() + .map(|page| PdfPage { + index: page.index, + markdown: page.markdown, + needs_ocr: page.needs_ocr, + ocr_reason: page.ocr_reason, + }) + .collect()) +} + /// Convert documents to GitHub-Flavored Markdown. #[pymodule] fn _anydoc(m: &Bound<'_, PyModule>) -> PyResult<()> { @@ -191,6 +233,8 @@ fn _anydoc(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_function(wrap_pyfunction!(to_markdown, m)?)?; m.add_function(wrap_pyfunction!(to_markdown_bytes, m)?)?; m.add_function(wrap_pyfunction!(to_document, m)?)?; + m.add_function(wrap_pyfunction!(pdf_pages, m)?)?; + m.add_class::()?; m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/python/tests/test_anydoc.py b/python/tests/test_anydoc.py index 1328d44a..9c384e25 100644 --- a/python/tests/test_anydoc.py +++ b/python/tests/test_anydoc.py @@ -12,6 +12,7 @@ OUTLINE = FIXTURES / "docx" / "handmade-outline.docx" RICH = FIXTURES / "docx" / "handmade-rich.docx" CSV = FIXTURES / "csv" / "sheet.csv" +PDF = FIXTURES / "pdf" / "text.pdf" ENCRYPTED = FIXTURES / "malformed" / "encrypted--errors.odt" ZIPBOMB = FIXTURES / "abuse" / "zipbomb--errors.docx" @@ -48,6 +49,19 @@ def test_to_document_carries_embedded_assets_as_bytes(self): self.assertGreater(len(image.data), 0) self.assertEqual(image.id, document.assets.index(image)) + def test_pdf_pages_returns_a_page_at_a_time_with_its_ocr_verdict(self): + pages = anydoc.pdf_pages(PDF.read_bytes()) + self.assertEqual([page.index for page in pages], [0, 1]) + self.assertIn("Fixture Document", pages[0].markdown) + # Every page of this fixture has an extractable text layer, including + # the thin second one that the document-level flag reports otherwise. + self.assertEqual([page.index for page in pages if page.needs_ocr], []) + self.assertIsNone(pages[0].ocr_reason) + + def test_pdf_pages_refuses_anything_that_is_not_a_pdf(self): + with self.assertRaises(anydoc.MalformedError): + anydoc.pdf_pages(RICH.read_bytes()) + def test_format_detection_reads_content_extension_and_path(self): self.assertEqual(anydoc.format_from_bytes(RICH.read_bytes()), "docx") # CSV carries no signature: only the extension names it. diff --git a/src/formats/pdf.rs b/src/formats/pdf.rs index 74a36e11..9e35a4b2 100644 --- a/src/formats/pdf.rs +++ b/src/formats/pdf.rs @@ -1,16 +1,67 @@ //! PDF via [pdf-inspector]: classification plus direct Markdown extraction. //! //! Unlike the other frontends, pdf-inspector emits Markdown itself, so PDFs -//! bypass the document model and the shared GFM writer. Scanned and -//! image-only PDFs need OCR, which is out of scope here; they error as -//! unsupported. Pages flagged for OCR in an otherwise text-based document -//! degrade with a log, consistent with the crate-wide recovery policy. +//! bypass the document model and the shared GFM writer. +//! +//! A PDF is the one input whose pages fail independently: a scanned page in an +//! otherwise text-based document extracts to nothing while its neighbours +//! extract cleanly. [`to_markdown`] returns one string and has nowhere to put +//! that verdict, so it degrades with a log, consistent with the crate-wide +//! recovery policy. [`pages`] returns the verdict per page, which is what a +//! caller able to OCR the remainder needs. +//! +//! **The two entry points do not share a routing verdict, and must not.** +//! [`to_markdown`] reads `process_pdf_mem`'s document-level `pages_needing_ocr`; +//! [`pages`] reads `extract_pages_markdown_mem`'s per-page `needs_ocr`. They +//! disagree, measured on this crate's own `tests/fixtures/pdf/text.pdf`: the +//! document-level call flags page 2, whose text layer in fact yields +//! `"i Endnote body text."` and which the per-page call correctly passes. Route +//! pages by [`pages`] alone — the per-page API is the one documented for it, +//! and the document-level flag sends text pages to an OCR engine. //! //! [pdf-inspector]: https://github.com/firecrawl/pdf-inspector use crate::error::ConvertError; use pdf_inspector::PdfError; +/// One page of a PDF, as its own text layer answered for it. +#[derive(Debug, Clone)] +pub struct Page { + /// 0-indexed page number, in document order. + pub index: u32, + /// Markdown extracted from this page's text layer. Empty when the text + /// layer answered for nothing. + pub markdown: String, + /// `true` when the text layer cannot be trusted here: no text at all, + /// GID-encoded fonts, broken encodings, or garbage output. + pub needs_ocr: bool, + /// Machine-readable reason for `needs_ocr`, where the cause is known. + pub ocr_reason: Option, +} + +/// Extract a PDF page by page, keeping the per-page OCR verdict. +/// +/// Font statistics are computed across the whole document, so heading +/// thresholds do not depend on how the caller later slices the result. The +/// per-page Markdown is not the same as [`to_markdown`]'s: the whole-document +/// path sees structure across a page break that a page on its own cannot, so +/// joining these strings produces a flatter document than [`to_markdown`] does. +/// Use this to decide what to OCR and to keep what extracted, not to reproduce +/// [`to_markdown`]. +pub fn pages(bytes: &[u8]) -> Result, ConvertError> { + let extracted = pdf_inspector::extract_pages_markdown_mem(bytes, None).map_err(map_error)?; + Ok(extracted + .pages + .into_iter() + .map(|page| Page { + index: page.page, + markdown: page.markdown, + needs_ocr: page.needs_ocr, + ocr_reason: page.ocr_reason, + }) + .collect()) +} + pub fn to_markdown(bytes: &[u8]) -> Result { let result = pdf_inspector::process_pdf_mem(bytes).map_err(map_error)?; if !result.pages_needing_ocr.is_empty() { diff --git a/src/lib.rs b/src/lib.rs index efba6ff8..523f269e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,6 +15,7 @@ mod render; mod shared; pub use error::ConvertError; +pub use formats::pdf::Page as PdfPage; use render::markdown::document_to_markdown; @@ -137,6 +138,18 @@ pub fn to_document( formats::parse(bytes, resolve_format(bytes, format.into())?) } +/// Extract a PDF page by page, keeping the per-page verdict on whether that +/// page's text layer can be trusted. +/// +/// [`to_markdown_bytes`] returns one string and cannot express a document +/// whose pages did not all extract, so it refuses one. This returns the pages +/// that did extract alongside the ones that did not, which is what a caller +/// able to OCR the remainder needs. The format is not detected and not passed: +/// this entry point is for PDFs only, and a non-PDF fails as malformed. +pub fn pdf_pages(bytes: &[u8]) -> Result, ConvertError> { + formats::pdf::pages(bytes) +} + fn resolve_format(bytes: &[u8], format: Option) -> Result { format.or_else(|| Format::from_bytes(bytes)).ok_or_else(|| { ConvertError::Unsupported("unrecognized file content: name the format explicitly".into()) diff --git a/tests/pdf_pages.rs b/tests/pdf_pages.rs new file mode 100644 index 00000000..ca12916d --- /dev/null +++ b/tests/pdf_pages.rs @@ -0,0 +1,50 @@ +//! `anydoc::pdf_pages` — the per-page entry point a caller routes OCR with. +//! +//! The load-bearing assertion here is the LAST one. `process_pdf_mem`, which +//! `to_markdown` reads, reports this fixture's page 2 as needing OCR; the +//! per-page API does not, and the per-page API is right — that page's text +//! layer yields real text. A caller that routed on the document-level flag +//! would send a text page to an OCR engine. If a pdf-inspector upgrade ever +//! makes these two agree, this test says so rather than leaving the +//! disagreement to be rediscovered. + +// The fixture path is built here rather than through `tests/common`: this +// binary uses only the root, and importing the shared module would pull in its +// unused `walk` and fail the `-D warnings` gate. +fn text_pdf() -> Vec { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/pdf/text.pdf"); + std::fs::read(path).expect("the pdf fixture") +} + +#[test] +fn pages_come_back_in_document_order_from_zero() { + let pages = anydoc::pdf_pages(&text_pdf()).expect("a readable pdf"); + assert_eq!(pages.len(), 2); + let indices: Vec = pages.iter().map(|page| page.index).collect(); + assert_eq!(indices, vec![0, 1]); +} + +#[test] +fn a_page_carries_the_markdown_of_its_own_text_layer() { + let pages = anydoc::pdf_pages(&text_pdf()).expect("a readable pdf"); + assert!(pages[0].markdown.contains("Fixture Document"), "{}", pages[0].markdown); + assert!(pages[1].markdown.contains("Endnote body text"), "{}", pages[1].markdown); +} + +#[test] +fn a_non_pdf_is_malformed_rather_than_unsupported() { + let error = anydoc::pdf_pages(b"PK\x03\x04not a pdf at all").expect_err("not a pdf"); + assert_eq!(error.code(), "malformed", "{error}"); +} + +/// See the module docstring: this is the disagreement, pinned. +#[test] +fn a_thin_but_real_text_page_is_not_routed_to_ocr() { + let pages = anydoc::pdf_pages(&text_pdf()).expect("a readable pdf"); + let flagged: Vec = + pages.iter().filter(|page| page.needs_ocr).map(|page| page.index).collect(); + assert!( + flagged.is_empty(), + "every page of this fixture has an extractable text layer, but {flagged:?} was flagged" + ); +} diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index 78c66a76..28377876 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -4,7 +4,7 @@ # only keeps it off crates.io. [package] name = "anydoc-wasm" -version = "2026.8.16" +version = "2026.8.18" edition = "2024" description = "WebAssembly bindings for anydoc: convert documents (doc, docx, odt, rtf, epub, pdf, presentations, spreadsheets, csv) to GitHub-Flavored Markdown in the browser" license = "MIT"