Skip to content

feat: Add Markdown plain text projection #12

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Fuyeors merged 1 commit into from
Aug 28, 2026
Merged

feat: Add Markdown plain text projection #12

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions fixtures/ffm.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,51 +4,63 @@
"cases": [
{
"input": "| 属性 | 类型 | 说明 |\n| :--- | :---: | ---: |\n| name | string | 用户名 |",
"html": "<table>\n<thead>\n<tr>\n<th align=\"left\">属性</th>\n<th align=\"center\">类型</th>\n<th align=\"right\">说明</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">name</td>\n<td align=\"center\">string</td>\n<td align=\"right\">用户名</td>\n</tr>\n</tbody>\n</table>"
"html": "<table>\n<thead>\n<tr>\n<th align=\"left\">属性</th>\n<th align=\"center\">类型</th>\n<th align=\"right\">说明</th>\n</tr>\n</thead>\n<tbody>\n<tr>\n<td align=\"left\">name</td>\n<td align=\"center\">string</td>\n<td align=\"right\">用户名</td>\n</tr>\n</tbody>\n</table>",
"plain_text": "属性\n类型\n说明\nname\nstring\n用户名"
},
{
"input": "| :--- | ---: |\n| Ray ID | a2c5ac427b7aa727 |\n| IP 地址 | 172.214.47.18 |",
"html": "<table>\n<tbody>\n<tr>\n<td align=\"left\">Ray ID</td>\n<td align=\"right\">a2c5ac427b7aa727</td>\n</tr>\n<tr>\n<td align=\"left\">IP 地址</td>\n<td align=\"right\">172.214.47.18</td>\n</tr>\n</tbody>\n</table>"
"html": "<table>\n<tbody>\n<tr>\n<td align=\"left\">Ray ID</td>\n<td align=\"right\">a2c5ac427b7aa727</td>\n</tr>\n<tr>\n<td align=\"left\">IP 地址</td>\n<td align=\"right\">172.214.47.18</td>\n</tr>\n</tbody>\n</table>",
"plain_text": "Ray ID\na2c5ac427b7aa727\nIP 地址\n172.214.47.18"
},
{
"input": "[x](javascript:alert(1))",
"html": "<p>[x](javascript:alert(1))</p>"
"html": "<p>[x](javascript:alert(1))</p>",
"plain_text": "[x](javascript:alert(1))"
},
{
"input": "--Fuyeor--",
"html": "<p><del>Fuyeor</del></p>"
"html": "<p><del>Fuyeor</del></p>",
"plain_text": "Fuyeor"
},
{
"input": "__Fuyeor__",
"html": "<p><u>Fuyeor</u></p>"
"html": "<p><u>Fuyeor</u></p>",
"plain_text": "Fuyeor"
},
{
"input": "`#ff0000`",
"html": "<p><code class=\"ffm-color-code\"><span class=\"ffm-color-swatch\" style=\"display:inline-block;width:0.8em;height:0.8em;border-radius:50%;background-color:#ff0000;vertical-align:middle;margin-right:0.3em;border:1px solid #00000030;\"></span>#ff0000</code></p>"
"html": "<p><code class=\"ffm-color-code\"><span class=\"ffm-color-swatch\" style=\"display:inline-block;width:0.8em;height:0.8em;border-radius:50%;background-color:#ff0000;vertical-align:middle;margin-right:0.3em;border:1px solid #00000030;\"></span>#ff0000</code></p>",
"plain_text": "#ff0000"
},
{
"input": "```quote\n## Title\n```",
"html": "<blockquote>\n<h2>Title</h2>\n</blockquote>"
"html": "<blockquote>\n<h2>Title</h2>\n</blockquote>",
"plain_text": "Title"
},
{
"input": "```accordion\n**One**\nContent\n```",
"html": "<div class=\"ffm-accordion\"><details name=\"acc-0\"><summary>One</summary><div class=\"accordion-content\"><p>Content</p>\n</div></details></div>"
"html": "<div class=\"ffm-accordion\"><details name=\"acc-0\"><summary>One</summary><div class=\"accordion-content\"><p>Content</p>\n</div></details></div>",
"plain_text": "One\nContent"
},
{
"input": "```chain\n**[x] Done**\nBody\n```",
"html": "<div class=\"chain-container\"><div class=\"chain-item is-completed\"><div class=\"chain-marker\"></div><div class=\"chain-content-wrapper\"><div class=\"chain-title\">Done</div><div class=\"chain-body\"><p>Body</p>\n</div></div></div></div>"
"html": "<div class=\"chain-container\"><div class=\"chain-item is-completed\"><div class=\"chain-marker\"></div><div class=\"chain-content-wrapper\"><div class=\"chain-title\">Done</div><div class=\"chain-body\"><p>Body</p>\n</div></div></div></div>",
"plain_text": "Done\nBody"
},
{
"input": "```slide\nFirst\n\n---\n\nSecond\n```",
"html": "<div class=\"slide-container-wrapper\"><div class=\"slide-container\"><div class=\"slide-item\"><p>First</p>\n</div><div class=\"slide-item\"><p>Second</p>\n</div></div></div>"
"html": "<div class=\"slide-container-wrapper\"><div class=\"slide-container\"><div class=\"slide-item\"><p>First</p>\n</div><div class=\"slide-item\"><p>Second</p>\n</div></div></div>",
"plain_text": "First\nSecond"
},
{
"input": "[x](data:text/html,<script>alert(1)</script>)",
"html": "<p>[x](data:text/html,&lt;script&gt;alert(1)&lt;/script&gt;)</p>"
"html": "<p>[x](data:text/html,&lt;script&gt;alert(1)&lt;/script&gt;)</p>",
"plain_text": "[x](data:text/html,<script>alert(1)</script>)"
},
{
"input": "[x](file:///etc/passwd)",
"html": "<p>[x](file:///etc/passwd)</p>"
"html": "<p>[x](file:///etc/passwd)</p>",
"plain_text": "[x](file:///etc/passwd)"
},
{
"input": "Visit fuyeor.xn--p1ai",
Expand All @@ -66,7 +78,8 @@
"path": "/0/children/1/children/0/content",
"value": "fuyeor.рф"
}
]
],
"plain_text": "Visit fuyeor.рф"
},
{
"input": "1. example\n - example",
Expand All @@ -79,7 +92,8 @@
"path": "/0/children/0/children/1/type",
"value": "list"
}
]
],
"plain_text": "example\nexample"
},
{
"input": "1. example\n - example",
Expand All @@ -92,7 +106,8 @@
"path": "/0/children/0/children/1/type",
"value": "list"
}
]
],
"plain_text": "example\nexample"
},
{
"input": "1. root\n - level 1\n - level 2",
Expand All @@ -105,14 +120,16 @@
"path": "/0/children/0/children/1/children/0/children/1/type",
"value": "list"
}
]
],
"plain_text": "root\nlevel 1\nlevel 2"
},
{
"input": ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> value",
"options": {
"max_nesting_depth": 8
},
"no_throw": true
"no_throw": true,
"plain_text": ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> value"
},
{
"options": {
Expand All @@ -139,7 +156,8 @@
"path": "/1/children/0/type",
"value": "heading"
}
]
],
"plain_text": "under and strike\nnested"
},
{
"input": "```slide\nFirst\n\n --- \n\nSecond\n```",
Expand All @@ -152,7 +170,8 @@
"path": "/0/children",
"length": 2
}
]
],
"plain_text": "First\nSecond"
},
{
"input": "```accordion\n**First**\nBody\n\n**Second**\nMore\n```",
Expand All @@ -169,7 +188,8 @@
"path": "/0/children/0/name",
"value": "acc-0"
}
]
],
"plain_text": "First\nBody\nSecond\nMore"
},
{
"input": "```chain\n**[x] Done**\nBody\n```",
Expand All @@ -186,7 +206,8 @@
"path": "/0/children/0/hasCheckbox",
"value": true
}
]
],
"plain_text": "Done\nBody"
}
]
}
23 changes: 17 additions & 6 deletions fixtures/markdown.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,23 @@
"cases": [
{
"input": "## Hello **World**",
"html": "<h2>Hello <strong>World</strong></h2>"
"html": "<h2>Hello <strong>World</strong></h2>",
"plain_text": "Hello World"
},
{
"input": "```ts\nconst a = 1;\n```",
"html": "<div class=\"code-block-wrapper\"><div class=\"code-lang\">ts</div><pre><code class=\"language-ts\">const a = 1;\n</code></pre></div>"
"html": "<div class=\"code-block-wrapper\"><div class=\"code-lang\">ts</div><pre><code class=\"language-ts\">const a = 1;\n</code></pre></div>",
"plain_text": "const a = 1;"
},
{
"input": "visit www.fuyeor.com or [click here](https://fuyeor.com)",
"html": "<p>visit <a href=\"https://www.fuyeor.com\">www.fuyeor.com</a> or <a href=\"https://fuyeor.com\">click here</a></p>"
"html": "<p>visit <a href=\"https://www.fuyeor.com\">www.fuyeor.com</a> or <a href=\"https://fuyeor.com\">click here</a></p>",
"plain_text": "visit www.fuyeor.com or click here"
},
{
"input": "line\\\nbreak",
"html": "<p>line<br />\nbreak</p>"
"html": "<p>line<br />\nbreak</p>",
"plain_text": "line\nbreak"
},
{
"input": "text <script>alert(1)</script>",
Expand All @@ -30,7 +34,8 @@
"path": "/0/children/0/content",
"value": "text <script>alert(1)</script>"
}
]
],
"plain_text": "text <script>alert(1)</script>"
},
{
"input": "\n| title 1 | title 2 |\n|---|---|\n| content 1 | content 2 |\n",
Expand All @@ -51,7 +56,13 @@
"path": "/0/children/0/type",
"value": "table_row"
}
]
],
"plain_text": "title 1\ntitle 2\ncontent 1\ncontent 2"
},
{
"input": "<style>.x { color: red; }</style>",
"html": "<p>&lt;style&gt;.x { color: red; }&lt;/style&gt;</p>",
"plain_text": "<style>.x { color: red; }</style>"
}
]
}
11 changes: 11 additions & 0 deletions packages/markdown-parser-rust/src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
mod ast;
mod linkify;
mod parser;
mod plain_text;
mod render;
mod rules;
mod safety;
Expand All @@ -17,6 +18,7 @@ mod state;
pub use ast::{Alignment, AstNode, NodeType};
pub use linkify::{LinkMatch, linkify};
pub use parser::{Linkifier, MarkdownParser, ParserContext, ParserError, ParserOptions};
pub use plain_text::to_plain_text;
pub use render::{render, render_optional};
pub use safety::{is_safe_color_value, is_safe_link_url};
pub use state::{BlockState, InlineState};
Expand All @@ -39,6 +41,7 @@ mod cross_language_fixtures {
use super::{
AstNode, MarkdownParser, ParserOptions, create_fuyeor_markdown_parser,
create_markdown_parser, is_safe_color_value, is_safe_link_url, linkify, render,
to_plain_text,
};

#[derive(Deserialize)]
Expand All @@ -51,6 +54,7 @@ mod cross_language_fixtures {
struct MarkdownCase {
input: Option<String>,
html: Option<String>,
plain_text: Option<String>,
#[serde(default)]
assert: Vec<MarkdownAssertion>,
error: Option<String>,
Expand Down Expand Up @@ -203,6 +207,13 @@ mod cross_language_fixtures {
if let Some(expected_html) = &case.html {
assert_eq!(render(&ast).trim(), expected_html, "Markdown fixture");
}
if let Some(expected_plain_text) = &case.plain_text {
assert_eq!(
to_plain_text(&ast),
*expected_plain_text,
"Markdown fixture"
);
}
let normalized = Value::Array(ast.iter().map(normalize_node).collect());
for assertion in &case.assert {
let actual = read_json_pointer(&normalized, &assertion.path)
Expand Down
110 changes: 110 additions & 0 deletions packages/markdown-parser-rust/src/plain_text.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
// packages/markdown-parser-rust/src/plain_text.rs
use std::fmt::Write;

use crate::ast::{AstNode, NodeType};

/// Converts Markdown/FFM AST nodes into the plain text used by search indexing.
///
/// The projection preserves visible text and code content, omits link destinations
/// and presentation metadata, and treats HTML/CSS text as ordinary node content.
pub fn to_plain_text(nodes: &[AstNode]) -> String {
let mut output = String::new();
append_block_nodes(nodes, &mut output);
output.trim_end_matches('\n').to_owned()
}

/// Appends block-level nodes with one newline between adjacent blocks.
fn append_block_nodes(nodes: &[AstNode], output: &mut String) {
for (index, node) in nodes.iter().enumerate() {
if index > 0 {
push_separator(output);
}
append_block_node(node, output);
}
}

/// Appends one node using its Markdown/FFM text semantics.
fn append_block_node(node: &AstNode, output: &mut String) {
match &node.node_type {
NodeType::Text | NodeType::InlineCode | NodeType::ColorCode => {
if let Some(content) = &node.content {
output.push_str(content);
}
}
NodeType::Heading
| NodeType::Paragraph
| NodeType::Bold
| NodeType::Italic
| NodeType::Underline
| NodeType::Strike
| NodeType::Link => append_inline_nodes(&node.children, output),
NodeType::CodeBlock => {
if let Some(content) = &node.content {
output.push_str(content);
}
}
NodeType::Blockquote
| NodeType::List
| NodeType::ListItem
| NodeType::Slide
| NodeType::SlideItem
| NodeType::Accordion
| NodeType::Chain => append_block_nodes(&node.children, output),
NodeType::AccordionItem => {
append_inline_nodes(node.title.as_deref().unwrap_or_default(), output);
if !output.is_empty() && !output.ends_with('\n') && !node.children.is_empty() {
output.push('\n');
}
append_block_nodes(&node.children, output);
}
NodeType::ChainItem => {
if let Some(title) = &node.title {
append_inline_nodes(title, output);
if !output.is_empty() && !output.ends_with('\n') && !node.children.is_empty() {
output.push('\n');
}
}
append_block_nodes(&node.children, output);
}
NodeType::Table => {
if let Some(headers) = &node.headers {
append_table_cells(headers, output);
}
if !node.children.is_empty() {
if !output.is_empty() {
push_separator(output);
}
append_block_nodes(&node.children, output);
}
}
NodeType::TableRow => append_table_cells(&node.children, output),
NodeType::TableCell => append_inline_nodes(&node.children, output),
NodeType::Hardbreak => output.push('\n'),
NodeType::Hr => {}
NodeType::Root | NodeType::Custom(_) => append_block_nodes(&node.children, output),
}
}

/// Appends inline children without inserting separators between formatting nodes.
fn append_inline_nodes(nodes: &[AstNode], output: &mut String) {
for node in nodes {
append_block_node(node, output);
}
}

/// Appends table cells with a newline boundary so adjacent values remain distinct.
fn append_table_cells(cells: &[AstNode], output: &mut String) {
for (index, cell) in cells.iter().enumerate() {
if index > 0 {
push_separator(output);
}
append_block_node(cell, output);
}
}

/// Adds a single logical newline without accumulating duplicate separators.
fn push_separator(output: &mut String) {
if !output.is_empty() && !output.ends_with('\n') {
let _ = writeln!(output);
}
}
Loading
Loading