diff --git a/fixtures/ffm.json b/fixtures/ffm.json
index d7eb8b4..8e7539a 100644
--- a/fixtures/ffm.json
+++ b/fixtures/ffm.json
@@ -4,51 +4,63 @@
"cases": [
{
"input": "| 属性 | 类型 | 说明 |\n| :--- | :---: | ---: |\n| name | string | 用户名 |",
- "html": "\n\n\n| 属性 | \n类型 | \n说明 | \n
\n\n\n\n| name | \nstring | \n用户名 | \n
\n\n
"
+ "html": "\n\n\n| 属性 | \n类型 | \n说明 | \n
\n\n\n\n| name | \nstring | \n用户名 | \n
\n\n
",
+ "plain_text": "属性\n类型\n说明\nname\nstring\n用户名"
},
{
"input": "| :--- | ---: |\n| Ray ID | a2c5ac427b7aa727 |\n| IP 地址 | 172.214.47.18 |",
- "html": "\n\n\n| Ray ID | \na2c5ac427b7aa727 | \n
\n\n| IP 地址 | \n172.214.47.18 | \n
\n\n
"
+ "html": "\n\n\n| Ray ID | \na2c5ac427b7aa727 | \n
\n\n| IP 地址 | \n172.214.47.18 | \n
\n\n
",
+ "plain_text": "Ray ID\na2c5ac427b7aa727\nIP 地址\n172.214.47.18"
},
{
"input": "[x](javascript:alert(1))",
- "html": "[x](javascript:alert(1))
"
+ "html": "[x](javascript:alert(1))
",
+ "plain_text": "[x](javascript:alert(1))"
},
{
"input": "--Fuyeor--",
- "html": "Fuyeor
"
+ "html": "Fuyeor
",
+ "plain_text": "Fuyeor"
},
{
"input": "__Fuyeor__",
- "html": "Fuyeor
"
+ "html": "Fuyeor
",
+ "plain_text": "Fuyeor"
},
{
"input": "`#ff0000`",
- "html": "#ff0000
"
+ "html": "#ff0000
",
+ "plain_text": "#ff0000"
},
{
"input": "```quote\n## Title\n```",
- "html": "\nTitle
\n
"
+ "html": "\nTitle
\n
",
+ "plain_text": "Title"
},
{
"input": "```accordion\n**One**\nContent\n```",
- "html": ""
+ "html": "",
+ "plain_text": "One\nContent"
},
{
"input": "```chain\n**[x] Done**\nBody\n```",
- "html": ""
+ "html": "",
+ "plain_text": "Done\nBody"
},
{
"input": "```slide\nFirst\n\n---\n\nSecond\n```",
- "html": ""
+ "html": "",
+ "plain_text": "First\nSecond"
},
{
"input": "[x](data:text/html,)",
- "html": "[x](data:text/html,<script>alert(1)</script>)
"
+ "html": "[x](data:text/html,<script>alert(1)</script>)
",
+ "plain_text": "[x](data:text/html,)"
},
{
"input": "[x](file:///etc/passwd)",
- "html": "[x](file:///etc/passwd)
"
+ "html": "[x](file:///etc/passwd)
",
+ "plain_text": "[x](file:///etc/passwd)"
},
{
"input": "Visit fuyeor.xn--p1ai",
@@ -66,7 +78,8 @@
"path": "/0/children/1/children/0/content",
"value": "fuyeor.рф"
}
- ]
+ ],
+ "plain_text": "Visit fuyeor.рф"
},
{
"input": "1. example\n - example",
@@ -79,7 +92,8 @@
"path": "/0/children/0/children/1/type",
"value": "list"
}
- ]
+ ],
+ "plain_text": "example\nexample"
},
{
"input": "1. example\n - example",
@@ -92,7 +106,8 @@
"path": "/0/children/0/children/1/type",
"value": "list"
}
- ]
+ ],
+ "plain_text": "example\nexample"
},
{
"input": "1. root\n - level 1\n - level 2",
@@ -105,14 +120,16 @@
"path": "/0/children/0/children/1/children/0/children/1/type",
"value": "list"
}
- ]
+ ],
+ "plain_text": "root\nlevel 1\nlevel 2"
},
{
"input": ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> value",
"options": {
"max_nesting_depth": 8
},
- "no_throw": true
+ "no_throw": true,
+ "plain_text": ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> value"
},
{
"options": {
@@ -139,7 +156,8 @@
"path": "/1/children/0/type",
"value": "heading"
}
- ]
+ ],
+ "plain_text": "under and strike\nnested"
},
{
"input": "```slide\nFirst\n\n --- \n\nSecond\n```",
@@ -152,7 +170,8 @@
"path": "/0/children",
"length": 2
}
- ]
+ ],
+ "plain_text": "First\nSecond"
},
{
"input": "```accordion\n**First**\nBody\n\n**Second**\nMore\n```",
@@ -169,7 +188,8 @@
"path": "/0/children/0/name",
"value": "acc-0"
}
- ]
+ ],
+ "plain_text": "First\nBody\nSecond\nMore"
},
{
"input": "```chain\n**[x] Done**\nBody\n```",
@@ -186,7 +206,8 @@
"path": "/0/children/0/hasCheckbox",
"value": true
}
- ]
+ ],
+ "plain_text": "Done\nBody"
}
]
}
diff --git a/fixtures/markdown.json b/fixtures/markdown.json
index 08abd59..96348b3 100644
--- a/fixtures/markdown.json
+++ b/fixtures/markdown.json
@@ -4,19 +4,23 @@
"cases": [
{
"input": "## Hello **World**",
- "html": "Hello World
"
+ "html": "Hello World
",
+ "plain_text": "Hello World"
},
{
"input": "```ts\nconst a = 1;\n```",
- "html": ""
+ "html": "",
+ "plain_text": "const a = 1;"
},
{
"input": "visit www.fuyeor.com or [click here](https://fuyeor.com)",
- "html": "visit www.fuyeor.com or click here
"
+ "html": "visit www.fuyeor.com or click here
",
+ "plain_text": "visit www.fuyeor.com or click here"
},
{
"input": "line\\\nbreak",
- "html": "line
\nbreak
"
+ "html": "line
\nbreak
",
+ "plain_text": "line\nbreak"
},
{
"input": "text ",
@@ -30,7 +34,8 @@
"path": "/0/children/0/content",
"value": "text "
}
- ]
+ ],
+ "plain_text": "text "
},
{
"input": "\n| title 1 | title 2 |\n|---|---|\n| content 1 | content 2 |\n",
@@ -51,7 +56,13 @@
"path": "/0/children/0/type",
"value": "table_row"
}
- ]
+ ],
+ "plain_text": "title 1\ntitle 2\ncontent 1\ncontent 2"
+ },
+ {
+ "input": "",
+ "html": "<style>.x { color: red; }</style>
",
+ "plain_text": ""
}
]
}
diff --git a/packages/markdown-parser-rust/src/lib.rs b/packages/markdown-parser-rust/src/lib.rs
index 18e625b..e795c1d 100644
--- a/packages/markdown-parser-rust/src/lib.rs
+++ b/packages/markdown-parser-rust/src/lib.rs
@@ -9,6 +9,7 @@
mod ast;
mod linkify;
mod parser;
+mod plain_text;
mod render;
mod rules;
mod safety;
@@ -17,6 +18,7 @@ mod state;
pub use ast::{Alignment, AstNode, NodeType};
pub use linkify::{LinkMatch, linkify};
pub use parser::{Linkifier, MarkdownParser, ParserContext, ParserError, ParserOptions};
+pub use plain_text::to_plain_text;
pub use render::{render, render_optional};
pub use safety::{is_safe_color_value, is_safe_link_url};
pub use state::{BlockState, InlineState};
@@ -39,6 +41,7 @@ mod cross_language_fixtures {
use super::{
AstNode, MarkdownParser, ParserOptions, create_fuyeor_markdown_parser,
create_markdown_parser, is_safe_color_value, is_safe_link_url, linkify, render,
+ to_plain_text,
};
#[derive(Deserialize)]
@@ -51,6 +54,7 @@ mod cross_language_fixtures {
struct MarkdownCase {
input: Option,
html: Option,
+ plain_text: Option,
#[serde(default)]
assert: Vec,
error: Option,
@@ -203,6 +207,13 @@ mod cross_language_fixtures {
if let Some(expected_html) = &case.html {
assert_eq!(render(&ast).trim(), expected_html, "Markdown fixture");
}
+ if let Some(expected_plain_text) = &case.plain_text {
+ assert_eq!(
+ to_plain_text(&ast),
+ *expected_plain_text,
+ "Markdown fixture"
+ );
+ }
let normalized = Value::Array(ast.iter().map(normalize_node).collect());
for assertion in &case.assert {
let actual = read_json_pointer(&normalized, &assertion.path)
diff --git a/packages/markdown-parser-rust/src/plain_text.rs b/packages/markdown-parser-rust/src/plain_text.rs
new file mode 100644
index 0000000..0d44eaa
--- /dev/null
+++ b/packages/markdown-parser-rust/src/plain_text.rs
@@ -0,0 +1,110 @@
+// packages/markdown-parser-rust/src/plain_text.rs
+use std::fmt::Write;
+
+use crate::ast::{AstNode, NodeType};
+
+/// Converts Markdown/FFM AST nodes into the plain text used by search indexing.
+///
+/// The projection preserves visible text and code content, omits link destinations
+/// and presentation metadata, and treats HTML/CSS text as ordinary node content.
+pub fn to_plain_text(nodes: &[AstNode]) -> String {
+ let mut output = String::new();
+ append_block_nodes(nodes, &mut output);
+ output.trim_end_matches('\n').to_owned()
+}
+
+/// Appends block-level nodes with one newline between adjacent blocks.
+fn append_block_nodes(nodes: &[AstNode], output: &mut String) {
+ for (index, node) in nodes.iter().enumerate() {
+ if index > 0 {
+ push_separator(output);
+ }
+ append_block_node(node, output);
+ }
+}
+
+/// Appends one node using its Markdown/FFM text semantics.
+fn append_block_node(node: &AstNode, output: &mut String) {
+ match &node.node_type {
+ NodeType::Text | NodeType::InlineCode | NodeType::ColorCode => {
+ if let Some(content) = &node.content {
+ output.push_str(content);
+ }
+ }
+ NodeType::Heading
+ | NodeType::Paragraph
+ | NodeType::Bold
+ | NodeType::Italic
+ | NodeType::Underline
+ | NodeType::Strike
+ | NodeType::Link => append_inline_nodes(&node.children, output),
+ NodeType::CodeBlock => {
+ if let Some(content) = &node.content {
+ output.push_str(content);
+ }
+ }
+ NodeType::Blockquote
+ | NodeType::List
+ | NodeType::ListItem
+ | NodeType::Slide
+ | NodeType::SlideItem
+ | NodeType::Accordion
+ | NodeType::Chain => append_block_nodes(&node.children, output),
+ NodeType::AccordionItem => {
+ append_inline_nodes(node.title.as_deref().unwrap_or_default(), output);
+ if !output.is_empty() && !output.ends_with('\n') && !node.children.is_empty() {
+ output.push('\n');
+ }
+ append_block_nodes(&node.children, output);
+ }
+ NodeType::ChainItem => {
+ if let Some(title) = &node.title {
+ append_inline_nodes(title, output);
+ if !output.is_empty() && !output.ends_with('\n') && !node.children.is_empty() {
+ output.push('\n');
+ }
+ }
+ append_block_nodes(&node.children, output);
+ }
+ NodeType::Table => {
+ if let Some(headers) = &node.headers {
+ append_table_cells(headers, output);
+ }
+ if !node.children.is_empty() {
+ if !output.is_empty() {
+ push_separator(output);
+ }
+ append_block_nodes(&node.children, output);
+ }
+ }
+ NodeType::TableRow => append_table_cells(&node.children, output),
+ NodeType::TableCell => append_inline_nodes(&node.children, output),
+ NodeType::Hardbreak => output.push('\n'),
+ NodeType::Hr => {}
+ NodeType::Root | NodeType::Custom(_) => append_block_nodes(&node.children, output),
+ }
+}
+
+/// Appends inline children without inserting separators between formatting nodes.
+fn append_inline_nodes(nodes: &[AstNode], output: &mut String) {
+ for node in nodes {
+ append_block_node(node, output);
+ }
+}
+
+/// Appends table cells with a newline boundary so adjacent values remain distinct.
+fn append_table_cells(cells: &[AstNode], output: &mut String) {
+ for (index, cell) in cells.iter().enumerate() {
+ if index > 0 {
+ push_separator(output);
+ }
+ append_block_node(cell, output);
+ }
+}
+
+/// Adds a single logical newline without accumulating duplicate separators.
+fn push_separator(output: &mut String) {
+ if !output.is_empty() && !output.ends_with('\n') {
+ let _ = writeln!(output);
+ }
+}
diff --git a/packages/markdown-parser/src/core/plain-text.ts b/packages/markdown-parser/src/core/plain-text.ts
new file mode 100644
index 0000000..1b62e3c
--- /dev/null
+++ b/packages/markdown-parser/src/core/plain-text.ts
@@ -0,0 +1,88 @@
+// @fuyeor/markdown-parser/src/core/plain-text.ts
+import type { ASTNode } from '../types';
+
+/// Converts Markdown/FFM AST nodes into search-oriented plain text.
+export function toPlainText(nodes: readonly ASTNode[]): string {
+ return blockTextList(nodes).replace(/\n+$/u, '');
+}
+
+/// Converts one AST node without interpreting HTML or CSS content.
+function blockText(node: ASTNode): string {
+ switch (node.type) {
+ case 'text':
+ case 'inline_code':
+ case 'color_code':
+ return node.content ?? '';
+ case 'code_block':
+ return node.content ?? '';
+ case 'heading':
+ case 'paragraph':
+ case 'bold':
+ case 'italic':
+ case 'underline':
+ case 'strike':
+ case 'link':
+ case 'table_cell':
+ return inlineText(node.children ?? []);
+ case 'blockquote':
+ case 'list':
+ case 'list_item':
+ case 'slide':
+ case 'slide_item':
+ case 'accordion':
+ case 'chain':
+ case 'root':
+ return blockTextList(node.children ?? []);
+ case 'accordion_item':
+ return joinTitleAndBody(node.title ?? [], node.children ?? []);
+ case 'chain_item':
+ return joinTitleAndBody(node.title ?? [], node.children ?? []);
+ case 'table': {
+ const header = node.headers === undefined ? '' : blockTextList(node.headers);
+ const body = blockTextList(node.children ?? []);
+ return header.length > 0 && body.length > 0
+ ? `${header}\n${body}`
+ : header || body;
+ }
+ case 'table_row':
+ return joinNodes(node.children ?? []);
+ case 'hardbreak':
+ return '\n';
+ case 'hr':
+ return '';
+ default:
+ return blockTextList(node.children ?? []);
+ }
+}
+
+/// Converts inline children without inserting formatting boundaries.
+function inlineText(nodes: readonly ASTNode[]): string {
+ return nodes.map(blockText).join('');
+}
+
+/// Converts block children with a stable newline boundary.
+function blockTextList(nodes: readonly ASTNode[]): string {
+ let output = '';
+ for (const [index, node] of nodes.entries()) {
+ if (index > 0 && output.length > 0 && !output.endsWith('\n')) output += '\n';
+ output += blockText(node);
+ }
+ return output;
+}
+
+/// Joins an FFM item's title and body without exposing presentation metadata.
+function joinTitleAndBody(
+ title: readonly ASTNode[],
+ body: readonly ASTNode[],
+): string {
+ const titleText = inlineText(title);
+ const bodyText = blockTextList(body);
+ return titleText.length > 0 && bodyText.length > 0
+ ? `${titleText}\n${bodyText}`
+ : titleText || bodyText;
+}
+
+/// Joins table cells with boundaries that preserve searchable word separation.
+function joinNodes(nodes: readonly ASTNode[]): string {
+ return nodes.map(blockText).join('\n');
+}
diff --git a/packages/markdown-parser/src/cross-language-fixtures.spec.ts b/packages/markdown-parser/src/cross-language-fixtures.spec.ts
index 71472bf..1100248 100644
--- a/packages/markdown-parser/src/cross-language-fixtures.spec.ts
+++ b/packages/markdown-parser/src/cross-language-fixtures.spec.ts
@@ -6,6 +6,7 @@ import { isSafeColorValue } from './core/color';
import { MarkdownParser } from './core/parser';
import { isSafeLinkUrl } from './core/url';
import { render } from './core/render';
+import { toPlainText } from './core/plain-text';
import { createFuyeorMarkdownParser, createMarkdownParser } from './default';
type MarkdownAssertion = {
@@ -17,6 +18,7 @@ type MarkdownAssertion = {
type MarkdownCase = {
input?: string;
html?: string;
+ plain_text?: string;
assert?: MarkdownAssertion[];
error?: 'invalid_nesting_depth';
no_throw?: boolean;
@@ -84,6 +86,8 @@ function executeMarkdownCase(fixture: MarkdownCase, ffm: boolean): void {
: createMarkdownParser(options);
const ast = parser(fixture.input ?? '');
if (fixture.html !== undefined) expect(render(ast).trim()).toBe(fixture.html);
+ if (fixture.plain_text !== undefined)
+ expect(toPlainText(ast)).toBe(fixture.plain_text);
for (const assertion of fixture.assert ?? []) {
const value = readJsonPointer(ast, assertion.path);
diff --git a/packages/markdown-parser/src/index.ts b/packages/markdown-parser/src/index.ts
index aaa247b..4c72f91 100644
--- a/packages/markdown-parser/src/index.ts
+++ b/packages/markdown-parser/src/index.ts
@@ -1,6 +1,7 @@
// @fuyeor/markdown-parser/src/index.ts
export { MarkdownParser } from './core/parser';
export { render } from './core/render';
+export { toPlainText } from './core/plain-text';
export { isSafeColorValue } from './core/color';
export { isSafeLinkUrl } from './core/url';
export {