From f5bfba3368b5d60e4ce4d8be9585158388c8e5f4 Mon Sep 17 00:00:00 2001 From: Nick Babcock Date: Mon, 28 Jul 2025 20:21:45 -0500 Subject: [PATCH 1/3] ... --- Cargo.toml | 2 + src/text/logos_lexer.rs | 468 ++++++++++++++++++++++++++++++++++++++++ src/text/mod.rs | 5 + 3 files changed, 475 insertions(+) create mode 100644 src/text/logos_lexer.rs diff --git a/Cargo.toml b/Cargo.toml index a50e21b..29e77f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,9 +21,11 @@ members = ["jomini_derive"] flate2 = { version = "1.1.5", optional = true, default-features = false, features = ["zlib-rs"] } itoa = { version = "1.0", optional = true } jomini_derive = { path = "jomini_derive", version = "^0.4.0", optional = true } +logos = "0.15.0" rawzip = { version = "0.4.2", optional = true } serde = { version = "1.0.195", optional = true } serde_json = { version = "1.0.114", optional = true } +smallvec = { version = "1.15.0", features = ["union", "const_generics"] } [features] default = ["derive", "faster_writer"] diff --git a/src/text/logos_lexer.rs b/src/text/logos_lexer.rs new file mode 100644 index 0000000..4a2efac --- /dev/null +++ b/src/text/logos_lexer.rs @@ -0,0 +1,468 @@ +use logos::{Lexer, Logos}; +use smallvec::SmallVec; +use crate::{Error, ErrorKind}; + +type ExpressionList = SmallVec<[ExpressionToken; 16]>; + +/// A lossless token that captures all syntax elements including trivia +#[derive(Logos, Debug, PartialEq, Clone)] +pub enum LosslessToken { + // Operators from existing parser + /// Exact equality operator `==` + #[token(b"==")] + Exact, + /// Less than or equal operator `<=` + #[token(b"<=")] + LessThanEqual, + /// Greater than or equal operator `>=` + #[token(b">=")] + GreaterThanEqual, + /// Not equal operator `!=` + #[token(b"!=")] + NotEqual, + /// Exists operator `?=` + #[token(b"?=")] + Exists, + /// Equal operator `=` + #[token(b"=")] + Equal, + /// Less than operator `<` + #[token(b"<")] + LessThan, + /// Greater than operator `>` + #[token(b">")] + GreaterThan, + + // Structural tokens + /// Left brace `{` + #[token(b"{")] + LBrace, + /// Right brace `}` + #[token(b"}")] + RBrace, + /// Left bracket `[` + #[token(b"[")] + LBracket, + /// Right bracket `]` + #[token(b"]")] + RBracket, + + // String literals - handle escape sequences properly + /// Quoted string literal + #[regex(br#""([^"\\]|\\.)*""#)] + Quoted, + + // Expression tokens for @[...] syntax + /// Expression in @[...] syntax + #[token(b"@[", parse_expression)] + Expression(ExpressionList), + + // Variable references starting with @ + /// Variable reference starting with @ + #[regex(br"@[a-zA-Z_][a-zA-Z0-9_]*")] + Variable, + + /// Undefined parameter marker ! (for [[!var_name] syntax) + #[token(b"!")] + UndefinedParameter, + + // Unquoted tokens - most common case + /// Unquoted identifier or value + #[regex(br"[a-zA-Z0-9_\-.:/|]+")] + Unquoted, + + // Trivia tokens for lossless parsing + /// Whitespace (spaces, tabs, newlines, semicolons) + #[regex(br"[ \t\r\n;]+")] + Whitespace, + + /// Comment starting with # + #[regex(br"#[^\r\n]*")] + Comment, +} + +/// Tokens for expressions inside @[...] constructs +#[derive(Debug, Copy, Clone, PartialEq, Logos)] +pub enum ExpressionToken { + /// Subtraction operator `-` + #[token(b"-")] + Subtract, + /// Addition operator `+` + #[token(b"+")] + Add, + /// Multiplication operator `*` + #[token(b"*")] + Multiply, + /// Division operator `/` + #[token(b"/")] + Divide, + /// Left parenthesis `(` + #[token(b"(")] + LParen, + /// Right parenthesis `)` + #[token(b")")] + RParen, + /// Right bracket `]` - marks end of expression + #[token(b"]")] + RBracket, + + /// Integer literal + #[regex(br"-?[0-9]+")] + Integer, + + /// Float literal (with optional 'f' suffix) + #[regex(br"-?[0-9]*\.[0-9]+f?")] + Float, + + /// Identifier (includes variables with @ prefix) + #[regex(br"@?[a-zA-Z_][a-zA-Z0-9_]*")] + Identifier, + + /// Whitespace within expressions + #[regex(br"[ \t]+")] + Whitespace, +} + + +/// Parse expression tokens between @[ and ] +fn parse_expression(lex: &mut Lexer) -> Result { + let remaining = lex.remainder(); + let mut expr_lexer = ExpressionToken::lexer(remaining); + let mut tokens = SmallVec::new(); + + // Parse tokens until we hit RBracket (which marks the end) + while let Some(token) = expr_lexer.next() { + match token { + Ok(ExpressionToken::RBracket) => { + // Found the end - advance main lexer past the ] + lex.bump(expr_lexer.span().end); + return Ok(tokens); + } + Ok(t) => tokens.push(t), + Err(_) => { + // Skip invalid tokens - could be improved + continue; + } + } + } + + // If we get here, we didn't find a closing bracket + Err(()) +} + +/// Lossless lexer for jomini text format +pub struct LosslessLexer<'a> { + lexer: Lexer<'a, LosslessToken>, + input: &'a [u8], + original_input: &'a [u8], + bom_offset: usize, +} + +impl<'a> LosslessLexer<'a> { + /// Create a new lossless lexer from input bytes + pub fn new(input: &'a [u8]) -> Self { + // Strip UTF-8 BOM if present + let (stripped_input, bom_offset) = if input.starts_with(&[0xef, 0xbb, 0xbf]) { + (&input[3..], 3) + } else { + (input, 0) + }; + + Self { + lexer: LosslessToken::lexer(stripped_input), + input: stripped_input, + original_input: input, + bom_offset, + } + } + + /// Get the next token from the input + pub fn next_token(&mut self) -> Option> { + match self.lexer.next() { + Some(Ok(token)) => Some(Ok(token)), + Some(Err(_)) => { + // Handle error - could be invalid token + let span = self.lexer.span(); + Some(Err(Error::new(ErrorKind::InvalidSyntax { + msg: "Invalid token".to_string(), + offset: span.start + }))) + } + None => None, + } + } + + /// Get current position in the input (adjusted for BOM) + pub fn position(&self) -> usize { + self.lexer.span().start + self.bom_offset + } + + /// Get the current span being processed (adjusted for BOM) + pub fn span(&self) -> std::ops::Range { + let span = self.lexer.span(); + (span.start + self.bom_offset)..(span.end + self.bom_offset) + } + + /// Get remaining input + pub fn remainder(&self) -> &'a [u8] { + self.lexer.remainder() + } + + /// Get the text for a token at the current span + pub fn token_text(&self) -> &'a [u8] { + let span = self.lexer.span(); + &self.input[span] + } + + /// Get the text for a specific span in the original input + pub fn span_text(&self, span: std::ops::Range) -> &'a [u8] { + &self.original_input[span] + } +} + +/// Iterator implementation for the lossless lexer +impl<'a> Iterator for LosslessLexer<'a> { + type Item = Result; + + fn next(&mut self) -> Option { + self.next_token() + } +} + +/// Token with position information for lossless reconstruction +#[derive(Debug, Clone, PartialEq)] +pub struct TokenWithSpan { + /// The token type + pub token: LosslessToken, + /// The byte span in the original input where this token occurs + pub span: std::ops::Range, +} + +/// Collect all tokens with their spans for lossless processing +pub fn collect_tokens_with_spans(input: &[u8]) -> Result, Error> { + let mut lexer = LosslessLexer::new(input); + let mut tokens = Vec::new(); + + while let Some(result) = lexer.next_token() { + match result { + Ok(token) => { + let span = lexer.span(); + tokens.push(TokenWithSpan { token, span }); + } + Err(e) => return Err(e), + } + } + + Ok(tokens) +} + +/// Verify lossless property by reconstructing input from tokens with spans +pub fn reconstruct_input(original_input: &[u8], tokens: &[TokenWithSpan]) -> Vec { + let mut result = Vec::new(); + + for token_with_span in tokens { + let text = &original_input[token_with_span.span.clone()]; + result.extend_from_slice(text); + } + + result +} + +#[cfg(test)] +mod tests { + use super::*; + use rstest::*; + + #[rstest] + #[case(b"foo = bar", vec![ + LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Equal, + LosslessToken::Whitespace, + LosslessToken::Unquoted, + ])] + fn test_basic_tokens(#[case] input: &[u8], #[case] expected_tokens: Vec) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); + + assert_eq!(tokens, expected_tokens); + + // Test lossless property + let reconstructed = reconstruct_input(input, &tokens_with_spans); + assert_eq!(reconstructed, input); + } + + #[rstest] + #[case(b"a==b c<=d e>=f g!=h i?=j kn", vec![ + LosslessToken::Unquoted, LosslessToken::Exact, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::LessThanEqual, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::GreaterThanEqual, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::NotEqual, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::Exists, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::LessThan, LosslessToken::Unquoted, + LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::GreaterThan, LosslessToken::Unquoted, + ])] + fn test_operators(#[case] input: &[u8], #[case] expected_tokens: Vec) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); + + assert_eq!(tokens, expected_tokens); + + // Test lossless property + let reconstructed = reconstruct_input(input, &tokens_with_spans); + assert_eq!(reconstructed, input); + } + + // Helper function to test lossless property for any input + fn test_lossless(input: &[u8]) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let reconstructed = reconstruct_input(input, &tokens_with_spans); + assert_eq!(reconstructed, input, "Failed lossless test for: {:?}", + std::str::from_utf8(input).unwrap_or("invalid utf8")); + } + + #[rstest] + #[case(br#"name = "hello \"world\"""#, vec![ + LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Equal, + LosslessToken::Whitespace, LosslessToken::Quoted + ])] + fn test_quoted_strings(#[case] input: &[u8], #[case] expected_tokens: Vec) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); + + assert_eq!(tokens, expected_tokens); + test_lossless(input); + } + + #[rstest] + #[case(b"# This is a comment\nfoo = bar\t# Another comment\n", vec![ + LosslessToken::Comment, LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Equal, LosslessToken::Whitespace, + LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Comment, LosslessToken::Whitespace + ])] + fn test_comments_and_whitespace(#[case] input: &[u8], #[case] expected_tokens: Vec) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); + + assert_eq!(tokens, expected_tokens); + test_lossless(input); + } + + #[rstest] + #[case(b"@planet_standard_scale = @default_window_name", vec![ + LosslessToken::Variable, LosslessToken::Whitespace, LosslessToken::Equal, + LosslessToken::Whitespace, LosslessToken::Variable + ])] + fn test_variables(#[case] input: &[u8], #[case] expected_tokens: Vec) { + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); + + assert_eq!(tokens, expected_tokens); + test_lossless(input); + } + + #[test] + fn test_expressions() { + let input = b"position = { @[1-leopard_x] @leopard_y }"; + test_lossless(input); + + // Verify we have an Expression token + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let has_expression = tokens_with_spans.iter().any(|t| matches!(t.token, LosslessToken::Expression(_))); + assert!(has_expression, "Should contain an Expression token"); + } + + #[test] + fn test_complex_expressions() { + let input = b"my_calc = @[(-half-half)*half]"; + test_lossless(input); + + // Verify we have an Expression token + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let has_expression = tokens_with_spans.iter().any(|t| matches!(t.token, LosslessToken::Expression(_))); + assert!(has_expression, "Should contain an Expression token"); + } + + #[test] + fn test_bom_stripping() { + let input = b"\xef\xbb\xbf# UTF-8 BOM test\nfoo = bar"; + let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); + let reconstructed = reconstruct_input(input, &tokens_with_spans); + + // BOM is stripped during lexing, so reconstructed input won't include BOM + // This is expected behavior since we removed LosslessToken::Bom variant + let expected_without_bom = b"# UTF-8 BOM test\nfoo = bar"; + assert_eq!(reconstructed, expected_without_bom, "Reconstructed should match input without BOM"); + + // The first token should be Comment (BOM was stripped before lexing) + assert_eq!(tokens_with_spans[0].token, LosslessToken::Comment); + assert_eq!(tokens_with_spans[0].span, 3..19); // BOM offset handled in span + + // Verify lexer properly handles BOM without producing separate token + let input_without_bom = b"# UTF-8 BOM test\nfoo = bar"; + let tokens_without_bom = collect_tokens_with_spans(input_without_bom).unwrap(); + + // Both inputs should produce the same tokens (just different spans) + assert_eq!(tokens_with_spans.len(), tokens_without_bom.len()); + for (with_bom, without_bom) in tokens_with_spans.iter().zip(tokens_without_bom.iter()) { + assert_eq!(with_bom.token, without_bom.token); + } + } + + #[test] + fn test_lossless_token_size() { + // Test that LosslessToken has a reasonable size + // This is important for memory efficiency since we create many tokens + let size = std::mem::size_of::(); + + // Should be reasonable size - exact size depends on SmallVec internal layout + // but should be significantly smaller than a Vec would be + println!("LosslessToken size: {} bytes", size); + + // Ensure it's not unreasonably large (less than 64 bytes) + assert!(size <= 64, "LosslessToken size ({} bytes) should be <= 64 bytes", size); + + // Also test ExpressionToken size + let expr_size = std::mem::size_of::(); + println!("ExpressionToken size: {} bytes", expr_size); + + // ExpressionToken should be small since it's Copy + assert!(expr_size <= 8, "ExpressionToken size ({} bytes) should be <= 8 bytes", expr_size); + } + + #[rstest] + #[case(b"[[scaled_skill] code here ] [[!var_name] other code ]")] + #[case(b"stats={{id=0 type=general} {id=1 type=admiral}}")] + fn test_lossless_samples(#[case] input: &[u8]) { + test_lossless(input); + } + + #[rstest] + #[case(b"foo = bar")] + #[case(b"open={1 2}")] + #[case(b"field1=-100.535")] + #[case(br#""foo"="bar" "3"="1444.11.11""#)] + #[case(br#"custom_name="THE !@#$%^&*( '\"LEGION\"')""#)] + #[case(b"foo{bar=qux}")] + #[case(b"foo=abc#def\nbar=qux")] + #[case(b"flavor_tur.8=yes")] + #[case(b"dashed-identifier=yes")] + #[case(b"province_id = event_target:agenda_province")] + #[case(b"mult = value:job_weights_research_modifier|JOB|head_researcher|")] + #[case(b"@planet_standard_scale = 11")] + #[case(b"window_name = @default_window_name")] + #[case(b"value=\"win\"; a=b")] + #[case(b"foo = 0.3;")] + #[case(b"a = 1; b = 2;; c = 3;")] + #[case(b";;;key = value;;;")] + #[case(b"age > 16")] + fn test_comprehensive_syntax(#[case] input: &[u8]) { + test_lossless(input); + } +} \ No newline at end of file diff --git a/src/text/mod.rs b/src/text/mod.rs index 71c0e05..fdb1bfc 100644 --- a/src/text/mod.rs +++ b/src/text/mod.rs @@ -25,6 +25,7 @@ pub mod de; mod dom; mod fnv; +mod logos_lexer; mod operator; mod reader; mod tape; @@ -37,6 +38,10 @@ pub use self::dom::{ ArrayReader, FieldGroupsIter, FieldsIter, GroupEntry, GroupEntryIter, ObjectReader, Reader, ScalarReader, ValueReader, ValuesIter, }; +pub use self::logos_lexer::{ + ExpressionToken, LosslessLexer, LosslessToken, TokenWithSpan, + collect_tokens_with_spans, reconstruct_input +}; pub use self::operator::*; pub use self::tape::{TextTape, TextTapeParser, TextToken}; pub use self::writer::*; From 266d982a75f04a8c2c38ad11f3694b601bc188c2 Mon Sep 17 00:00:00 2001 From: Nick Babcock Date: Sat, 13 Jun 2026 11:31:19 -0500 Subject: [PATCH 2/3] LSP --- Cargo.toml | 1 - bench/src/benchmarks/text.rs | 18 +- examples/lint.rs | 169 + .../mod/common/buildings/01_mod_buildings.txt | 18 + .../mod-demo/mod/events/broken_events.txt | 8 + examples/mod-demo/mod/events/mod_events.txt | 8 + .../vanilla/common/buildings/00_buildings.txt | 18 + .../mod-demo/vanilla/events/court_events.txt | 9 + examples/syntax.rs | 109 + jomini-lsp/.gitignore | 2 + jomini-lsp/Cargo.toml | 23 + jomini-lsp/README.md | 112 + jomini-lsp/src/convert.rs | 53 + jomini-lsp/src/lib.rs | 124 + jomini-lsp/src/line_index.rs | 203 ++ jomini-lsp/src/main.rs | 12 + jomini-lsp/src/server.rs | 523 +++ jomini-lsp/tests/smoke.rs | 258 ++ src/text/lint.rs | 1274 +++++++ src/text/logos_lexer.rs | 468 --- src/text/mod.rs | 7 +- src/text/syntax.rs | 2979 +++++++++++++++++ 22 files changed, 5921 insertions(+), 475 deletions(-) create mode 100644 examples/lint.rs create mode 100644 examples/mod-demo/mod/common/buildings/01_mod_buildings.txt create mode 100644 examples/mod-demo/mod/events/broken_events.txt create mode 100644 examples/mod-demo/mod/events/mod_events.txt create mode 100644 examples/mod-demo/vanilla/common/buildings/00_buildings.txt create mode 100644 examples/mod-demo/vanilla/events/court_events.txt create mode 100644 examples/syntax.rs create mode 100644 jomini-lsp/.gitignore create mode 100644 jomini-lsp/Cargo.toml create mode 100644 jomini-lsp/README.md create mode 100644 jomini-lsp/src/convert.rs create mode 100644 jomini-lsp/src/lib.rs create mode 100644 jomini-lsp/src/line_index.rs create mode 100644 jomini-lsp/src/main.rs create mode 100644 jomini-lsp/src/server.rs create mode 100644 jomini-lsp/tests/smoke.rs create mode 100644 src/text/lint.rs delete mode 100644 src/text/logos_lexer.rs create mode 100644 src/text/syntax.rs diff --git a/Cargo.toml b/Cargo.toml index 29e77f0..8b56331 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,6 @@ members = ["jomini_derive"] flate2 = { version = "1.1.5", optional = true, default-features = false, features = ["zlib-rs"] } itoa = { version = "1.0", optional = true } jomini_derive = { path = "jomini_derive", version = "^0.4.0", optional = true } -logos = "0.15.0" rawzip = { version = "0.4.2", optional = true } serde = { version = "1.0.195", optional = true } serde_json = { version = "1.0.114", optional = true } diff --git a/bench/src/benchmarks/text.rs b/bench/src/benchmarks/text.rs index 8f7a812..49ab048 100644 --- a/bench/src/benchmarks/text.rs +++ b/bench/src/benchmarks/text.rs @@ -18,6 +18,12 @@ pub(crate) fn tape_parse(data: &[u8]) -> usize { black_box(tape.tokens().len()) } +#[inline(never)] +pub(crate) fn syntax_parse(data: &[u8]) -> usize { + let tree = jomini::text::syntax::parse(data); + black_box(tree.root().children().count()) +} + #[inline(never)] pub(crate) fn reader_count_equals(data: &[u8]) -> i32 { let mut reader = jomini::text::TokenReader::from_slice(data); @@ -91,6 +97,9 @@ pub mod criterion_benches { group.bench_function(BenchmarkId::new("reader", game), |b| { b.iter(|| reader_count_equals(data)) }); + group.bench_function(BenchmarkId::new("syntax", game), |b| { + b.iter(|| syntax_parse(data)) + }); } group.finish(); } @@ -137,6 +146,13 @@ pub mod gungraun_benches { reader_count_equals(data) } + #[library_benchmark] + #[bench::eu4(setup = setup_eu4)] + #[bench::ck3(setup = setup_ck3)] + fn syntax(data: &[u8]) -> usize { + syntax_parse(data) + } + #[library_benchmark] #[bench::meta(setup = setup_meta)] fn deserialize(data: &[u8]) -> Meta { @@ -152,6 +168,6 @@ pub mod gungraun_benches { library_benchmark_group!( name = text_benches, - benchmarks = [tape, reader, deserialize, compressed_read,] + benchmarks = [tape, reader, syntax, deserialize, compressed_read,] ); } diff --git a/examples/lint.rs b/examples/lint.rs new file mode 100644 index 0000000..b55b8d2 --- /dev/null +++ b/examples/lint.rs @@ -0,0 +1,169 @@ +//! A **cross-file** linter for Clausewitz game files, built on the lossless +//! syntax tree's new project/semantic layer ([`jomini::text::lint`]). +//! +//! This is the thing a single-file parser fundamentally *cannot* do: it loads a +//! whole "mod" plus its "vanilla" base, builds a project-wide symbol table, and +//! then resolves every building reference against it. A reference to a building +//! defined in no file — possibly a typo — becomes an error with a precise +//! location *and a "did you mean?" fix*; a mod definition that shadows a vanilla +//! one is noted; a building defined twice in one layer is a warning; and a +//! reference inside a block with a syntax error is suppressed instead of +//! generating cascade noise. +//! +//! Because the tree is **lossless**, fixes are byte-faithful: the typo is +//! rewritten in place and everything else is preserved exactly — something a +//! lossy validator structurally cannot offer. +//! +//! ```sh +//! cargo run --example lint # lint the bundled examples/mod-demo +//! cargo run --example lint -- --fix # also show machine-applicable fixes +//! cargo run --example lint -- --fix --write # ...and write them to disk +//! cargo run --example lint -- --no-color +//! cargo run --example lint -- +//! ``` + +use jomini::text::lint::{ + apply_fixes, line_col, lints, render, FileKind, Fileset, Fix, Linter, Schema, Severity, +}; +use std::collections::BTreeMap; +use std::path::{Path, PathBuf}; + +fn main() { + let mut color = true; + let mut fix_mode = false; + let mut write_mode = false; + let mut dirs: Vec = Vec::new(); + for arg in std::env::args().skip(1) { + match arg.as_str() { + "--no-color" => color = false, + "--fix" => fix_mode = true, + "--write" => { + fix_mode = true; + write_mode = true; + } + _ => dirs.push(PathBuf::from(arg)), + } + } + + // Default to the bundled two-layer demo corpus (resolved relative to the + // crate so `cargo run --example lint` works from anywhere). + let manifest = Path::new(env!("CARGO_MANIFEST_DIR")); + let (vanilla_dir, mod_dir) = match dirs.as_slice() { + [] => ( + manifest.join("examples/mod-demo/vanilla"), + manifest.join("examples/mod-demo/mod"), + ), + [v, m] => (v.clone(), m.clone()), + _ => { + eprintln!("usage: lint [--no-color] [--fix] [--write] [ ]"); + std::process::exit(2); + } + }; + + // 1. Project model: load each directory tree as a load-order layer. + let mut fileset = Fileset::new(); + let vanilla = fileset.load_dir(&vanilla_dir, FileKind::Vanilla).unwrap_or_else(|e| { + eprintln!("failed to read {}: {e}", vanilla_dir.display()); + std::process::exit(1); + }); + let modded = fileset.load_dir(&mod_dir, FileKind::Mod(0)).unwrap_or_else(|e| { + eprintln!("failed to read {}: {e}", mod_dir.display()); + std::process::exit(1); + }); + + // 2. Schema: the (hand-written) rules of what defines and references a + // building. A real linter would carry a per-game rule set here. + let mut schema = Schema::new(); + schema + .define_dir("buildings", "building") + .reference_key("has_building", "building") + .reference_key("add_building", "building") + .reference_key("remove_building", "building") + .reference_key("upgrades_from", "building"); + + // 3. Two-pass lint: collect all definitions, then resolve all references. + let diagnostics = Linter::new(schema).run(&fileset); + + let bold = if color { "\x1b[1m" } else { "" }; + let dim = if color { "\x1b[2m" } else { "" }; + let reset = if color { "\x1b[0m" } else { "" }; + + println!( + "{bold}Linting {} vanilla + {} mod file(s){reset} {dim}(one combined project){reset}\n", + vanilla.len(), + modded.len(), + ); + + for diag in &diagnostics { + print!("{}", render(diag, &fileset, color)); + println!(); + } + + // Summary, broken down by the Severity axis. + let count = |s: Severity| diagnostics.iter().filter(|d| d.severity == s).count(); + let errors = count(Severity::Error); + let warnings = count(Severity::Warning); + let notes = count(Severity::Tip); + println!("{bold}Summary:{reset} {errors} error(s), {warnings} warning(s), {notes} note(s)"); + + // The punchline: the undefined-reference catch is impossible per-file. + if diagnostics.iter().any(|d| d.id == lints::UNDEFINED_REFERENCE) { + println!( + "{dim}The undefined-reference error spans files: the reference is in one file, \n\ + its (missing) definition would live in another — only a project-wide pass finds it.{reset}" + ); + } + + // 4. Autofix: apply each diagnostic's machine-applicable fix. The lossless + // tree makes this byte-faithful — only the offending token changes. + if fix_mode { + apply_and_report(&fileset, &diagnostics, write_mode, bold, dim, reset); + } + + std::process::exit(if errors > 0 && !write_mode { 1 } else { 0 }); +} + +/// Group the fixes by file, show each as `old -> new` at its location, and — +/// with `--write` — splice them into the file on disk via `apply_fixes`. +fn apply_and_report( + fileset: &Fileset, + diagnostics: &[jomini::text::lint::Diagnostic], + write: bool, + bold: &str, + dim: &str, + reset: &str, +) { + let mut by_file: BTreeMap<_, Vec<&Fix>> = BTreeMap::new(); + for d in diagnostics { + if let Some(fix) = &d.fix { + by_file.entry(d.file).or_default().push(fix); + } + } + + if by_file.is_empty() { + println!("\n{dim}No automatic fixes available.{reset}"); + return; + } + + println!("\n{bold}Fixes:{reset}"); + for (file, fixes) in &by_file { + let src = fileset.source(*file); + let path = fileset.path(*file); + for f in fixes { + let (l, c) = line_col(src, f.range.0); + let old = String::from_utf8_lossy(&src[f.range.0 as usize..f.range.1 as usize]); + println!(" {}:{l}:{c} `{old}` -> `{}`", path.display(), f.replacement); + } + if write { + let owned: Vec = fixes.iter().map(|f| (*f).clone()).collect(); + let fixed = apply_fixes(src, &owned); + match std::fs::write(path, &fixed) { + Ok(()) => println!(" {dim}wrote {}{reset}", path.display()), + Err(e) => eprintln!(" failed to write {}: {e}", path.display()), + } + } + } + if !write { + println!("\n{dim}(dry run — re-run with --write to apply these fixes to disk){reset}"); + } +} diff --git a/examples/mod-demo/mod/common/buildings/01_mod_buildings.txt b/examples/mod-demo/mod/common/buildings/01_mod_buildings.txt new file mode 100644 index 0000000..62d8339 --- /dev/null +++ b/examples/mod-demo/mod/common/buildings/01_mod_buildings.txt @@ -0,0 +1,18 @@ +# The mod layer. Loaded after vanilla, so same-named entities override it. + +# Overrides the vanilla temple (mod wins) — the linter notes the override. +temple = { + cost = 80 +} + +# A new building. Its upgrades_from points at a *vanilla* building, defined in +# a different file in a different layer — a cross-file, cross-layer reference. +fortress = { + cost = 400 + upgrades_from = castle +} + +# Oops: fortress is defined twice in the same layer — a duplicate definition. +fortress = { + cost = 401 +} diff --git a/examples/mod-demo/mod/events/broken_events.txt b/examples/mod-demo/mod/events/broken_events.txt new file mode 100644 index 0000000..04a4a82 --- /dev/null +++ b/examples/mod-demo/mod/events/broken_events.txt @@ -0,0 +1,8 @@ +# This file has a syntax error: the outer block is never closed. The reference +# to `ghost_building` is undefined, but because it sits in a malformed subtree +# the linter SUPPRESSES the undefined-reference (and surfaces the syntax error +# instead) rather than piling a cascade of semantic noise on top of it. +broken_event = { + effect = { + add_building = ghost_building + } diff --git a/examples/mod-demo/mod/events/mod_events.txt b/examples/mod-demo/mod/events/mod_events.txt new file mode 100644 index 0000000..cfa4d49 --- /dev/null +++ b/examples/mod-demo/mod/events/mod_events.txt @@ -0,0 +1,8 @@ +# Mod events. Mixes valid and invalid building references. +mod_event = { + effect = { + add_building = fortress # ok: defined in the mod + add_building = baracks # ERROR: typo for the vanilla `barracks` + remove_building = temple # ok: resolves to the mod's override + } +} diff --git a/examples/mod-demo/vanilla/common/buildings/00_buildings.txt b/examples/mod-demo/vanilla/common/buildings/00_buildings.txt new file mode 100644 index 0000000..3899cae --- /dev/null +++ b/examples/mod-demo/vanilla/common/buildings/00_buildings.txt @@ -0,0 +1,18 @@ +# Base game buildings (the "vanilla" layer). +temple = { + cost = 100 + desc = "A place of worship" +} + +castle = { + cost = 250 + desc = "A fortified keep" +} + +city = { + cost = 500 +} + +barracks = { + cost = 150 +} diff --git a/examples/mod-demo/vanilla/events/court_events.txt b/examples/mod-demo/vanilla/events/court_events.txt new file mode 100644 index 0000000..bc6b3ed --- /dev/null +++ b/examples/mod-demo/vanilla/events/court_events.txt @@ -0,0 +1,9 @@ +# Base game events. Every building referenced here is defined in vanilla. +court_event = { + trigger = { + has_building = castle + } + effect = { + add_building = temple + } +} diff --git a/examples/syntax.rs b/examples/syntax.rs new file mode 100644 index 0000000..9c546ed --- /dev/null +++ b/examples/syntax.rs @@ -0,0 +1,109 @@ +//! Inspect a Clausewitz text file through the **lossless** syntax tree +//! ([`jomini::text::syntax`]). +//! +//! Default mode highlights the source by token kind — and because the tree's +//! leaves tile the input exactly, the highlighted bytes (sans the ANSI codes) +//! *are* the input, which is the whole point of a lossless tree. With `--tree` +//! it instead dumps the parse tree as an indented S-expression. With `--format` +//! it reprints the file in the formatter's house style. Either way it prints a +//! footer (to stderr) reporting the round-trip check and diagnostics. +//! +//! ```sh +//! cargo run --example syntax -- tests/fixtures/ck3-header.txt +//! cargo run --example syntax -- --tree tests/fixtures/meta.txt +//! cargo run --example syntax -- --format tests/fixtures/meta.txt +//! printf 'rate = @[ (1 - x) * 2 ]\n' | cargo run --example syntax +//! ``` + +use jomini::text::syntax::{format, parse, SyntaxKind}; +use std::io::{Read, Write}; + +fn main() { + let mut path: Option = None; + let mut tree_mode = false; + let mut format_mode = false; + let mut color = true; + for arg in std::env::args().skip(1) { + match arg.as_str() { + "--tree" => tree_mode = true, + "--format" => format_mode = true, + "--no-color" => color = false, + "-h" | "--help" => { + eprintln!( + "usage: syntax [--tree] [--format] [--no-color] [FILE] \ + (reads stdin if no FILE)" + ); + return; + } + _ => path = Some(arg), + } + } + + let source = match &path { + Some(p) => std::fs::read(p).unwrap_or_else(|e| { + eprintln!("error reading {p}: {e}"); + std::process::exit(1); + }), + None => { + let mut buf = Vec::new(); + std::io::stdin().read_to_end(&mut buf).expect("read stdin"); + buf + } + }; + + let tree = parse(&source); + + if format_mode { + let _ = std::io::stdout().lock().write_all(&format(&source)); + } else if tree_mode { + print!("{}", tree.debug_tree()); + } else { + // Re-emit every leaf in order, wrapping it in its kind's color. With + // --no-color, stdout is byte-for-byte identical to the input. + let mut out = std::io::stdout().lock(); + for tok in tree.tokens() { + match (color, ansi(tok.kind())) { + (true, Some(sgr)) => { + let _ = write!(out, "\x1b[{sgr}m"); + let _ = out.write_all(tok.text()); + let _ = write!(out, "\x1b[0m"); + } + _ => { + let _ = out.write_all(tok.text()); + } + } + } + let _ = out.flush(); + } + + // Footer on stderr, so a piped stdout stays clean. + let lossless = tree.reconstruct() == source; + eprintln!( + "\n\x1b[90m— {} bytes · {} tokens · round-trip {} · {} error(s)\x1b[0m", + source.len(), + tree.tokens().count(), + if lossless { "✓" } else { "✗ MISMATCH" }, + tree.errors().len(), + ); + for e in tree.errors() { + eprintln!("\x1b[31m [{}..{}] {}\x1b[0m", e.range.0, e.range.1, e.message); + } +} + +/// ANSI SGR parameter for a token kind, or `None` to print it uncolored +/// (bare identifiers/numbers and whitespace are left in the terminal default). +fn ansi(kind: SyntaxKind) -> Option<&'static str> { + use SyntaxKind::*; + Some(match kind { + Comment | Bom => "90", // grey + Quoted => "32", // green + Variable | CalcOpen | CalcClose => "36", // cyan: @-things + MacroParam | Plus | Minus | Star | Slash => "35", // magenta: substitutions & calc ops + Operator | Number => "33", // yellow + CalcIdent => "34", // blue: calc operands + OpenBrace | CloseBrace | OpenParen | CloseParen => "1", // bold: grouping + OpenBracket | CloseBracket | Bang => "31", // red: param brackets + Error => "41", // red background + _ => return None, // Unquoted, Whitespace + }) +} diff --git a/jomini-lsp/.gitignore b/jomini-lsp/.gitignore new file mode 100644 index 0000000..96ef6c0 --- /dev/null +++ b/jomini-lsp/.gitignore @@ -0,0 +1,2 @@ +/target +Cargo.lock diff --git a/jomini-lsp/Cargo.toml b/jomini-lsp/Cargo.toml new file mode 100644 index 0000000..f684c01 --- /dev/null +++ b/jomini-lsp/Cargo.toml @@ -0,0 +1,23 @@ +[package] +publish = false +name = "jomini-lsp" +version = "0.0.1" +authors = ["Nick Babcock "] +description = "Language server for Clausewitz (Paradox) game files, built on jomini's lossless syntax tree + cross-file lint layer." +homepage = "https://github.com/rakaly/jomini/tree/master/jomini-lsp" +repository = "https://github.com/rakaly/jomini" +license = "MIT" +edition = "2024" + +# Its own workspace root (mirrors `bench/`) so the LSP/transport dependency tree +# never touches the published `jomini` crate's graph or lockfile. +[workspace] + +[dependencies] +crossbeam-channel = "0.5.15" +jomini = { path = "..", default-features = false } +lsp-server = "0.7.9" +lsp-types = "0.97.0" +serde = { version = "1.0.228", features = ["derive"] } +serde_json = "1.0.150" +url = "2.5.8" diff --git a/jomini-lsp/README.md b/jomini-lsp/README.md new file mode 100644 index 0000000..0dbd9cb --- /dev/null +++ b/jomini-lsp/README.md @@ -0,0 +1,112 @@ +# jomini-lsp + +A [Language Server Protocol](https://microsoft.github.io/language-server-protocol/) +server for Clausewitz (Paradox) game files — EU4 / CK3 / HOI4 / Vic3 / Imperator +and friends — built on jomini's **lossless** text syntax tree (`jomini::text::syntax`) +and its **cross-file lint layer** (`jomini::text::lint`). + +This is the editor-facing payoff of the lossless tree: a single retained +[`Analysis`](../src/text/lint.rs) answers every query, and because the tree +preserves every byte, the quick-fixes it produces are byte-faithful. + +It is a synchronous server on rust-analyzer's own stack +([`lsp-server`](https://crates.io/crates/lsp-server) + `lsp-types`), kept in its +**own workspace** (like `bench/`) so its transport dependencies never touch the +published `jomini` crate's graph. + +## Features (first cut) + +| LSP request | Backed by | +|---|---| +| `textDocument/publishDiagnostics` | `Linter::analyze` — undefined references, overrides, duplicates, syntax errors, suppressed-in-error-subtree | +| `textDocument/formatting` | `syntax::format` (whole-document; honors `tabSize`/`insertSpaces`) | +| `textDocument/definition` | `Analysis::definition_for` — resolves a reference to the *winning* definition, across files & load-order layers | +| `textDocument/references` | `Analysis::references_to` — every use site of the name under the cursor | +| `textDocument/documentSymbol` | per-file `FileSummary.defs` | +| `workspace/symbol` | `Index::definitions` (winning defs, fuzzy-matched by query) | +| `textDocument/codeAction` | each `Diagnostic.fix` → a `QuickFix` `TextEdit` (e.g. the "did you mean `barracks`?" rename) | + +Cross-file is the point: an `add_building = baracks` typo is flagged in the file +it appears in, even though its (missing) definition would live in another file — +something a per-file checker fundamentally cannot do. + +## Build & run + +```sh +# from this directory (it is a standalone workspace) +cargo build --release +# the server speaks LSP over stdio: +./target/release/jomini-lsp +``` + +### Wiring into an editor + +The server talks stdio and advertises its capabilities on `initialize`. Point any +generic LSP client at the binary for the `clausewitz` / plaintext file type. + +**Neovim** (`nvim-lspconfig`-style, using a custom config): + +```lua +vim.lsp.start({ + name = 'jomini-lsp', + cmd = { '/path/to/jomini-lsp' }, + root_dir = vim.fs.dirname(vim.fs.find({ 'descriptor.mod', '.git' }, { upward = true })[1]), + init_options = { vanilla = '/path/to/game/install/game' }, -- optional vanilla base +}) +``` + +**VS Code**: a thin extension that launches the binary as a stdio server +(`vscode-languageclient`, `TransportKind.stdio`). (No extension is bundled yet.) + +### Configuration + +`initializationOptions`: + +| key | meaning | +|---|---| +| `vanilla` | filesystem path to a base-game directory, loaded as the `Vanilla` layer beneath the workspace (which loads as `Mod(0)`). Mod definitions then override vanilla, matching Paradox load order. | + +Position encoding is negotiated: UTF-8 when the client offers it, else UTF-16. + +## Architecture + +``` +main.rs stdio transport → jomini_lsp::serve +lib.rs serve(): initialize handshake (+ encoding negotiation) and the dispatch loop +server.rs Server state (Fileset overlay + retained Analysis) and one handler per feature +line_index byte offset ↔ LSP Position (UTF-16/UTF-8); the one mapping the tree leaves to us +convert jomini lint types → lsp_types +``` + +The server keeps one `Fileset` (whose bytes double as the open-document overlay) +and one `Analysis`. Every edit re-runs `Linter::analyze` from scratch and +republishes — correct and simple. The library additions that make this possible +(`Analysis`, the reverse reference index, offset hit-testing, `Fileset::set_source`/ +`id_for_path`) live in `jomini::text::lint`, the rust-analyzer-style split of +reusable analysis (`ide-db`) from the LSP shell. + +## Known limitations / next steps + +- **Prototype schema.** The rule set (what defines/references a "building") is the + hand-written demo from `examples/lint.rs`, hardcoded in `server.rs`. A real + per-game schema (or a `.cwt`-config front end feeding the same `Index`) is the + follow-up. +- **Full re-analysis per edit.** A one-file edit can flip override winners + project-wide, so the first cut re-summarizes everything on each change. + Incremental reparse + incremental re-summarize is the planned optimization. +- **Whole-document formatting only** (no range/on-type formatting). +- **Encoding.** Positions assume UTF-8 document bytes (what the client sends). + Windows-1252 files on disk are ASCII-exact but not yet fully encoding-aware — the + same follow-up the lint layer notes. +- **Not yet implemented:** hover, completion, semantic-token highlighting, rename + (the `Index` + `Fix` infra make these the natural next features). + +## Testing + +```sh +cargo test # line_index unit tests + an in-process end-to-end smoke test +``` + +`tests/smoke.rs` drives the real `serve` loop over an in-memory `Connection` +against `../examples/mod-demo`, exercising diagnostics, symbols, go-to-definition, +find-references, code actions, and formatting without a subprocess. diff --git a/jomini-lsp/src/convert.rs b/jomini-lsp/src/convert.rs new file mode 100644 index 0000000..a037235 --- /dev/null +++ b/jomini-lsp/src/convert.rs @@ -0,0 +1,53 @@ +//! Conversions between jomini's `lint` types and `lsp_types`. +//! +//! Everything here is a pure function of owned data plus a [`LineIndex`] (to turn +//! byte ranges into LSP line/character ranges), so it has no dependency on a live +//! parse tree. + +use crate::line_index::LineIndex; +use jomini::text::lint::{Diagnostic, Severity}; +use lsp_types::{DiagnosticSeverity, NumberOrString, Range, SymbolKind}; + +/// A jomini half-open byte range → an LSP range, positioned with the target +/// document's [`LineIndex`]. +pub fn to_range(li: &LineIndex, range: (u32, u32)) -> Range { + Range { start: li.position(range.0), end: li.position(range.1) } +} + +/// Map a lint [`Severity`] onto LSP's four-level scale. jomini has two extra +/// rungs (`Untidy`, `Fatal`); they fold onto the nearest LSP level. +pub fn to_severity(sev: Severity) -> DiagnosticSeverity { + match sev { + Severity::Tip => DiagnosticSeverity::HINT, + Severity::Untidy => DiagnosticSeverity::INFORMATION, + Severity::Warning => DiagnosticSeverity::WARNING, + Severity::Error | Severity::Fatal => DiagnosticSeverity::ERROR, + } +} + +/// A lint [`Diagnostic`] → an LSP diagnostic, positioned via `li` (the +/// [`LineIndex`] of the file the diagnostic belongs to). The stable lint id +/// becomes the `code`; the `help` note — which LSP has no dedicated field for — +/// is appended to the message. +pub fn to_lsp_diagnostic(li: &LineIndex, d: &Diagnostic) -> lsp_types::Diagnostic { + let mut message = d.message.clone(); + if let Some(help) = &d.help { + message.push('\n'); + message.push_str(help); + } + lsp_types::Diagnostic { + range: to_range(li, d.range), + severity: Some(to_severity(d.severity)), + code: Some(NumberOrString::String(d.id.to_string())), + source: Some("jomini".to_string()), + message, + ..Default::default() + } +} + +/// The LSP symbol kind for a lint entity kind. Definitions are name-keyed game +/// entities (buildings, etc.); `OBJECT` is the neutral choice. A richer per-game +/// schema could map specific kinds (e.g. event → `EVENT`). +pub fn symbol_kind(_kind: &str) -> SymbolKind { + SymbolKind::OBJECT +} diff --git a/jomini-lsp/src/lib.rs b/jomini-lsp/src/lib.rs new file mode 100644 index 0000000..f5b87bb --- /dev/null +++ b/jomini-lsp/src/lib.rs @@ -0,0 +1,124 @@ +//! `jomini-lsp` — a Language Server Protocol server for Clausewitz (Paradox) game +//! files, built on jomini's lossless syntax tree and cross-file lint layer. +//! +//! A synchronous server on rust-analyzer's [`lsp_server`] stack. The first cut +//! serves: publish-diagnostics, document formatting, go-to-definition, +//! find-references, document & workspace symbols, and quick-fix code actions — +//! all reading off a single retained [`jomini::text::lint::Analysis`]. +//! +//! [`serve`] runs the protocol against any [`Connection`] (stdio in the binary, an +//! in-memory pair in tests), so the whole server is driveable without a subprocess. + +mod convert; +mod line_index; +mod server; + +use line_index::PositionEncoding; +use lsp_server::{Connection, Message, Response, ResponseError}; +use lsp_types::notification::Notification as _; +use lsp_types::{ + CodeActionProviderCapability, InitializeParams, InitializeResult, OneOf, PositionEncodingKind, + ServerCapabilities, ServerInfo, TextDocumentSyncCapability, TextDocumentSyncKind, +}; +use server::Server; + +pub type DynError = Box; + +// JSON-RPC `InvalidParams` (lsp-server 0.7 exposes no error-code enum). +const INVALID_PARAMS: i32 = -32602; + +/// Run the full LSP lifecycle over `connection`: the `initialize` handshake (with +/// position-encoding negotiation and capability advertisement) followed by the +/// request/notification dispatch loop, until `shutdown`/`exit`. +pub fn serve(connection: &Connection) -> Result<(), DynError> { + let (id, init_value) = connection.initialize_start()?; + // `initialize_start` returns the id without responding (that is + // `initialize_finish`'s job). If the params are malformed we must still answer + // the request — otherwise the one request that matters most hangs unanswered. + let init_params: InitializeParams = match serde_json::from_value(init_value) { + Ok(params) => params, + Err(e) => { + let err = ResponseError { + code: INVALID_PARAMS, + message: format!("invalid initialize params: {e}"), + data: None, + }; + let _ = connection.sender.send(Message::Response(Response { + id, + result: None, + error: Some(err), + })); + return Err(e.into()); + } + }; + let encoding = negotiate_encoding(&init_params); + let init_result = InitializeResult { + capabilities: server_capabilities(encoding), + server_info: Some(ServerInfo { + name: "jomini-lsp".to_string(), + version: Some(env!("CARGO_PKG_VERSION").to_string()), + }), + }; + connection.initialize_finish(id, serde_json::to_value(init_result)?)?; + + let mut server = Server::new(&init_params, connection.sender.clone(), encoding); + // `initialize_finish` already consumed the `initialized` notification, so this + // is the first point we may push messages: publish the initial diagnostics. + server.publish_all(); + main_loop(connection, &mut server) +} + +fn main_loop(connection: &Connection, server: &mut Server) -> Result<(), DynError> { + for msg in &connection.receiver { + match msg { + Message::Request(req) => { + // `handle_shutdown` answers a `shutdown` request and returns true + // once the following `exit` arrives (it consumes that `exit` + // itself), at which point we stop cleanly. + if connection.handle_shutdown(&req)? { + return Ok(()); + } + server.on_request(req); + } + Message::Notification(not) => { + // A bare `exit` (no preceding `shutdown` — that path is consumed by + // `handle_shutdown` above) is the protocol's abrupt-termination + // signal: stop, and report it as an error so the process exits + // non-zero per the spec. + if not.method == lsp_types::notification::Exit::METHOD { + return Err("received `exit` without a prior `shutdown`".into()); + } + server.on_notification(not); + } + // We issue no server→client requests yet, so any response is ignored. + Message::Response(_) => {} + } + } + Ok(()) +} + +/// Prefer UTF-8 columns when the client supports them (a straight byte mapping); +/// otherwise the LSP default, UTF-16. +fn negotiate_encoding(init: &InitializeParams) -> PositionEncoding { + let supports_utf8 = init + .capabilities + .general + .as_ref() + .and_then(|g| g.position_encodings.as_ref()) + .is_some_and(|encs| encs.contains(&PositionEncodingKind::UTF8)); + if supports_utf8 { PositionEncoding::Utf8 } else { PositionEncoding::Utf16 } +} + +fn server_capabilities(encoding: PositionEncoding) -> ServerCapabilities { + ServerCapabilities { + position_encoding: Some(encoding.to_lsp()), + text_document_sync: Some(TextDocumentSyncCapability::Kind(TextDocumentSyncKind::FULL)), + document_formatting_provider: Some(OneOf::Left(true)), + definition_provider: Some(OneOf::Left(true)), + references_provider: Some(OneOf::Left(true)), + document_symbol_provider: Some(OneOf::Left(true)), + workspace_symbol_provider: Some(OneOf::Left(true)), + code_action_provider: Some(CodeActionProviderCapability::Simple(true)), + ..Default::default() + } +} diff --git a/jomini-lsp/src/line_index.rs b/jomini-lsp/src/line_index.rs new file mode 100644 index 0000000..32b3c54 --- /dev/null +++ b/jomini-lsp/src/line_index.rs @@ -0,0 +1,203 @@ +//! Byte-offset ↔ LSP [`Position`] conversion — the one mapping the lossless tree +//! deliberately leaves to the editor layer. +//! +//! jomini's ranges are raw half-open **byte** offsets over `&[u8]`. LSP positions +//! are 0-based `(line, character)` where `character` counts UTF-16 code units (the +//! protocol default) or, when the client agrees during `initialize` negotiation, +//! UTF-8 bytes. We own both directions here. +//! +//! **Encoding caveat.** We treat document bytes as UTF-8 — which is what a client +//! sends in a `textDocument/did{Open,Change}`. Game files on disk may be +//! Windows-1252; for the ASCII-dominated content of keys, numbers, and tags the +//! conversion is exact, and a fully encoding-aware mapping is a follow-up (the +//! lint layer punts on encoding the same way today). + +use lsp_types::{Position, PositionEncodingKind}; + +/// Which unit an LSP `character` column counts. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +pub enum PositionEncoding { + /// LSP's default: `character` counts UTF-16 code units. + Utf16, + /// Negotiated when the client advertises it: `character` counts UTF-8 bytes, + /// so the mapping is a straight subtraction (rust-analyzer prefers this). + Utf8, +} + +impl PositionEncoding { + /// The matching LSP capability value to echo back in `initialize`. + pub fn to_lsp(self) -> PositionEncodingKind { + match self { + PositionEncoding::Utf16 => PositionEncodingKind::UTF16, + PositionEncoding::Utf8 => PositionEncodingKind::UTF8, + } + } +} + +/// A precomputed line-start table for one document plus its bytes, supporting +/// offset → [`Position`] and [`Position`] → offset. +pub struct LineIndex<'a> { + text: &'a [u8], + /// Byte offset of the start of each line; `line_starts[0] == 0`. + line_starts: Vec, + encoding: PositionEncoding, +} + +impl<'a> LineIndex<'a> { + pub fn new(text: &'a [u8], encoding: PositionEncoding) -> Self { + let mut line_starts = vec![0u32]; + for (i, &b) in text.iter().enumerate() { + if b == b'\n' { + line_starts.push(i as u32 + 1); + } + } + LineIndex { text, line_starts, encoding } + } + + /// The [`Position`] of byte `offset` (clamped to the document end). + pub fn position(&self, offset: u32) -> Position { + let offset = offset.min(self.text.len() as u32); + // The line is the last line-start ≤ offset. + let line = match self.line_starts.binary_search(&offset) { + Ok(exact) => exact, // offset is exactly a line start + Err(next) => next - 1, // between starts: the preceding line + }; + let line_start = self.line_starts[line] as usize; + let character = self.encode_col(&self.text[line_start..offset as usize]); + Position { line: line as u32, character } + } + + /// The byte offset of `position` (clamped into range). + pub fn offset(&self, position: Position) -> u32 { + let line = position.line as usize; + if line >= self.line_starts.len() { + return self.text.len() as u32; + } + let line_start = self.line_starts[line] as usize; + let mut line_end = + self.line_starts.get(line + 1).map(|&s| s as usize).unwrap_or(self.text.len()); + // Exclude the line terminator from the slice, so a column past the visible + // content clamps to end-of-line rather than jumping onto the next line's + // first byte (the rust-analyzer convention). + if line_end > line_start && self.text[line_end - 1] == b'\n' { + line_end -= 1; + if line_end > line_start && self.text[line_end - 1] == b'\r' { + line_end -= 1; + } + } + let col_bytes = self.decode_col(&self.text[line_start..line_end], position.character); + (line_start + col_bytes) as u32 + } + + /// The end-of-document position — the `end` of a whole-document range. + pub fn end_position(&self) -> Position { + self.position(self.text.len() as u32) + } + + /// Code units (per the encoding) spanned by `slice` — i.e. the column of the + /// offset at the end of `slice` within its line. + fn encode_col(&self, slice: &[u8]) -> u32 { + match self.encoding { + PositionEncoding::Utf8 => slice.len() as u32, + // For non-UTF-8 bytes the replacement char is one UTF-16 unit, which + // keeps columns sane for the ASCII-dominated content we target. + PositionEncoding::Utf16 => { + String::from_utf8_lossy(slice).chars().map(|c| c.len_utf16() as u32).sum() + } + } + } + + /// Inverse of [`encode_col`]: the byte length of the prefix of `line` spanning + /// `character` code units (clamped to the line; a column landing inside a + /// multi-unit char rounds up to that char's end). + fn decode_col(&self, line: &[u8], character: u32) -> usize { + match self.encoding { + PositionEncoding::Utf8 => (character as usize).min(line.len()), + PositionEncoding::Utf16 => { + let text = String::from_utf8_lossy(line); + let mut units = 0u32; + let mut bytes = 0usize; + for c in text.chars() { + if units >= character { + break; + } + units += c.len_utf16() as u32; + bytes += c.len_utf8(); + } + bytes.min(line.len()) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn pos(line: u32, character: u32) -> Position { + Position { line, character } + } + + #[test] + fn ascii_offsets_and_positions_round_trip() { + let li = LineIndex::new(b"abc\ndef", PositionEncoding::Utf16); + assert_eq!(li.position(0), pos(0, 0)); + assert_eq!(li.position(3), pos(0, 3)); // the '\n' + assert_eq!(li.position(4), pos(1, 0)); // 'd' + assert_eq!(li.position(6), pos(1, 2)); + assert_eq!(li.offset(pos(1, 0)), 4); + assert_eq!(li.offset(pos(1, 2)), 6); + // Past-end positions clamp. + assert_eq!(li.offset(pos(99, 99)), 7); + assert_eq!(li.position(999), pos(1, 3)); + } + + #[test] + fn utf16_counts_code_units_not_bytes() { + // "é" is 2 UTF-8 bytes but 1 UTF-16 unit. + let text = "é = 1".as_bytes(); + let li = LineIndex::new(text, PositionEncoding::Utf16); + // After 'é' (byte offset 2) the UTF-16 column is 1. + assert_eq!(li.position(2), pos(0, 1)); + assert_eq!(li.offset(pos(0, 1)), 2); + } + + #[test] + fn utf8_counts_bytes() { + let text = "é = 1".as_bytes(); + let li = LineIndex::new(text, PositionEncoding::Utf8); + assert_eq!(li.position(2), pos(0, 2)); + assert_eq!(li.offset(pos(0, 2)), 2); + } + + #[test] + fn utf16_handles_astral_surrogate_pairs() { + // "😀" is 4 UTF-8 bytes and 2 UTF-16 units. + let text = "😀x".as_bytes(); + let li = LineIndex::new(text, PositionEncoding::Utf16); + assert_eq!(li.position(4), pos(0, 2)); // after the emoji + assert_eq!(li.offset(pos(0, 2)), 4); + assert_eq!(li.position(5), pos(0, 3)); // after 'x' + } + + #[test] + fn crlf_line_starts() { + let li = LineIndex::new(b"a\r\nb", PositionEncoding::Utf16); + assert_eq!(li.position(0), pos(0, 0)); + assert_eq!(li.position(1), pos(0, 1)); // '\r' stays on line 0 + assert_eq!(li.position(3), pos(1, 0)); // 'b' on line 1 + } + + #[test] + fn out_of_range_column_clamps_to_end_of_line_not_next_line() { + // A column past the visible content of line 0 must land at end-of-content + // (byte 1, after 'a'), NOT at the start of line 1 (byte 3, the 'b'). The + // line terminator (\r\n) is excluded from the clamp. + let li = LineIndex::new(b"a\r\nb", PositionEncoding::Utf16); + assert_eq!(li.offset(pos(0, 99)), 1); + assert_eq!(li.offset(pos(0, 1)), 1); // in-range columns are unchanged + // A '\n'-only line behaves the same. + let li = LineIndex::new(b"abc\ndef", PositionEncoding::Utf16); + assert_eq!(li.offset(pos(0, 99)), 3); // end of "abc", before '\n' + } +} diff --git a/jomini-lsp/src/main.rs b/jomini-lsp/src/main.rs new file mode 100644 index 0000000..c676208 --- /dev/null +++ b/jomini-lsp/src/main.rs @@ -0,0 +1,12 @@ +//! The `jomini-lsp` binary: wire the [`jomini_lsp::serve`] loop to stdio. + +use lsp_server::Connection; + +fn main() -> Result<(), jomini_lsp::DynError> { + eprintln!("jomini-lsp: starting (stdio)"); + let (connection, io_threads) = Connection::stdio(); + jomini_lsp::serve(&connection)?; + io_threads.join()?; + eprintln!("jomini-lsp: stopped"); + Ok(()) +} diff --git a/jomini-lsp/src/server.rs b/jomini-lsp/src/server.rs new file mode 100644 index 0000000..46b043b --- /dev/null +++ b/jomini-lsp/src/server.rs @@ -0,0 +1,523 @@ +//! The language server: project state plus one handler per LSP feature. +//! +//! State is a single owned [`Analysis`] (the lint snapshot) over a [`Fileset`] +//! whose bytes double as the open-document overlay. Every edit re-runs +//! [`Linter::analyze`] from scratch — correct and simple; incremental re-analysis +//! is a deliberate follow-up (a one-file edit can flip override winners +//! project-wide, so a naive single-file update would be wrong). + +use crate::convert; +use crate::line_index::{LineIndex, PositionEncoding}; +use crossbeam_channel::Sender; +use jomini::text::lint::{Analysis, FileId, FileKind, Fileset, Linter, Schema}; +use jomini::text::syntax::{self, FormatOptions}; +use lsp_server::{Message, Notification, Request, RequestId, Response, ResponseError}; +use lsp_types::notification::Notification as NotificationTrait; +use lsp_types::request::Request as RequestTrait; +use lsp_types::{ + CodeAction, CodeActionKind, CodeActionOrCommand, CodeActionParams, CodeActionResponse, + DidChangeTextDocumentParams, DidCloseTextDocumentParams, DidOpenTextDocumentParams, + DocumentFormattingParams, DocumentSymbol, DocumentSymbolParams, DocumentSymbolResponse, + GotoDefinitionParams, GotoDefinitionResponse, InitializeParams, Location, Position, + PublishDiagnosticsParams, Range, ReferenceParams, SymbolInformation, TextEdit, Uri, + WorkspaceEdit, WorkspaceSymbolParams, WorkspaceSymbolResponse, +}; +use std::collections::{HashMap, HashSet}; +use std::path::{Path, PathBuf}; + +// JSON-RPC error codes (LSP reserves the JSON-RPC range). +const METHOD_NOT_FOUND: i32 = -32601; +const INVALID_PARAMS: i32 = -32602; + +pub struct Server { + fileset: Fileset, + linter: Linter, + analysis: Analysis, + encoding: PositionEncoding, + sender: Sender, + /// Canonicalized path → [`FileId`], for resolving an incoming document URL to + /// its handle robustly (independent of `/tmp` vs `/private/tmp`-style aliases). + path_to_id: HashMap, + /// Files currently open in the editor — so a close can revert the overlay. + open: HashSet, + /// Files we last published non-empty diagnostics for, so stale sets get cleared. + published: HashSet, +} + +impl Server { + /// Build the server: load the workspace (and an optional vanilla base) into a + /// [`Fileset`] and run the first [`Linter::analyze`]. No messages are sent + /// yet — that waits for the `initialized` notification (per the LSP spec). + pub fn new(init: &InitializeParams, sender: Sender, encoding: PositionEncoding) -> Self { + let mut fileset = Fileset::new(); + + // The mod layer: every workspace folder (or the deprecated root_uri). + for root in workspace_roots(init) { + let _ = fileset.load_dir(&root, FileKind::Mod(0)); + } + + // An optional vanilla base, from initializationOptions: { "vanilla": "" }. + if let Some(vanilla) = init + .initialization_options + .as_ref() + .and_then(|v| v.get("vanilla")) + .and_then(|v| v.as_str()) + { + let _ = fileset.load_dir(vanilla, FileKind::Vanilla); + } + + let mut path_to_id = HashMap::new(); + for id in fileset.ids() { + path_to_id.insert(canonical(fileset.path(id)), id); + } + + let linter = Linter::new(demo_schema()); + let analysis = linter.analyze(&fileset); + Server { + fileset, + linter, + analysis, + encoding, + sender, + path_to_id, + open: HashSet::new(), + published: HashSet::new(), + } + } + + // --- request / notification dispatch ----------------------------------- + + pub fn on_request(&mut self, req: Request) { + match req.method.as_str() { + lsp_types::request::Formatting::METHOD => { + self.dispatch::(req, |s, p| s.formatting(p)) + } + lsp_types::request::GotoDefinition::METHOD => { + self.dispatch::(req, |s, p| s.goto_definition(p)) + } + lsp_types::request::References::METHOD => { + self.dispatch::(req, |s, p| s.references(p)) + } + lsp_types::request::DocumentSymbolRequest::METHOD => self + .dispatch::(req, |s, p| { + s.document_symbol(p) + }), + lsp_types::request::WorkspaceSymbolRequest::METHOD => self + .dispatch::(req, |s, p| { + s.workspace_symbol(p) + }), + lsp_types::request::CodeActionRequest::METHOD => { + self.dispatch::(req, |s, p| s.code_action(p)) + } + other => { + self.respond_err(req.id, METHOD_NOT_FOUND, format!("unhandled request: {other}")) + } + } + } + + pub fn on_notification(&mut self, not: Notification) { + match not.method.as_str() { + lsp_types::notification::DidOpenTextDocument::METHOD => { + if let Ok(p) = serde_json::from_value::(not.params) { + self.did_open(p); + } + } + lsp_types::notification::DidChangeTextDocument::METHOD => { + if let Ok(p) = serde_json::from_value::(not.params) { + self.did_change(p); + } + } + lsp_types::notification::DidCloseTextDocument::METHOD => { + if let Ok(p) = serde_json::from_value::(not.params) { + self.did_close(p); + } + } + // FULL sync means the latest text already arrived via didChange. + _ => {} + } + } + + fn dispatch(&mut self, req: Request, handler: impl FnOnce(&mut Self, R::Params) -> R::Result) + where + R: RequestTrait, + { + let Request { id, params, .. } = req; + match serde_json::from_value::(params) { + Ok(params) => { + let result = handler(self, params); + self.respond(id, result); + } + Err(e) => self.respond_err(id, INVALID_PARAMS, format!("invalid params: {e}")), + } + } + + // --- document lifecycle ------------------------------------------------- + + fn did_open(&mut self, params: DidOpenTextDocumentParams) { + let doc = params.text_document; + let bytes = doc.text.into_bytes(); + let id = match self.file_for_url(&doc.uri) { + Some(id) => { + self.fileset.set_source(id, bytes); + id + } + // A file outside the loaded set (or never saved): add it as a mod file. + None => { + let Some(path) = uri_to_path(&doc.uri) else { return }; + let id = self.fileset.add(path.clone(), FileKind::Mod(0), bytes); + self.path_to_id.insert(canonical(&path), id); + id + } + }; + self.open.insert(id); + self.reanalyze_and_publish(); + } + + fn did_change(&mut self, params: DidChangeTextDocumentParams) { + let Some(id) = self.file_for_url(¶ms.text_document.uri) else { return }; + // FULL sync: the last content change carries the whole document. + if let Some(change) = params.content_changes.into_iter().next_back() { + self.fileset.set_source(id, change.text.into_bytes()); + self.reanalyze_and_publish(); + } + } + + fn did_close(&mut self, params: DidCloseTextDocumentParams) { + let Some(id) = self.file_for_url(¶ms.text_document.uri) else { return }; + self.open.remove(&id); + // Drop the editor overlay: revert to disk content, or to empty bytes when + // the file is gone or never existed (a never-saved buffer; there is no + // Fileset removal API yet). Re-analyze and re-publish *unconditionally* so + // any diagnostics for the now-closed buffer are cleared rather than left + // stale — the server, not the client, owns clearing diagnostics. + let path = self.fileset.path(id).to_path_buf(); + let reverted = std::fs::read(&path).unwrap_or_default(); + self.fileset.set_source(id, reverted); + self.reanalyze_and_publish(); + } + + fn reanalyze_and_publish(&mut self) { + self.analysis = self.linter.analyze(&self.fileset); + self.publish_all(); + } + + // --- diagnostics -------------------------------------------------------- + + /// Publish diagnostics for every file that has any, and clear the set for any + /// file that had diagnostics last time but no longer does. Publishing for + /// *all* files (not just open ones) is the point of a cross-file linter — an + /// undefined reference often lives in a file the user has not opened. + /// + /// Called once after the `initialize` handshake (which `lsp_server` consumes + /// the `initialized` notification within) and again after every edit. + pub fn publish_all(&mut self) { + let (payloads, now_published) = { + let mut grouped: HashMap> = HashMap::new(); + for d in self.analysis.diagnostics() { + grouped.entry(d.file).or_default().push(d); + } + + let mut payloads: Vec<(Uri, Vec)> = Vec::new(); + let mut now_published: HashSet = HashSet::new(); + for (&file, diags) in &grouped { + let Some(url) = self.url_for_file(file) else { continue }; + let li = LineIndex::new(self.fileset.source(file), self.encoding); + let lsp_diags = + diags.iter().map(|d| convert::to_lsp_diagnostic(&li, d)).collect(); + now_published.insert(file); + payloads.push((url, lsp_diags)); + } + // Clear files whose diagnostics went away. + for &file in &self.published { + if !now_published.contains(&file) + && let Some(url) = self.url_for_file(file) + { + payloads.push((url, Vec::new())); + } + } + (payloads, now_published) + }; + + self.published = now_published; + for (uri, diagnostics) in payloads { + self.notify::(PublishDiagnosticsParams { + uri, + diagnostics, + version: None, + }); + } + } + + // --- features ----------------------------------------------------------- + + /// `textDocument/formatting` → one whole-document edit from [`syntax::format`]. + /// Formatting a file with syntax errors is safe: the formatter round-trips + /// every token losslessly, regenerating only whitespace. + fn formatting(&mut self, params: DocumentFormattingParams) -> Option> { + let id = self.file_for_url(¶ms.text_document.uri)?; + let src = self.fileset.source(id); + let indent = if params.options.insert_spaces { + " ".repeat(params.options.tab_size as usize) + } else { + "\t".to_string() + }; + let formatted = syntax::parse(src).format(&FormatOptions { indent }); + if formatted == src { + return Some(Vec::new()); + } + // Until encoding-aware formatting lands, refuse to emit an edit that would + // lose non-UTF-8 (e.g. Windows-1252) bytes through a lossy decode — a + // "format" must never corrupt content. Both sides valid UTF-8 ⇒ the + // whole-document replace is safe; otherwise make it a no-op. + let new_text = match (std::str::from_utf8(&formatted), std::str::from_utf8(src)) { + (Ok(text), Ok(_)) => text.to_owned(), + _ => return Some(Vec::new()), + }; + let li = LineIndex::new(src, self.encoding); + Some(vec![TextEdit { + range: Range { start: Position { line: 0, character: 0 }, end: li.end_position() }, + new_text, + }]) + } + + /// `textDocument/definition` → the winning [`Definition`] for the reference + /// (or definition) under the cursor, possibly in another file. + fn goto_definition(&mut self, params: GotoDefinitionParams) -> Option { + let tdp = params.text_document_position_params; + let id = self.file_for_url(&tdp.text_document.uri)?; + let offset = LineIndex::new(self.fileset.source(id), self.encoding).offset(tdp.position); + let def = self.analysis.definition_for(id, offset)?; + let (file, range) = (def.file, def.range); + let url = self.url_for_file(file)?; + let li = LineIndex::new(self.fileset.source(file), self.encoding); + Some(GotoDefinitionResponse::Scalar(Location { uri: url, range: convert::to_range(&li, range) })) + } + + /// `textDocument/references` → every use site of the `(kind, name)` under the + /// cursor across the project, plus the declaration if requested. + fn references(&mut self, params: ReferenceParams) -> Option> { + let tdp = params.text_document_position; + let id = self.file_for_url(&tdp.text_document.uri)?; + let offset = LineIndex::new(self.fileset.source(id), self.encoding).offset(tdp.position); + + // The name under the cursor — from a reference, or from a definition site. + let (kind, name) = if let Some(r) = self.analysis.reference_at(id, offset) { + (r.kind.clone(), r.name.clone()) + } else if let Some(d) = self.analysis.definition_at(id, offset) { + (d.kind.clone(), d.name.clone()) + } else { + return Some(Vec::new()); + }; + + // Gather (file, byte range) for every use site, plus — if requested — + // *every* declaration site of the name. Paradox names share a single + // namespace, so an override or same-layer duplicate are all declarations of + // the symbol; report them all, not just the winner. + let mut ranges_by_file: HashMap> = HashMap::new(); + for (file, r) in self.analysis.references_to(&kind, &name) { + ranges_by_file.entry(file).or_default().push(r.range); + } + if params.context.include_declaration { + for def in self.analysis.definitions() { + if def.kind == kind && def.name == name { + ranges_by_file.entry(def.file).or_default().push(def.range); + } + } + } + + // One [`LineIndex`] per distinct file, not one per result. + let mut locations = Vec::new(); + for (file, ranges) in ranges_by_file { + let Some(uri) = self.url_for_file(file) else { continue }; + let li = LineIndex::new(self.fileset.source(file), self.encoding); + for range in ranges { + locations.push(Location { uri: uri.clone(), range: convert::to_range(&li, range) }); + } + } + Some(locations) + } + + /// `textDocument/documentSymbol` → the file's top-level definitions. + fn document_symbol(&mut self, params: DocumentSymbolParams) -> Option { + let id = self.file_for_url(¶ms.text_document.uri)?; + let li = LineIndex::new(self.fileset.source(id), self.encoding); + let summary = self.analysis.summary(id)?; + let symbols = summary + .defs + .iter() + .map(|d| { + let range = convert::to_range(&li, d.range); + #[allow(deprecated)] + DocumentSymbol { + name: d.name.clone(), + detail: Some(d.kind.clone()), + kind: convert::symbol_kind(&d.kind), + tags: None, + deprecated: None, + range, + selection_range: range, + children: None, + } + }) + .collect(); + Some(DocumentSymbolResponse::Nested(symbols)) + } + + /// `workspace/symbol` → every winning definition whose name matches `query`. + fn workspace_symbol(&mut self, params: WorkspaceSymbolParams) -> Option { + let query = params.query.to_lowercase(); + let mut infos = Vec::new(); + for def in self.analysis.definitions() { + // One entry per winning name (skip shadowed duplicates). + if !def.winner { + continue; + } + if !query.is_empty() && !def.name.to_lowercase().contains(&query) { + continue; + } + let Some(location) = self.location(def.file, def.range) else { continue }; + #[allow(deprecated)] + infos.push(SymbolInformation { + name: def.name.clone(), + kind: convert::symbol_kind(&def.kind), + tags: None, + deprecated: None, + location, + container_name: Some(def.kind.clone()), + }); + } + Some(WorkspaceSymbolResponse::Flat(infos)) + } + + /// `textDocument/codeAction` → a quick fix for each fixable diagnostic that + /// overlaps the requested range. The lint [`Fix`] maps 1:1 to an LSP + /// `TextEdit`, and because the tree is lossless the edit is byte-faithful. + // `WorkspaceEdit.changes` is keyed by `Uri`, whose `fluent-uri` internals carry + // a cache `Cell` (interior mutability) that never affects hashing — the key + // type is mandated by lsp-types, so the lint is a false positive here. + #[allow(clippy::mutable_key_type)] + fn code_action(&mut self, params: CodeActionParams) -> Option { + let id = self.file_for_url(¶ms.text_document.uri)?; + let uri = params.text_document.uri.clone(); + let li = LineIndex::new(self.fileset.source(id), self.encoding); + let sel_start = li.offset(params.range.start); + let sel_end = li.offset(params.range.end); + + let mut actions = Vec::new(); + for d in self.analysis.diagnostics() { + if d.file != id { + continue; + } + let Some(fix) = &d.fix else { continue }; + // Keep only fixes whose diagnostic overlaps the requested range. + if d.range.1 < sel_start || d.range.0 > sel_end { + continue; + } + let edit = TextEdit { + range: convert::to_range(&li, fix.range), + new_text: fix.replacement.clone(), + }; + let mut changes = HashMap::new(); + changes.insert(uri.clone(), vec![edit]); + let title = d + .help + .clone() + .unwrap_or_else(|| format!("Replace with `{}`", fix.replacement)); + #[allow(deprecated)] + actions.push(CodeActionOrCommand::CodeAction(CodeAction { + title, + kind: Some(CodeActionKind::QUICKFIX), + diagnostics: Some(vec![convert::to_lsp_diagnostic(&li, d)]), + edit: Some(WorkspaceEdit { changes: Some(changes), ..Default::default() }), + is_preferred: Some(true), + ..Default::default() + })); + } + Some(actions) + } + + // --- helpers ------------------------------------------------------------ + + /// A [`Location`] for a byte range within `file`, building that file's + /// [`LineIndex`] from its current (overlaid) bytes. + fn location(&self, file: FileId, range: (u32, u32)) -> Option { + let url = self.url_for_file(file)?; + let li = LineIndex::new(self.fileset.source(file), self.encoding); + Some(Location { uri: url, range: convert::to_range(&li, range) }) + } + + fn file_for_url(&self, uri: &Uri) -> Option { + let path = uri_to_path(uri)?; + self.path_to_id.get(&canonical(&path)).copied() + } + + fn url_for_file(&self, file: FileId) -> Option { + path_to_uri(self.fileset.path(file)) + } + + fn respond(&self, id: RequestId, result: T) { + let result = serde_json::to_value(result).unwrap_or(serde_json::Value::Null); + let _ = self.sender.send(Message::Response(Response { id, result: Some(result), error: None })); + } + + fn respond_err(&self, id: RequestId, code: i32, message: String) { + let response = Response { id, result: None, error: Some(ResponseError { code, message, data: None }) }; + let _ = self.sender.send(Message::Response(response)); + } + + fn notify(&self, params: N::Params) { + let not = Notification { + method: N::METHOD.to_string(), + params: serde_json::to_value(params).unwrap_or(serde_json::Value::Null), + }; + let _ = self.sender.send(Message::Notification(not)); + } +} + +/// The workspace folders to load as the mod layer: every `workspace_folder`, or +/// the deprecated `root_uri` as a fallback. +fn workspace_roots(init: &InitializeParams) -> Vec { + if let Some(folders) = &init.workspace_folders { + return folders.iter().filter_map(|f| uri_to_path(&f.uri)).collect(); + } + #[allow(deprecated)] + if let Some(root) = &init.root_uri + && let Some(p) = uri_to_path(root) + { + return vec![p]; + } + Vec::new() +} + +/// The prototype rule set: which directories define entities and which field keys +/// reference them. This stands in for a real per-game schema (a `.cwt`-config or +/// richer hand-written front-end is the follow-up); it mirrors `examples/lint.rs` +/// so the bundled `examples/mod-demo` corpus lights up end-to-end in an editor. +fn demo_schema() -> Schema { + let mut schema = Schema::new(); + schema + .define_dir("buildings", "building") + .reference_key("has_building", "building") + .reference_key("add_building", "building") + .reference_key("remove_building", "building") + .reference_key("upgrades_from", "building"); + schema +} + +/// `lsp_types::Uri` → filesystem path. lsp-types 0.97 dropped the `url` crate for +/// `fluent-uri` (which has no path conversion), so we bridge through `url`. +fn uri_to_path(uri: &Uri) -> Option { + url::Url::parse(uri.as_str()).ok()?.to_file_path().ok() +} + +/// Filesystem path → a `file://` `lsp_types::Uri`, via the `url` crate. +fn path_to_uri(path: &Path) -> Option { + let url = url::Url::from_file_path(path).ok()?; + url.as_str().parse::().ok() +} + +/// Canonicalize a path for stable URL↔file matching, falling back to the path as +/// given when it does not exist on disk (e.g. an unsaved buffer). +fn canonical(path: &Path) -> PathBuf { + std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()) +} diff --git a/jomini-lsp/tests/smoke.rs b/jomini-lsp/tests/smoke.rs new file mode 100644 index 0000000..0bc284c --- /dev/null +++ b/jomini-lsp/tests/smoke.rs @@ -0,0 +1,258 @@ +//! End-to-end smoke test: drive the real server (via [`jomini_lsp::serve`]) over +//! an in-memory [`Connection`] against the bundled `examples/mod-demo` corpus — +//! the same two-layer vanilla+mod project `examples/lint.rs` uses. No subprocess, +//! no wire framing: the client and server share a process over channel pairs. + +use lsp_server::{Connection, Message, Notification, Request, RequestId, Response}; +use serde_json::{json, Value}; +use std::path::{Path, PathBuf}; +use std::time::Duration; + +/// A tiny client over the memory connection: sends requests/notifications and +/// reads responses, stashing any server notifications (e.g. publishDiagnostics) +/// seen along the way for later assertions. +struct Client { + conn: Connection, + notifications: Vec, + next_id: i32, +} + +impl Client { + fn request(&mut self, method: &str, params: Value) -> Value { + let id = self.next_id; + self.next_id += 1; + self.conn + .sender + .send(Message::Request(Request { id: RequestId::from(id), method: method.into(), params })) + .unwrap(); + self.read_until_response(RequestId::from(id)) + } + + fn notify(&self, method: &str, params: Value) { + self.conn + .sender + .send(Message::Notification(Notification { method: method.into(), params })) + .unwrap(); + } + + fn read_until_response(&mut self, id: RequestId) -> Value { + loop { + match self.conn.receiver.recv_timeout(Duration::from_secs(15)) { + Ok(Message::Response(Response { id: rid, result, error })) if rid == id => { + assert!(error.is_none(), "request {id} errored: {error:?}"); + return result.unwrap_or(Value::Null); + } + Ok(Message::Notification(not)) => self.notifications.push(not), + Ok(_) => {} // a stray response/request — ignore + Err(e) => panic!("timed out waiting for response to {id}: {e}"), + } + } + } + + fn published_diagnostics(&self) -> Vec<(String, Value)> { + self.notifications + .iter() + .filter(|n| n.method == "textDocument/publishDiagnostics") + .map(|n| { + let uri = n.params["uri"].as_str().unwrap_or_default().to_string(); + (uri, n.params["diagnostics"].clone()) + }) + .collect() + } +} + +fn demo_paths() -> (PathBuf, PathBuf) { + let repo = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap().to_path_buf(); + let demo = repo.join("examples/mod-demo"); + (demo.join("mod"), demo.join("vanilla")) +} + +fn file_uri(path: &Path) -> String { + url::Url::from_file_path(path).unwrap().to_string() +} + +/// The LSP position (line, UTF-16 character) of the first occurrence of `needle` +/// in `text`. Target tokens live on ASCII lines, so UTF-16 = bytes there. +fn pos_of(text: &str, needle: &str) -> Value { + let off = text.find(needle).unwrap_or_else(|| panic!("`{needle}` not found")); + let prefix = &text[..off]; + let line = prefix.matches('\n').count(); + let line_start = prefix.rfind('\n').map(|i| i + 1).unwrap_or(0); + let character: usize = text[line_start..off].chars().map(|c| c.len_utf16()).sum(); + json!({ "line": line, "character": character }) +} + +#[test] +fn server_serves_the_mod_demo_project() { + let (mod_dir, vanilla_dir) = demo_paths(); + let (server_conn, client_conn) = Connection::memory(); + + // Run the server loop on a background thread; the test drives the client end. + let handle = std::thread::spawn(move || jomini_lsp::serve(&server_conn)); + let mut client = Client { conn: client_conn, notifications: Vec::new(), next_id: 1 }; + + // --- initialize: workspace = the mod dir, with the vanilla base as an option. + let init = client.request( + "initialize", + json!({ + "processId": null, + "rootUri": file_uri(&mod_dir), + "capabilities": {}, + "initializationOptions": { "vanilla": vanilla_dir.to_str().unwrap() } + }), + ); + let caps = &init["capabilities"]; + assert_eq!(caps["definitionProvider"], json!(true)); + assert_eq!(caps["referencesProvider"], json!(true)); + assert_eq!(caps["documentSymbolProvider"], json!(true)); + assert!(caps["codeActionProvider"] != Value::Null); + client.notify("initialized", json!({})); + + let mod_events = mod_dir.join("events/mod_events.txt"); + let mod_buildings = mod_dir.join("common/buildings/01_mod_buildings.txt"); + let vanilla_buildings = vanilla_dir.join("common/buildings/00_buildings.txt"); + let mod_events_text = std::fs::read_to_string(&mod_events).unwrap(); + let mod_buildings_text = std::fs::read_to_string(&mod_buildings).unwrap(); + + // --- document symbols: the vanilla buildings file defines four buildings. + let syms = client.request( + "textDocument/documentSymbol", + json!({ "textDocument": { "uri": file_uri(&vanilla_buildings) } }), + ); + let names: Vec = syms + .as_array() + .unwrap() + .iter() + .map(|s| s["name"].as_str().unwrap().to_string()) + .collect(); + assert_eq!(names, ["temple", "castle", "city", "barracks"], "document symbols"); + + // --- the cross-file diagnostic: `baracks` is undefined (published, by now). + let diags = client.published_diagnostics(); + let baracks = diags.iter().find_map(|(uri, ds)| { + ds.as_array()?.iter().find(|d| { + d["code"] == json!("undefined-reference") + && d["message"].as_str().unwrap_or("").contains("baracks") + })?; + Some(uri.clone()) + }); + let baracks_uri = baracks.expect("an undefined-reference diagnostic for `baracks`"); + assert!(baracks_uri.ends_with("mod_events.txt"), "in the events file: {baracks_uri}"); + // The malformed-subtree reference is suppressed, not a hard error. + let has_ghost_error = diags.iter().any(|(_, ds)| { + ds.as_array().unwrap().iter().any(|d| { + d["code"] == json!("undefined-reference") + && d["message"].as_str().unwrap_or("").contains("ghost_building") + }) + }); + assert!(!has_ghost_error, "ghost_building must be suppressed, not a hard undefined-reference"); + + // --- go-to-definition: `fortress` (used in mod_events) is defined in the mod + // buildings file — a cross-file jump. + let def = client.request( + "textDocument/definition", + json!({ + "textDocument": { "uri": file_uri(&mod_events) }, + "position": pos_of(&mod_events_text, "fortress"), + }), + ); + let def_uri = def["uri"].as_str().expect("a single definition Location"); + assert!(def_uri.ends_with("01_mod_buildings.txt"), "jumps to the mod buildings file: {def_uri}"); + + // --- find-references: `castle` is referenced in the mod (upgrades_from) and + // in vanilla (has_building); with the declaration that is at least three. + let refs = client.request( + "textDocument/references", + json!({ + "textDocument": { "uri": file_uri(&mod_buildings) }, + "position": pos_of(&mod_buildings_text, "castle"), + "context": { "includeDeclaration": true }, + }), + ); + assert!(refs.as_array().unwrap().len() >= 3, "castle references: {refs}"); + + // `temple` is overridden (declared in vanilla AND the mod). With + // includeDeclaration, references must return EVERY declaration site — not just + // the winning one — so the shadowed vanilla declaration appears. Vanilla's + // buildings file contains no *use* of temple, so its presence here proves the + // all-declarations behavior (use "temple = {" to skip the comment mentions). + let temple_refs = client.request( + "textDocument/references", + json!({ + "textDocument": { "uri": file_uri(&mod_buildings) }, + "position": pos_of(&mod_buildings_text, "temple = {"), + "context": { "includeDeclaration": true }, + }), + ); + let temple_uris: Vec<&str> = + temple_refs.as_array().unwrap().iter().map(|l| l["uri"].as_str().unwrap()).collect(); + assert!( + temple_uris.iter().any(|u| u.ends_with("00_buildings.txt")), + "the shadowed vanilla temple declaration must be included: {temple_uris:?}" + ); + assert!( + temple_uris.iter().any(|u| u.ends_with("01_mod_buildings.txt")), + "the winning mod temple declaration must be included: {temple_uris:?}" + ); + + // --- code action: a quick fix on the `baracks` typo offers `barracks`. + let start = pos_of(&mod_events_text, "baracks"); + let end = { + let mut e = start.clone(); + e["character"] = json!(start["character"].as_u64().unwrap() + "baracks".len() as u64); + e + }; + let actions = client.request( + "textDocument/codeAction", + json!({ + "textDocument": { "uri": file_uri(&mod_events) }, + "range": { "start": start, "end": end }, + "context": { "diagnostics": [] }, + }), + ); + let has_fix = actions.as_array().unwrap().iter().any(|a| { + a["title"].as_str().unwrap_or("").contains("barracks") && a["edit"]["changes"] != Value::Null + }); + assert!(has_fix, "a quick fix offering `barracks`: {actions}"); + + // --- formatting: returns an edit list (possibly empty) without erroring. + let fmt = client.request( + "textDocument/formatting", + json!({ + "textDocument": { "uri": file_uri(&vanilla_buildings) }, + "options": { "tabSize": 4, "insertSpaces": false } + }), + ); + assert!(fmt.is_array(), "formatting returns a TextEdit list: {fmt}"); + + // --- shutdown / exit: the loop returns and the thread joins cleanly. + let _ = client.request("shutdown", Value::Null); + client.notify("exit", Value::Null); + handle.join().unwrap().expect("server loop exits Ok"); +} + +#[test] +fn bare_exit_without_shutdown_terminates_the_server() { + let (mod_dir, vanilla_dir) = demo_paths(); + let (server_conn, client_conn) = Connection::memory(); + let handle = std::thread::spawn(move || jomini_lsp::serve(&server_conn)); + let mut client = Client { conn: client_conn, notifications: Vec::new(), next_id: 1 }; + + client.request( + "initialize", + json!({ + "processId": null, + "rootUri": file_uri(&mod_dir), + "capabilities": {}, + "initializationOptions": { "vanilla": vanilla_dir.to_str().unwrap() } + }), + ); + client.notify("initialized", json!({})); + + // A bare `exit` (no preceding `shutdown`) is the protocol's abrupt-termination + // path. The loop must still stop — otherwise a persistent transport hangs — + // and `serve` reports it as an error (non-zero process exit per the spec). + client.notify("exit", Value::Null); + let result = handle.join().expect("server thread did not panic / hang"); + assert!(result.is_err(), "bare exit terminates the loop with a non-Ok status"); +} diff --git a/src/text/lint.rs b/src/text/lint.rs new file mode 100644 index 0000000..d48743f --- /dev/null +++ b/src/text/lint.rs @@ -0,0 +1,1274 @@ +//! **Experimental** cross-file linting layer over the lossless [`syntax`] tree. +//! +//! A single [`GreenTree`](syntax::GreenTree) describes *one* file. But the +//! interesting bugs in a Paradox mod are *cross-file*: a building that references +//! a culture defined in another file, or in no file at all. Catching those needs +//! a project-wide view, so this module adds the layers a per-file parser cannot +//! provide — mirroring how rust-analyzer, Roslyn, swift-syntax, and Carbon all +//! keep name resolution in a layer *above* a purely syntactic tree: +//! +//! 1. [`Fileset`] — the project model. Every source file plus a [`FileKind`] +//! load-order layer, so a mod definition overrides a vanilla one (rust-analyzer's +//! `vfs`/`FileId` shape plus Paradox override semantics). +//! 2. [`FileSummary`] — an *ItemTree-style* summary. Each file is parsed once and +//! lowered into a small, **owned**, position-independent list of definitions +//! and references (with source ranges). The borrowing [`GreenTree`] is then +//! dropped, which sidesteps the "thousands of trees, each borrowing its source" +//! lifetime problem entirely. +//! 3. [`Index`] — the symbol table. A flat arena of [`Definition`]s keyed by +//! `(kind, name)`, with override/duplicate resolution by load order. +//! 4. [`Resolution`] / [`Linter`] — the two-pass resolver. Collect *all* +//! definitions, *then* resolve every reference; a miss is an undefined-reference +//! [`Diagnostic`]. Resolution is a three-way [`Resolution`] (Carbon's +//! `Found`/`NotFound`/`Poisoned` shape) so a reference inside a subtree the +//! parser already flagged as malformed is *suppressed* rather than piling a +//! semantic error on top of a syntactic one. +//! +//! Diagnostics carry both a [`Severity`] and a [`Confidence`] (tiger's two-axis +//! model) so a young rule set can ship its uncertain findings without drowning +//! the user. See `examples/lint.rs` for an end-to-end demo. +//! +//! This is a deliberately small **prototype**: the "schema" of what defines and +//! what references an entity is a handful of hand-written rules ([`Schema`]), +//! not a full per-game rule set, and names are compared as lossy-UTF-8 strings +//! rather than encoding-aware (a documented follow-up). + +#![allow(missing_docs)] // experimental surface; docs land as the API stabilizes + +use crate::text::syntax::{self, AstNode, Field, Flavor, SyntaxKind, SyntaxNode, Value}; +use smallvec::SmallVec; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +// --------------------------------------------------------------------------- +// Project model: files + load order +// --------------------------------------------------------------------------- + +/// An opaque handle to a file in a [`Fileset`]. Cheap to copy and hash; the +/// path and bytes are recovered through the [`Fileset`] (never carried around, +/// mirroring rust-analyzer's `FileId`). +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord)] +pub struct FileId(pub u32); + +/// Where a file sits in the Paradox load order. Ordered so that a later layer +/// overrides an earlier one for the same `(kind, name)`: `Vanilla < Dlc < Mod` +/// (and `Mod(0) < Mod(1)` for sub-mods). This is the minimal slice of tiger's +/// richer `FileKind` overlay that the prototype needs. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum FileKind { + /// Base game files. + Vanilla, + /// DLC content, layered above vanilla by index. + Dlc(u16), + /// A loaded mod, layered above DLC by index (load order). + Mod(u16), +} + +struct FileEntry { + path: PathBuf, + kind: FileKind, + source: Vec, +} + +/// The set of source files under consideration, each tagged with its load-order +/// [`FileKind`]. **Owns** the file bytes, so the parsed trees (which borrow +/// their source) can come and go while the project model stays alive. +#[derive(Default)] +pub struct Fileset { + files: Vec, +} + +impl Fileset { + /// An empty fileset. + pub fn new() -> Self { + Fileset::default() + } + + /// Add one file's bytes at `path` in load-order layer `kind`, returning its + /// [`FileId`]. + pub fn add(&mut self, path: impl Into, kind: FileKind, source: Vec) -> FileId { + let id = FileId(self.files.len() as u32); + self.files.push(FileEntry { path: path.into(), kind, source }); + id + } + + /// Recursively add every `*.txt` file under `root` to layer `kind`. Files + /// are added in **globally path-sorted** order, so that [`FileId`] assignment + /// — which same-layer override resolution uses as the load order (last wins) + /// — matches Paradox's lexicographic load order, regardless of directory + /// traversal. Directory **symlinks are not followed** (no cycle risk). The + /// stored path is the real filesystem path, used for display in diagnostics. + pub fn load_dir(&mut self, root: impl AsRef, kind: FileKind) -> std::io::Result> { + // Collect first, then sort the whole set, then assign ids — rather than + // letting the DFS traversal order decide ids. + let mut paths: Vec = Vec::new(); + let mut stack = vec![root.as_ref().to_path_buf()]; + while let Some(dir) = stack.pop() { + for entry in std::fs::read_dir(&dir)? { + let entry = entry?; + // `file_type()` comes from the directory entry and does not + // traverse symlinks, so a symlinked directory is skipped. + let file_type = entry.file_type()?; + let path = entry.path(); + if file_type.is_dir() { + stack.push(path); + } else if file_type.is_file() && path.extension().is_some_and(|e| e == "txt") { + paths.push(path); + } + } + } + paths.sort(); + + let mut ids = Vec::with_capacity(paths.len()); + for path in paths { + let source = std::fs::read(&path)?; + ids.push(self.add(path, kind, source)); + } + Ok(ids) + } + + /// The raw source bytes of a file. + pub fn source(&self, id: FileId) -> &[u8] { + &self.files[id.0 as usize].source + } + + /// The display path of a file. + pub fn path(&self, id: FileId) -> &Path { + &self.files[id.0 as usize].path + } + + /// The load-order layer of a file. + pub fn kind(&self, id: FileId) -> FileKind { + self.files[id.0 as usize].kind + } + + /// Every [`FileId`] in the set. + pub fn ids(&self) -> impl Iterator + '_ { + (0..self.files.len() as u32).map(FileId) + } + + /// The [`FileId`] previously assigned to `path` (by exact path match), if the + /// file is in the set. An editor uses this to map an on-disk path back to its + /// handle so it can overlay the unsaved buffer with + /// [`set_source`](Fileset::set_source) instead of rebuilding the whole set. + pub fn id_for_path(&self, path: impl AsRef) -> Option { + let path = path.as_ref(); + self.files.iter().position(|f| f.path == path).map(|i| FileId(i as u32)) + } + + /// Replace a file's source bytes in place, keeping its [`FileId`], path, and + /// layer. This is the editor-overlay hook: an LSP server swaps in the unsaved + /// buffer on a keystroke and re-analyzes, without disturbing any other file's + /// handle. Re-run [`Linter::analyze`] afterwards to refresh the snapshot. + pub fn set_source(&mut self, id: FileId, source: Vec) { + self.files[id.0 as usize].source = source; + } +} + +// --------------------------------------------------------------------------- +// Schema: the (tiny, hand-written) notion of what defines / references an entity +// --------------------------------------------------------------------------- + +/// The entity kind of a definition or reference, e.g. `"building"`. A +/// real linter would intern these; the prototype keeps them as owned strings +/// for clarity. +pub type Kind = String; + +/// A minimal, hand-written description of which files *define* entities and +/// which field keys *reference* them — the prototype's stand-in for a full +/// per-game rule set (the "rules-as-code" v1 from the design analysis). +#[derive(Default)] +pub struct Schema { + /// Directory component (e.g. `"buildings"`) → the [`Kind`] that each + /// top-level key in a file under that directory defines. + definition_dirs: HashMap, + /// Field key (e.g. `"add_building"`) → the [`Kind`] its value references. + reference_keys: HashMap, +} + +impl Schema { + /// An empty schema. + pub fn new() -> Self { + Schema::default() + } + + /// Declare that top-level keys in any file located under a `dir` directory + /// component define an entity of kind `kind` (e.g. `("buildings", + /// "building")`). + pub fn define_dir(&mut self, dir: &str, kind: &str) -> &mut Self { + self.definition_dirs.insert(dir.to_string(), kind.to_string()); + self + } + + /// Declare that a field whose key is `key` references an entity of kind + /// `kind` (e.g. `("add_building", "building")`). + pub fn reference_key(&mut self, key: &str, kind: &str) -> &mut Self { + self.reference_keys.insert(key.to_string(), kind.to_string()); + self + } + + /// The kind that a file at `path` defines, if any directory component of + /// the path is a known definition directory. + fn definition_kind(&self, path: &Path) -> Option<&Kind> { + path.components() + .filter_map(|c| c.as_os_str().to_str()) + .find_map(|c| self.definition_dirs.get(c)) + } +} + +// --------------------------------------------------------------------------- +// Per-file summary (the "ItemTree": owned, position-independent, tree-free) +// --------------------------------------------------------------------------- + +/// A definition extracted from a file: a `(kind, name)` plus where it sits. +#[derive(Debug, Clone)] +pub struct DefItem { + pub kind: Kind, + pub name: String, + pub range: (u32, u32), +} + +/// A reference extracted from a file: a `(kind, name)` use site, plus whether +/// it lives inside a subtree the parser flagged as malformed (so resolution can +/// suppress a cascade — this is what the new [`NodeFlags`](syntax::NodeFlags) +/// `HAS_ERROR` bit buys us, looked up in O(1)). +#[derive(Debug, Clone)] +pub struct RefItem { + pub kind: Kind, + pub name: String, + pub range: (u32, u32), + pub in_error_subtree: bool, +} + +/// The owned, tree-free summary of one file: its definitions, references, and +/// any syntax errors. Produced by [`summarize`], after which the parsed +/// [`GreenTree`](syntax::GreenTree) is dropped — the summary outlives it. This +/// is the invalidation barrier rust-analyzer's `ItemTree` provides: it depends +/// only on the items in the file, not on byte offsets within value bodies. +pub struct FileSummary { + pub file: FileId, + pub defs: Vec, + pub refs: Vec, + pub syntax_errors: Vec<(String, (u32, u32))>, +} + +fn decode(s: crate::Scalar) -> String { + String::from_utf8_lossy(s.as_bytes()).into_owned() +} + +/// Parse one file and lower it into a [`FileSummary`]. The [`GreenTree`] is +/// local to this function — every datum kept is owned — so no tree outlives the +/// call, and the whole project's worth of summaries can be held at once without +/// the borrow-one-slice lifetime headache. +pub fn summarize(fileset: &Fileset, schema: &Schema, file: FileId) -> FileSummary { + let source = fileset.source(file); + let tree = syntax::parse_with(source, Flavor::default()); + + let mut defs = Vec::new(); + // Top-level keys define entities only when the file lives in a known + // definition directory. + if let Some(kind) = schema.definition_kind(fileset.path(file)) { + for field in tree.ast().fields() { + if let Some(key) = field.key() { + defs.push(DefItem { + kind: kind.clone(), + name: decode(key.as_scalar()), + range: key.text_range(), + }); + } + } + } + + // References can appear at any depth, so walk the whole tree. + let mut refs = Vec::new(); + collect_refs(tree.root(), schema, false, &mut refs); + + let syntax_errors = + tree.errors().iter().map(|e| (e.message.clone(), e.range)).collect(); + + FileSummary { file, defs, refs, syntax_errors } +} + +/// Recursively collect reference uses. `in_error` carries down whether any +/// enclosing subtree was flagged malformed by the parser, so a reference inside +/// a broken block is recorded as suppressible. The O(1) [`SyntaxNode::has_error`] +/// check means we learn this without re-walking the subtree. +fn collect_refs(node: SyntaxNode<'_, '_>, schema: &Schema, in_error: bool, out: &mut Vec) { + // A reference is "in an error subtree" when the parser had trouble here: + // either an explicit recovery element (the O(1) HAS_ERROR flag covers stray + // `}` Bogus nodes), or an *unclosed* block — which is only ever a diagnostic, + // creating no error element, so it must be detected structurally (no + // `CloseBrace` child). Everything inside such a region is suspect, so a + // failed name lookup there is suppressed rather than reported as a hard miss. + let unclosed_block = node.kind() == SyntaxKind::Block + && !node.child_tokens().any(|t| t.kind() == SyntaxKind::CloseBrace); + let in_error = in_error || node.has_error() || unclosed_block; + + if let Some(field) = Field::cast(node) + && let (Some(key), Some(value)) = (field.key(), field.value()) + && let Some(kind) = schema.reference_keys.get(&decode(key.as_scalar())) + && let Value::Scalar(tok) = value + { + out.push(RefItem { + kind: kind.clone(), + name: decode(tok.as_scalar()), + range: tok.text_range(), + in_error_subtree: in_error, + }); + } + + for child in node.child_nodes() { + collect_refs(child, schema, in_error, out); + } +} + +// --------------------------------------------------------------------------- +// Symbol table (the Index): a flat arena of definitions, keyed by (kind, name) +// --------------------------------------------------------------------------- + +/// An index into [`Index::definitions`]; a Carbon-style typed handle into a flat +/// arena rather than a pointer. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct DefId(pub u32); + +/// A registered definition: a `(kind, name)` plus the file and range it came +/// from. `winner` marks the definition that wins after override resolution. +#[derive(Debug, Clone)] +pub struct Definition { + pub kind: Kind, + pub name: String, + pub file: FileId, + pub range: (u32, u32), + pub winner: bool, +} + +#[derive(PartialEq, Eq, Hash, Clone)] +struct Key { + kind: Kind, + name: String, +} + +/// The outcome of resolving a reference against the [`Index`]. The third arm — +/// Carbon's `Poisoned` — distinguishes "genuinely undefined" from "the +/// reference sits in a region the parser already marked broken", which the +/// resolver suppresses to avoid cascade noise. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Resolution { + /// The name resolves to a definition. + Found(DefId), + /// No definition of this `(kind, name)` exists in any loaded file. + NotFound, + /// Unresolved, but inside a malformed subtree — suppress the diagnostic. + UpstreamError, +} + +/// The project-wide symbol table: a flat arena of [`Definition`]s plus a +/// `(kind, name)` → definitions map. Built by registering every definition from +/// every [`FileSummary`], then [`finalize`](Index::finalize)d to pick winners. +#[derive(Default)] +pub struct Index { + definitions: Vec, + by_key: HashMap>, + winners: HashMap, +} + +impl Index { + /// An empty index. + pub fn new() -> Self { + Index::default() + } + + /// Register one definition, returning its [`DefId`]. + pub fn define(&mut self, kind: Kind, name: String, file: FileId, range: (u32, u32)) -> DefId { + let id = self.definitions.len() as u32; + let key = Key { kind: kind.clone(), name: name.clone() }; + self.definitions.push(Definition { kind, name, file, range, winner: false }); + self.by_key.entry(key).or_default().push(id); + DefId(id) + } + + /// All registered definitions (read-only). + pub fn definitions(&self) -> &[Definition] { + &self.definitions + } + + /// Resolve a reference. A present `(kind, name)` is [`Resolution::Found`]; + /// an absent one is [`Resolution::NotFound`], or [`Resolution::UpstreamError`] + /// when the use site is inside a malformed subtree. + pub fn resolve(&self, kind: &str, name: &str, in_error_subtree: bool) -> Resolution { + let key = Key { kind: kind.to_string(), name: name.to_string() }; + match self.winners.get(&key) { + Some(&id) => Resolution::Found(DefId(id)), + None if in_error_subtree => Resolution::UpstreamError, + None => Resolution::NotFound, + } + } + + /// The closest *winning* definition name of the same `kind` within a small + /// edit distance of `name` — the "did you mean?" candidate for an undefined + /// reference. Returns `None` if nothing is close enough. Only winners are + /// suggested, so the proposed name is one that would actually resolve. Must + /// run after [`finalize`](Index::finalize). + pub fn suggest(&self, kind: &str, name: &str) -> Option<&str> { + // A length-scaled threshold: tight for short identifiers, looser (capped + // at 3) for long ones, so `baracks`→`barracks` (1) is offered but + // unrelated names are not. + let threshold = (name.len() / 3).clamp(1, 3); + self.definitions + .iter() + .filter(|d| d.winner && d.kind == kind) + .map(|d| (levenshtein(name.as_bytes(), d.name.as_bytes()), d.name.as_str())) + .filter(|&(dist, _)| dist <= threshold) + .min_by_key(|&(dist, candidate)| (dist, candidate.len())) + .map(|(_, candidate)| candidate) + } + + /// Resolve override/duplicate conflicts across the load order and pick a + /// winner per `(kind, name)`. Emits informational diagnostics for mod-over- + /// vanilla overrides and warnings for duplicate definitions within one layer. + /// + /// This is the closest the prototype comes to cwtools' fixpoint index build, + /// kept as a clean single pass (tiger-style) since no subtype-style + /// conditional classification is involved. + pub fn finalize(&mut self, fileset: &Fileset) -> Vec { + let mut diags = Vec::new(); + + // Collect keys first to avoid borrowing `self.by_key` while mutating. + let keys: Vec = self.by_key.keys().cloned().collect(); + for key in keys { + let ids = self.by_key[&key].clone(); + + // The winning layer is the highest FileKind among the definitions. + let max_layer = ids.iter().map(|&id| fileset.kind(self.definitions[id as usize].file)).max().unwrap(); + + // Within the winning layer, the last-loaded definition wins + // (Paradox load order). Earlier same-layer definitions are + // duplicates; lower-layer definitions are overridden. + let winners_in_layer: Vec = ids + .iter() + .copied() + .filter(|&id| fileset.kind(self.definitions[id as usize].file) == max_layer) + .collect(); + let winner = *winners_in_layer.last().unwrap(); + self.definitions[winner as usize].winner = true; + self.winners.insert(key.clone(), winner); + + for &id in &ids { + let def = &self.definitions[id as usize]; + let layer = fileset.kind(def.file); + if id == winner { + // If the winner shadows lower layers, note the override. + if ids.iter().any(|&o| fileset.kind(self.definitions[o as usize].file) < max_layer) { + diags.push(Diagnostic { + id: lints::OVERRIDE, + severity: Severity::Tip, + confidence: Confidence::Strong, + file: def.file, + range: def.range, + message: format!( + "{} `{}` overrides a lower-layer definition", + def.kind, def.name + ), + help: None, + fix: None, + }); + } + } else if layer == max_layer { + // Same top layer but not the winner: a real duplicate. + diags.push(Diagnostic { + id: lints::DUPLICATE, + severity: Severity::Warning, + confidence: Confidence::Strong, + file: def.file, + range: def.range, + message: format!( + "duplicate definition of {} `{}` in the same layer", + def.kind, def.name + ), + help: None, + fix: None, + }); + } + } + } + + diags.sort_by_key(|d| (d.file, d.range.0)); + diags + } +} + +/// Levenshtein edit distance between two byte strings (the classic two-row DP). +/// Used by [`Index::suggest`]; inputs are short identifiers, so the O(n·m) cost +/// is negligible. +fn levenshtein(a: &[u8], b: &[u8]) -> usize { + if a.is_empty() { + return b.len(); + } + let mut prev: Vec = (0..=b.len()).collect(); + let mut curr = vec![0usize; b.len() + 1]; + for (i, &ca) in a.iter().enumerate() { + curr[0] = i + 1; + for (j, &cb) in b.iter().enumerate() { + let cost = if ca == cb { 0 } else { 1 }; + curr[j + 1] = (prev[j] + cost).min(prev[j + 1] + 1).min(curr[j] + 1); + } + std::mem::swap(&mut prev, &mut curr); + } + prev[b.len()] +} + +// --------------------------------------------------------------------------- +// Diagnostics: stable id + Severity + Confidence (Roslyn descriptor + tiger axes) +// --------------------------------------------------------------------------- + +/// Stable lint ids. A real linter would attach a description/category/default- +/// severity to each (Roslyn's `DiagnosticDescriptor`); the prototype keeps just +/// the stable string id that users would reference to configure or suppress. +pub mod lints { + pub const UNDEFINED_REFERENCE: &str = "undefined-reference"; + pub const SUPPRESSED_REFERENCE: &str = "undefined-reference-suppressed"; + pub const SYNTAX_ERROR: &str = "syntax-error"; + pub const DUPLICATE: &str = "duplicate-definition"; + pub const OVERRIDE: &str = "override"; +} + +/// How serious a finding is. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Severity { + /// An informational note. + Tip, + /// A style/tidiness nit. + Untidy, + /// Probably wrong. + Warning, + /// Almost certainly wrong. + Error, + /// Fatal — the file cannot be meaningfully processed further. + Fatal, +} + +/// How sure the linter is that a finding is real — tiger's second axis, the +/// dial that lets a young rule set surface uncertain findings without crying +/// wolf. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum Confidence { + /// Likely a false positive; show only when asked. + Weak, + /// A reasonable finding. + Reasonable, + /// High confidence. + Strong, +} + +/// A machine-applicable fix: replace the bytes in `range` with `replacement`. +/// The range comes from a syntax element (a [`SyntaxToken`](syntax::SyntaxToken) +/// the rule points at), so a fix is "rewrite this token" expressed as a text +/// edit — the practical, conflict-checkable form of "splice the tree, then +/// reprint". Apply with [`apply_fixes`]; the result is guaranteed to reparse, +/// and you can run [`format`](syntax::format) over it for the house style. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Fix { + /// The half-open byte range to replace. + pub range: (u32, u32), + /// The text to substitute in. + pub replacement: String, +} + +/// One finding: a stable id, the two axes, and a located message. Carries a +/// [`FileId`] + range rather than a tree reference, so it long outlives the +/// parse and is rendered later against the [`Fileset`]. May also carry a +/// human-readable [`help`](Diagnostic::help) note and a machine-applicable +/// [`fix`](Diagnostic::fix). +#[derive(Debug, Clone)] +pub struct Diagnostic { + pub id: &'static str, + pub severity: Severity, + pub confidence: Confidence, + pub file: FileId, + pub range: (u32, u32), + pub message: String, + /// An optional secondary hint (e.g. "did you mean `barracks`?"). + pub help: Option, + /// An optional machine-applicable fix. + pub fix: Option, +} + +// --------------------------------------------------------------------------- +// The linter: the two-pass orchestration +// --------------------------------------------------------------------------- + +/// The cross-file linter: a [`Schema`] plus the two-pass `collect → resolve` +/// run. Holds no per-file state — everything flows through the [`Fileset`] and +/// the [`Index`]. +pub struct Linter { + schema: Schema, +} + +impl Linter { + /// A linter driven by `schema`. + pub fn new(schema: Schema) -> Self { + Linter { schema } + } + + /// Lint a whole [`Fileset`], returning just the diagnostics (sorted by file + /// then position). A convenience wrapper over [`analyze`](Linter::analyze) for + /// batch callers who only want the findings; an editor/LSP should call + /// `analyze` instead and keep the [`Analysis`] around to answer navigation + /// queries off the same data. + pub fn run(&self, fileset: &Fileset) -> Vec { + self.analyze(fileset).into_diagnostics() + } + + /// Lint a whole [`Fileset`] and **retain** the working state — the [`Index`] + /// and every per-file [`FileSummary`] — in a queryable [`Analysis`]. + /// + /// Pass 1 summarizes every file and builds the [`Index`]; pass 2 surfaces + /// syntax errors, resolves every reference, and (unlike [`run`](Linter::run)) + /// records each use site in a `(kind, name)` → use-sites reverse map. The + /// diagnostics produced are identical to `run`; the difference is that the + /// definitions, reference use-sites, and that reverse map survive the call — + /// so go-to-definition / find-references / symbol queries cost a lookup + /// rather than a re-lint. + pub fn analyze(&self, fileset: &Fileset) -> Analysis { + // Pass 1: lower every file to a summary, then register definitions. + let summaries: Vec = + fileset.ids().map(|id| summarize(fileset, &self.schema, id)).collect(); + + let mut index = Index::new(); + for s in &summaries { + for d in &s.defs { + index.define(d.kind.clone(), d.name.clone(), s.file, d.range); + } + } + let mut diags = index.finalize(fileset); + + // Pass 2: surface syntax errors, resolve references, and build the + // reverse use-site map that find-references reads. + let mut refs_by_name: HashMap> = HashMap::new(); + for s in &summaries { + for (message, range) in &s.syntax_errors { + diags.push(Diagnostic { + id: lints::SYNTAX_ERROR, + severity: Severity::Error, + confidence: Confidence::Strong, + file: s.file, + range: *range, + message: message.clone(), + help: None, + fix: None, + }); + } + for (ref_index, r) in s.refs.iter().enumerate() { + refs_by_name + .entry(Key { kind: r.kind.clone(), name: r.name.clone() }) + .or_default() + .push((s.file, ref_index as u32)); + + match index.resolve(&r.kind, &r.name, r.in_error_subtree) { + Resolution::Found(_) => {} + Resolution::NotFound => { + // Offer the nearest defined name of the same kind as a + // "did you mean?" hint, and — since a reference is a + // single token — a machine-applicable rename fix. + let (help, fix) = match index.suggest(&r.kind, &r.name) { + Some(name) => ( + Some(format!("did you mean `{name}`?")), + Some(Fix { range: r.range, replacement: name.to_string() }), + ), + None => (None, None), + }; + diags.push(Diagnostic { + id: lints::UNDEFINED_REFERENCE, + severity: Severity::Error, + confidence: Confidence::Strong, + file: s.file, + range: r.range, + message: format!( + "no {} named `{}` is defined in any loaded file", + r.kind, r.name + ), + help, + fix, + }); + } + Resolution::UpstreamError => diags.push(Diagnostic { + id: lints::SUPPRESSED_REFERENCE, + severity: Severity::Tip, + confidence: Confidence::Weak, + file: s.file, + range: r.range, + message: format!( + "`{}` may be an undefined {}, but its block has a syntax error — suppressed", + r.name, r.kind + ), + help: None, + fix: None, + }), + } + } + } + + diags.sort_by_key(|d| (d.file, d.range.0, d.range.1)); + Analysis { index, summaries, diagnostics: diags, refs_by_name } + } +} + +// --------------------------------------------------------------------------- +// Analysis: the retained, queryable result an editor/LSP reads +// --------------------------------------------------------------------------- + +/// A queryable snapshot of a linted project: the diagnostics, plus the retained +/// [`Index`] and per-file [`FileSummary`]s that produced them. Built by +/// [`Linter::analyze`]. +/// +/// This is the editor/LSP substrate. [`Linter::run`] discards its working state; +/// `Analysis` keeps it, so go-to-definition, find-references, and document/ +/// workspace symbols are all reads off this one structure. Like [`FileSummary`] +/// it is fully **owned** — it holds no borrow on any parse tree or on the +/// [`Fileset`] — so it can live in a long-running server and be replaced wholesale +/// when a file changes (re-run [`Linter::analyze`] on the updated [`Fileset`]). +pub struct Analysis { + index: Index, + /// `summaries[i]` is the summary of `FileId(i)` (built in `fileset.ids()` + /// order, which is `0..len`). + summaries: Vec, + diagnostics: Vec, + /// `(kind, name)` → the use sites of that name, each `(file, index into that + /// file's `summary.refs`)`. The reverse of the forward resolution the + /// [`Index`] does, and the thing find-references needs. + refs_by_name: HashMap>, +} + +impl Analysis { + /// All diagnostics, sorted by file then position (identical to what + /// [`Linter::run`] returns). + pub fn diagnostics(&self) -> &[Diagnostic] { + &self.diagnostics + } + + /// Consume the analysis, yielding just its diagnostics. + pub fn into_diagnostics(self) -> Vec { + self.diagnostics + } + + /// The project-wide symbol table. + pub fn index(&self) -> &Index { + &self.index + } + + /// Every registered [`Definition`] — the workspace-symbol source. + pub fn definitions(&self) -> &[Definition] { + self.index.definitions() + } + + /// The owned [`FileSummary`] of `file` (its defs, refs, and syntax errors), + /// or `None` if `file` is not in the analyzed set. The per-file + /// document-symbol source is `summary(file).defs`. + pub fn summary(&self, file: FileId) -> Option<&FileSummary> { + self.summaries.get(file.0 as usize) + } + + /// The reference use site whose byte range covers `offset` in `file`, if any. + /// Ranges are matched inclusively at both ends so a cursor resting just after + /// the last character of a token still hits it (references are sparse, so this + /// never double-matches). The hit-test is a linear scan — fine for the small + /// game files this layer targets; a spatial index is a later optimization. + pub fn reference_at(&self, file: FileId, offset: u32) -> Option<&RefItem> { + let summary = self.summary(file)?; + summary.refs.iter().find(|r| r.range.0 <= offset && offset <= r.range.1) + } + + /// The definition site (its name token) whose byte range covers `offset` in + /// `file`, if any. Inclusive at both ends, like [`reference_at`]. + pub fn definition_at(&self, file: FileId, offset: u32) -> Option<&DefItem> { + let summary = self.summary(file)?; + summary.defs.iter().find(|d| d.range.0 <= offset && offset <= d.range.1) + } + + /// Go-to-definition for the token at `offset` in `file`: resolve a reference + /// there to its winning [`Definition`], or — if `offset` rests on a + /// definition's own name — return that entity's winner (so jumping from a + /// shadowed definition lands on the one that actually wins). `None` when there + /// is nothing resolvable under the cursor (plain syntax, or an *undefined* + /// reference, which resolves to no definition). + pub fn definition_for(&self, file: FileId, offset: u32) -> Option<&Definition> { + if let Some(r) = self.reference_at(file, offset) { + return self.winning_definition(&r.kind, &r.name); + } + if let Some(d) = self.definition_at(file, offset) { + return self.winning_definition(&d.kind, &d.name); + } + None + } + + /// The winning [`Definition`] for `(kind, name)`, if the name is defined. + fn winning_definition(&self, kind: &str, name: &str) -> Option<&Definition> { + match self.index.resolve(kind, name, false) { + Resolution::Found(DefId(id)) => self.index.definitions().get(id as usize), + _ => None, + } + } + + /// Find-references: every use site of `(kind, name)` across the whole project, + /// as `(file, &RefItem)`. Because overrides share a single name, every use of + /// that name conceptually targets the winning definition, so this is exactly + /// the set find-references should return. Empty if the name is used nowhere. + pub fn references_to<'s>( + &'s self, + kind: &str, + name: &str, + ) -> impl Iterator + 's { + let key = Key { kind: kind.to_string(), name: name.to_string() }; + self.refs_by_name.get(&key).into_iter().flatten().filter_map(move |&(file, idx)| { + let r = self.summaries.get(file.0 as usize)?.refs.get(idx as usize)?; + Some((file, r)) + }) + } +} + +// --------------------------------------------------------------------------- +// Rendering: lazy line/column + a compiler-style console report +// --------------------------------------------------------------------------- + +/// Convert a byte `offset` into 1-based `(line, column)` by scanning newlines. +/// Carbon's lazy location model: positions are stored as plain offsets and +/// resolved to line/column only when a diagnostic is actually rendered. +pub fn line_col(source: &[u8], offset: u32) -> (usize, usize) { + let offset = (offset as usize).min(source.len()); + let mut line = 1; + let mut col = 1; + for &b in &source[..offset] { + if b == b'\n' { + line += 1; + col = 1; + } else { + col += 1; + } + } + (line, col) +} + +/// The bytes of the line containing `offset`, and the offset of that line start. +fn line_bytes(source: &[u8], offset: u32) -> (&[u8], usize) { + let offset = (offset as usize).min(source.len()); + let start = source[..offset].iter().rposition(|&b| b == b'\n').map_or(0, |p| p + 1); + let end = source[start..].iter().position(|&b| b == b'\n').map_or(source.len(), |p| start + p); + (&source[start..end], start) +} + +fn severity_label(s: Severity) -> &'static str { + match s { + Severity::Tip => "tip", + Severity::Untidy => "untidy", + Severity::Warning => "warning", + Severity::Error => "error", + Severity::Fatal => "fatal", + } +} + +fn severity_color(s: Severity) -> &'static str { + match s { + Severity::Tip => "\x1b[36m", // cyan + Severity::Untidy | Severity::Warning => "\x1b[33m", // yellow + Severity::Error | Severity::Fatal => "\x1b[31m", // red + } +} + +/// Render one diagnostic as a multi-line, compiler-style report against the +/// fileset's source. With `color`, severity labels and the caret are ANSI- +/// colored. This is the separate *rendering* layer (Roslyn / rust-analyzer keep +/// diagnostic production and rendering apart): it touches only owned data + the +/// source bytes, never a parsed tree. +pub fn render(diag: &Diagnostic, fileset: &Fileset, color: bool) -> String { + let source = fileset.source(diag.file); + let (line, col) = line_col(source, diag.range.0); + let (line_src, line_start) = line_bytes(source, diag.range.0); + let line_text = String::from_utf8_lossy(line_src); + + let (reset, bold, dim) = + if color { ("\x1b[0m", "\x1b[1m", "\x1b[2m") } else { ("", "", "") }; + let sev = severity_color(diag.severity); + let sevc = if color { sev } else { "" }; + + let conf = match diag.confidence { + Confidence::Weak => " (weak)", + Confidence::Reasonable => "", + Confidence::Strong => "", + }; + + // Caret span within the line, clamped to the line's end. + let caret_col = diag.range.0 as usize - line_start; + // `saturating_sub` guards a hand-built `Diagnostic` with an inverted range + // (`Diagnostic`'s fields are public): a plain `-` would underflow-panic in + // debug and balloon `"^".repeat(span)` in release. + let span = (diag.range.1.saturating_sub(diag.range.0)).max(1) as usize; + let span = span.min(line_src.len().saturating_sub(caret_col).max(1)); + let pad: String = line_src[..caret_col.min(line_src.len())] + .iter() + .map(|&b| if b == b'\t' { '\t' } else { ' ' }) + .collect(); + let carets = "^".repeat(span); + + let gutter = format!("{:>4}", line); + let blank: String = " ".repeat(4); + + let mut out = format!( + "{sevc}{bold}{sev_label}{reset}{bold}[{id}]{reset}: {msg}{conf}\n\ + {blank} {dim}-->{reset} {path}:{line}:{col}\n\ + {blank} {dim}|{reset}\n\ + {gutter} {dim}|{reset} {line_text}\n\ + {blank} {dim}|{reset} {pad}{sevc}{carets}{reset}\n", + sev_label = severity_label(diag.severity), + id = diag.id, + msg = diag.message, + path = fileset.path(diag.file).display(), + ); + // A "did you mean?" hint, and a marker when an automatic fix is available. + if let Some(help) = &diag.help { + let fixable = if diag.fix.is_some() { " (fixable with --fix)" } else { "" }; + out.push_str(&format!("{blank} {dim}={reset} help: {help}{fixable}\n")); + } + out +} + +// --------------------------------------------------------------------------- +// Autofix: apply machine-applicable fixes as a text-edit splice +// --------------------------------------------------------------------------- + +/// Apply [`Fix`]es to `source`, returning the rewritten bytes. Fixes are sorted +/// by position and applied left-to-right; any fix that **overlaps** one already +/// applied is skipped (a fix-all must never emit conflicting edits). The result +/// is plain text that reparses cleanly — run [`format`](syntax::format) over it +/// if you also want the house style ("splice, then reprint"). +/// +/// This is the deliberately simple, robust form of tree rewriting: because each +/// fix targets a single element's byte range (e.g. a reference token), splicing +/// text is equivalent to rebuilding the green subtree but needs no width +/// recomputation — and it is exactly the LSP `TextEdit` model. +pub fn apply_fixes(source: &[u8], fixes: &[Fix]) -> Vec { + let mut sorted: Vec<&Fix> = fixes.iter().collect(); + sorted.sort_by_key(|f| (f.range.0, f.range.1)); + + let mut out = Vec::with_capacity(source.len()); + let mut cursor = 0u32; // bytes of `source` already copied + for fix in sorted { + let (start, end) = fix.range; + // Skip an out-of-order/overlapping or inverted edit rather than corrupt + // the output. + if start < cursor || end < start || end as usize > source.len() { + continue; + } + out.extend_from_slice(&source[cursor as usize..start as usize]); + out.extend_from_slice(fix.replacement.as_bytes()); + cursor = end; + } + out.extend_from_slice(&source[cursor as usize..]); + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Build a tiny two-layer fileset: vanilla buildings + a mod that overrides + /// and references them, plus a broken file. Returns the fileset and the + /// demo schema. + fn demo() -> (Fileset, Schema) { + let mut fs = Fileset::new(); + fs.add( + "vanilla/common/buildings/00_buildings.txt", + FileKind::Vanilla, + b"temple = { cost = 100 }\ncastle = { cost = 250 }\ncity = { cost = 500 }\nbarracks = { cost = 150 }\n".to_vec(), + ); + fs.add( + "mod/common/buildings/01_mod.txt", + FileKind::Mod(0), + b"temple = { cost = 80 }\nfortress = { upgrades_from = castle }\nfortress = { cost = 401 }\n" + .to_vec(), + ); + // `baracks` is a typo for the vanilla `barracks` (edit distance 1). + fs.add( + "mod/events/mod_events.txt", + FileKind::Mod(0), + b"e = { effect = { add_building = fortress\n add_building = baracks\n remove_building = temple } }\n".to_vec(), + ); + fs.add( + "mod/events/broken.txt", + FileKind::Mod(0), + b"broken = { effect = { add_building = ghost_building\n}\n".to_vec(), + ); + + let mut schema = Schema::new(); + schema + .define_dir("buildings", "building") + .reference_key("add_building", "building") + .reference_key("remove_building", "building") + .reference_key("upgrades_from", "building"); + (fs, schema) + } + + #[test] + fn detects_cross_file_undefined_reference() { + let (fs, schema) = demo(); + let diags = Linter::new(schema).run(&fs); + + // `baracks` is referenced but defined nowhere → an undefined-reference. + let undef: Vec<_> = + diags.iter().filter(|d| d.id == lints::UNDEFINED_REFERENCE).collect(); + assert_eq!(undef.len(), 1, "exactly one undefined reference: {diags:#?}"); + assert!(undef[0].message.contains("baracks")); + assert_eq!(undef[0].severity, Severity::Error); + } + + #[test] + fn suggests_and_offers_fix_for_typo() { + let (fs, schema) = demo(); + let diags = Linter::new(schema).run(&fs); + let d = diags + .iter() + .find(|d| d.id == lints::UNDEFINED_REFERENCE) + .expect("the undefined-reference diagnostic"); + + // The nearest defined building name is suggested as a help + a fix. + assert!(d.help.as_deref().unwrap_or("").contains("barracks"), "help: {:?}", d.help); + let fix = d.fix.as_ref().expect("a machine-applicable fix"); + assert_eq!(fix.replacement, "barracks"); + + // Applying the fix rewrites the typo in place; the result parses clean + // and the bad token is gone (`barracks` now resolves against the index). + let fixed = apply_fixes(fs.source(d.file), std::slice::from_ref(fix)); + let fixed_text = String::from_utf8_lossy(&fixed); + assert!(fixed_text.contains("add_building = barracks")); + assert!(!fixed_text.contains("baracks\n") && !fixed_text.contains("baracks ")); // no stray typo + // The fix reparses without errors (a rename of one token stays well-formed). + assert!(syntax::parse(&fixed).errors().is_empty()); + } + + #[test] + fn apply_fixes_splices_and_skips_overlap() { + // A single in-range rename. + let renamed = apply_fixes( + b"add_building = baracks", + &[Fix { range: (15, 22), replacement: "barracks".into() }], + ); + assert_eq!(renamed, b"add_building = barracks"); + + // Overlapping/out-of-order edits are skipped, never corrupting output. + let out = apply_fixes( + b"abcdef", + &[ + Fix { range: (0, 3), replacement: "X".into() }, + Fix { range: (2, 5), replacement: "Y".into() }, // overlaps the first + ], + ); + assert_eq!(out, b"Xdef"); + } + + #[test] + fn resolves_cross_layer_definitions() { + let (fs, schema) = demo(); + let diags = Linter::new(schema).run(&fs); + // `fortress` (mod), `castle` (vanilla, from the mod's upgrades_from), and + // `temple` (mod override) all resolve, so none appear as undefined. + for name in ["fortress", "castle", "temple"] { + assert!( + !diags.iter().any(|d| d.id == lints::UNDEFINED_REFERENCE && d.message.contains(name)), + "`{name}` should resolve across files/layers" + ); + } + } + + #[test] + fn flags_override_and_duplicate() { + let (fs, schema) = demo(); + let diags = Linter::new(schema).run(&fs); + assert!( + diags.iter().any(|d| d.id == lints::OVERRIDE && d.message.contains("temple")), + "mod temple overrides vanilla temple" + ); + assert!( + diags.iter().any(|d| d.id == lints::DUPLICATE && d.message.contains("fortress")), + "fortress is defined twice in the mod layer" + ); + } + + #[test] + fn suppresses_reference_in_broken_subtree() { + let (fs, schema) = demo(); + let diags = Linter::new(schema).run(&fs); + // `ghost_building` is undefined, but its block is unterminated, so the + // reference is suppressed (Weak) rather than reported as a hard error. + assert!( + !diags.iter().any(|d| d.id == lints::UNDEFINED_REFERENCE && d.message.contains("ghost_building")), + "a reference in a malformed subtree must not be a hard undefined-reference" + ); + assert!( + diags.iter().any(|d| d.id == lints::SUPPRESSED_REFERENCE && d.message.contains("ghost_building")), + "it should instead be a suppressed/weak finding" + ); + // And the underlying syntax error is still surfaced. + assert!(diags.iter().any(|d| d.id == lints::SYNTAX_ERROR)); + } + + #[test] + fn resolution_enum_three_ways() { + let mut index = Index::new(); + let fs = { + let mut fs = Fileset::new(); + fs.add("v/common/buildings/a.txt", FileKind::Vanilla, b"castle = {}".to_vec()); + fs + }; + index.define("building".into(), "castle".into(), FileId(0), (0, 6)); + index.finalize(&fs); + assert!(matches!(index.resolve("building", "castle", false), Resolution::Found(_))); + assert_eq!(index.resolve("building", "ghost", false), Resolution::NotFound); + assert_eq!(index.resolve("building", "ghost", true), Resolution::UpstreamError); + } + + #[test] + fn render_is_panic_free_on_odd_ranges() { + let mut fs = Fileset::new(); + let f = fs.add("a.txt", FileKind::Vanilla, b"a = 1\n".to_vec()); + // An inverted range, a past-EOF range, and a zero-width range must all + // render without panicking (Diagnostic fields are public, so callers + // can construct any range). + for range in [(5, 2), (100, 200), (3, 3)] { + let d = Diagnostic { + id: lints::SYNTAX_ERROR, + severity: Severity::Error, + confidence: Confidence::Strong, + file: f, + range, + message: "x".into(), + help: None, + fix: None, + }; + let _ = render(&d, &fs, false); + } + } + + #[test] + fn line_col_is_one_based() { + let src = b"a = 1\nbb = 2\n"; + assert_eq!(line_col(src, 0), (1, 1)); + assert_eq!(line_col(src, 6), (2, 1)); // first byte of line 2 + assert_eq!(line_col(src, 9), (2, 4)); + } + + #[test] + fn analyze_diagnostics_match_run() { + // `analyze` must produce exactly the diagnostics `run` does — `run` is now + // a thin wrapper over it. + let (fs, schema) = demo(); + let via_run = Linter::new(schema).run(&fs); + let (fs2, schema2) = demo(); + let via_analyze = Linter::new(schema2).analyze(&fs2).into_diagnostics(); + assert_eq!(via_run.len(), via_analyze.len()); + for (a, b) in via_run.iter().zip(&via_analyze) { + assert_eq!((a.id, a.file, a.range), (b.id, b.file, b.range)); + } + } + + #[test] + fn goto_definition_jumps_cross_file_to_the_winner() { + let (fs, schema) = demo(); + let analysis = Linter::new(schema).analyze(&fs); + + // The events file (FileId 2) references building `fortress`. + let events = FileId(2); + let fref = analysis + .reference_at(events, 35) // inside `fortress` (the add_building value) + .expect("a reference under the cursor"); + assert_eq!((fref.kind.as_str(), fref.name.as_str()), ("building", "fortress")); + + // Go-to-definition lands on the *winning* fortress definition, which lives + // in the mod buildings file (FileId 1) — a jump no single-file view affords. + let def = analysis.definition_for(events, 35).expect("a definition to jump to"); + assert_eq!((def.kind.as_str(), def.name.as_str()), ("building", "fortress")); + assert_eq!(def.file, FileId(1)); + assert!(def.winner); + + // An undefined reference (`baracks`, the typo) resolves to no definition. + let baracks = analysis + .reference_at(events, 60) + .filter(|r| r.name == "baracks") + .expect("the baracks reference"); + assert_eq!(baracks.name, "baracks"); + assert!(analysis.definition_for(events, 60).is_none()); + } + + #[test] + fn goto_definition_on_a_definition_resolves_the_override_winner() { + let (fs, schema) = demo(); + let analysis = Linter::new(schema).analyze(&fs); + // Cursor on the *vanilla* `temple` definition key (FileId 0, offset 2) + // jumps to whichever `temple` actually wins — the mod override (FileId 1). + let def = analysis.definition_for(FileId(0), 2).expect("temple resolves"); + assert_eq!(def.name, "temple"); + assert!(def.winner); + assert_eq!(fs.kind(def.file), FileKind::Mod(0)); + } + + #[test] + fn find_references_collects_use_sites_across_the_project() { + let (fs, schema) = demo(); + let analysis = Linter::new(schema).analyze(&fs); + + // `fortress` is *defined* twice (FileId 1) but *referenced* once: the + // `add_building = fortress` in the events file. + let uses: Vec<_> = analysis.references_to("building", "fortress").collect(); + assert_eq!(uses.len(), 1, "one use site for fortress: {uses:?}"); + let (file, r) = uses[0]; + assert_eq!(file, FileId(2)); + assert_eq!(r.name, "fortress"); + + // `castle` is referenced once (the mod's `upgrades_from = castle`). + assert_eq!(analysis.references_to("building", "castle").count(), 1); + // A name used nowhere yields nothing. + assert_eq!(analysis.references_to("building", "nonesuch").count(), 0); + } + + #[test] + fn document_symbols_come_from_per_file_summaries() { + let (fs, schema) = demo(); + let analysis = Linter::new(schema).analyze(&fs); + // The vanilla buildings file defines four buildings, in source order. + let names: Vec<&str> = + analysis.summary(FileId(0)).unwrap().defs.iter().map(|d| d.name.as_str()).collect(); + assert_eq!(names, ["temple", "castle", "city", "barracks"]); + } + + #[test] + fn overlay_set_source_then_reanalyze_resolves_the_typo() { + let (mut fs, schema) = demo(); + let linter = Linter::new(schema); + + // Path → id round-trip, and a miss for an unknown path. + let buildings = fs + .id_for_path("vanilla/common/buildings/00_buildings.txt") + .expect("the vanilla buildings file is in the set"); + assert_eq!(buildings, FileId(0)); + assert!(fs.id_for_path("nope.txt").is_none()); + + // Before: `baracks` (typo) is referenced but defined nowhere. + let before = linter.analyze(&fs); + assert!(before + .diagnostics() + .iter() + .any(|d| d.id == lints::UNDEFINED_REFERENCE && d.message.contains("baracks"))); + + // Overlay an edit that *defines* `baracks`, as an editor would on a + // keystroke, then re-analyze the same fileset. + let mut edited = fs.source(buildings).to_vec(); + edited.extend_from_slice(b"baracks = { cost = 1 }\n"); + fs.set_source(buildings, edited); + let after = linter.analyze(&fs); + + // The undefined reference is gone — the overlay fed straight into the index. + assert!(!after + .diagnostics() + .iter() + .any(|d| d.id == lints::UNDEFINED_REFERENCE && d.message.contains("baracks"))); + } +} + diff --git a/src/text/logos_lexer.rs b/src/text/logos_lexer.rs deleted file mode 100644 index 4a2efac..0000000 --- a/src/text/logos_lexer.rs +++ /dev/null @@ -1,468 +0,0 @@ -use logos::{Lexer, Logos}; -use smallvec::SmallVec; -use crate::{Error, ErrorKind}; - -type ExpressionList = SmallVec<[ExpressionToken; 16]>; - -/// A lossless token that captures all syntax elements including trivia -#[derive(Logos, Debug, PartialEq, Clone)] -pub enum LosslessToken { - // Operators from existing parser - /// Exact equality operator `==` - #[token(b"==")] - Exact, - /// Less than or equal operator `<=` - #[token(b"<=")] - LessThanEqual, - /// Greater than or equal operator `>=` - #[token(b">=")] - GreaterThanEqual, - /// Not equal operator `!=` - #[token(b"!=")] - NotEqual, - /// Exists operator `?=` - #[token(b"?=")] - Exists, - /// Equal operator `=` - #[token(b"=")] - Equal, - /// Less than operator `<` - #[token(b"<")] - LessThan, - /// Greater than operator `>` - #[token(b">")] - GreaterThan, - - // Structural tokens - /// Left brace `{` - #[token(b"{")] - LBrace, - /// Right brace `}` - #[token(b"}")] - RBrace, - /// Left bracket `[` - #[token(b"[")] - LBracket, - /// Right bracket `]` - #[token(b"]")] - RBracket, - - // String literals - handle escape sequences properly - /// Quoted string literal - #[regex(br#""([^"\\]|\\.)*""#)] - Quoted, - - // Expression tokens for @[...] syntax - /// Expression in @[...] syntax - #[token(b"@[", parse_expression)] - Expression(ExpressionList), - - // Variable references starting with @ - /// Variable reference starting with @ - #[regex(br"@[a-zA-Z_][a-zA-Z0-9_]*")] - Variable, - - /// Undefined parameter marker ! (for [[!var_name] syntax) - #[token(b"!")] - UndefinedParameter, - - // Unquoted tokens - most common case - /// Unquoted identifier or value - #[regex(br"[a-zA-Z0-9_\-.:/|]+")] - Unquoted, - - // Trivia tokens for lossless parsing - /// Whitespace (spaces, tabs, newlines, semicolons) - #[regex(br"[ \t\r\n;]+")] - Whitespace, - - /// Comment starting with # - #[regex(br"#[^\r\n]*")] - Comment, -} - -/// Tokens for expressions inside @[...] constructs -#[derive(Debug, Copy, Clone, PartialEq, Logos)] -pub enum ExpressionToken { - /// Subtraction operator `-` - #[token(b"-")] - Subtract, - /// Addition operator `+` - #[token(b"+")] - Add, - /// Multiplication operator `*` - #[token(b"*")] - Multiply, - /// Division operator `/` - #[token(b"/")] - Divide, - /// Left parenthesis `(` - #[token(b"(")] - LParen, - /// Right parenthesis `)` - #[token(b")")] - RParen, - /// Right bracket `]` - marks end of expression - #[token(b"]")] - RBracket, - - /// Integer literal - #[regex(br"-?[0-9]+")] - Integer, - - /// Float literal (with optional 'f' suffix) - #[regex(br"-?[0-9]*\.[0-9]+f?")] - Float, - - /// Identifier (includes variables with @ prefix) - #[regex(br"@?[a-zA-Z_][a-zA-Z0-9_]*")] - Identifier, - - /// Whitespace within expressions - #[regex(br"[ \t]+")] - Whitespace, -} - - -/// Parse expression tokens between @[ and ] -fn parse_expression(lex: &mut Lexer) -> Result { - let remaining = lex.remainder(); - let mut expr_lexer = ExpressionToken::lexer(remaining); - let mut tokens = SmallVec::new(); - - // Parse tokens until we hit RBracket (which marks the end) - while let Some(token) = expr_lexer.next() { - match token { - Ok(ExpressionToken::RBracket) => { - // Found the end - advance main lexer past the ] - lex.bump(expr_lexer.span().end); - return Ok(tokens); - } - Ok(t) => tokens.push(t), - Err(_) => { - // Skip invalid tokens - could be improved - continue; - } - } - } - - // If we get here, we didn't find a closing bracket - Err(()) -} - -/// Lossless lexer for jomini text format -pub struct LosslessLexer<'a> { - lexer: Lexer<'a, LosslessToken>, - input: &'a [u8], - original_input: &'a [u8], - bom_offset: usize, -} - -impl<'a> LosslessLexer<'a> { - /// Create a new lossless lexer from input bytes - pub fn new(input: &'a [u8]) -> Self { - // Strip UTF-8 BOM if present - let (stripped_input, bom_offset) = if input.starts_with(&[0xef, 0xbb, 0xbf]) { - (&input[3..], 3) - } else { - (input, 0) - }; - - Self { - lexer: LosslessToken::lexer(stripped_input), - input: stripped_input, - original_input: input, - bom_offset, - } - } - - /// Get the next token from the input - pub fn next_token(&mut self) -> Option> { - match self.lexer.next() { - Some(Ok(token)) => Some(Ok(token)), - Some(Err(_)) => { - // Handle error - could be invalid token - let span = self.lexer.span(); - Some(Err(Error::new(ErrorKind::InvalidSyntax { - msg: "Invalid token".to_string(), - offset: span.start - }))) - } - None => None, - } - } - - /// Get current position in the input (adjusted for BOM) - pub fn position(&self) -> usize { - self.lexer.span().start + self.bom_offset - } - - /// Get the current span being processed (adjusted for BOM) - pub fn span(&self) -> std::ops::Range { - let span = self.lexer.span(); - (span.start + self.bom_offset)..(span.end + self.bom_offset) - } - - /// Get remaining input - pub fn remainder(&self) -> &'a [u8] { - self.lexer.remainder() - } - - /// Get the text for a token at the current span - pub fn token_text(&self) -> &'a [u8] { - let span = self.lexer.span(); - &self.input[span] - } - - /// Get the text for a specific span in the original input - pub fn span_text(&self, span: std::ops::Range) -> &'a [u8] { - &self.original_input[span] - } -} - -/// Iterator implementation for the lossless lexer -impl<'a> Iterator for LosslessLexer<'a> { - type Item = Result; - - fn next(&mut self) -> Option { - self.next_token() - } -} - -/// Token with position information for lossless reconstruction -#[derive(Debug, Clone, PartialEq)] -pub struct TokenWithSpan { - /// The token type - pub token: LosslessToken, - /// The byte span in the original input where this token occurs - pub span: std::ops::Range, -} - -/// Collect all tokens with their spans for lossless processing -pub fn collect_tokens_with_spans(input: &[u8]) -> Result, Error> { - let mut lexer = LosslessLexer::new(input); - let mut tokens = Vec::new(); - - while let Some(result) = lexer.next_token() { - match result { - Ok(token) => { - let span = lexer.span(); - tokens.push(TokenWithSpan { token, span }); - } - Err(e) => return Err(e), - } - } - - Ok(tokens) -} - -/// Verify lossless property by reconstructing input from tokens with spans -pub fn reconstruct_input(original_input: &[u8], tokens: &[TokenWithSpan]) -> Vec { - let mut result = Vec::new(); - - for token_with_span in tokens { - let text = &original_input[token_with_span.span.clone()]; - result.extend_from_slice(text); - } - - result -} - -#[cfg(test)] -mod tests { - use super::*; - use rstest::*; - - #[rstest] - #[case(b"foo = bar", vec![ - LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Equal, - LosslessToken::Whitespace, - LosslessToken::Unquoted, - ])] - fn test_basic_tokens(#[case] input: &[u8], #[case] expected_tokens: Vec) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); - - assert_eq!(tokens, expected_tokens); - - // Test lossless property - let reconstructed = reconstruct_input(input, &tokens_with_spans); - assert_eq!(reconstructed, input); - } - - #[rstest] - #[case(b"a==b c<=d e>=f g!=h i?=j kn", vec![ - LosslessToken::Unquoted, LosslessToken::Exact, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::LessThanEqual, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::GreaterThanEqual, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::NotEqual, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::Exists, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::LessThan, LosslessToken::Unquoted, - LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::GreaterThan, LosslessToken::Unquoted, - ])] - fn test_operators(#[case] input: &[u8], #[case] expected_tokens: Vec) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); - - assert_eq!(tokens, expected_tokens); - - // Test lossless property - let reconstructed = reconstruct_input(input, &tokens_with_spans); - assert_eq!(reconstructed, input); - } - - // Helper function to test lossless property for any input - fn test_lossless(input: &[u8]) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let reconstructed = reconstruct_input(input, &tokens_with_spans); - assert_eq!(reconstructed, input, "Failed lossless test for: {:?}", - std::str::from_utf8(input).unwrap_or("invalid utf8")); - } - - #[rstest] - #[case(br#"name = "hello \"world\"""#, vec![ - LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Equal, - LosslessToken::Whitespace, LosslessToken::Quoted - ])] - fn test_quoted_strings(#[case] input: &[u8], #[case] expected_tokens: Vec) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); - - assert_eq!(tokens, expected_tokens); - test_lossless(input); - } - - #[rstest] - #[case(b"# This is a comment\nfoo = bar\t# Another comment\n", vec![ - LosslessToken::Comment, LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Equal, LosslessToken::Whitespace, - LosslessToken::Unquoted, LosslessToken::Whitespace, LosslessToken::Comment, LosslessToken::Whitespace - ])] - fn test_comments_and_whitespace(#[case] input: &[u8], #[case] expected_tokens: Vec) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); - - assert_eq!(tokens, expected_tokens); - test_lossless(input); - } - - #[rstest] - #[case(b"@planet_standard_scale = @default_window_name", vec![ - LosslessToken::Variable, LosslessToken::Whitespace, LosslessToken::Equal, - LosslessToken::Whitespace, LosslessToken::Variable - ])] - fn test_variables(#[case] input: &[u8], #[case] expected_tokens: Vec) { - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let tokens: Vec<_> = tokens_with_spans.iter().map(|t| t.token.clone()).collect(); - - assert_eq!(tokens, expected_tokens); - test_lossless(input); - } - - #[test] - fn test_expressions() { - let input = b"position = { @[1-leopard_x] @leopard_y }"; - test_lossless(input); - - // Verify we have an Expression token - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let has_expression = tokens_with_spans.iter().any(|t| matches!(t.token, LosslessToken::Expression(_))); - assert!(has_expression, "Should contain an Expression token"); - } - - #[test] - fn test_complex_expressions() { - let input = b"my_calc = @[(-half-half)*half]"; - test_lossless(input); - - // Verify we have an Expression token - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let has_expression = tokens_with_spans.iter().any(|t| matches!(t.token, LosslessToken::Expression(_))); - assert!(has_expression, "Should contain an Expression token"); - } - - #[test] - fn test_bom_stripping() { - let input = b"\xef\xbb\xbf# UTF-8 BOM test\nfoo = bar"; - let tokens_with_spans = collect_tokens_with_spans(input).unwrap(); - let reconstructed = reconstruct_input(input, &tokens_with_spans); - - // BOM is stripped during lexing, so reconstructed input won't include BOM - // This is expected behavior since we removed LosslessToken::Bom variant - let expected_without_bom = b"# UTF-8 BOM test\nfoo = bar"; - assert_eq!(reconstructed, expected_without_bom, "Reconstructed should match input without BOM"); - - // The first token should be Comment (BOM was stripped before lexing) - assert_eq!(tokens_with_spans[0].token, LosslessToken::Comment); - assert_eq!(tokens_with_spans[0].span, 3..19); // BOM offset handled in span - - // Verify lexer properly handles BOM without producing separate token - let input_without_bom = b"# UTF-8 BOM test\nfoo = bar"; - let tokens_without_bom = collect_tokens_with_spans(input_without_bom).unwrap(); - - // Both inputs should produce the same tokens (just different spans) - assert_eq!(tokens_with_spans.len(), tokens_without_bom.len()); - for (with_bom, without_bom) in tokens_with_spans.iter().zip(tokens_without_bom.iter()) { - assert_eq!(with_bom.token, without_bom.token); - } - } - - #[test] - fn test_lossless_token_size() { - // Test that LosslessToken has a reasonable size - // This is important for memory efficiency since we create many tokens - let size = std::mem::size_of::(); - - // Should be reasonable size - exact size depends on SmallVec internal layout - // but should be significantly smaller than a Vec would be - println!("LosslessToken size: {} bytes", size); - - // Ensure it's not unreasonably large (less than 64 bytes) - assert!(size <= 64, "LosslessToken size ({} bytes) should be <= 64 bytes", size); - - // Also test ExpressionToken size - let expr_size = std::mem::size_of::(); - println!("ExpressionToken size: {} bytes", expr_size); - - // ExpressionToken should be small since it's Copy - assert!(expr_size <= 8, "ExpressionToken size ({} bytes) should be <= 8 bytes", expr_size); - } - - #[rstest] - #[case(b"[[scaled_skill] code here ] [[!var_name] other code ]")] - #[case(b"stats={{id=0 type=general} {id=1 type=admiral}}")] - fn test_lossless_samples(#[case] input: &[u8]) { - test_lossless(input); - } - - #[rstest] - #[case(b"foo = bar")] - #[case(b"open={1 2}")] - #[case(b"field1=-100.535")] - #[case(br#""foo"="bar" "3"="1444.11.11""#)] - #[case(br#"custom_name="THE !@#$%^&*( '\"LEGION\"')""#)] - #[case(b"foo{bar=qux}")] - #[case(b"foo=abc#def\nbar=qux")] - #[case(b"flavor_tur.8=yes")] - #[case(b"dashed-identifier=yes")] - #[case(b"province_id = event_target:agenda_province")] - #[case(b"mult = value:job_weights_research_modifier|JOB|head_researcher|")] - #[case(b"@planet_standard_scale = 11")] - #[case(b"window_name = @default_window_name")] - #[case(b"value=\"win\"; a=b")] - #[case(b"foo = 0.3;")] - #[case(b"a = 1; b = 2;; c = 3;")] - #[case(b";;;key = value;;;")] - #[case(b"age > 16")] - fn test_comprehensive_syntax(#[case] input: &[u8]) { - test_lossless(input); - } -} \ No newline at end of file diff --git a/src/text/mod.rs b/src/text/mod.rs index fdb1bfc..af53eb1 100644 --- a/src/text/mod.rs +++ b/src/text/mod.rs @@ -25,9 +25,10 @@ pub mod de; mod dom; mod fnv; -mod logos_lexer; +pub mod lint; mod operator; mod reader; +pub mod syntax; mod tape; mod writer; @@ -38,10 +39,6 @@ pub use self::dom::{ ArrayReader, FieldGroupsIter, FieldsIter, GroupEntry, GroupEntryIter, ObjectReader, Reader, ScalarReader, ValueReader, ValuesIter, }; -pub use self::logos_lexer::{ - ExpressionToken, LosslessLexer, LosslessToken, TokenWithSpan, - collect_tokens_with_spans, reconstruct_input -}; pub use self::operator::*; pub use self::tape::{TextTape, TextTapeParser, TextToken}; pub use self::writer::*; diff --git a/src/text/syntax.rs b/src/text/syntax.rs new file mode 100644 index 0000000..6b9030c --- /dev/null +++ b/src/text/syntax.rs @@ -0,0 +1,2979 @@ +//! **Experimental** lossless syntax tree for the Clausewitz text format. +//! +//! This is the foundation for tooling — formatting, linting, highlighting, and +//! transformations — and (later) ergonomic deserialization. Unlike [`TextTape`], +//! which is a fast, lossy tape that discards trivia and elides the `=` operator, +//! this tree preserves **every byte** of the source: comments, whitespace, the +//! `=`, even bytes that match no rule. The defining invariant is: +//! +//! ```text +//! concat(text of every leaf token, in order) == source +//! ``` +//! +//! # Representation +//! +//! The tree is stored in a single flat arena ([`GreenTree::tape`]) in pre-order, +//! mirroring jomini's tape philosophy for cache locality. Following rowan's +//! green/red split, the stored data is **position-independent**: each node and +//! token records only its *relative width* in bytes, never an absolute offset. +//! Absolute offsets and parent links are derived once and the lightweight +//! [`SyntaxNode`]/[`SyntaxToken`] cursors ("red layer") read them on demand. +//! Keeping widths relative is what makes future incremental reparse (reusing +//! untouched subtrees) tractable. +//! +//! # Games (one superset parser) +//! +//! A single permissive parser handles every PDS title. The lexer accepts the +//! *union* of all games' syntax (`@vars`, `@[calc]`, `$macros$`, `[[params]]`, +//! broad identifier bytes); game-appropriateness is a concern for a later lint +//! layer, not the parser. The handful of genuine lexer forks are toggled by +//! [`Flavor`] — currently just HoI4's newline-terminated strings. +//! +//! On top of the green tree sit lightweight [`SyntaxNode`]/[`SyntaxToken`] +//! cursors and an ungrammar-style **typed AST** ([`AstNode`], [`Field`], +//! [`Block`], [`HeaderedBlock`], [`Calc`], …) with value coercions +//! ([`Value::to_f64`], [`HeaderedBlock::as_color`]). A [`format`](fn@format) pass reprints +//! the tree in a normalized house style, preserving every comment and +//! significant token. +//! +//! This is a Phase-1 module: `[[param]]` blocks are not yet given dedicated +//! structure (they round-trip losslessly as loose tokens, surfaced as +//! [`Item::Other`]). The internals of `@[calc]` *are* parsed into a real +//! expression subtree ([`SyntaxKind::Calc`] / [`SyntaxKind::BinaryExpr`] / +//! [`SyntaxKind::UnaryExpr`] / [`SyntaxKind::ParenExpr`]). A depth guard caps +//! recursion on pathological nesting (see [`SyntaxError`]). +//! +//! [`TextTape`]: crate::TextTape +#![allow(missing_docs)] // experimental surface; docs land as the API stabilizes + +use crate::{text::Operator, Scalar}; + +/// The kind of every node (interior) and token (leaf) in a [`GreenTree`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[repr(u16)] +pub enum SyntaxKind { + // ===== tokens (leaves) ===== + /// A leading UTF-8 byte-order-mark (`EF BB BF`). + Bom, + /// A run of spaces, tabs, newlines, carriage returns, or semicolons. + Whitespace, + /// A `# ...` comment running to end of line. + Comment, + /// A bare scalar: identifier, number, date, `yes`/`no`, etc. + Unquoted, + /// A `"..."` quoted scalar (raw text, including the quotes). + Quoted, + /// A `@name` reader-variable reference. + Variable, + /// A `$NAME$` macro parameter. + MacroParam, + /// `{` + OpenBrace, + /// `}` + CloseBrace, + /// `[` + OpenBracket, + /// `]` + CloseBracket, + /// `!`, the undefined-parameter marker in `[[!name] ...]`. + Bang, + /// An operator: `=` `==` `?=` `!=` `<` `<=` `>` `>=`. + Operator, + /// Reserved for explicit error tokens (the current lexer classifies every + /// byte, worst case as [`SyntaxKind::Unquoted`], so it is not emitted yet). + Error, + + // ===== calc tokens (interior of `@[ ... ]`) ===== + /// `@[`, opening a parse-time calculation. + CalcOpen, + /// `]`, closing a parse-time calculation. + CalcClose, + /// `+` inside a calc. + Plus, + /// `-` inside a calc. + Minus, + /// `*` inside a calc. + Star, + /// `/` inside a calc. + Slash, + /// `(` inside a calc. + OpenParen, + /// `)` inside a calc. + CloseParen, + /// A numeric literal inside a calc (e.g. `1`, `10.0`, `10.0f`). + Number, + /// An operand identifier inside a calc (e.g. `tier`, `leopard_x`, `@var`). + CalcIdent, + + // ===== nodes (interior) ===== + /// The whole document. + Root, + /// A `key value` field. + Field, + /// A `{ ... }` block. + Block, + /// A tagged block such as `rgb { 1 2 3 }` (a header scalar plus a block). + HeaderedBlock, + /// A `@[ ... ]` parse-time calculation wrapping an arithmetic expression. + Calc, + /// A binary arithmetic expression `lhs rhs` inside a [`SyntaxKind::Calc`]. + BinaryExpr, + /// A prefix `-`/`+` expression inside a [`SyntaxKind::Calc`]. + UnaryExpr, + /// A parenthesized expression `( ... )` inside a [`SyntaxKind::Calc`]. + ParenExpr, + /// An error-recovery wrapper around tokens that could not be placed. + Bogus, +} + +impl SyntaxKind { + /// Whitespace, comments, and the BOM — insignificant to structure but + /// preserved for losslessness. + pub fn is_trivia(self) -> bool { + matches!(self, SyntaxKind::Whitespace | SyntaxKind::Comment | SyntaxKind::Bom) + } + + /// A token that can stand as a scalar value (or an object key). This is the + /// single definition of "scalar" shared by key detection, [`Value::Scalar`], + /// and the formatter: `$macro$` parameters count, since in template files + /// they appear in key, value, and array-element position alike. + pub fn is_scalar(self) -> bool { + matches!( + self, + SyntaxKind::Unquoted + | SyntaxKind::Quoted + | SyntaxKind::Variable + | SyntaxKind::MacroParam + ) + } + + /// Whether this kind labels an interior node (rather than a leaf token). + pub fn is_node(self) -> bool { + matches!( + self, + SyntaxKind::Root + | SyntaxKind::Field + | SyntaxKind::Block + | SyntaxKind::HeaderedBlock + | SyntaxKind::Calc + | SyntaxKind::BinaryExpr + | SyntaxKind::UnaryExpr + | SyntaxKind::ParenExpr + | SyntaxKind::Bogus + ) + } +} + +// --------------------------------------------------------------------------- +// Lexing: a hand-rolled, error-tolerant, BOM-preserving scanner. It classifies +// every byte (worst case as Unquoted) so the tokens always tile the input with +// no gaps or overlaps, and every branch advances by at least one byte. +// --------------------------------------------------------------------------- + +/// Game-specific lexer configuration. +/// +/// [`Flavor::default`] is the most permissive superset and is correct for every +/// game except where a genuine lexer fork exists. [`Flavor::hoi4`] enables +/// HoI4's newline-terminated strings and treats `@`/`$` as ordinary identifier +/// bytes (HoI4 has no reader variables or macros). +#[derive(Debug, Clone, Copy)] +pub struct Flavor { + /// A quoted string ends at the first newline if it has no closing quote. + pub newline_terminated_strings: bool, + /// Recognize `@name` reader variables and `@[ ... ]` calculations. + pub variables: bool, + /// Recognize `$NAME$` macro parameters. + pub macros: bool, +} + +impl Default for Flavor { + fn default() -> Self { + Flavor { newline_terminated_strings: false, variables: true, macros: true } + } +} + +impl Flavor { + /// The permissive cross-game superset (same as [`Flavor::default`]). + pub fn superset() -> Self { + Flavor::default() + } + + /// Hearts of Iron IV: newline-terminated strings, no `@vars`/`$macros$`. + pub fn hoi4() -> Self { + Flavor { newline_terminated_strings: true, variables: false, macros: false } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct Tok { + kind: SyntaxKind, + start: u32, + len: u32, +} + +#[inline] +fn is_ws(b: u8) -> bool { + matches!(b, b' ' | b'\t' | b'\r' | b'\n' | 0x0b | 0x0c | b';') +} + +/// Bytes that terminate an unquoted run. These either start their own token +/// (braces, brackets, operators, quote, comment) or are whitespace. Notably +/// `@`, `$`, `!`, `?`, `:`, `.`, `-`, `/`, `|` and high bytes are *not* here, so +/// they are absorbed mid-identifier and only dispatch specially as a first byte. +#[inline] +fn is_stop(b: u8) -> bool { + is_ws(b) + || matches!( + b, + b'{' | b'}' | b'[' | b']' | b'=' | b'<' | b'>' | b'"' | b'#' + ) +} + +/// Find the closing `$` of a macro parameter starting just after the opening +/// `$`, or `None` if a terminator is hit first (so it isn't really a macro). +fn macro_close(source: &[u8], from: usize) -> Option { + let mut j = from; + while j < source.len() { + match source[j] { + b'$' => return Some(j), + b if is_stop(b) => return None, + _ => j += 1, + } + } + None +} + +fn lex(source: &[u8], flavor: Flavor) -> Vec { + let mut out: Vec = Vec::new(); + let n = source.len(); + + let mut i = 0usize; + if source.starts_with(&[0xEF, 0xBB, 0xBF]) { + out.push(Tok { kind: SyntaxKind::Bom, start: 0, len: 3 }); + i = 3; + } + + while i < n { + let b = source[i]; + + // `@[ ... ]` calc: emit structured interior tokens (CalcOpen, operands, + // operators, interior whitespace as trivia, CalcClose) rather than one + // opaque span, so the arithmetic expression can be parsed into a subtree. + if flavor.variables && b == b'@' && i + 1 < n && source[i + 1] == b'[' { + i = lex_calc(source, i, &mut out); + continue; + } + + let start = i; + let kind; + + if is_ws(b) { + i += 1; + while i < n && is_ws(source[i]) { + i += 1; + } + kind = SyntaxKind::Whitespace; + } else if b == b'#' { + i += 1; + while i < n && source[i] != b'\n' && source[i] != b'\r' { + i += 1; + } + kind = SyntaxKind::Comment; + } else if b == b'"' { + i += 1; + loop { + if i >= n { + break; + } + match source[i] { + b'\\' => i = (i + 2).min(n), + b'"' => { + i += 1; + break; + } + b'\n' | b'\r' if flavor.newline_terminated_strings => break, + _ => i += 1, + } + } + kind = SyntaxKind::Quoted; + } else if b == b'{' { + i += 1; + kind = SyntaxKind::OpenBrace; + } else if b == b'}' { + i += 1; + kind = SyntaxKind::CloseBrace; + } else if b == b'[' { + i += 1; + kind = SyntaxKind::OpenBracket; + } else if b == b']' { + i += 1; + kind = SyntaxKind::CloseBracket; + } else if matches!(b, b'=' | b'<' | b'>') { + i += 1; + if i < n && source[i] == b'=' { + i += 1; // ==, <=, >= + } + kind = SyntaxKind::Operator; + } else if matches!(b, b'!' | b'?') && i + 1 < n && source[i + 1] == b'=' { + i += 2; // != or ?= + kind = SyntaxKind::Operator; + } else if b == b'!' { + i += 1; + kind = SyntaxKind::Bang; + } else if b == b'@' + && flavor.variables + && i + 1 < n + && !is_stop(source[i + 1]) + && source[i + 1] != b'@' + { + // @name reader variable + i += 1; + while i < n && !is_stop(source[i]) { + i += 1; + } + kind = SyntaxKind::Variable; + } else if b == b'$' && flavor.macros && macro_close(source, i + 1).is_some() { + let close = macro_close(source, i + 1).unwrap(); + i = close + 1; + kind = SyntaxKind::MacroParam; + } else { + // Ordinary unquoted run: consume up to the next terminator. This + // catch-all is what keeps the lexer total (every byte classified). + i += 1; + while i < n && !is_stop(source[i]) { + i += 1; + } + kind = SyntaxKind::Unquoted; + } + + out.push(Tok { kind, start: start as u32, len: (i - start) as u32 }); + } + + out +} + +/// A byte that ends a calc operand identifier or number: whitespace, an +/// arithmetic operator, a parenthesis, or the closing `]`. A bare `[` is *not* a +/// stop, so a stray one (only seen in malformed input) is absorbed rather than +/// stalling the lexer; a real calc never contains one. +#[inline] +fn is_calc_stop(b: u8) -> bool { + is_ws(b) || matches!(b, b'(' | b')' | b'+' | b'-' | b'*' | b'/' | b']') +} + +/// Scan a calc numeric literal beginning at `i` (`source[i]` is a digit or a `.` +/// directly followed by a digit). Consumes digits, an optional fractional part, +/// and an optional `f` suffix (e.g. `1`, `10.0`, `.5`, `10.0f`). +fn scan_number(source: &[u8], mut i: usize) -> usize { + let n = source.len(); + while i < n && source[i].is_ascii_digit() { + i += 1; + } + if i < n && source[i] == b'.' { + i += 1; + while i < n && source[i].is_ascii_digit() { + i += 1; + } + } + if i < n && source[i] == b'f' { + i += 1; + } + i +} + +/// Lex a `@[ ... ]` calc region. `i` points at the leading `@`. Emits a +/// `CalcOpen` token, then structured interior tokens (operands, operators, and +/// interior whitespace as trivia) until the matching `]` (emitted as +/// `CalcClose`) or end of input, and returns the index just past what it +/// consumed. Every byte is classified, so the calc region tiles the input. +fn lex_calc(source: &[u8], mut i: usize, out: &mut Vec) -> usize { + let n = source.len(); + out.push(Tok { kind: SyntaxKind::CalcOpen, start: i as u32, len: 2 }); + i += 2; // past `@[` + + while i < n { + let b = source[i]; + let start = i; + let kind; + + if b == b']' { + i += 1; + out.push(Tok { kind: SyntaxKind::CalcClose, start: start as u32, len: 1 }); + return i; // leave calc mode + } else if is_ws(b) { + i += 1; + while i < n && is_ws(source[i]) { + i += 1; + } + kind = SyntaxKind::Whitespace; + } else if b == b'(' { + i += 1; + kind = SyntaxKind::OpenParen; + } else if b == b')' { + i += 1; + kind = SyntaxKind::CloseParen; + } else if b == b'+' { + i += 1; + kind = SyntaxKind::Plus; + } else if b == b'-' { + i += 1; + kind = SyntaxKind::Minus; + } else if b == b'*' { + i += 1; + kind = SyntaxKind::Star; + } else if b == b'/' { + i += 1; + kind = SyntaxKind::Slash; + } else if b.is_ascii_digit() || (b == b'.' && i + 1 < n && source[i + 1].is_ascii_digit()) { + i = scan_number(source, i); + kind = SyntaxKind::Number; + } else { + // An operand identifier (`tier`, `leopard_x`, `@var`) — or, in + // malformed input, any other non-stop byte. Always consumes >= 1. + i += 1; + while i < n && !is_calc_stop(source[i]) { + i += 1; + } + kind = SyntaxKind::CalcIdent; + } + + out.push(Tok { kind, start: start as u32, len: (i - start) as u32 }); + } + + i // reached EOF without a closing `]` (unterminated calc) +} + +// --------------------------------------------------------------------------- +// Diagnostics +// --------------------------------------------------------------------------- + +/// A non-fatal problem found while parsing. Parsing never fails — the tree is +/// always lossless — but recoverable issues are collected here for tooling. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SyntaxError { + /// Human-readable description. + pub message: String, + /// The half-open byte range the problem covers. + pub range: (u32, u32), +} + +// --------------------------------------------------------------------------- +// Green tree: flat pre-order arena. Nodes store a relative `len` (width in +// bytes) and an `end` index delimiting their subtree; tokens store only `len`. +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy)] +enum Green { + /// A leaf. Its bytes are `source[offset .. offset + len]`, where `offset` + /// is derived by the tree (never stored on the node itself). + Token { kind: SyntaxKind, len: u32 }, + /// An interior node. Its children are the elements in `[self_index+1, end)`. + Node { kind: SyntaxKind, len: u32, end: u32 }, +} + +impl Green { + fn kind(&self) -> SyntaxKind { + match *self { + Green::Token { kind, .. } | Green::Node { kind, .. } => kind, + } + } + + fn len(&self) -> u32 { + match *self { + Green::Token { len, .. } | Green::Node { len, .. } => len, + } + } +} + +/// Precomputed per-subtree summary bits, one byte per element, filled in a +/// single reverse pass of [`Builder::finish`]. An element's flags are the union +/// of its own kind's bits and *every* descendant's, so a consumer can skip a +/// whole subtree with one O(1) test — "does this block contain a comment? a +/// syntax error? a calc? a macro parameter?" — instead of walking it. This +/// mirrors swift-syntax's `RecursiveRawSyntaxFlags` and Carbon's per-node +/// `has_error` bit, and is the cheap accelerator a linter or a future +/// incremental engine leans on. Stored in a side `Vec` (not on [`Green`]) so the +/// hot build loop is untouched — consistent with the offsets/parents design. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct NodeFlags(u8); + +impl NodeFlags { + /// The subtree contains at least one [`SyntaxKind::Comment`]. + pub const HAS_COMMENT: NodeFlags = NodeFlags(1 << 0); + /// The subtree contains an error-recovery [`SyntaxKind::Bogus`] node or an + /// explicit [`SyntaxKind::Error`] token. + pub const HAS_ERROR: NodeFlags = NodeFlags(1 << 1); + /// The subtree contains a `@[ ... ]` [`SyntaxKind::Calc`]. + pub const HAS_CALC: NodeFlags = NodeFlags(1 << 2); + /// The subtree contains a `$macro$` [`SyntaxKind::MacroParam`]. + pub const HAS_MACRO: NodeFlags = NodeFlags(1 << 3); + + /// Whether every bit set in `other` is also set in `self`. + pub fn contains(self, other: NodeFlags) -> bool { + self.0 & other.0 == other.0 + } + + fn insert(&mut self, other: NodeFlags) { + self.0 |= other.0; + } +} + +/// The flag bits a single element contributes on its own, before the upward +/// union of its descendants' bits. +fn own_flag_bits(kind: SyntaxKind) -> NodeFlags { + match kind { + SyntaxKind::Comment => NodeFlags::HAS_COMMENT, + SyntaxKind::Error | SyntaxKind::Bogus => NodeFlags::HAS_ERROR, + // The `Calc` node covers a well-formed calc; `CalcOpen` covers the rare + // flattened case where the depth guard tripped before the node formed. + SyntaxKind::Calc | SyntaxKind::CalcOpen => NodeFlags::HAS_CALC, + SyntaxKind::MacroParam => NodeFlags::HAS_MACRO, + _ => NodeFlags(0), + } +} + +/// A lossless syntax tree borrowing its source bytes. +pub struct GreenTree<'a> { + source: &'a [u8], + tape: Vec, + /// Absolute byte offset of each element, derived from leaf widths. + offsets: Vec, + /// Parent element index per element (`u32::MAX` for the root). + parents: Vec, + /// Per-subtree [`NodeFlags`] for each element (see [`NodeFlags`]). + flags: Vec, + errors: Vec, +} + +/// Parse `source` into a lossless [`GreenTree`] using the permissive superset. +pub fn parse(source: &[u8]) -> GreenTree<'_> { + parse_with(source, Flavor::default()) +} + +/// Parse `source` into a lossless [`GreenTree`] with a specific [`Flavor`]. +pub fn parse_with(source: &[u8], flavor: Flavor) -> GreenTree<'_> { + let tokens = lex(source, flavor); + let builder = Builder::with_capacity(tokens.len()); + let mut p = Parser { tokens: &tokens, pos: 0, builder, errors: Vec::new(), depth: 0 }; + p.builder.start_node(SyntaxKind::Root); + p.parse_items(false); + p.builder.finish_node(); + let Parser { builder, errors, .. } = p; + builder.finish(source, errors) +} + +impl<'a> GreenTree<'a> { + /// The full source the tree was parsed from. + pub fn source(&self) -> &'a [u8] { + self.source + } + + /// The root [`SyntaxNode`] (always [`SyntaxKind::Root`]). + pub fn root(&self) -> SyntaxNode<'_, 'a> { + SyntaxNode { tree: self, idx: 0 } + } + + /// Recoverable problems found during parsing (empty for clean input). + pub fn errors(&self) -> &[SyntaxError] { + &self.errors + } + + /// Every leaf [`SyntaxToken`] in document order. + /// + /// Because the green tape is stored in pre-order, the leaves are *already* + /// sequenced — this merely skips the interior nodes, so it is O(n) with no + /// tree walking. Concatenating [`SyntaxToken::text`] over this iterator + /// reproduces [`source`](GreenTree::source): the lossless invariant in + /// iterator form. Ideal for highlighters and other leaf-oriented tooling. + pub fn tokens(&self) -> impl Iterator> + '_ { + (0..self.tape.len() as u32).filter_map(move |i| match self.tape[i as usize] { + Green::Token { .. } => Some(SyntaxToken { tree: self, idx: i }), + Green::Node { .. } => None, + }) + } + + /// Reconstruct the source by walking the tree's leaves via the cursor API. + /// + /// Equals [`GreenTree::source`] for any tree; the lossless invariant. + pub fn reconstruct(&self) -> Vec { + fn collect(node: SyntaxNode<'_, '_>, out: &mut Vec) { + for el in node.children() { + match el { + SyntaxElement::Token(t) => out.extend_from_slice(t.text()), + SyntaxElement::Node(n) => collect(n, out), + } + } + } + let mut out = Vec::with_capacity(self.source.len()); + collect(self.root(), &mut out); + out + } + + /// An indented S-expression dump of the tree, for debugging and tests. + pub fn debug_tree(&self) -> String { + fn go(el: SyntaxElement<'_, '_>, depth: usize, out: &mut String) { + for _ in 0..depth { + out.push_str(" "); + } + match el { + SyntaxElement::Node(n) => { + out.push_str(&format!("{:?}\n", n.kind())); + for c in n.children() { + go(c, depth + 1, out); + } + } + SyntaxElement::Token(t) => { + out.push_str(&format!( + "{:?} {:?}\n", + t.kind(), + String::from_utf8_lossy(t.text()) + )); + } + } + } + let mut out = String::new(); + go(SyntaxElement::Node(self.root()), 0, &mut out); + out + } + + fn element(&self, idx: u32) -> SyntaxElement<'_, 'a> { + match self.tape[idx as usize] { + Green::Node { .. } => SyntaxElement::Node(SyntaxNode { tree: self, idx }), + Green::Token { .. } => SyntaxElement::Token(SyntaxToken { tree: self, idx }), + } + } +} + +/// Builds the flat arena. The hot path pushes only to a single `tape` vector; +/// the derived `offsets`/`parents` caches are computed in two tight linear +/// passes in [`Builder::finish`]. (Filling those caches incrementally during +/// the build measured ~6% slower — three interleaved vector pushes hurt the +/// hot loop more than two cache-friendly sequential passes cost.) +struct Builder { + tape: Vec, + stack: Vec, + len_stack: Vec, +} + +impl Builder { + fn with_capacity(tokens: usize) -> Self { + // Elements = tokens + interior nodes; nodes are a fraction of tokens. + Builder { + tape: Vec::with_capacity(tokens + tokens / 2), + stack: Vec::new(), + len_stack: Vec::new(), + } + } + + fn start_node(&mut self, kind: SyntaxKind) { + let idx = self.tape.len(); + self.tape.push(Green::Node { kind, len: 0, end: 0 }); + self.stack.push(idx); + self.len_stack.push(0); + } + + fn token(&mut self, kind: SyntaxKind, len: u32) { + self.tape.push(Green::Token { kind, len }); + if let Some(top) = self.len_stack.last_mut() { + *top += len; + } + } + + fn finish_node(&mut self) { + let idx = self.stack.pop().expect("finish_node without start_node"); + let len = self.len_stack.pop().unwrap(); + let end = self.tape.len() as u32; + if let Green::Node { len: l, end: e, .. } = &mut self.tape[idx] { + *l = len; + *e = end; + } + if let Some(top) = self.len_stack.last_mut() { + *top += len; + } + } + + fn finish(self, source: &[u8], errors: Vec) -> GreenTree<'_> { + debug_assert!(self.stack.is_empty(), "unfinished nodes remain"); + let tape = self.tape; + + // Absolute offset of each element = sum of leaf widths preceding it. + let mut offsets = vec![0u32; tape.len()]; + let mut acc = 0u32; + for (i, g) in tape.iter().enumerate() { + offsets[i] = acc; + if let Green::Token { len, .. } = g { + acc += *len; + } + } + debug_assert_eq!(acc as usize, source.len(), "leaf widths do not cover the source"); + + // Nearest-enclosing-node parent links via an end-delimited stack. + let mut parents = vec![u32::MAX; tape.len()]; + let mut stack: Vec<(u32, u32)> = Vec::new(); // (node idx, end) + for (i, g) in tape.iter().enumerate() { + while let Some(&(_, end)) = stack.last() { + if end as usize <= i { + stack.pop(); + } else { + break; + } + } + if let Some(&(p, _)) = stack.last() { + parents[i] = p; + } + if let Green::Node { end, .. } = g { + stack.push((i as u32, *end)); + } + } + + // Per-subtree flag bits. Seed each element with its own bits, then fold + // each element's flags into its parent. Pre-order means every descendant + // has a higher index than its ancestors, so iterating high→low finalizes + // a node's flags (all descendants already folded in) before it folds into + // its own parent — one linear pass, no extra tree walk. (The root at + // index 0 has no parent, so the range starts at 1.) + let mut flags: Vec = tape.iter().map(|g| own_flag_bits(g.kind())).collect(); + for i in (1..tape.len()).rev() { + let p = parents[i]; + if p != u32::MAX { + let child = flags[i]; + flags[p as usize].insert(child); + } + } + + GreenTree { source, tape, offsets, parents, flags, errors } + } +} + +/// Maximum nesting depth before the parser stops recursing, flattens the +/// remainder into flat leaf tokens, and records a diagnostic. This guards +/// against a stack overflow on pathological input like `{{{{…}}}}` or +/// `@[((((…))))]` thousands deep, while sitting far above any real game file's +/// nesting. It applies independently to block nesting and to calc-expression +/// nesting (each has its own recursion). +const MAX_DEPTH: u32 = 256; + +struct Parser<'t> { + tokens: &'t [Tok], + pos: usize, + builder: Builder, + errors: Vec, + /// Current block-nesting depth, compared against [`MAX_DEPTH`]. + depth: u32, +} + +impl Parser<'_> { + fn peek(&self) -> Option { + self.tokens.get(self.pos).map(|t| t.kind) + } + + fn bump(&mut self) { + let t = self.tokens[self.pos]; + self.builder.token(t.kind, t.len); + self.pos += 1; + } + + fn bump_trivia(&mut self) { + while matches!(self.peek(), Some(k) if k.is_trivia()) { + self.bump(); + } + } + + /// Kind of the first non-trivia token at or after `from`. + fn next_significant(&self, from: usize) -> Option { + self.tokens[from..].iter().map(|t| t.kind).find(|k| !k.is_trivia()) + } + + fn parse_items(&mut self, in_block: bool) { + loop { + match self.peek() { + None => break, + Some(SyntaxKind::CloseBrace) if in_block => break, // caller eats `}` + Some(SyntaxKind::CloseBrace) => { + // Unmatched `}` at the top level: record and wrap in Bogus. + let t = self.tokens[self.pos]; + self.errors.push(SyntaxError { + message: "unmatched '}'".into(), + range: (t.start, t.start + t.len), + }); + self.builder.start_node(SyntaxKind::Bogus); + self.bump(); + self.builder.finish_node(); + } + Some(k) if k.is_trivia() => self.bump(), + Some(_) => self.parse_item(), + } + } + } + + fn parse_item(&mut self) { + match self.peek() { + Some(k) if k.is_scalar() => { + if self.next_significant(self.pos + 1) == Some(SyntaxKind::Operator) { + // key value + self.builder.start_node(SyntaxKind::Field); + self.bump(); // key + self.bump_trivia(); + self.bump(); // operator + self.bump_trivia(); + self.parse_value(); + self.builder.finish_node(); + } else { + // bare scalar (array element / loose value) + self.bump(); + } + } + Some(SyntaxKind::OpenBrace) => self.parse_block(), + Some(SyntaxKind::CalcOpen) => self.parse_calc(), + // Operators, brackets, bang, macros in item position: keep verbatim. + Some(_) => self.bump(), + None => {} + } + } + + fn parse_value(&mut self) { + match self.peek() { + Some(SyntaxKind::OpenBrace) => self.parse_block(), + Some(SyntaxKind::CalcOpen) => self.parse_calc(), + Some(SyntaxKind::Unquoted) + if self.next_significant(self.pos + 1) == Some(SyntaxKind::OpenBrace) => + { + // headered block: `rgb { ... }`, `hsv { ... }`, tag { ... } + self.builder.start_node(SyntaxKind::HeaderedBlock); + self.bump(); // header scalar + self.bump_trivia(); + self.parse_block(); + self.builder.finish_node(); + } + Some(k) if !k.is_trivia() && k != SyntaxKind::CloseBrace => self.bump(), + // missing value (e.g. `a =` at EOF, or `a = }`): emit nothing. + _ => {} + } + } + + fn parse_block(&mut self) { + let open = self.tokens[self.pos]; + self.builder.start_node(SyntaxKind::Block); + self.bump(); // `{` + self.depth += 1; + if self.depth >= MAX_DEPTH { + // Pathologically deep nesting. Rather than recurse (and risk a + // stack overflow), consume the rest of this block — including + // everything nested inside it — as flat leaf tokens. The tree + // stays lossless; it just loses structure past this point. + self.errors.push(SyntaxError { + message: "maximum nesting depth exceeded; structure flattened".into(), + range: (open.start, open.start + open.len), + }); + self.flatten_to_block_close(); + } else { + self.parse_items(true); + if self.peek() == Some(SyntaxKind::CloseBrace) { + self.bump(); // `}` + } else { + self.errors.push(SyntaxError { + message: "unclosed '{'".into(), + range: (open.start, open.start + open.len), + }); + } + } + self.depth -= 1; + self.builder.finish_node(); + } + + /// Consume the remainder of the current block — including any nested braces + /// — as flat leaf tokens, stopping just after the matching `}` (or at EOF). + /// The opening `{` has already been consumed, so brace depth starts at 1. + /// Used by the depth guard to bound recursion without sacrificing + /// losslessness (every token is still emitted, just unstructured). + fn flatten_to_block_close(&mut self) { + let mut balance = 1u32; + while let Some(k) = self.peek() { + match k { + SyntaxKind::OpenBrace => balance += 1, + SyntaxKind::CloseBrace => balance -= 1, + _ => {} + } + self.bump(); + if balance == 0 { + return; // emitted the matching `}` + } + } + // EOF before the block closed — the flatten diagnostic already covers + // it, so no separate "unclosed" diagnostic is added here. + } + + /// Parse a `@[ ... ]` calc. The current token is [`SyntaxKind::CalcOpen`]. + /// + /// The lexer has already split the interior into calc tokens, so the whole + /// region is a contiguous run `CalcOpen, , [CalcClose]`. The + /// interior is parsed (precedence-climbing) into a temporary element tree by + /// [`parse_calc_interior`], then emitted into the builder in source order — + /// no checkpoints needed, since a lossless tree always emits leaves in order + /// and only adds node boundaries. + fn parse_calc(&mut self) { + let open = self.tokens[self.pos]; + self.builder.start_node(SyntaxKind::Calc); + self.bump(); // `@[` + + // The interior is everything up to the matching `]` (or EOF). The lexer + // only ever emits calc tokens between a CalcOpen and its CalcClose, so + // no calc token can leak past this slice into the outer parser. + let from = self.pos; + while !matches!(self.peek(), Some(SyntaxKind::CalcClose) | None) { + self.pos += 1; + } + let (elems, overflowed) = parse_calc_interior(&self.tokens[from..self.pos]); + for el in &elems { + emit_calc(&mut self.builder, el); + } + if overflowed { + self.errors.push(SyntaxError { + message: "maximum calc nesting depth exceeded; structure flattened".into(), + range: (open.start, open.start + open.len), + }); + } + + if self.peek() == Some(SyntaxKind::CalcClose) { + self.bump(); // `]` + } else { + self.errors.push(SyntaxError { + message: "unclosed '@['".into(), + range: (open.start, open.start + open.len), + }); + } + self.builder.finish_node(); + } +} + +// --------------------------------------------------------------------------- +// Calc expression grammar (interior of `@[ ... ]`). +// +// operands : Number | CalcIdent | `(` expr `)` | (`-`|`+`) operand +// binary : `+` `-` (looser) and `*` `/` (tighter), left-associative +// +// Parsed precedence-climbing into a temporary `CalcElem` tree, which is then +// walked in pre-order to emit green tokens/nodes. Interior whitespace rides +// along as trivia leaves, attached to whichever node encloses it, so the region +// round-trips byte-for-byte. +// --------------------------------------------------------------------------- + +/// A node in the temporary calc tree (see [`parse_calc_interior`]). +enum CalcElem { + /// A leaf carrying a lexer token's kind and width. + Leaf { kind: SyntaxKind, len: u32 }, + /// An interior expression node with children in source order. + Node { kind: SyntaxKind, children: Vec }, +} + +/// Walk a [`CalcElem`] in pre-order, emitting its tokens/nodes into `b`. +fn emit_calc(b: &mut Builder, el: &CalcElem) { + match el { + CalcElem::Leaf { kind, len } => b.token(*kind, *len), + CalcElem::Node { kind, children } => { + b.start_node(*kind); + for c in children { + emit_calc(b, c); + } + b.finish_node(); + } + } +} + +/// Infix binding powers; `* /` bind tighter than `+ -`. The left power being +/// below the right power makes equal-precedence chains left-associative. +fn infix_bp(kind: SyntaxKind) -> Option<(u8, u8)> { + match kind { + SyntaxKind::Plus | SyntaxKind::Minus => Some((1, 2)), + SyntaxKind::Star | SyntaxKind::Slash => Some((3, 4)), + _ => None, + } +} + +/// Parse the interior tokens of a calc (everything between `@[` and `]`) into a +/// flat list of [`CalcElem`]s: leading trivia, the expression, then any trailing +/// trivia / unconsumed (malformed) tokens. Total over any token slice. The +/// returned bool is `true` if the [`MAX_DEPTH`] guard tripped (pathologically +/// deep parens/unary), in which case parsing stopped descending but every token +/// is still emitted as a flat leaf. +fn parse_calc_interior(toks: &[Tok]) -> (Vec, bool) { + let mut c = CalcCursor { toks, pos: 0, depth: 0, overflowed: false }; + let mut out = Vec::new(); + c.eat_trivia(&mut out); + if c.has_more() { + out.push(c.parse_expr(0)); + } + // Trailing trivia plus, for malformed input like `@[1 2]`, any leftover + // tokens the expression grammar did not consume — kept as leaves so the + // region still tiles its source. + while c.has_more() { + out.push(c.leaf()); + } + (out, c.overflowed) +} + +/// A cursor over a calc's interior tokens used by [`parse_calc_interior`]. +struct CalcCursor<'t> { + toks: &'t [Tok], + pos: usize, + /// Current operand-nesting depth, compared against [`MAX_DEPTH`]. + depth: u32, + /// Set once the depth guard trips (see [`CalcCursor::parse_operand`]). + overflowed: bool, +} + +impl CalcCursor<'_> { + fn has_more(&self) -> bool { + self.pos < self.toks.len() + } + + fn peek(&self) -> Option { + self.toks.get(self.pos).map(|t| t.kind) + } + + /// Consume one token as a leaf, advancing the cursor. + fn leaf(&mut self) -> CalcElem { + let t = self.toks[self.pos]; + self.pos += 1; + CalcElem::Leaf { kind: t.kind, len: t.len } + } + + /// Push any run of trivia tokens at the cursor onto `out`. + fn eat_trivia(&mut self, out: &mut Vec) { + while matches!(self.peek(), Some(k) if k.is_trivia()) { + out.push(self.leaf()); + } + } + + /// Parse an operand: a unary expression, a parenthesized expression, a + /// number/identifier leaf, or (for malformed input) whatever leaf is here. + fn parse_operand(&mut self) -> CalcElem { + // Depth guard: `@[((((…))))]` and `@[----…x]` recurse through here. + // Past the limit, stop descending and hand the current token back as a + // leaf; the remaining interior tokens then fall through to flat leaves + // in `parse_calc_interior`. Every operand frame increments `depth`, so + // this bounds the whole calc recursion while staying lossless. (Flat + // chains like `1+1+1` do not nest — `parse_expr` handles those in its + // loop — so they never approach the limit.) + if self.depth >= MAX_DEPTH { + self.overflowed = true; + return self.leaf(); + } + self.depth += 1; + let elem = self.parse_operand_inner(); + self.depth -= 1; + elem + } + + fn parse_operand_inner(&mut self) -> CalcElem { + match self.peek() { + Some(SyntaxKind::Minus) | Some(SyntaxKind::Plus) => { + let mut children = vec![self.leaf()]; // unary operator + self.eat_trivia(&mut children); + if self.has_more() && self.peek() != Some(SyntaxKind::CloseParen) { + children.push(self.parse_operand()); + } + CalcElem::Node { kind: SyntaxKind::UnaryExpr, children } + } + Some(SyntaxKind::OpenParen) => { + let mut children = vec![self.leaf()]; // `(` + self.eat_trivia(&mut children); + if self.has_more() && self.peek() != Some(SyntaxKind::CloseParen) { + children.push(self.parse_expr(0)); + } + self.eat_trivia(&mut children); + if self.peek() == Some(SyntaxKind::CloseParen) { + children.push(self.leaf()); // `)` + } + CalcElem::Node { kind: SyntaxKind::ParenExpr, children } + } + // Number, CalcIdent, or — in malformed input — a stray operator. + Some(_) => self.leaf(), + None => unreachable!("parse_operand called at end of interior"), + } + } + + /// Precedence-climbing parse of a (sub)expression with binding power floor + /// `min_bp`. Trivia before a candidate operator is held speculatively and + /// either folded into the binary node or rewound to the enclosing context. + fn parse_expr(&mut self, min_bp: u8) -> CalcElem { + let mut lhs = self.parse_operand(); + // Length of the left-associative chain folded in this call. Unlike + // parens/unary, a flat chain (`@[1+1+1+…]`) is folded *iteratively* + // here, so `parse_operand`'s depth counter never sees it — yet it still + // builds a `BinaryExpr` tree nested `folds` deep, which `emit_calc` and + // the tree's recursive `Drop` would later walk. Cap it the same way so + // no calc shape can overflow the stack; the tail falls to flat leaves. + let mut folds = 0u32; + loop { + // Peek past trivia for the next operator; rewind if it isn't one + // (or binds too loosely) so that trivia stays with the outer node. + let save = self.pos; + let mut trivia = Vec::new(); + self.eat_trivia(&mut trivia); + let bp = self.peek().and_then(infix_bp); + match bp { + Some((l_bp, r_bp)) if l_bp >= min_bp && folds < MAX_DEPTH => { + folds += 1; + let mut children = vec![lhs]; + children.append(&mut trivia); + children.push(self.leaf()); // operator + self.eat_trivia(&mut children); + if self.has_more() && self.peek() != Some(SyntaxKind::CloseParen) { + children.push(self.parse_expr(r_bp)); + } + lhs = CalcElem::Node { kind: SyntaxKind::BinaryExpr, children }; + } + _ => { + // If a foldable operator is present and we are stopping only + // because the fold cap was hit, flag the overflow (the tail + // becomes flat leaves in `parse_calc_interior`). + if matches!(bp, Some((l_bp, _)) if l_bp >= min_bp) { + self.overflowed = true; + } + self.pos = save; + break; + } + } + } + lhs + } +} + +// --------------------------------------------------------------------------- +// Cursor ("red layer"): lightweight handles into the tree that compute +// positions on demand. `'t` borrows the tree, `'a` is the source lifetime. +// --------------------------------------------------------------------------- + +/// A handle to an interior node in a [`GreenTree`]. +#[derive(Clone, Copy)] +pub struct SyntaxNode<'t, 'a> { + tree: &'t GreenTree<'a>, + idx: u32, +} + +/// A handle to a leaf token in a [`GreenTree`]. +#[derive(Clone, Copy)] +pub struct SyntaxToken<'t, 'a> { + tree: &'t GreenTree<'a>, + idx: u32, +} + +/// Either a [`SyntaxNode`] or a [`SyntaxToken`]. +#[derive(Clone, Copy)] +pub enum SyntaxElement<'t, 'a> { + Node(SyntaxNode<'t, 'a>), + Token(SyntaxToken<'t, 'a>), +} + +impl<'t, 'a> SyntaxNode<'t, 'a> { + /// This node's kind. + pub fn kind(&self) -> SyntaxKind { + self.tree.tape[self.idx as usize].kind() + } + + /// The half-open byte range this node spans in the source. + pub fn text_range(&self) -> (u32, u32) { + let start = self.tree.offsets[self.idx as usize]; + (start, start + self.tree.tape[self.idx as usize].len()) + } + + /// The source bytes this node spans (a lossless slice of the subtree). + pub fn text(&self) -> &'a [u8] { + let (s, e) = self.text_range(); + &self.tree.source[s as usize..e as usize] + } + + /// The parent node, or `None` for the root. + pub fn parent(&self) -> Option> { + let p = self.tree.parents[self.idx as usize]; + (p != u32::MAX).then_some(SyntaxNode { tree: self.tree, idx: p }) + } + + /// The precomputed [`NodeFlags`] summarizing this node's whole subtree (see + /// [`NodeFlags`]). An O(1) lookup — no subtree walk. + pub fn flags(&self) -> NodeFlags { + self.tree.flags[self.idx as usize] + } + + /// Whether this subtree contains a [`SyntaxKind::Bogus`]/[`SyntaxKind::Error`] + /// recovery element. Lets a linter cheaply skip — or downgrade — semantic + /// checks over a region the parser already flagged as malformed. + pub fn has_error(&self) -> bool { + self.flags().contains(NodeFlags::HAS_ERROR) + } + + /// Whether this subtree contains a comment (O(1); see [`NodeFlags`]). + pub fn has_comment(&self) -> bool { + self.flags().contains(NodeFlags::HAS_COMMENT) + } + + /// Whether this subtree contains a `@[ ... ]` calc (O(1); see [`NodeFlags`]). + pub fn contains_calc(&self) -> bool { + self.flags().contains(NodeFlags::HAS_CALC) + } + + /// Whether this subtree contains a `$macro$` parameter (O(1); see [`NodeFlags`]). + pub fn contains_macro(&self) -> bool { + self.flags().contains(NodeFlags::HAS_MACRO) + } + + /// All direct children, nodes and tokens, in source order. + pub fn children(&self) -> Children<'t, 'a> { + let end = match self.tree.tape[self.idx as usize] { + Green::Node { end, .. } => end, + Green::Token { .. } => self.idx + 1, + }; + Children { tree: self.tree, next: self.idx + 1, end } + } + + /// Direct child nodes only. + pub fn child_nodes(&self) -> impl Iterator> + 't { + self.children().filter_map(SyntaxElement::into_node) + } + + /// Direct child tokens only. + pub fn child_tokens(&self) -> impl Iterator> + 't { + self.children().filter_map(SyntaxElement::into_token) + } +} + +impl<'t, 'a> SyntaxToken<'t, 'a> { + /// This token's kind. + pub fn kind(&self) -> SyntaxKind { + self.tree.tape[self.idx as usize].kind() + } + + /// The half-open byte range this token spans in the source. + pub fn text_range(&self) -> (u32, u32) { + let start = self.tree.offsets[self.idx as usize]; + (start, start + self.tree.tape[self.idx as usize].len()) + } + + /// The raw source bytes of this token. + pub fn text(&self) -> &'a [u8] { + let (s, e) = self.text_range(); + &self.tree.source[s as usize..e as usize] + } + + /// The parent node. + pub fn parent(&self) -> Option> { + let p = self.tree.parents[self.idx as usize]; + (p != u32::MAX).then_some(SyntaxNode { tree: self.tree, idx: p }) + } + + /// If this is an [`SyntaxKind::Operator`] token, the concrete [`Operator`]. + pub fn operator(&self) -> Option { + if self.kind() != SyntaxKind::Operator { + return None; + } + Some(match self.text() { + b"<" => Operator::LessThan, + b"<=" => Operator::LessThanEqual, + b">" => Operator::GreaterThan, + b">=" => Operator::GreaterThanEqual, + b"==" => Operator::Exact, + b"=" => Operator::Equal, + b"!=" => Operator::NotEqual, + b"?=" => Operator::Exists, + _ => return None, + }) + } +} + +impl<'t, 'a> SyntaxElement<'t, 'a> { + /// This element's kind. + pub fn kind(&self) -> SyntaxKind { + match self { + SyntaxElement::Node(n) => n.kind(), + SyntaxElement::Token(t) => t.kind(), + } + } + + /// The source bytes this element spans (its subtree, for a node). + pub fn text(&self) -> &'a [u8] { + match self { + SyntaxElement::Node(n) => n.text(), + SyntaxElement::Token(t) => t.text(), + } + } + + /// Unwrap to a node, or `None` if this is a token. + pub fn into_node(self) -> Option> { + match self { + SyntaxElement::Node(n) => Some(n), + SyntaxElement::Token(_) => None, + } + } + + /// Unwrap to a token, or `None` if this is a node. + pub fn into_token(self) -> Option> { + match self { + SyntaxElement::Token(t) => Some(t), + SyntaxElement::Node(_) => None, + } + } +} + +/// Iterator over a node's direct children (see [`SyntaxNode::children`]). +pub struct Children<'t, 'a> { + tree: &'t GreenTree<'a>, + next: u32, + end: u32, +} + +impl<'t, 'a> Iterator for Children<'t, 'a> { + type Item = SyntaxElement<'t, 'a>; + + fn next(&mut self) -> Option { + if self.next >= self.end { + return None; + } + let idx = self.next; + // Skip past this child's whole subtree to reach the next sibling. + self.next = match self.tree.tape[idx as usize] { + Green::Node { end, .. } => end, + Green::Token { .. } => idx + 1, + }; + Some(self.tree.element(idx)) + } +} + +// --------------------------------------------------------------------------- +// Typed AST: "ungrammar-style" strongly-typed views over the red layer. Each +// wrapper is a thin newtype around a `SyntaxNode` of a fixed `SyntaxKind`, with +// accessors that locate children by kind/position. They are zero-copy `Copy` +// views computed on demand, so constructing one is free; `syntax()` always +// recovers the untyped node for ranges, text, or raw traversal. +// --------------------------------------------------------------------------- + +/// A typed view over a [`SyntaxNode`] of a particular [`SyntaxKind`]. +/// +/// Mirrors rust-analyzer's generated AST: [`AstNode::cast`] succeeds only when +/// the node's kind matches, and [`AstNode::syntax`] recovers the untyped node. +pub trait AstNode<'t, 'a>: Sized { + /// Wrap `node` if its kind matches this type, otherwise `None`. + fn cast(node: SyntaxNode<'t, 'a>) -> Option; + /// The underlying untyped node. + fn syntax(&self) -> SyntaxNode<'t, 'a>; +} + +macro_rules! ast_nodes { + ($($(#[$m:meta])* $name:ident => $kind:ident),+ $(,)?) => {$( + $(#[$m])* + #[derive(Clone, Copy)] + pub struct $name<'t, 'a>(SyntaxNode<'t, 'a>); + + impl<'t, 'a> AstNode<'t, 'a> for $name<'t, 'a> { + fn cast(node: SyntaxNode<'t, 'a>) -> Option { + if node.kind() == SyntaxKind::$kind { + Some($name(node)) + } else { + None + } + } + fn syntax(&self) -> SyntaxNode<'t, 'a> { + self.0 + } + } + )+}; +} + +ast_nodes! { + /// The whole document ([`SyntaxKind::Root`]). + Root => Root, + /// A `key value` field ([`SyntaxKind::Field`]). + Field => Field, + /// A `{ ... }` block ([`SyntaxKind::Block`]). + Block => Block, + /// A tagged block such as `rgb { 1 2 3 }` ([`SyntaxKind::HeaderedBlock`]). + HeaderedBlock => HeaderedBlock, + /// A `@[ ... ]` parse-time calculation ([`SyntaxKind::Calc`]). + Calc => Calc, + /// A binary arithmetic expression inside a [`Calc`] ([`SyntaxKind::BinaryExpr`]). + BinaryExpr => BinaryExpr, + /// A prefix `-`/`+` expression inside a [`Calc`] ([`SyntaxKind::UnaryExpr`]). + UnaryExpr => UnaryExpr, + /// A parenthesized expression inside a [`Calc`] ([`SyntaxKind::ParenExpr`]). + ParenExpr => ParenExpr, +} + +/// An entry within a [`Root`] or [`Block`]. +#[derive(Clone, Copy)] +pub enum Item<'t, 'a> { + /// A `key value` field. + Field(Field<'t, 'a>), + /// A bare value: an array element or a loose value. + Value(Value<'t, 'a>), + /// A significant element that is neither a field nor a value — e.g. a stray + /// operator/bracket, a [`SyntaxKind::Bogus`] node, or the still-ungrouped + /// tokens of a `[[param]]` block. Surfaced so the typed view drops nothing. + Other(SyntaxElement<'t, 'a>), +} + +/// A value in value position (the RHS of a [`Field`] or a bare array element). +#[derive(Clone, Copy)] +pub enum Value<'t, 'a> { + /// A scalar token: [`SyntaxKind::Unquoted`], [`SyntaxKind::Quoted`], + /// [`SyntaxKind::Variable`], or [`SyntaxKind::MacroParam`]. + Scalar(SyntaxToken<'t, 'a>), + /// A `{ ... }` block. + Block(Block<'t, 'a>), + /// A tagged block such as `rgb { 1 2 3 }`. + Headered(HeaderedBlock<'t, 'a>), + /// A `@[ ... ]` calculation. + Calc(Calc<'t, 'a>), +} + +/// An arithmetic expression inside a [`Calc`]. +#[derive(Clone, Copy)] +pub enum Expr<'t, 'a> { + /// `lhs rhs`. + Binary(BinaryExpr<'t, 'a>), + /// `-operand` / `+operand`. + Unary(UnaryExpr<'t, 'a>), + /// `( inner )`. + Paren(ParenExpr<'t, 'a>), + /// A numeric literal ([`SyntaxKind::Number`]). + Number(SyntaxToken<'t, 'a>), + /// An operand identifier ([`SyntaxKind::CalcIdent`]). + Ident(SyntaxToken<'t, 'a>), +} + +/// A color value parsed from a [`HeaderedBlock`] such as `rgb { 255 0 128 }`. +#[derive(Debug, Clone, Copy, PartialEq)] +pub enum Color { + /// `rgb { r g b }`, with an optional fourth alpha channel. + Rgb { r: u8, g: u8, b: u8, a: Option }, + /// `hsv { h s v }`, channels nominally in `0.0..=1.0`. + Hsv { h: f64, s: f64, v: f64 }, + /// `hsv360 { h s v }`, hue in `0..=360`, saturation/value in `0..=100`. + Hsv360 { h: f64, s: f64, v: f64 }, +} + +/// Strip a leading and/or trailing `"` from a quoted token's raw bytes. Handles +/// the unterminated case (only a leading quote) and the degenerate `"`/`""`. +fn strip_quotes(b: &[u8]) -> &[u8] { + let b = b.strip_prefix(b"\"").unwrap_or(b); + b.strip_suffix(b"\"").unwrap_or(b) +} + +impl<'t, 'a> SyntaxToken<'t, 'a> { + /// The semantic scalar value of this token: its raw bytes, but with the + /// surrounding quotes stripped for a [`SyntaxKind::Quoted`] token (escapes + /// inside the quotes are left as-is). This is the right input for the + /// numeric/boolean coercions on [`Scalar`]. + pub fn as_scalar(&self) -> Scalar<'a> { + let text = self.text(); + let bytes = if self.kind() == SyntaxKind::Quoted { + strip_quotes(text) + } else { + text + }; + Scalar::new(bytes) + } +} + +/// First significant child element of `node` after skipping leading trivia and +/// (for blocks) the delimiting braces — used to classify entries. +fn child_items<'t, 'a>(node: SyntaxNode<'t, 'a>) -> impl Iterator> + 't { + node.children().filter_map(|el| { + let kind = el.kind(); + if kind.is_trivia() || matches!(kind, SyntaxKind::OpenBrace | SyntaxKind::CloseBrace) { + return None; + } + match el { + SyntaxElement::Node(n) if kind == SyntaxKind::Field => Field::cast(n).map(Item::Field), + _ => Some(match Value::cast_element(el) { + Some(v) => Item::Value(v), + None => Item::Other(el), + }), + } + }) +} + +impl<'t, 'a> Root<'t, 'a> { + /// Every top-level entry, in source order (trivia skipped). + pub fn items(&self) -> impl Iterator> + 't { + child_items(self.syntax()) + } + + /// The top-level `key = value` fields. + pub fn fields(&self) -> impl Iterator> + 't { + self.items().filter_map(Item::into_field) + } +} + +impl<'t, 'a> Field<'t, 'a> { + /// The key scalar token (the left-hand side). + pub fn key(&self) -> Option> { + self.syntax() + .child_tokens() + .find(|t| t.kind().is_scalar()) + } + + /// The operator token (`=`, `==`, `?=`, `<`, …). + pub fn op_token(&self) -> Option> { + self.syntax() + .child_tokens() + .find(|t| t.kind() == SyntaxKind::Operator) + } + + /// The concrete [`Operator`], re-derived from the operator token's text. + pub fn op(&self) -> Option { + self.op_token().and_then(|t| t.operator()) + } + + /// The value (the right-hand side), or `None` if it is missing. + pub fn value(&self) -> Option> { + let mut after_op = false; + for el in self.syntax().children() { + if el.kind().is_trivia() { + continue; + } + if !after_op { + after_op = el.kind() == SyntaxKind::Operator; + continue; + } + return Value::cast_element(el); + } + None + } +} + +impl<'t, 'a> Block<'t, 'a> { + /// Every entry between the braces, in source order (trivia skipped). + pub fn entries(&self) -> impl Iterator> + 't { + child_items(self.syntax()) + } + + /// The `key = value` fields directly inside this block. + pub fn fields(&self) -> impl Iterator> + 't { + self.entries().filter_map(Item::into_field) + } + + /// The bare values (array elements) directly inside this block. + pub fn values(&self) -> impl Iterator> + 't { + self.entries().filter_map(Item::into_value) + } + + /// Whether the block has no entries (only braces, whitespace, comments). + pub fn is_empty(&self) -> bool { + self.entries().next().is_none() + } +} + +impl<'t, 'a> HeaderedBlock<'t, 'a> { + /// The header scalar (e.g. `rgb`, `hsv`, a tag). + pub fn header(&self) -> Option> { + self.syntax() + .child_tokens() + .find(|t| t.kind().is_scalar()) + } + + /// The block that follows the header. + pub fn block(&self) -> Option> { + self.syntax().child_nodes().find_map(Block::cast) + } + + /// Interpret an `rgb`/`hsv`/`hsv360` headered block as a [`Color`], or + /// `None` if the header is unrecognized or the channels do not parse. + pub fn as_color(&self) -> Option { + let header = self.header()?; + let block = self.block()?; + let chans: Vec> = block + .syntax() + .child_tokens() + .filter(|t| t.kind() == SyntaxKind::Unquoted) + .map(|t| t.as_scalar()) + .collect(); + let byte = |s: &Scalar<'a>| u8::try_from(s.to_u64().ok()?).ok(); + match header.text() { + b"rgb" | b"RGB" => { + if !(3..=4).contains(&chans.len()) { + return None; + } + Some(Color::Rgb { + r: byte(&chans[0])?, + g: byte(&chans[1])?, + b: byte(&chans[2])?, + a: chans.get(3).and_then(byte), + }) + } + b"hsv" | b"HSV" => { + let [h, s, v] = chans.as_slice() else { return None }; + Some(Color::Hsv { h: h.to_f64().ok()?, s: s.to_f64().ok()?, v: v.to_f64().ok()? }) + } + b"hsv360" => { + let [h, s, v] = chans.as_slice() else { return None }; + Some(Color::Hsv360 { h: h.to_f64().ok()?, s: s.to_f64().ok()?, v: v.to_f64().ok()? }) + } + _ => None, + } + } +} + +impl<'t, 'a> Calc<'t, 'a> { + /// The wrapped arithmetic expression (between `@[` and `]`). + pub fn expr(&self) -> Option> { + self.syntax().children().find_map(Expr::cast_element) + } +} + +impl<'t, 'a> BinaryExpr<'t, 'a> { + /// The left operand. + pub fn lhs(&self) -> Option> { + self.syntax().children().find_map(Expr::cast_element) + } + + /// The operator token (`+`, `-`, `*`, `/`). + pub fn op_token(&self) -> Option> { + self.syntax().child_tokens().find(|t| { + matches!( + t.kind(), + SyntaxKind::Plus | SyntaxKind::Minus | SyntaxKind::Star | SyntaxKind::Slash + ) + }) + } + + /// The right operand. + pub fn rhs(&self) -> Option> { + self.syntax().children().filter_map(Expr::cast_element).nth(1) + } +} + +impl<'t, 'a> UnaryExpr<'t, 'a> { + /// The prefix operator token (`-` or `+`). + pub fn op_token(&self) -> Option> { + self.syntax() + .child_tokens() + .find(|t| matches!(t.kind(), SyntaxKind::Plus | SyntaxKind::Minus)) + } + + /// The operand the prefix applies to. + pub fn operand(&self) -> Option> { + self.syntax().children().find_map(Expr::cast_element) + } +} + +impl<'t, 'a> ParenExpr<'t, 'a> { + /// The expression between the parentheses. + pub fn inner(&self) -> Option> { + self.syntax().children().find_map(Expr::cast_element) + } +} + +impl<'t, 'a> Item<'t, 'a> { + /// The [`Field`] if this entry is one. + pub fn into_field(self) -> Option> { + match self { + Item::Field(f) => Some(f), + _ => None, + } + } + + /// The [`Value`] if this entry is a bare value. + pub fn into_value(self) -> Option> { + match self { + Item::Value(v) => Some(v), + _ => None, + } + } +} + +impl<'t, 'a> Value<'t, 'a> { + /// Classify a child element as a value, or `None` if it cannot be one. + fn cast_element(el: SyntaxElement<'t, 'a>) -> Option { + match el { + SyntaxElement::Token(t) if t.kind().is_scalar() => Some(Value::Scalar(t)), + SyntaxElement::Token(_) => None, + SyntaxElement::Node(n) => match n.kind() { + SyntaxKind::Block => Block::cast(n).map(Value::Block), + SyntaxKind::HeaderedBlock => HeaderedBlock::cast(n).map(Value::Headered), + SyntaxKind::Calc => Calc::cast(n).map(Value::Calc), + _ => None, + }, + } + } + + /// The scalar token, if this is [`Value::Scalar`]. + pub fn as_scalar(&self) -> Option> { + match self { + Value::Scalar(t) => Some(t.as_scalar()), + _ => None, + } + } + + /// The block, if this is [`Value::Block`]. + pub fn as_block(&self) -> Option> { + match self { + Value::Block(b) => Some(*b), + _ => None, + } + } + + /// The headered block, if this is [`Value::Headered`]. + pub fn as_headered(&self) -> Option> { + match self { + Value::Headered(h) => Some(*h), + _ => None, + } + } + + /// The calc, if this is [`Value::Calc`]. + pub fn as_calc(&self) -> Option> { + match self { + Value::Calc(c) => Some(*c), + _ => None, + } + } + + /// Coerce a scalar value to `f64` (e.g. `1.000`, `-5.7`, `10.0f`). + pub fn to_f64(&self) -> Option { + self.as_scalar()?.to_f64().ok() + } + + /// Coerce a scalar value to `i64`. + pub fn to_i64(&self) -> Option { + self.as_scalar()?.to_i64().ok() + } + + /// Coerce a scalar value to `u64`. + pub fn to_u64(&self) -> Option { + self.as_scalar()?.to_u64().ok() + } + + /// Coerce a scalar value to `bool` (`yes`/`no`). + pub fn to_bool(&self) -> Option { + self.as_scalar()?.to_bool().ok() + } + + /// Interpret an `rgb`/`hsv`/`hsv360` value as a [`Color`]. + pub fn as_color(&self) -> Option { + self.as_headered()?.as_color() + } +} + +impl<'t, 'a> Expr<'t, 'a> { + /// Classify a child element as a calc expression, or `None`. + fn cast_element(el: SyntaxElement<'t, 'a>) -> Option { + match el { + SyntaxElement::Node(n) => match n.kind() { + SyntaxKind::BinaryExpr => BinaryExpr::cast(n).map(Expr::Binary), + SyntaxKind::UnaryExpr => UnaryExpr::cast(n).map(Expr::Unary), + SyntaxKind::ParenExpr => ParenExpr::cast(n).map(Expr::Paren), + _ => None, + }, + SyntaxElement::Token(t) => match t.kind() { + SyntaxKind::Number => Some(Expr::Number(t)), + SyntaxKind::CalcIdent => Some(Expr::Ident(t)), + _ => None, + }, + } + } +} + +impl<'a> GreenTree<'a> { + /// The typed [`Root`] of the document. + pub fn ast(&self) -> Root<'_, 'a> { + Root::cast(self.root()).expect("the root node is always SyntaxKind::Root") + } +} + +// --------------------------------------------------------------------------- +// Formatter: walk the tree and re-emit it with normalized whitespace. Only +// whitespace is regenerated — every significant token, comment, and the BOM is +// reproduced byte-for-byte — so reparsing the output yields the same tree of +// significant tokens, and formatting is idempotent. +// +// House style: +// * one entry per line, indented by `FormatOptions::indent` per nesting level; +// * `key = value`, a single space around the operator; +// * a block of only bare scalars stays inline (`{ 1 2 3 }`); any block with +// fields, nested blocks, or comments breaks onto multiple lines; an empty +// block collapses to `{}`; +// * comments are preserved and always end their line, so a comment can never +// swallow a following token; an author's blank line between entries is kept +// (collapsed to a single blank line); +// * the document ends in exactly one newline. +// +// Constructs the parser does not yet structure (`[[param]]` blocks, `Bogus` +// recovery nodes, calc internals) are reflowed conservatively: calc and Bogus +// nodes are reprinted verbatim, and loose tokens land one per line. Content is +// always preserved; only the layout of those rare constructs is rough. +// --------------------------------------------------------------------------- + +/// Configuration for [`GreenTree::format`]. +#[derive(Debug, Clone)] +pub struct FormatOptions { + /// The string emitted once per nesting level of indentation. Defaults to a + /// single tab (the Paradox convention); set it to spaces if preferred. + pub indent: String, +} + +impl Default for FormatOptions { + fn default() -> Self { + FormatOptions { indent: String::from("\t") } + } +} + +/// Parse `source` and reformat it with the default [`FormatOptions`]. +/// +/// The result reparses to the same significant tokens as `source` and is +/// idempotent (`format(format(x)) == format(x)`). +pub fn format(source: &[u8]) -> Vec { + parse(source).format(&FormatOptions::default()) +} + +impl<'a> GreenTree<'a> { + /// Reformat the tree with the given [`FormatOptions`]. See the module-level + /// formatter notes for the house style. + pub fn format(&self, opts: &FormatOptions) -> Vec { + let mut f = Fmt { + out: Vec::with_capacity(self.source.len()), + opts, + comment_open: false, + quote_open: false, + }; + f.fmt_root(self.root()); + f.finish() + } +} + +/// Number of newline bytes in `bytes` (used to detect line breaks and the +/// author's blank lines within a whitespace run). +fn count_newlines(bytes: &[u8]) -> usize { + bytes.iter().filter(|&&b| b == b'\n').count() +} + +/// Whether a quoted token's bytes (`text` begins with `"`) are properly closed +/// by a `"` before running out — mirroring the lexer's escape handling. An +/// *un*closed quote (it ran to end of input) would absorb any byte the +/// formatter emits after it, so the formatter must not follow it with a newline. +fn quote_is_closed(text: &[u8]) -> bool { + if text.first() != Some(&b'"') { + return true; // not an opening quote; nothing to absorb + } + let mut i = 1; + while i < text.len() { + match text[i] { + b'\\' => i += 2, + b'"' => return true, + _ => i += 1, + } + } + false +} + +/// Whether `el` is a bare scalar token eligible to keep its block inline. +fn is_inline_scalar(el: SyntaxElement<'_, '_>) -> bool { + el.kind().is_scalar() +} + +/// Collect the comments that sit *inline* within a field/headered-block line — +/// directly inside the node, or inside a nested headered block (a field's value +/// can be `rgb # c { … }`). Comments inside a [`SyntaxKind::Block`] are *not* +/// gathered: those belong to the block's own layout. Such inline comments are +/// hoisted ahead of the line so they can never sit mid-line or fall inside a +/// multi-line block value (which would break idempotence). +fn collect_inline_comments<'t, 'a>(node: SyntaxNode<'t, 'a>, out: &mut Vec>) { + for el in node.children() { + match el { + SyntaxElement::Token(t) if t.kind() == SyntaxKind::Comment => out.push(t), + SyntaxElement::Node(n) + if matches!(n.kind(), SyntaxKind::Field | SyntaxKind::HeaderedBlock) => + { + collect_inline_comments(n, out); + } + _ => {} + } + } +} + +struct Fmt<'o> { + out: Vec, + opts: &'o FormatOptions, + /// True when the current output line ends in a comment, so the next thing + /// emitted must start on a new line (a comment runs to end of line). + comment_open: bool, + /// True when the last token emitted is an unterminated quoted string, which + /// would absorb a following newline; suppresses the trailing newline. + quote_open: bool, +} + +impl Fmt<'_> { + fn finish(mut self) -> Vec { + // Exactly one trailing newline when there is content — unless the last + // token is an unterminated quote, which would swallow it. + if !self.out.is_empty() && self.out.last() != Some(&b'\n') && !self.quote_open { + self.out.push(b'\n'); + } + self.out + } + + /// Emit raw layout bytes (braces, spaces, indentation). These reset both + /// "open" flags: the last thing on the line is now ordinary content. + fn push(&mut self, bytes: &[u8]) { + self.out.extend_from_slice(bytes); + self.comment_open = false; + self.quote_open = false; + } + + fn push_byte(&mut self, b: u8) { + self.out.push(b); + self.comment_open = false; + self.quote_open = false; + } + + /// Emit a token verbatim, tracking whether it leaves a quote open. + fn emit_text(&mut self, text: &[u8]) { + self.push(text); + self.quote_open = !quote_is_closed(text); + } + + /// Emit a comment verbatim, marking the line as comment-closed. + fn emit_comment(&mut self, text: &[u8]) { + self.push(text); + self.comment_open = true; + } + + /// Emit a newline (two for a preserved blank line), reopening the line. + fn line_break(&mut self, blank: bool) { + self.out.push(b'\n'); + if blank { + self.out.push(b'\n'); + } + self.comment_open = false; + self.quote_open = false; + } + + fn write_indent(&mut self, level: usize) { + for _ in 0..level { + self.out.extend_from_slice(self.opts.indent.as_bytes()); + } + } + + fn fmt_root(&mut self, root: SyntaxNode<'_, '_>) { + let children: Vec> = root.children().collect(); + // A leading BOM is reproduced verbatim at the very top of the file. + let start = match children.first() { + Some(el) if el.kind() == SyntaxKind::Bom => { + self.push(el.text()); + 1 + } + _ => 0, + }; + self.fmt_items(&children[start..], 0, true); + } + + /// Format a run of container children (significant items interleaved with + /// trivia), one significant item per line at `indent`. `suppress_first` + /// omits the break before the first item — true for the document root + /// (no leading blank line), false for a block (the break follows its `{`). + fn fmt_items(&mut self, items: &[SyntaxElement<'_, '_>], indent: usize, suppress_first: bool) { + let mut first = true; + let mut pending_blank = false; + let mut ws_had_newline = true; // a fresh container behaves like a new line + + for &el in items { + match el.kind() { + SyntaxKind::Whitespace => { + let nls = count_newlines(el.text()); + ws_had_newline = nls >= 1; + if nls >= 2 && !first { + pending_blank = true; + } + } + // A stray BOM mid-stream cannot occur (the lexer only emits it + // first), but reproduce it verbatim if it ever does. + SyntaxKind::Bom => self.push(el.text()), + SyntaxKind::Comment => { + // Keep it trailing only if the previous item is on this very + // line and that line is not already closed by a comment. + let trailing = !first && !ws_had_newline && !self.comment_open; + if trailing { + self.push_byte(b' '); + } else { + if !(first && suppress_first) { + self.line_break(pending_blank); + } + self.write_indent(indent); + } + self.emit_comment(el.text()); + pending_blank = false; + ws_had_newline = true; + first = false; + } + _ => { + if !(first && suppress_first) { + self.line_break(pending_blank); + } + self.write_indent(indent); + self.hoist_inline_comments(el, indent); + self.fmt_item(el, indent); + pending_blank = false; + ws_had_newline = false; + first = false; + } + } + } + } + + /// Emit any inline comments of a field/headered-block item as their own + /// leading lines (see [`collect_inline_comments`]); a no-op otherwise. The + /// caller has already written this line's indent. + fn hoist_inline_comments(&mut self, el: SyntaxElement<'_, '_>, indent: usize) { + let node = match el { + SyntaxElement::Node(n) + if matches!(n.kind(), SyntaxKind::Field | SyntaxKind::HeaderedBlock) => + { + n + } + _ => return, + }; + let mut leading = Vec::new(); + collect_inline_comments(node, &mut leading); + for c in leading { + self.emit_comment(c.text()); + self.line_break(false); + self.write_indent(indent); + } + } + + /// Format one significant element (already positioned at the line start). + fn fmt_item(&mut self, el: SyntaxElement<'_, '_>, indent: usize) { + match el { + SyntaxElement::Node(n) => match n.kind() { + SyntaxKind::Field | SyntaxKind::HeaderedBlock => self.fmt_spaced(n, indent), + SyntaxKind::Block => self.fmt_block(n, indent), + // A calc is a self-contained expression and a Bogus node wraps + // unstructured bytes — reprint either verbatim. + _ => self.emit_text(n.text()), + }, + // A loose scalar / operator / bracket / bang token: verbatim. + SyntaxElement::Token(t) => self.emit_text(t.text()), + } + } + + /// Format a [`SyntaxKind::Field`] or [`SyntaxKind::HeaderedBlock`]: their + /// significant children joined by single spaces (`key = value`, + /// `rgb { … }`). Interior comments are *not* emitted here — the caller has + /// already hoisted them ahead of the line via [`Fmt::hoist_inline_comments`] + /// — so this only lays out significant tokens. + fn fmt_spaced(&mut self, node: SyntaxNode<'_, '_>, indent: usize) { + let mut first = true; + for el in node.children() { + match el.kind() { + SyntaxKind::Whitespace | SyntaxKind::Bom | SyntaxKind::Comment => {} + _ => { + if !first { + self.push_byte(b' '); + } + first = false; + self.fmt_item(el, indent); + } + } + } + } + + /// Format a [`SyntaxKind::Block`]. Inline when it holds only bare scalars; + /// `{}` when empty; multiline otherwise. The closing `}` is emitted only if + /// the block actually has one (an unclosed block keeps its token count). + fn fmt_block(&mut self, block: SyntaxNode<'_, '_>, indent: usize) { + self.push_byte(b'{'); + + let children: Vec> = block.children().collect(); + let has_close = matches!(children.last(), Some(el) if el.kind() == SyntaxKind::CloseBrace); + // children[0] is always the opening `{`; drop it and the closing `}`. + let inner_end = if has_close { children.len() - 1 } else { children.len() }; + let inner = &children[1..inner_end]; + + let has_comment = inner.iter().any(|el| el.kind() == SyntaxKind::Comment); + let sig: Vec> = + inner.iter().copied().filter(|el| !el.kind().is_trivia()).collect(); + + if sig.is_empty() && !has_comment { + if has_close { + self.push_byte(b'}'); + } + return; + } + + if has_close && !has_comment && sig.iter().all(|el| is_inline_scalar(*el)) { + self.push_byte(b' '); + for (i, el) in sig.iter().enumerate() { + if i > 0 { + self.push_byte(b' '); + } + self.emit_text(el.text()); + } + self.push(b" }"); + return; + } + + self.fmt_items(inner, indent + 1, false); + if has_close { + self.line_break(false); + self.write_indent(indent); + self.push_byte(b'}'); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use quickcheck_macros::quickcheck; + + /// Assert the lexer tiles the input: contiguous spans covering `[0, len)`. + fn assert_tiles(data: &[u8], flavor: Flavor) { + let toks = lex(data, flavor); + let mut at = 0u32; + for t in &toks { + assert_eq!(t.start, at, "gap/overlap at {} in {:?}", at, data); + at += t.len; + } + assert_eq!(at as usize, data.len(), "tokens do not reach end of input"); + } + + /// Parse, then assert byte-exact round-trip via the cursor traversal. + fn rt(data: &[u8]) { + assert_tiles(data, Flavor::default()); + assert_tiles(data, Flavor::hoi4()); + let tree = parse(data); + assert_eq!( + tree.reconstruct(), + data, + "round-trip mismatch\n--- tree ---\n{}", + tree.debug_tree() + ); + assert_eq!(parse_with(data, Flavor::hoi4()).reconstruct(), data); + } + + #[test] + fn round_trip_comprehensive() { + let cases: &[&[u8]] = &[ + b"", + b" ", + b"\n\t ;; ", + b"foo = bar", + b"foo=bar", + b"open={1 2}", + b"field1=-100.535", + br#""foo"="bar" "3"="1444.11.11""#, + br#"custom_name="THE !@#$%^&*( '\"LEGION\"')""#, + b"foo{bar=qux}", + b"foo=abc#def\nbar=qux", + b"flavor_tur.8=yes", + b"dashed-identifier=yes", + b"province_id = event_target:agenda_province", + b"mult = value:job_weights_research_modifier|JOB|head_researcher|", + b"@planet_standard_scale = 11", + b"window_name = @default_window_name", + b"value=\"win\"; a=b", + b"foo = 0.3;", + b"a = 1; b = 2;; c = 3;", + b";;;key = value;;;", + b"age > 16", + b"a==b c<=d e>=f g!=h i?=j kn", + b"position = { @[1-leopard_x] @leopard_y }", + b"my_calc = @[(-half-half)*half]", + b"a = @[ tier + 1 ]", + b"a = @[1+2*3]", + b"a = @[ (1 - leopard_x) ]", + b"a = @[10.0f / 2.5]", + b"a = @[ -x ]", + b"a = @[((@var))]", + b"a = @[]", + b"a = @[ ]", + b"a = @[1 2 3]", // malformed: loose operands, still lossless + b"a = @[1 +", // unterminated calc + b"a = @[)*/]", // malformed operators, still lossless + b"[[scaled_skill] code here ] [[!var_name] other code ]", + b"stats={{id=0 type=general} {id=1 type=admiral}}", + b"868416617618464 = { 11777 4108 { 5632 4187=1089 } 0=1089 }", + b"weird = $MACRO$ ( ) * +", + b"name = $TIER|capital$", + b"color = rgb { 255 0 255 }", + b"hsv_color = hsv360 { 180 10 60 }", + b"\xef\xbb\xbf# BOM\nfoo = bar", + b"name = \"J\xc3\xa5hk\xc3\xa5m\xc3\xa5hkke\"", + b"\xa7GRichard\xa7", + b"a =", + b"a = }", + b"} stray", + b"{{{{}}}}", + b"a = { b = c", + ]; + for c in cases { + rt(c); + } + } + + + #[test] + fn round_trip_fixtures() { + let fixtures: &[&[u8]] = &[ + include_bytes!("../../tests/fixtures/meta.txt"), + include_bytes!("../../tests/fixtures/ck3-header.txt"), + include_bytes!("../../tests/fixtures/campaign_stats.txt"), + include_bytes!("../../tests/fixtures/string-array.txt"), + include_bytes!("../../tests/fixtures/nested-hidden-obj.txt"), + ]; + for f in fixtures { + rt(f); + } + } + + #[test] + fn structure_field() { + let tree = parse(b"a = b"); + let root = tree.root(); + assert_eq!(root.kind(), SyntaxKind::Root); + + let fields: Vec<_> = root.child_nodes().collect(); + assert_eq!(fields.len(), 1); + let field = fields[0]; + assert_eq!(field.kind(), SyntaxKind::Field); + + let kinds: Vec<_> = field.child_tokens().map(|t| t.kind()).collect(); + assert_eq!( + kinds, + [ + SyntaxKind::Unquoted, + SyntaxKind::Whitespace, + SyntaxKind::Operator, + SyntaxKind::Whitespace, + SyntaxKind::Unquoted + ] + ); + + let toks: Vec<_> = field.child_tokens().collect(); + assert_eq!(toks[0].text(), b"a"); + assert_eq!(toks[2].operator(), Some(Operator::Equal)); + assert_eq!(toks[4].text(), b"b"); + assert_eq!(field.text(), b"a = b"); + assert!(tree.errors().is_empty()); + } + + #[test] + fn structure_block_value() { + let tree = parse(b"a = { b c }"); + let field = tree.root().child_nodes().next().unwrap(); + assert_eq!(field.kind(), SyntaxKind::Field); + + let block = field.child_nodes().next().unwrap(); + assert_eq!(block.kind(), SyntaxKind::Block); + assert_eq!(block.text(), b"{ b c }"); + + let scalars: Vec<_> = block + .child_tokens() + .filter(|t| t.kind() == SyntaxKind::Unquoted) + .map(|t| t.text()) + .collect(); + assert_eq!(scalars, [b"b", b"c"]); + } + + #[test] + fn headered_color_block() { + let tree = parse(b"color = rgb { 1 2 3 }"); + let field = tree.root().child_nodes().next().unwrap(); + let hb = field.child_nodes().next().unwrap(); + assert_eq!(hb.kind(), SyntaxKind::HeaderedBlock); + assert_eq!(hb.text(), b"rgb { 1 2 3 }"); + + let header = hb + .child_tokens() + .find(|t| t.kind() == SyntaxKind::Unquoted) + .unwrap(); + assert_eq!(header.text(), b"rgb"); + + let block = hb.child_nodes().next().unwrap(); + assert_eq!(block.kind(), SyntaxKind::Block); + } + + #[test] + fn macro_param_token() { + let tree = parse(b"x = $TIER|capital$"); + let field = tree.root().child_nodes().next().unwrap(); + let m = field + .child_tokens() + .find(|t| t.kind() == SyntaxKind::MacroParam) + .unwrap(); + assert_eq!(m.text(), b"$TIER|capital$"); + } + + #[test] + fn non_equal_operator_preserved() { + let tree = parse(b"age > 16"); + let field = tree.root().child_nodes().next().unwrap(); + let op = field + .child_tokens() + .find(|t| t.kind() == SyntaxKind::Operator) + .unwrap(); + assert_eq!(op.operator(), Some(Operator::GreaterThan)); + } + + #[test] + fn flavor_newline_terminated_strings() { + let src = b"x=\"ab\ncd\""; + + // Default: the newline is inside the string; one Quoted token. + let toks = lex(src, Flavor::default()); + let quoted: Vec<_> = toks.iter().filter(|t| t.kind == SyntaxKind::Quoted).collect(); + assert_eq!(quoted.len(), 1); + assert_eq!(quoted[0].len as usize, src.len() - 2); // `"ab\ncd"` + + // HoI4: the string ends at the newline. + let toks = lex(src, Flavor::hoi4()); + let first = toks.iter().find(|t| t.kind == SyntaxKind::Quoted).unwrap(); + let text = &src[first.start as usize..(first.start + first.len) as usize]; + assert_eq!(text, b"\"ab"); + + // Both round-trip. + assert_eq!(parse_with(src, Flavor::default()).reconstruct(), src); + assert_eq!(parse_with(src, Flavor::hoi4()).reconstruct(), src); + } + + #[test] + fn flavor_hoi4_treats_at_as_identifier() { + // With HoI4, `@` is an ordinary identifier byte, not a variable marker. + let toks = lex(b"@foo", Flavor::hoi4()); + assert_eq!(toks.len(), 1); + assert_eq!(toks[0].kind, SyntaxKind::Unquoted); + + let toks = lex(b"@foo", Flavor::default()); + assert_eq!(toks[0].kind, SyntaxKind::Variable); + } + + #[test] + fn diagnostics_unclosed_block() { + let tree = parse(b"a = { b"); + assert!(tree.errors().iter().any(|e| e.message.contains("unclosed"))); + assert_eq!(tree.reconstruct(), b"a = { b"); // still lossless + } + + #[test] + fn diagnostics_unmatched_brace() { + let tree = parse(b"x } y"); + assert!(tree.errors().iter().any(|e| e.message.contains("unmatched"))); + assert_eq!(tree.reconstruct(), b"x } y"); + // The stray brace lands in a Bogus node. + assert!(tree.root().child_nodes().any(|n| n.kind() == SyntaxKind::Bogus)); + } + + #[test] + fn clean_input_has_no_errors() { + assert!(parse(b"a = { b = c }").errors().is_empty()); + } + + #[test] + #[cfg_attr(miri, ignore)] // the deep input is slow under miri + fn deeply_nested_blocks_do_not_overflow() { + // Analogous to TextTape's `test_too_heavily_nested`: tens of thousands + // of open braces would blow a naive recursive descent's stack. The + // depth guard caps recursion, records a diagnostic, and the tree still + // round-trips byte-for-byte. + let mut data = Vec::new(); + data.extend_from_slice(b"foo="); + data.resize(data.len() + 100_000, b'{'); + let tree = parse(&data); + assert_eq!(tree.reconstruct(), data); + assert!(tree + .errors() + .iter() + .any(|e| e.message.contains("maximum nesting depth"))); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn deeply_nested_balanced_blocks_round_trip() { + // Balanced this time, so flattening must still pair every brace. + let mut data = vec![b'{'; 50_000]; + data.resize(100_000, b'}'); + let tree = parse(&data); + assert_eq!(tree.reconstruct(), data); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn deeply_nested_calc_parens_do_not_overflow() { + // Calc expressions recurse through `parse_operand`/`parse_expr`; the + // same guard covers `@[((((…))))]`. + let mut data = Vec::new(); + data.extend_from_slice(b"x = @["); + data.resize(data.len() + 100_000, b'('); + data.push(b'1'); + data.resize(data.len() + 100_000, b')'); + data.push(b']'); + let tree = parse(&data); + assert_eq!(tree.reconstruct(), data); + assert!(tree.errors().iter().any(|e| e.message.contains("calc nesting"))); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn deeply_nested_calc_unary_do_not_overflow() { + // Unary `-` also recurses through `parse_operand`. + let mut data = Vec::new(); + data.extend_from_slice(b"x = @["); + data.resize(data.len() + 100_000, b'-'); + data.extend_from_slice(b"x]"); + let tree = parse(&data); + assert_eq!(tree.reconstruct(), data); + assert!(tree.errors().iter().any(|e| e.message.contains("calc nesting"))); + } + + #[test] + #[cfg_attr(miri, ignore)] + fn deeply_nested_calc_binary_chain_does_not_overflow() { + // A flat chain `@[1+1+1+…]` is folded iteratively, but yields a + // left-nested BinaryExpr tree of depth = chain length that emit_calc and + // the tree's recursive Drop walk — so the fold cap must bound it too. + let mut data = Vec::new(); + data.extend_from_slice(b"x = @[1"); + for _ in 0..100_000 { + data.extend_from_slice(b"+1"); + } + data.push(b']'); + let tree = parse(&data); + assert_eq!(tree.reconstruct(), data); + assert!(tree.errors().iter().any(|e| e.message.contains("calc nesting"))); + // A pure `*` chain folds the same way. + let mut data = Vec::new(); + data.extend_from_slice(b"x = @[1"); + for _ in 0..100_000 { + data.extend_from_slice(b"*1"); + } + data.push(b']'); + assert_eq!(parse(&data).reconstruct(), data); + } + + #[test] + fn macro_param_is_a_scalar_key_and_value() { + // `$P$` counts as a scalar everywhere (template files use it as a key, + // value, and array element), so `$P$ = $Q$` is a Field. + let tree = parse(b"$P$ = $Q$"); + let field = tree.ast().fields().next().expect("a Field"); + assert_eq!(field.key().unwrap().text(), b"$P$"); + assert_eq!(field.value().unwrap().as_scalar().unwrap().as_bytes(), b"$Q$"); + } + + // ---- typed AST ---- + + #[test] + fn ast_field_key_op_value() { + let tree = parse(b"name = \"Ragusa\""); + let field = tree.ast().fields().next().unwrap(); + assert_eq!(field.key().unwrap().text(), b"name"); + assert_eq!(field.op(), Some(Operator::Equal)); + let value = field.value().unwrap(); + // Quoted value: as_scalar() strips the surrounding quotes. + assert_eq!(value.as_scalar().unwrap().as_bytes(), b"Ragusa"); + } + + #[test] + fn ast_value_coercions() { + let tree = parse(b"a = 1.5 b = -3 c = 42 d = yes"); + let vals: Vec<_> = tree.ast().fields().map(|f| f.value().unwrap()).collect(); + assert_eq!(vals[0].to_f64(), Some(1.5)); + assert_eq!(vals[1].to_i64(), Some(-3)); + assert_eq!(vals[2].to_u64(), Some(42)); + assert_eq!(vals[3].to_bool(), Some(true)); + } + + #[test] + fn ast_block_entries_fields_and_values() { + let tree = parse(b"obj = { a = 1 b = 2 } arr = { x y z }"); + let mut fields = tree.ast().fields(); + let obj = fields.next().unwrap().value().unwrap().as_block().unwrap(); + assert_eq!(obj.fields().count(), 2); + assert!(obj.values().next().is_none()); + assert!(!obj.is_empty()); + + let arr = fields.next().unwrap().value().unwrap().as_block().unwrap(); + let elems: Vec<_> = arr + .values() + .filter_map(|v| v.as_scalar()) + .map(|s| s.as_bytes().to_vec()) + .collect(); + assert_eq!(elems, [b"x", b"y", b"z"]); + assert!(arr.fields().next().is_none()); + } + + #[test] + fn ast_empty_block_is_empty() { + let tree = parse(b"a = {}"); + let block = tree.ast().fields().next().unwrap().value().unwrap().as_block().unwrap(); + assert!(block.is_empty()); + } + + #[test] + fn ast_headered_block_as_color() { + // The color of the first field's value. + fn color(src: &[u8]) -> Option { + parse(src).ast().fields().next()?.value()?.as_color() + } + assert_eq!( + color(b"c = rgb { 255 0 128 }"), + Some(Color::Rgb { r: 255, g: 0, b: 128, a: None }) + ); + assert_eq!( + color(b"c = rgb { 1 2 3 4 }"), + Some(Color::Rgb { r: 1, g: 2, b: 3, a: Some(4) }) + ); + assert_eq!( + color(b"c = hsv { 0.3 0.2 0.8 }"), + Some(Color::Hsv { h: 0.3, s: 0.2, v: 0.8 }) + ); + assert_eq!( + color(b"c = hsv360 { 180 50 80 }"), + Some(Color::Hsv360 { h: 180.0, s: 50.0, v: 80.0 }) + ); + // Out-of-range channel (>255) and wrong arity reject. + assert_eq!(color(b"c = rgb { 300 0 0 }"), None); + assert_eq!(color(b"c = rgb { 1 2 }"), None); + assert_eq!(color(b"c = tag { 1 2 3 }"), None); + } + + #[test] + fn ast_calc_expr_structure() { + let tree = parse(b"x = @[1 + tier * 2]"); + let calc = tree.ast().fields().next().unwrap().value().unwrap().as_calc().unwrap(); + // 1 + (tier * 2): top is a binary `+`. + let Some(Expr::Binary(add)) = calc.expr() else { + panic!("expected a binary expression"); + }; + assert_eq!(add.op_token().unwrap().text(), b"+"); + assert!(matches!(add.lhs(), Some(Expr::Number(_)))); + // rhs is the tighter `tier * 2`. + let Some(Expr::Binary(mul)) = add.rhs() else { + panic!("expected a nested binary expression"); + }; + assert_eq!(mul.op_token().unwrap().text(), b"*"); + assert!(matches!(mul.lhs(), Some(Expr::Ident(_)))); + assert!(matches!(mul.rhs(), Some(Expr::Number(_)))); + } + + #[test] + fn ast_calc_unary_and_paren() { + let tree = parse(b"x = @[ -(a) ]"); + let calc = tree.ast().fields().next().unwrap().value().unwrap().as_calc().unwrap(); + let Some(Expr::Unary(neg)) = calc.expr() else { + panic!("expected a unary expression"); + }; + assert_eq!(neg.op_token().unwrap().text(), b"-"); + let Some(Expr::Paren(paren)) = neg.operand() else { + panic!("expected a parenthesized operand"); + }; + assert!(matches!(paren.inner(), Some(Expr::Ident(_)))); + } + + #[test] + fn ast_item_other_preserves_loose_tokens() { + // `[[param]]` tokens are not yet grouped, so they surface as Item::Other + // rather than being silently dropped from the typed entry view. + let tree = parse(b"[[scaled_skill] body ]"); + let others = tree.ast().items().filter(|i| matches!(i, Item::Other(_))).count(); + assert!(others >= 1, "loose bracket tokens should appear as Item::Other"); + } + + // ---- formatter ---- + + fn fmt(src: &[u8]) -> String { + String::from_utf8(format(src)).unwrap() + } + + /// The significant-token stream (everything but whitespace and comments) as + /// `(kind, text)` pairs. The formatter must preserve this exactly, in order. + fn significant(src: &[u8]) -> Vec<(SyntaxKind, Vec)> { + parse(src) + .tokens() + .filter(|t| !matches!(t.kind(), SyntaxKind::Whitespace | SyntaxKind::Comment)) + .map(|t| (t.kind(), t.text().to_vec())) + .collect() + } + + /// The multiset of comment texts (comments may be relocated, never dropped, + /// merged, or altered). + fn comment_texts(src: &[u8]) -> Vec> { + let mut v: Vec> = parse(src) + .tokens() + .filter(|t| t.kind() == SyntaxKind::Comment) + .map(|t| t.text().to_vec()) + .collect(); + v.sort(); + v + } + + #[test] + fn fmt_normalizes_field_spacing() { + assert_eq!(fmt(b"a=b"), "a = b\n"); + assert_eq!(fmt(b"a =\tb"), "a = b\n"); + assert_eq!(fmt(b"a == b"), "a == b\n"); // operator text kept verbatim + assert_eq!(fmt(b"age>16"), "age > 16\n"); + } + + #[test] + fn fmt_inline_scalar_blocks() { + assert_eq!(fmt(b"color=rgb{255 0 128}"), "color = rgb { 255 0 128 }\n"); + assert_eq!(fmt(b"arr = {1 2 3}"), "arr = { 1 2 3 }\n"); + } + + #[test] + fn fmt_empty_block_collapses() { + assert_eq!(fmt(b"a = { }"), "a = {}\n"); + assert_eq!(fmt(b"a={\n\n}"), "a = {}\n"); + } + + #[test] + fn fmt_nested_blocks_indent_with_tabs() { + assert_eq!(fmt(b"a={b={c=d}}"), "a = {\n\tb = {\n\t\tc = d\n\t}\n}\n"); + } + + #[test] + fn fmt_block_with_fields_is_multiline() { + assert_eq!(fmt(b"o={x=1 y=2}"), "o = {\n\tx = 1\n\ty = 2\n}\n"); + } + + #[test] + fn fmt_indent_option_spaces() { + let opts = FormatOptions { indent: " ".into() }; + let out = String::from_utf8(parse(b"a={b=c}").format(&opts)).unwrap(); + assert_eq!(out, "a = {\n b = c\n}\n"); + } + + #[test] + fn fmt_comments_leading_trailing_and_in_block() { + assert_eq!(fmt(b"a = b # note"), "a = b # note\n"); + assert_eq!(fmt(b"# header\na = b"), "# header\na = b\n"); + assert_eq!(fmt(b"o = {\n# inner\nx = 1\n}"), "o = {\n\t# inner\n\tx = 1\n}\n"); + } + + #[test] + fn fmt_relocated_interior_comment_is_safe() { + // A comment between `=` and the value is hoisted to its own line before + // the field so it cannot comment out the value; both tokens survive. + let out = fmt(b"a = # oops\n b"); + assert_eq!(out, "# oops\na = b\n"); + assert_eq!(significant(b"a = # oops\n b"), significant(out.as_bytes())); + } + + #[test] + fn fmt_preserves_single_blank_line() { + assert_eq!(fmt(b"a = 1\n\n\n\nb = 2"), "a = 1\n\nb = 2\n"); + } + + #[test] + fn fmt_calc_is_verbatim() { + assert_eq!(fmt(b"x=@[1+2]"), "x = @[1+2]\n"); + assert_eq!(fmt(b"x = @[ a + b ]"), "x = @[ a + b ]\n"); + } + + #[test] + fn fmt_unterminated_quote_keeps_no_trailing_newline() { + // The trailing newline would be absorbed into the open string, so it is + // suppressed and the (significant) token stream is preserved. + assert_eq!(format(b"a = \"abc"), b"a = \"abc"); + assert_eq!(significant(b"a = \"abc"), significant(&format(b"a = \"abc"))); + } + + #[test] + fn fmt_bom_preserved_at_top() { + assert_eq!(format(b"\xef\xbb\xbfa=b"), b"\xef\xbb\xbfa = b\n"); + } + + #[test] + fn fmt_empty_input() { + assert_eq!(format(b""), b""); + assert_eq!(format(b" \n\t "), b""); + } + + #[test] + fn fmt_unclosed_block_keeps_brace_count() { + // No phantom `}` is invented for an unclosed block. + let out = fmt(b"a = { b = c"); + assert_eq!(significant(b"a = { b = c"), significant(out.as_bytes())); + assert_eq!(out.matches('}').count(), 0); + } + + #[quickcheck] + fn prop_format_idempotent(data: Vec) -> bool { + let once = format(&data); + let twice = format(&once); + once == twice + } + + #[quickcheck] + fn prop_format_preserves_significant_and_comments(data: Vec) -> bool { + let f = format(&data); + significant(&data) == significant(&f) && comment_texts(&data) == comment_texts(&f) + } + + /// Stress the formatter on a dense alphabet of structural bytes (random + /// input almost never forms blocks/calcs/comments otherwise). + #[quickcheck] + fn prop_format_alphabet(data: Vec) -> bool { + const ALPHABET: &[u8] = b"{}[]()=<># \n\t\"@$.0123456789abc-+*/"; + let m: Vec = data.iter().map(|b| ALPHABET[*b as usize % ALPHABET.len()]).collect(); + let f = format(&m); + significant(&m) == significant(&f) + && comment_texts(&m) == comment_texts(&f) + && format(&f) == f + } + + #[test] + fn tokens_iter_is_ordered_leaves_and_lossless() { + let src = b"a = { b = @[1+2] } # c"; + let tree = parse(src); + // Only leaves, never nodes. + assert!(tree.tokens().all(|t| !t.kind().is_node())); + // In document order, contiguous, covering the whole source. + let mut at = 0u32; + let mut joined = Vec::new(); + for t in tree.tokens() { + assert_eq!(t.text_range().0, at); + at = t.text_range().1; + joined.extend_from_slice(t.text()); + } + assert_eq!(joined, src); // the lossless invariant, via tokens() + } + + #[test] + fn node_flags_summarize_subtrees() { + // `outer` holds a comment + a macro; the nested `inner` block is clean; + // the calc lives only under sibling `c`. Each node's flags reflect its + // *whole* subtree, derived in finish() with no per-query walk. + let tree = parse(b"outer = { # hi\n a = $P$ inner = { b = c } } c = @[1+2]"); + let root = tree.root(); + assert!(root.has_comment() && root.contains_macro() && root.contains_calc()); + assert!(!root.has_error()); + + let mut fields = tree.ast().fields(); + let outer_block = fields.next().unwrap().value().unwrap().as_block().unwrap().syntax(); + assert!(outer_block.has_comment()); + assert!(outer_block.contains_macro()); + assert!(!outer_block.contains_calc(), "the calc is in a sibling, not here"); + + // The genuinely nested `inner = { b = c }` block carries none of the bits. + let inner = outer_block + .child_nodes() + .filter_map(|f| Field::cast(f)?.value()?.as_block()) + .map(|b| b.syntax()) + .next() + .expect("the `inner` block"); + assert!(!inner.has_comment() && !inner.contains_macro() && !inner.contains_calc()); + + // The calc sibling carries HAS_CALC and nothing spurious. + let calc_block = fields.next().unwrap().syntax(); + assert!(calc_block.contains_calc() && !calc_block.has_comment()); + + // A `Bogus` recovery node sets HAS_ERROR all the way to the root. + // (An unclosed `{`, by contrast, is only a diagnostic — it creates no + // error element — so it deliberately does not set the bit.) + let stray = parse(b"x } y"); + assert!(stray.root().has_error(), "the Bogus node sets HAS_ERROR"); + assert!(!parse(b"a = { b").root().has_error()); + } + + #[test] + fn parent_links_reach_root() { + let tree = parse(b"a = { b = c }"); + let field = tree.root().child_nodes().next().unwrap(); + let block = field.child_nodes().next().unwrap(); + let inner = block.child_nodes().next().unwrap(); + assert_eq!(inner.kind(), SyntaxKind::Field); + let c = inner + .child_tokens() + .filter(|t| t.kind() == SyntaxKind::Unquoted) + .last() + .unwrap(); + assert_eq!(c.text(), b"c"); + + let mut n = c.parent().unwrap(); + let mut depth = 0; + while let Some(p) = n.parent() { + n = p; + depth += 1; + } + assert_eq!(n.kind(), SyntaxKind::Root); + assert!(depth >= 2, "expected Field -> Block -> Root nesting"); + } + + /// Compact structural rendering of a subtree: interior nodes as + /// `Kind(child ...)`, significant tokens as their bare `Kind`, trivia + /// omitted. Terse enough to assert precedence and associativity directly. + fn shape(el: SyntaxElement<'_, '_>) -> String { + match el { + SyntaxElement::Node(n) => { + let kids: Vec = n + .children() + .filter(|c| !c.kind().is_trivia()) + .map(shape) + .collect(); + format!("{:?}({})", n.kind(), kids.join(" ")) + } + SyntaxElement::Token(t) => format!("{:?}", t.kind()), + } + } + + /// Parse `key = ` and return the structural shape of the `Calc` node. + fn calc_shape(src: &[u8]) -> String { + let tree = parse(src); + let field = tree.root().child_nodes().next().unwrap(); + let calc = field + .child_nodes() + .find(|n| n.kind() == SyntaxKind::Calc) + .expect("a Calc node"); + shape(SyntaxElement::Node(calc)) + } + + #[test] + fn calc_precedence_mul_binds_tighter() { + // `*` binds tighter than `+`: 1 + (2 * 3). + assert_eq!( + calc_shape(b"x = @[1+2*3]"), + "Calc(CalcOpen BinaryExpr(Number Plus BinaryExpr(Number Star Number)) CalcClose)" + ); + // Mirror: (1 * 2) + 3. + assert_eq!( + calc_shape(b"x = @[1*2+3]"), + "Calc(CalcOpen BinaryExpr(BinaryExpr(Number Star Number) Plus Number) CalcClose)" + ); + } + + #[test] + fn calc_addition_is_left_associative() { + // 1 - 2 - 3 parses as (1 - 2) - 3, not 1 - (2 - 3). + assert_eq!( + calc_shape(b"x = @[1-2-3]"), + "Calc(CalcOpen BinaryExpr(BinaryExpr(Number Minus Number) Minus Number) CalcClose)" + ); + } + + #[test] + fn calc_unary_minus() { + assert_eq!( + calc_shape(b"x = @[ -x ]"), + "Calc(CalcOpen UnaryExpr(Minus CalcIdent) CalcClose)" + ); + // Unary binds tighter than binary: (-half) - half. + assert_eq!( + calc_shape(b"x = @[-half-half]"), + "Calc(CalcOpen BinaryExpr(UnaryExpr(Minus CalcIdent) Minus CalcIdent) CalcClose)" + ); + } + + #[test] + fn calc_nested_parens() { + assert_eq!( + calc_shape(b"x = @[((1))]"), + "Calc(CalcOpen ParenExpr(OpenParen ParenExpr(OpenParen Number CloseParen) \ + CloseParen) CalcClose)" + ); + // A parenthesized sub-expression lowers `*` below it: (a + b) * c. + assert_eq!( + calc_shape(b"x = @[(a+b)*c]"), + "Calc(CalcOpen BinaryExpr(ParenExpr(OpenParen BinaryExpr(CalcIdent Plus CalcIdent) \ + CloseParen) Star CalcIdent) CalcClose)" + ); + } + + #[test] + fn calc_float_with_f_suffix_is_one_number() { + let tree = parse(b"x = @[10.0f / tier]"); + let field = tree.root().child_nodes().next().unwrap(); + let calc = field.child_nodes().next().unwrap(); + let bin = calc.child_nodes().next().unwrap(); + assert_eq!(bin.kind(), SyntaxKind::BinaryExpr); + let nums: Vec<_> = bin + .child_tokens() + .filter(|t| t.kind() == SyntaxKind::Number) + .map(|t| t.text()) + .collect(); + assert_eq!(nums, [b"10.0f"]); + let op = bin.child_tokens().find(|t| t.kind() == SyntaxKind::Slash).unwrap(); + assert_eq!(op.text(), b"/"); + } + + #[test] + fn calc_at_variable_operand() { + let tree = parse(b"x = @[@var + 1]"); + let field = tree.root().child_nodes().next().unwrap(); + let bin = field.child_nodes().next().unwrap().child_nodes().next().unwrap(); + let ident = bin + .child_tokens() + .find(|t| t.kind() == SyntaxKind::CalcIdent) + .unwrap(); + assert_eq!(ident.text(), b"@var"); + } + + #[test] + fn calc_in_array_position() { + // A bare calc as an array element (no `=`) is still grouped as a Calc. + let tree = parse(b"position = { @[1-leopard_x] @leopard_y }"); + let block = tree.root().child_nodes().next().unwrap().child_nodes().next().unwrap(); + assert_eq!(block.kind(), SyntaxKind::Block); + let calc = block.child_nodes().find(|n| n.kind() == SyntaxKind::Calc).unwrap(); + assert_eq!(calc.text(), b"@[1-leopard_x]"); + // The following `@leopard_y` is a Variable token, not folded into the calc. + assert!(block.child_tokens().any(|t| t.kind() == SyntaxKind::Variable)); + } + + #[test] + fn calc_empty() { + assert_eq!(calc_shape(b"x = @[]"), "Calc(CalcOpen CalcClose)"); + assert!(parse(b"x = @[]").errors().is_empty()); + } + + #[test] + fn calc_unterminated_diagnostic() { + let tree = parse(b"x = @[1 + 2"); + assert!( + tree.errors().iter().any(|e| e.message.contains("unclosed '@['")), + "expected an unclosed-calc diagnostic, got {:?}", + tree.errors() + ); + assert_eq!(tree.reconstruct(), b"x = @[1 + 2"); // still lossless + // No CalcClose token was emitted, but the expression still parsed. + let calc = tree + .root() + .child_nodes() + .next() + .unwrap() + .child_nodes() + .find(|n| n.kind() == SyntaxKind::Calc) + .unwrap(); + assert!(calc.child_tokens().all(|t| t.kind() != SyntaxKind::CalcClose)); + assert!(calc.child_nodes().any(|n| n.kind() == SyntaxKind::BinaryExpr)); + } + + #[test] + fn calc_hoi4_flavor_has_no_calc() { + // HoI4 has no reader variables, so `@[` is ordinary identifier bytes and + // no calc machinery (no `Calc`/`CalcOpen`/...) ever appears in the tree. + let tree = parse_with(b"x = @[1+2]", Flavor::hoi4()); + assert_eq!(tree.reconstruct(), b"x = @[1+2]"); + assert!(!tree.debug_tree().contains("Calc")); + } + + #[quickcheck] + fn prop_round_trip(data: Vec) -> bool { + parse(&data).reconstruct() == data + } + + #[quickcheck] + fn prop_round_trip_hoi4(data: Vec) -> bool { + parse_with(&data, Flavor::hoi4()).reconstruct() == data + } + + /// Round-trip over a calc-heavy alphabet: random bytes almost never form a + /// `@[ ... ]`, so map each byte onto the small set of calc-relevant tokens + /// to actually exercise the calc lexer/parser under quickcheck. + #[quickcheck] + fn prop_round_trip_calc_alphabet(data: Vec) -> bool { + const ALPHABET: &[u8] = b"@[]()+-*/ .0123456789fxy_"; + let mapped: Vec = data.iter().map(|b| ALPHABET[*b as usize % ALPHABET.len()]).collect(); + parse(&mapped).reconstruct() == mapped + } +} From 0d4998ed64409ade81bf04cbf858b4bc635bf9e3 Mon Sep 17 00:00:00 2001 From: Nick Babcock Date: Sun, 19 Jul 2026 20:33:16 -0500 Subject: [PATCH 3/3] ... --- src/text/syntax.rs | 166 +++++++++++++++++++++++---------------------- 1 file changed, 86 insertions(+), 80 deletions(-) diff --git a/src/text/syntax.rs b/src/text/syntax.rs index 6b9030c..7a8c20e 100644 --- a/src/text/syntax.rs +++ b/src/text/syntax.rs @@ -32,9 +32,8 @@ //! On top of the green tree sit lightweight [`SyntaxNode`]/[`SyntaxToken`] //! cursors and an ungrammar-style **typed AST** ([`AstNode`], [`Field`], //! [`Block`], [`HeaderedBlock`], [`Calc`], …) with value coercions -//! ([`Value::to_f64`], [`HeaderedBlock::as_color`]). A [`format`](fn@format) pass reprints -//! the tree in a normalized house style, preserving every comment and -//! significant token. +//! ([`Value::to_f64`]). A [`format`](fn@format) pass reprints the tree in a +//! normalized house style, preserving every comment and significant token. //! //! This is a Phase-1 module: `[[param]]` blocks are not yet given dedicated //! structure (they round-trip losslessly as loose tokens, surfaced as @@ -1440,17 +1439,6 @@ pub enum Expr<'t, 'a> { Ident(SyntaxToken<'t, 'a>), } -/// A color value parsed from a [`HeaderedBlock`] such as `rgb { 255 0 128 }`. -#[derive(Debug, Clone, Copy, PartialEq)] -pub enum Color { - /// `rgb { r g b }`, with an optional fourth alpha channel. - Rgb { r: u8, g: u8, b: u8, a: Option }, - /// `hsv { h s v }`, channels nominally in `0.0..=1.0`. - Hsv { h: f64, s: f64, v: f64 }, - /// `hsv360 { h s v }`, hue in `0..=360`, saturation/value in `0..=100`. - Hsv360 { h: f64, s: f64, v: f64 }, -} - /// Strip a leading and/or trailing `"` from a quoted token's raw bytes. Handles /// the unterminated case (only a leading quote) and the degenerate `"`/`""`. fn strip_quotes(b: &[u8]) -> &[u8] { @@ -1575,42 +1563,6 @@ impl<'t, 'a> HeaderedBlock<'t, 'a> { pub fn block(&self) -> Option> { self.syntax().child_nodes().find_map(Block::cast) } - - /// Interpret an `rgb`/`hsv`/`hsv360` headered block as a [`Color`], or - /// `None` if the header is unrecognized or the channels do not parse. - pub fn as_color(&self) -> Option { - let header = self.header()?; - let block = self.block()?; - let chans: Vec> = block - .syntax() - .child_tokens() - .filter(|t| t.kind() == SyntaxKind::Unquoted) - .map(|t| t.as_scalar()) - .collect(); - let byte = |s: &Scalar<'a>| u8::try_from(s.to_u64().ok()?).ok(); - match header.text() { - b"rgb" | b"RGB" => { - if !(3..=4).contains(&chans.len()) { - return None; - } - Some(Color::Rgb { - r: byte(&chans[0])?, - g: byte(&chans[1])?, - b: byte(&chans[2])?, - a: chans.get(3).and_then(byte), - }) - } - b"hsv" | b"HSV" => { - let [h, s, v] = chans.as_slice() else { return None }; - Some(Color::Hsv { h: h.to_f64().ok()?, s: s.to_f64().ok()?, v: v.to_f64().ok()? }) - } - b"hsv360" => { - let [h, s, v] = chans.as_slice() else { return None }; - Some(Color::Hsv360 { h: h.to_f64().ok()?, s: s.to_f64().ok()?, v: v.to_f64().ok()? }) - } - _ => None, - } - } } impl<'t, 'a> Calc<'t, 'a> { @@ -1747,11 +1699,6 @@ impl<'t, 'a> Value<'t, 'a> { pub fn to_bool(&self) -> Option { self.as_scalar()?.to_bool().ok() } - - /// Interpret an `rgb`/`hsv`/`hsv360` value as a [`Color`]. - pub fn as_color(&self) -> Option { - self.as_headered()?.as_color() - } } impl<'t, 'a> Expr<'t, 'a> { @@ -2509,31 +2456,90 @@ mod tests { } #[test] - fn ast_headered_block_as_color() { - // The color of the first field's value. - fn color(src: &[u8]) -> Option { - parse(src).ast().fields().next()?.value()?.as_color() - } - assert_eq!( - color(b"c = rgb { 255 0 128 }"), - Some(Color::Rgb { r: 255, g: 0, b: 128, a: None }) - ); - assert_eq!( - color(b"c = rgb { 1 2 3 4 }"), - Some(Color::Rgb { r: 1, g: 2, b: 3, a: Some(4) }) - ); - assert_eq!( - color(b"c = hsv { 0.3 0.2 0.8 }"), - Some(Color::Hsv { h: 0.3, s: 0.2, v: 0.8 }) - ); - assert_eq!( - color(b"c = hsv360 { 180 50 80 }"), - Some(Color::Hsv360 { h: 180.0, s: 50.0, v: 80.0 }) - ); - // Out-of-range channel (>255) and wrong arity reject. - assert_eq!(color(b"c = rgb { 300 0 0 }"), None); - assert_eq!(color(b"c = rgb { 1 2 }"), None); - assert_eq!(color(b"c = tag { 1 2 3 }"), None); + fn ast_headered_block_is_generic() { + let tree = parse(b"color = rgb { 1 0.5 0 } custom = tag { x = y }"); + let mut fields = tree.ast().fields(); + + let rgb = fields + .next() + .unwrap() + .value() + .unwrap() + .as_headered() + .unwrap(); + assert_eq!(rgb.header().unwrap().text(), b"rgb"); + let channels: Vec<_> = rgb + .block() + .unwrap() + .values() + .map(|value| value.to_f64()) + .collect(); + assert_eq!(channels, [Some(1.0), Some(0.5), Some(0.0)]); + + let custom = fields + .next() + .unwrap() + .value() + .unwrap() + .as_headered() + .unwrap(); + assert_eq!(custom.header().unwrap().text(), b"tag"); + assert_eq!(custom.block().unwrap().fields().count(), 1); + } + + #[test] + fn ast_reader_variable_definition_with_calc_value() { + let tree = parse(b"@half = @[1 / 2]"); + let field = tree.ast().fields().next().unwrap(); + + let key = field.key().unwrap(); + assert_eq!(key.kind(), SyntaxKind::Variable); + assert_eq!(key.text(), b"@half"); + assert_eq!(field.op(), Some(Operator::Equal)); + + let calc = field.value().unwrap().as_calc().unwrap(); + let Some(Expr::Binary(div)) = calc.expr() else { + panic!("expected a binary calculation"); + }; + assert_eq!(div.op_token().unwrap().kind(), SyntaxKind::Slash); + assert!(matches!(div.lhs(), Some(Expr::Number(_)))); + assert!(matches!(div.rhs(), Some(Expr::Number(_)))); + } + + #[test] + fn ast_headered_block_with_variables_and_calcs() { + let tree = parse(b"color = rgb { @red @[green / 2] 0 }"); + let headered = tree + .ast() + .fields() + .next() + .unwrap() + .value() + .unwrap() + .as_headered() + .unwrap(); + assert_eq!(headered.header().unwrap().text(), b"rgb"); + + let values: Vec<_> = headered.block().unwrap().values().collect(); + let Value::Scalar(red) = values[0] else { + panic!("expected a variable scalar"); + }; + assert_eq!(red.kind(), SyntaxKind::Variable); + assert_eq!(red.text(), b"@red"); + + let Value::Calc(calc) = values[1] else { + panic!("expected a calculation"); + }; + let Some(Expr::Binary(div)) = calc.expr() else { + panic!("expected a binary calculation"); + }; + assert!(matches!(div.lhs(), Some(Expr::Ident(_)))); + assert!(matches!(div.rhs(), Some(Expr::Number(_)))); + + let Value::Scalar(zero) = values[2] else { + panic!("expected a scalar channel"); + }; + assert_eq!(zero.text(), b"0"); } #[test]