[WIP] [Tinkering] A lossless parser

Cargo.toml

@@ -24,6 +24,7 @@ jomini_derive = { path = "jomini_derive", version = "^0.4.0", optional = true }
 rawzip = { version = "0.4.2", optional = true }
 serde = { version = "1.0.195", optional = true }
 serde_json = { version = "1.0.114", optional = true }
+smallvec = { version = "1.15.0", features = ["union", "const_generics"] }
 
 [features]
 default = ["derive", "faster_writer"]

bench/src/benchmarks/text.rs

@@ -18,6 +18,12 @@ pub(crate) fn tape_parse(data: &[u8]) -> usize {
     black_box(tape.tokens().len())
 }
 
+#[inline(never)]
+pub(crate) fn syntax_parse(data: &[u8]) -> usize {
+    let tree = jomini::text::syntax::parse(data);
+    black_box(tree.root().children().count())
+}
+
 #[inline(never)]
 pub(crate) fn reader_count_equals(data: &[u8]) -> i32 {
     let mut reader = jomini::text::TokenReader::from_slice(data);
@@ -91,6 +97,9 @@ pub mod criterion_benches {
             group.bench_function(BenchmarkId::new("reader", game), |b| {
                 b.iter(|| reader_count_equals(data))
             });
+            group.bench_function(BenchmarkId::new("syntax", game), |b| {
+                b.iter(|| syntax_parse(data))
+            });
         }
         group.finish();
     }
@@ -137,6 +146,13 @@ pub mod gungraun_benches {
         reader_count_equals(data)
     }
 
+    #[library_benchmark]
+    #[bench::eu4(setup = setup_eu4)]
+    #[bench::ck3(setup = setup_ck3)]
+    fn syntax(data: &[u8]) -> usize {
+        syntax_parse(data)
+    }
+
     #[library_benchmark]
     #[bench::meta(setup = setup_meta)]
     fn deserialize(data: &[u8]) -> Meta {
@@ -152,6 +168,6 @@ pub mod gungraun_benches {
 
     library_benchmark_group!(
         name = text_benches,
-        benchmarks = [tape, reader, deserialize, compressed_read,]
+        benchmarks = [tape, reader, syntax, deserialize, compressed_read,]
     );
 }

examples/lint.rs

@@ -0,0 +1,169 @@
+//! A **cross-file** linter for Clausewitz game files, built on the lossless
+//! syntax tree's new project/semantic layer ([`jomini::text::lint`]).
+//!
+//! This is the thing a single-file parser fundamentally *cannot* do: it loads a
+//! whole "mod" plus its "vanilla" base, builds a project-wide symbol table, and
+//! then resolves every building reference against it. A reference to a building
+//! defined in no file — possibly a typo — becomes an error with a precise
+//! location *and a "did you mean?" fix*; a mod definition that shadows a vanilla
+//! one is noted; a building defined twice in one layer is a warning; and a
+//! reference inside a block with a syntax error is suppressed instead of
+//! generating cascade noise.
+//!
+//! Because the tree is **lossless**, fixes are byte-faithful: the typo is
+//! rewritten in place and everything else is preserved exactly — something a
+//! lossy validator structurally cannot offer.
+//!
+//! ```sh
+//! cargo run --example lint                 # lint the bundled examples/mod-demo
+//! cargo run --example lint -- --fix        # also show machine-applicable fixes
+//! cargo run --example lint -- --fix --write  # ...and write them to disk
+//! cargo run --example lint -- --no-color
+//! cargo run --example lint -- <vanilla-dir> <mod-dir>
+//! ```
+
+use jomini::text::lint::{
+    apply_fixes, line_col, lints, render, FileKind, Fileset, Fix, Linter, Schema, Severity,
+};
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+
+fn main() {
+    let mut color = true;
+    let mut fix_mode = false;
+    let mut write_mode = false;
+    let mut dirs: Vec<PathBuf> = Vec::new();
+    for arg in std::env::args().skip(1) {
+        match arg.as_str() {
+            "--no-color" => color = false,
+            "--fix" => fix_mode = true,
+            "--write" => {
+                fix_mode = true;
+                write_mode = true;
+            }
+            _ => dirs.push(PathBuf::from(arg)),
+        }
+    }
+
+    // Default to the bundled two-layer demo corpus (resolved relative to the
+    // crate so `cargo run --example lint` works from anywhere).
+    let manifest = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let (vanilla_dir, mod_dir) = match dirs.as_slice() {
+        [] => (
+            manifest.join("examples/mod-demo/vanilla"),
+            manifest.join("examples/mod-demo/mod"),
+        ),
+        [v, m] => (v.clone(), m.clone()),
+        _ => {
+            eprintln!("usage: lint [--no-color] [--fix] [--write] [<vanilla-dir> <mod-dir>]");
+            std::process::exit(2);
+        }
+    };
+
+    // 1. Project model: load each directory tree as a load-order layer.
+    let mut fileset = Fileset::new();
+    let vanilla = fileset.load_dir(&vanilla_dir, FileKind::Vanilla).unwrap_or_else(|e| {
+        eprintln!("failed to read {}: {e}", vanilla_dir.display());
+        std::process::exit(1);
+    });
+    let modded = fileset.load_dir(&mod_dir, FileKind::Mod(0)).unwrap_or_else(|e| {
+        eprintln!("failed to read {}: {e}", mod_dir.display());
+        std::process::exit(1);
+    });
+
+    // 2. Schema: the (hand-written) rules of what defines and references a
+    //    building. A real linter would carry a per-game rule set here.
+    let mut schema = Schema::new();
+    schema
+        .define_dir("buildings", "building")
+        .reference_key("has_building", "building")
+        .reference_key("add_building", "building")
+        .reference_key("remove_building", "building")
+        .reference_key("upgrades_from", "building");
+
+    // 3. Two-pass lint: collect all definitions, then resolve all references.
+    let diagnostics = Linter::new(schema).run(&fileset);
+
+    let bold = if color { "\x1b[1m" } else { "" };
+    let dim = if color { "\x1b[2m" } else { "" };
+    let reset = if color { "\x1b[0m" } else { "" };
+
+    println!(
+        "{bold}Linting {} vanilla + {} mod file(s){reset} {dim}(one combined project){reset}\n",
+        vanilla.len(),
+        modded.len(),
+    );
+
+    for diag in &diagnostics {
+        print!("{}", render(diag, &fileset, color));
+        println!();
+    }
+
+    // Summary, broken down by the Severity axis.
+    let count = |s: Severity| diagnostics.iter().filter(|d| d.severity == s).count();
+    let errors = count(Severity::Error);
+    let warnings = count(Severity::Warning);
+    let notes = count(Severity::Tip);
+    println!("{bold}Summary:{reset} {errors} error(s), {warnings} warning(s), {notes} note(s)");
+
+    // The punchline: the undefined-reference catch is impossible per-file.
+    if diagnostics.iter().any(|d| d.id == lints::UNDEFINED_REFERENCE) {
+        println!(
+            "{dim}The undefined-reference error spans files: the reference is in one file, \n\
+             its (missing) definition would live in another — only a project-wide pass finds it.{reset}"
+        );
+    }
+
+    // 4. Autofix: apply each diagnostic's machine-applicable fix. The lossless
+    //    tree makes this byte-faithful — only the offending token changes.
+    if fix_mode {
+        apply_and_report(&fileset, &diagnostics, write_mode, bold, dim, reset);
+    }
+
+    std::process::exit(if errors > 0 && !write_mode { 1 } else { 0 });
+}
+
+/// Group the fixes by file, show each as `old -> new` at its location, and —
+/// with `--write` — splice them into the file on disk via `apply_fixes`.
+fn apply_and_report(
+    fileset: &Fileset,
+    diagnostics: &[jomini::text::lint::Diagnostic],
+    write: bool,
+    bold: &str,
+    dim: &str,
+    reset: &str,
+) {
+    let mut by_file: BTreeMap<_, Vec<&Fix>> = BTreeMap::new();
+    for d in diagnostics {
+        if let Some(fix) = &d.fix {
+            by_file.entry(d.file).or_default().push(fix);
+        }
+    }
+
+    if by_file.is_empty() {
+        println!("\n{dim}No automatic fixes available.{reset}");
+        return;
+    }
+
+    println!("\n{bold}Fixes:{reset}");
+    for (file, fixes) in &by_file {
+        let src = fileset.source(*file);
+        let path = fileset.path(*file);
+        for f in fixes {
+            let (l, c) = line_col(src, f.range.0);
+            let old = String::from_utf8_lossy(&src[f.range.0 as usize..f.range.1 as usize]);
+            println!("  {}:{l}:{c}  `{old}` -> `{}`", path.display(), f.replacement);
+        }
+        if write {
+            let owned: Vec<Fix> = fixes.iter().map(|f| (*f).clone()).collect();
+            let fixed = apply_fixes(src, &owned);
+            match std::fs::write(path, &fixed) {
+                Ok(()) => println!("  {dim}wrote {}{reset}", path.display()),
+                Err(e) => eprintln!("  failed to write {}: {e}", path.display()),
+            }
+        }
+    }
+    if !write {
+        println!("\n{dim}(dry run — re-run with --write to apply these fixes to disk){reset}");
+    }
+}

examples/mod-demo/mod/common/buildings/01_mod_buildings.txt

@@ -0,0 +1,18 @@
+# The mod layer. Loaded after vanilla, so same-named entities override it.
+
+# Overrides the vanilla temple (mod wins) — the linter notes the override.
+temple = {
+	cost = 80
+}
+
+# A new building. Its upgrades_from points at a *vanilla* building, defined in
+# a different file in a different layer — a cross-file, cross-layer reference.
+fortress = {
+	cost = 400
+	upgrades_from = castle
+}
+
+# Oops: fortress is defined twice in the same layer — a duplicate definition.
+fortress = {
+	cost = 401
+}

examples/mod-demo/mod/events/broken_events.txt

@@ -0,0 +1,8 @@
+# This file has a syntax error: the outer block is never closed. The reference
+# to `ghost_building` is undefined, but because it sits in a malformed subtree
+# the linter SUPPRESSES the undefined-reference (and surfaces the syntax error
+# instead) rather than piling a cascade of semantic noise on top of it.
+broken_event = {
+	effect = {
+		add_building = ghost_building
+	}

examples/mod-demo/mod/events/mod_events.txt

@@ -0,0 +1,8 @@
+# Mod events. Mixes valid and invalid building references.
+mod_event = {
+	effect = {
+		add_building = fortress      # ok: defined in the mod
+		add_building = baracks       # ERROR: typo for the vanilla `barracks`
+		remove_building = temple     # ok: resolves to the mod's override
+	}
+}

examples/mod-demo/vanilla/common/buildings/00_buildings.txt

@@ -0,0 +1,18 @@
+# Base game buildings (the "vanilla" layer).
+temple = {
+	cost = 100
+	desc = "A place of worship"
+}
+
+castle = {
+	cost = 250
+	desc = "A fortified keep"
+}
+
+city = {
+	cost = 500
+}
+
+barracks = {
+	cost = 150
+}

examples/mod-demo/vanilla/events/court_events.txt

@@ -0,0 +1,9 @@
+# Base game events. Every building referenced here is defined in vanilla.
+court_event = {
+	trigger = {
+		has_building = castle
+	}
+	effect = {
+		add_building = temple
+	}
+}

examples/syntax.rs

@@ -0,0 +1,109 @@
+//! Inspect a Clausewitz text file through the **lossless** syntax tree
+//! ([`jomini::text::syntax`]).
+//!
+//! Default mode highlights the source by token kind — and because the tree's
+//! leaves tile the input exactly, the highlighted bytes (sans the ANSI codes)
+//! *are* the input, which is the whole point of a lossless tree. With `--tree`
+//! it instead dumps the parse tree as an indented S-expression. With `--format`
+//! it reprints the file in the formatter's house style. Either way it prints a
+//! footer (to stderr) reporting the round-trip check and diagnostics.
+//!
+//! ```sh
+//! cargo run --example syntax -- tests/fixtures/ck3-header.txt
+//! cargo run --example syntax -- --tree tests/fixtures/meta.txt
+//! cargo run --example syntax -- --format tests/fixtures/meta.txt
+//! printf 'rate = @[ (1 - x) * 2 ]\n' | cargo run --example syntax
+//! ```
+
+use jomini::text::syntax::{format, parse, SyntaxKind};
+use std::io::{Read, Write};
+
+fn main() {
+    let mut path: Option<String> = None;
+    let mut tree_mode = false;
+    let mut format_mode = false;
+    let mut color = true;
+    for arg in std::env::args().skip(1) {
+        match arg.as_str() {
+            "--tree" => tree_mode = true,
+            "--format" => format_mode = true,
+            "--no-color" => color = false,
+            "-h" | "--help" => {
+                eprintln!(
+                    "usage: syntax [--tree] [--format] [--no-color] [FILE]   \
+                     (reads stdin if no FILE)"
+                );
+                return;
+            }
+            _ => path = Some(arg),
+        }
+    }
+
+    let source = match &path {
+        Some(p) => std::fs::read(p).unwrap_or_else(|e| {
+            eprintln!("error reading {p}: {e}");
+            std::process::exit(1);
+        }),
+        None => {
+            let mut buf = Vec::new();
+            std::io::stdin().read_to_end(&mut buf).expect("read stdin");
+            buf
+        }
+    };
+
+    let tree = parse(&source);
+
+    if format_mode {
+        let _ = std::io::stdout().lock().write_all(&format(&source));
+    } else if tree_mode {
+        print!("{}", tree.debug_tree());
+    } else {
+        // Re-emit every leaf in order, wrapping it in its kind's color. With
+        // --no-color, stdout is byte-for-byte identical to the input.
+        let mut out = std::io::stdout().lock();
+        for tok in tree.tokens() {
+            match (color, ansi(tok.kind())) {
+                (true, Some(sgr)) => {
+                    let _ = write!(out, "\x1b[{sgr}m");
+                    let _ = out.write_all(tok.text());
+                    let _ = write!(out, "\x1b[0m");
+                }
+                _ => {
+                    let _ = out.write_all(tok.text());
+                }
+            }
+        }
+        let _ = out.flush();
+    }
+
+    // Footer on stderr, so a piped stdout stays clean.
+    let lossless = tree.reconstruct() == source;
+    eprintln!(
+        "\n\x1b[90m— {} bytes · {} tokens · round-trip {} · {} error(s)\x1b[0m",
+        source.len(),
+        tree.tokens().count(),
+        if lossless { "✓" } else { "✗ MISMATCH" },
+        tree.errors().len(),
+    );
+    for e in tree.errors() {
+        eprintln!("\x1b[31m  [{}..{}] {}\x1b[0m", e.range.0, e.range.1, e.message);
+    }
+}
+
+/// ANSI SGR parameter for a token kind, or `None` to print it uncolored
+/// (bare identifiers/numbers and whitespace are left in the terminal default).
+fn ansi(kind: SyntaxKind) -> Option<&'static str> {
+    use SyntaxKind::*;
+    Some(match kind {
+        Comment | Bom => "90",                     // grey
+        Quoted => "32",                            // green
+        Variable | CalcOpen | CalcClose => "36",   // cyan: @-things
+        MacroParam | Plus | Minus | Star | Slash => "35", // magenta: substitutions & calc ops
+        Operator | Number => "33",                 // yellow
+        CalcIdent => "34",                         // blue: calc operands
+        OpenBrace | CloseBrace | OpenParen | CloseParen => "1", // bold: grouping
+        OpenBracket | CloseBracket | Bang => "31", // red: param brackets
+        Error => "41",                             // red background
+        _ => return None,                          // Unquoted, Whitespace
+    })
+}