From f50615b5493fd6c47a36a495e5cc98cd73592144 Mon Sep 17 00:00:00 2001 From: ViTeXFTW Date: Sun, 12 Jul 2026 21:31:25 +0200 Subject: [PATCH 1/4] perf: cache and parallelize asset indexing --- README.md | 5 + crates/analysis/Cargo.toml | 2 +- crates/analysis/src/index.rs | 7 +- crates/analysis/src/lib.rs | 3 +- crates/server/src/backend.rs | 828 ++++++++++++++++++++++++++++---- crates/server/tests/e2e.py | 36 ++ editors/vscode/package.json | 14 +- editors/vscode/src/extension.ts | 4 +- 8 files changed, 794 insertions(+), 105 deletions(-) diff --git a/README.md b/README.md index f44042e..af6e039 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,11 @@ For map/solo.ini diagnostics and W3D model/bone completions, set INI definitions are treated as already loaded before the map file; W3D assets are indexed for model and bone checks. +The server advertises the standard LSP workspace commands +`zerosyntax.clearIndexCache` and `zerosyntax.rebuildIndexCache`. Any editor can +invoke them through `workspace/executeCommand`; rebuilding refreshes the live +index and cache without restarting the language server. + ### Standalone language server Release assets also include the standalone `zerosyntax-lsp` binary. Any editor diff --git a/crates/analysis/Cargo.toml b/crates/analysis/Cargo.toml index a252f8c..1d7ce6f 100644 --- a/crates/analysis/Cargo.toml +++ b/crates/analysis/Cargo.toml @@ -8,9 +8,9 @@ license.workspace = true zerosyntax-schema.workspace = true zerosyntax-syntax.workspace = true rowan.workspace = true +serde.workspace = true [dev-dependencies] -serde.workspace = true toml.workspace = true criterion.workspace = true diff --git a/crates/analysis/src/index.rs b/crates/analysis/src/index.rs index 359bdb1..0b624d0 100644 --- a/crates/analysis/src/index.rs +++ b/crates/analysis/src/index.rs @@ -7,6 +7,7 @@ use std::collections::HashMap; +use serde::{Deserialize, Serialize}; use zerosyntax_schema::{RefKind, ValueType}; use zerosyntax_syntax::ast::{Block, Field, Module}; use zerosyntax_syntax::{Parse, SyntaxKind, SyntaxNode}; @@ -15,7 +16,7 @@ use crate::model::{scope_schema, ScopeSchema}; use crate::{Analyzer, Span}; /// Model data discovered from a W3D asset. -#[derive(Debug, Clone, PartialEq, Eq)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ModelAsset { pub name: String, pub members: Vec, @@ -29,7 +30,7 @@ pub struct Location { } /// A named definition discovered in a document. -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct Definition { pub name: String, pub kind: RefKind, @@ -37,7 +38,7 @@ pub struct Definition { } /// A place where a definition is *referenced* (a Reference-typed field value). -#[derive(Debug, Clone)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] pub struct ReferenceSite { pub name: String, pub kind: RefKind, diff --git a/crates/analysis/src/lib.rs b/crates/analysis/src/lib.rs index e367f85..0ee3bd6 100644 --- a/crates/analysis/src/lib.rs +++ b/crates/analysis/src/lib.rs @@ -8,6 +8,7 @@ use std::collections::{HashMap, HashSet}; +use serde::{Deserialize, Serialize}; use zerosyntax_schema::{BlockType, ModuleType, RefKind, Schema, ValueSet}; use zerosyntax_syntax::{parse, Edit, OpenerOracle, Parse, Strategy}; @@ -25,7 +26,7 @@ pub use diagnostics::{Diagnostic, Severity}; pub use index::WorkspaceIndex; /// A half-open byte range `[start, end)` into the source text. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] pub struct Span { pub start: u32, pub end: u32, diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index 97dc092..386440d 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -6,14 +6,17 @@ //! `didChange` deltas are applied to the rope and the document is re-parsed //! once per change batch; read-only requests reuse the cached parse. +use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; +use std::hash::{Hash, Hasher}; use std::io::{Read, Seek, SeekFrom}; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock, RwLock}; +use std::time::{Duration, Instant, UNIX_EPOCH}; use dashmap::DashMap; use ropey::Rope; -use serde::Deserialize; +use serde::{Deserialize, Serialize}; use tower_lsp::lsp_types::*; use tower_lsp::{jsonrpc::Result, Client, LanguageServer}; use zerosyntax_analysis::diagnostics::DiagnosticsCache; @@ -59,6 +62,8 @@ pub struct Backend { base_roots: Mutex>, /// Number of base INI files indexed from configured base roots. base_indexed_count: AtomicUsize, + /// Number of top-level files restored from the persistent cache last scan. + last_scan_cache_hits: AtomicUsize, /// Whether the initial workspace/base scan has completed at least once. scan_finished: AtomicBool, /// One-shot guard for the map/solo.ini configuration hint. @@ -91,6 +96,93 @@ type ScanEntry = ( Option>, ); +const INDEX_CACHE_VERSION: u32 = 1; +const MAX_SCAN_WORKERS: usize = 4; +const CLEAR_INDEX_CACHE_COMMAND: &str = "zerosyntax.clearIndexCache"; +const REBUILD_INDEX_CACHE_COMMAND: &str = "zerosyntax.rebuildIndexCache"; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct CachedScanEntry { + file: String, + definitions: Vec, + references: Vec, + tags: Vec<(String, String)>, + models: Vec, + text: Option, +} + +impl CachedScanEntry { + fn from_scan(entry: &ScanEntry) -> Self { + Self { + file: entry.0.clone(), + definitions: entry.1.clone(), + references: entry.2.clone(), + tags: entry.3.clone(), + models: entry.4.clone(), + text: entry.5.as_deref().map(str::to_string), + } + } + + fn into_scan(self) -> ScanEntry { + ( + self.file, + self.definitions, + self.references, + self.tags, + self.models, + self.text.map(Arc::from), + ) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct FileFingerprint { + len: u64, + modified_secs: u64, + modified_nanos: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CachedFile { + fingerprint: FileFingerprint, + entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexCache { + version: u32, + schema_hash: u64, + files: HashMap, +} + +impl IndexCache { + fn empty(schema_hash: u64) -> Self { + Self { + version: INDEX_CACHE_VERSION, + schema_hash, + files: HashMap::new(), + } + } +} + +#[derive(Debug, Clone)] +struct ScanPath { + path: PathBuf, + key: String, + is_base: bool, + fingerprint: FileFingerprint, +} + +#[derive(Debug, Default)] +struct ScanStats { + discovery: Duration, + cache_load: Duration, + parse: Duration, + cache_write: Duration, + hits: usize, + misses: usize, +} + #[derive(Deserialize)] pub struct VirtualFileParams { uri: String, @@ -110,6 +202,102 @@ fn read_lossy(path: &Path) -> Option { .map(|b| String::from_utf8_lossy(&b).into_owned()) } +fn schema_hash() -> u64 { + let mut hasher = DefaultHasher::new(); + zerosyntax_schema::EMBEDDED_SCHEMA_JSON.hash(&mut hasher); + hasher.finish() +} + +fn path_key(path: &Path) -> String { + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let key = path.to_string_lossy().replace('\\', "/"); + if cfg!(windows) { + key.to_ascii_lowercase() + } else { + key + } +} + +fn file_fingerprint(path: &Path) -> Option { + let metadata = std::fs::metadata(path).ok()?; + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .unwrap_or_default(); + Some(FileFingerprint { + len: metadata.len(), + modified_secs: modified.as_secs(), + modified_nanos: modified.subsec_nanos(), + }) +} + +fn cache_dir() -> PathBuf { + #[cfg(windows)] + if let Some(path) = std::env::var_os("LOCALAPPDATA") { + return PathBuf::from(path).join("zerosyntax"); + } + #[cfg(not(windows))] + { + if let Some(path) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(path).join("zerosyntax"); + } + if let Some(path) = std::env::var_os("HOME") { + return PathBuf::from(path).join(".cache/zerosyntax"); + } + } + std::env::temp_dir().join("zerosyntax") +} + +fn index_cache_path(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> PathBuf { + let mut roots: Vec<_> = workspace_roots + .iter() + .map(|root| format!("workspace:{}", path_key(root))) + .chain( + base_roots + .iter() + .map(|root| format!("base:{}", path_key(root))), + ) + .collect(); + roots.sort_unstable(); + let mut hasher = DefaultHasher::new(); + roots.hash(&mut hasher); + cache_dir().join(format!( + "index-v{INDEX_CACHE_VERSION}-{:016x}.json", + hasher.finish() + )) +} + +fn load_index_cache(path: &Path, expected_schema_hash: u64) -> IndexCache { + let Some(bytes) = std::fs::read(path).ok() else { + return IndexCache::empty(expected_schema_hash); + }; + let Ok(cache) = serde_json::from_slice::(&bytes) else { + return IndexCache::empty(expected_schema_hash); + }; + if cache.version != INDEX_CACHE_VERSION || cache.schema_hash != expected_schema_hash { + return IndexCache::empty(expected_schema_hash); + } + cache +} + +fn write_index_cache(path: &Path, cache: &IndexCache) -> std::io::Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + std::fs::create_dir_all(parent)?; + let bytes = serde_json::to_vec(cache).map_err(std::io::Error::other)?; + std::fs::write(path, bytes) +} + +fn remove_index_cache(path: &Path) -> std::io::Result { + match std::fs::remove_file(path) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + /// Normalise a client-supplied URI to the form [`Url::from_file_path`] produces. /// On Windows, VS Code sends `file:///c%3A/…` (percent-encoded colon, lowercase /// drive letter) while `from_file_path` produces `file:///C:/…`. The mismatch @@ -193,38 +381,36 @@ fn read_c_string(reader: &mut R) -> std::io::Result { Ok(String::from_utf8_lossy(&buf).into_owned()) } -fn big_entries(path: &Path) -> std::io::Result> { - let mut file = std::fs::File::open(path)?; +fn big_entries(file: &mut std::fs::File) -> std::io::Result> { let mut magic = [0u8; 4]; file.read_exact(&mut magic)?; if &magic != b"BIGF" { return Ok(Vec::new()); } - let _archive_size = read_u32_be(&mut file)?; - let count = read_u32_be(&mut file)?; + let _archive_size = read_u32_be(&mut *file)?; + let count = read_u32_be(&mut *file)?; file.seek(SeekFrom::Start(0x10))?; let mut entries = Vec::with_capacity(count as usize); for _ in 0..count { - let offset = read_u32_be(&mut file)? as u64; - let size = read_u32_be(&mut file)? as usize; - let name = read_c_string(&mut file)?.replace('\\', "/"); + let offset = read_u32_be(&mut *file)? as u64; + let size = read_u32_be(&mut *file)? as usize; + let name = read_c_string(&mut *file)?.replace('\\', "/"); entries.push(BigEntry { name, offset, size }); } Ok(entries) } -fn read_big_entry_bytes(path: &Path, entry: &BigEntry) -> Option> { - let mut file = std::fs::File::open(path).ok()?; +fn read_big_entry_bytes(file: &mut std::fs::File, entry: &BigEntry) -> Option> { file.seek(SeekFrom::Start(entry.offset)).ok()?; let mut bytes = vec![0; entry.size]; file.read_exact(&mut bytes).ok()?; Some(bytes) } -fn read_big_entry(path: &Path, entry: &BigEntry) -> Option { - let bytes = read_big_entry_bytes(path, entry)?; +fn read_big_entry(file: &mut std::fs::File, entry: &BigEntry) -> Option { + let bytes = read_big_entry_bytes(file, entry)?; Some(String::from_utf8_lossy(&bytes).into_owned()) } @@ -251,38 +437,28 @@ fn parse_w3d_models(bytes: &[u8], fallback_name: &str) -> Vec { names.push(fallback_name.to_string()); } walk_w3d_chunks(bytes, 0, bytes.len(), 0, &mut |kind, payload| match kind { - 0x0000_001F => { - if payload.len() >= 40 { - push_name(&mut members, read_fixed_name(&payload[8..24])); - push_name(&mut names, read_fixed_name(&payload[24..40])); - } + 0x0000_001F if payload.len() >= 40 => { + push_name(&mut members, read_fixed_name(&payload[8..24])); + push_name(&mut names, read_fixed_name(&payload[24..40])); } // HIERARCHY_HEADER, EMITTER_HEADER, AGGREGATE_HEADER: Version + Name[16]. - 0x0000_0101 | 0x0000_0501 | 0x0000_0601 => { - if payload.len() >= 20 { - push_name(&mut names, read_fixed_name(&payload[4..20])); - } + 0x0000_0101 | 0x0000_0501 | 0x0000_0601 if payload.len() >= 20 => { + push_name(&mut names, read_fixed_name(&payload[4..20])); } 0x0000_0102 => { for pivot in payload.chunks_exact(60) { push_name(&mut members, read_fixed_name(&pivot[..16])); } } - 0x0000_0701 => { - if payload.len() >= 40 { - push_name(&mut names, read_fixed_name(&payload[8..24])); - push_name(&mut names, read_fixed_name(&payload[24..40])); - } + 0x0000_0701 if payload.len() >= 40 => { + push_name(&mut names, read_fixed_name(&payload[8..24])); + push_name(&mut names, read_fixed_name(&payload[24..40])); } - 0x0000_0704 => { - if payload.len() >= 36 { - push_name(&mut members, read_fixed_name(&payload[4..36])); - } + 0x0000_0704 if payload.len() >= 36 => { + push_name(&mut members, read_fixed_name(&payload[4..36])); } - 0x0000_0740 => { - if payload.len() >= 40 { - push_name(&mut members, read_fixed_name(&payload[8..40])); - } + 0x0000_0740 if payload.len() >= 40 => { + push_name(&mut members, read_fixed_name(&payload[8..40])); } 0x0000_0750 if payload.len() >= 48 => { push_name(&mut members, read_fixed_name(&payload[16..48])) @@ -376,28 +552,31 @@ fn dedup_case_insensitive(values: &mut Vec) { fn scan_big(analyzer: &Analyzer, path: &Path) -> Vec { let mut out = Vec::new(); - let Ok(entries) = big_entries(path) else { + let Ok(mut file) = std::fs::File::open(path) else { + return out; + }; + let Ok(entries) = big_entries(&mut file) else { return out; }; for entry in entries { - let file = big_uri(path, &entry.name); + let uri = big_uri(path, &entry.name); if entry.name.ends_with(".ini") || entry.name.ends_with(".INI") { - let Some(text) = read_big_entry(path, &entry) else { + let Some(text) = read_big_entry(&mut file, &entry) else { continue; }; let parse = analyzer.parse(&text); - let defs = definitions_in(analyzer, &parse, &file); + let defs = definitions_in(analyzer, &parse, &uri); let refs = references_in(analyzer, &parse); let tags = module_tags_in(analyzer, &parse); - out.push((file, defs, refs, tags, Vec::new(), Some(Arc::from(text)))); + out.push((uri, defs, refs, tags, Vec::new(), Some(Arc::from(text)))); } else if entry.name.ends_with(".w3d") || entry.name.ends_with(".W3D") { - let Some(bytes) = read_big_entry_bytes(path, &entry) else { + let Some(bytes) = read_big_entry_bytes(&mut file, &entry) else { continue; }; let stem = file_stem_str(&entry.name); let models = parse_w3d_models(&bytes, &stem); if !models.is_empty() { - out.push((file, Vec::new(), Vec::new(), Vec::new(), models, None)); + out.push((uri, Vec::new(), Vec::new(), Vec::new(), models, None)); } } } @@ -405,7 +584,7 @@ fn scan_big(analyzer: &Analyzer, path: &Path) -> Vec { } /// Walk `roots` and index `.ini` files plus `.w3d` model assets in one call. -/// Production code goes through `collect_scan_paths` + `scan_files` so the +/// Production code goes through `collect_scan_plan` + `scan_with_cache` so the /// scan can report progress; this convenience wrapper serves the tests. #[cfg(test)] fn scan_roots(analyzer: &Analyzer, roots: &[PathBuf]) -> Vec { @@ -442,9 +621,73 @@ fn collect_scan_paths(roots: &[PathBuf]) -> Vec { out } +fn collect_scan_plan(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + // Base paths come first to preserve the existing base-then-workspace apply + // order. A path present in both groups belongs to the base group. + for (roots, is_base) in [(base_roots, true), (workspace_roots, false)] { + for path in collect_scan_paths(roots) { + let key = path_key(&path); + let Some(fingerprint) = file_fingerprint(&path) else { + continue; + }; + if seen.insert(key.clone()) { + out.push(ScanPath { + path, + key, + is_base, + fingerprint, + }); + } + } + } + out +} + +fn scan_path(analyzer: &Analyzer, path: &Path) -> Vec { + let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); + if ext.eq_ignore_ascii_case("big") { + return scan_big(analyzer, path); + } + if ext.eq_ignore_ascii_case("ini") { + if let (Some(text), Ok(uri)) = (read_lossy(path), Url::from_file_path(path)) { + let parse = analyzer.parse(&text); + return vec![( + uri.to_string(), + definitions_in(analyzer, &parse, uri.as_str()), + references_in(analyzer, &parse), + module_tags_in(analyzer, &parse), + Vec::new(), + None, + )]; + } + } else if ext.eq_ignore_ascii_case("w3d") { + if let (Ok(bytes), Ok(uri)) = (std::fs::read(path), Url::from_file_path(path)) { + let stem = path + .file_stem() + .and_then(|s| s.to_str()) + .unwrap_or_default(); + let models = parse_w3d_models(&bytes, stem); + if !models.is_empty() { + return vec![( + uri.to_string(), + Vec::new(), + Vec::new(), + Vec::new(), + models, + None, + )]; + } + } + } + Vec::new() +} + /// Phase 2 of the scan: parse/index each collected file, invoking /// `progress(done, total)` after each one (a `.big` archive counts as one /// unit of work regardless of how many entries it holds). +#[cfg(test)] fn scan_files( analyzer: &Analyzer, paths: &[PathBuf], @@ -452,39 +695,110 @@ fn scan_files( ) -> Vec { let mut out = Vec::new(); for (i, path) in paths.iter().enumerate() { - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); - if ext.eq_ignore_ascii_case("big") { - out.extend(scan_big(analyzer, path)); - } else if ext.eq_ignore_ascii_case("ini") { - if let (Some(text), Ok(uri)) = (read_lossy(path), Url::from_file_path(path)) { - let parse = analyzer.parse(&text); - let defs = definitions_in(analyzer, &parse, uri.as_str()); - let refs = references_in(analyzer, &parse); - let tags = module_tags_in(analyzer, &parse); - out.push((uri.to_string(), defs, refs, tags, Vec::new(), None)); + out.extend(scan_path(analyzer, path)); + progress(i + 1, paths.len()); + } + out +} + +fn scan_with_cache( + analyzer: &Analyzer, + paths: &[ScanPath], + cache_path: &Path, + workers: usize, + progress: &mut impl FnMut(usize, usize), +) -> (Vec<(bool, ScanEntry)>, ScanStats) { + let expected_schema_hash = schema_hash(); + let load_started = Instant::now(); + let mut old_cache = load_index_cache(cache_path, expected_schema_hash); + let cache_load = load_started.elapsed(); + let old_file_count = old_cache.files.len(); + let mut results: Vec>> = (0..paths.len()).map(|_| None).collect(); + let mut misses = Vec::new(); + let mut hits = 0; + let mut done = 0; + + for (i, path) in paths.iter().enumerate() { + match old_cache.files.remove(&path.key) { + Some(cached) if cached.fingerprint == path.fingerprint => { + results[i] = Some( + cached + .entries + .into_iter() + .map(CachedScanEntry::into_scan) + .collect(), + ); + hits += 1; + done += 1; + progress(done, paths.len()); } - } else if ext.eq_ignore_ascii_case("w3d") { - if let (Ok(bytes), Ok(uri)) = (std::fs::read(path), Url::from_file_path(path)) { - let stem = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let models = parse_w3d_models(&bytes, stem); - if !models.is_empty() { - out.push(( - uri.to_string(), - Vec::new(), - Vec::new(), - Vec::new(), - models, - None, - )); - } + _ => misses.push(i), + } + } + + let parse_started = Instant::now(); + if !misses.is_empty() { + let worker_count = workers.max(1).min(misses.len()); + let next = AtomicUsize::new(0); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + for _ in 0..worker_count { + let tx = tx.clone(); + let misses = &misses; + let next = &next; + scope.spawn(move || loop { + let work = next.fetch_add(1, Ordering::Relaxed); + let Some(&index) = misses.get(work) else { + break; + }; + let entries = scan_path(analyzer, &paths[index].path); + if tx.send((index, entries)).is_err() { + break; + } + }); + } + drop(tx); + for (index, entries) in rx { + results[index] = Some(entries); + done += 1; + progress(done, paths.len()); } + }); + } + let parse = parse_started.elapsed(); + + let mut cache = IndexCache::empty(expected_schema_hash); + let mut scanned = Vec::new(); + for (path, entries) in paths.iter().zip(results) { + let entries = entries.unwrap_or_default(); + cache.files.insert( + path.key.clone(), + CachedFile { + fingerprint: path.fingerprint.clone(), + entries: entries.iter().map(CachedScanEntry::from_scan).collect(), + }, + ); + scanned.extend(entries.into_iter().map(|entry| (path.is_base, entry))); + } + + let write_started = Instant::now(); + if !misses.is_empty() || old_file_count != paths.len() { + if let Err(error) = write_index_cache(cache_path, &cache) { + tracing::warn!(%error, path = %cache_path.display(), "could not write asset index cache"); } - progress(i + 1, paths.len()); } - out + let cache_write = write_started.elapsed(); + ( + scanned, + ScanStats { + cache_load, + parse, + cache_write, + hits, + misses: misses.len(), + ..ScanStats::default() + }, + ) } impl Backend { @@ -500,6 +814,7 @@ impl Backend { format_enabled: OnceLock::new(), base_roots: Mutex::new(Vec::new()), base_indexed_count: AtomicUsize::new(0), + last_scan_cache_hits: AtomicUsize::new(0), scan_finished: AtomicBool::new(false), base_roots_hint_shown: AtomicBool::new(false), client_base_ini_hint: OnceLock::new(), @@ -522,6 +837,20 @@ impl Backend { .fetch_add(1, std::sync::atomic::Ordering::Relaxed) } + fn current_index_cache_path(&self) -> PathBuf { + let roots = self + .roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + let base_roots = self + .base_roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + index_cache_path(&roots, &base_roots) + } + /// Update the cross-file index from the document's cached parse, run /// diagnostics (via the per-block cache), and publish. The parse itself is /// maintained synchronously by `did_open`/`did_change`. @@ -635,59 +964,65 @@ impl Backend { // each update as a progress report while waiting for the results. let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, usize)>(); let handle = tokio::task::spawn_blocking(move || { - let workspace_paths = collect_scan_paths(&roots); - let base_paths = collect_scan_paths(&base_roots); - let total = workspace_paths.len() + base_paths.len(); - let mut done = 0; + let discovery_started = Instant::now(); + let paths = collect_scan_plan(&roots, &base_roots); + let discovery = discovery_started.elapsed(); + let cache_path = index_cache_path(&roots, &base_roots); + let workers = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) + .min(MAX_SCAN_WORKERS); let mut last_percent = u32::MAX; // Throttle to whole-percent changes (plus the final count) so a // 10k-file scan sends ~100 notifications, not 10k. - let mut progress = |_done_in_batch: usize, _batch_total: usize| { - done += 1; + let mut progress = |done: usize, total: usize| { let percent = (done * 100 / total.max(1)) as u32; if percent != last_percent || done == total { last_percent = percent; let _ = tx.send((done, total)); } }; - let scanned = scan_files(&analyzer, &workspace_paths, &mut progress); - let base_scanned = scan_files(&analyzer, &base_paths, &mut progress); - (scanned, base_scanned) + let (scanned, mut stats) = + scan_with_cache(&analyzer, &paths, &cache_path, workers, &mut progress); + stats.discovery = discovery; + (scanned, stats) }); while let Some((done, total)) = rx.recv().await { - self.report_scan_progress(&progress_token, done, total).await; + self.report_scan_progress(&progress_token, done, total) + .await; } - let (scanned, base_scanned) = handle.await.unwrap_or_default(); - let base_ini_count = base_scanned + let (scanned, stats) = handle.await.unwrap_or_default(); + let base_ini_count = scanned .iter() - .filter(|(_, _, _, _, models, _)| models.is_empty()) + .filter(|(is_base, (_, _, _, _, models, _))| *is_base && models.is_empty()) .count(); self.base_indexed_count .store(base_ini_count, Ordering::Relaxed); + self.last_scan_cache_hits + .store(stats.hits, Ordering::Relaxed); self.scan_finished.store(true, Ordering::Relaxed); - let ini_total = base_ini_count - + scanned - .iter() - .filter(|(_, _, _, _, models, _)| models.is_empty()) - .count(); - let model_total: usize = base_scanned + let ini_total = scanned + .iter() + .filter(|(_, (_, _, _, _, models, _))| models.is_empty()) + .count(); + let model_total: usize = scanned .iter() - .chain(scanned.iter()) - .map(|(_, _, _, _, models, _)| models.len()) + .map(|(_, (_, _, _, _, models, _))| models.len()) .sum(); // Don't overwrite index entries for already-open documents with stale // disk content; `initialized` calls `refresh` for each open doc right // after this returns, so they will populate the index from live text. - let open: std::collections::HashSet = self + let open: HashSet = self .docs .iter() .map(|e| e.key().as_str().to_string()) .collect(); + let apply_started = Instant::now(); { let Ok(mut idx) = self.index.write() else { return; }; - for (uri, defs, refs, tags, models, text) in base_scanned.into_iter().chain(scanned) { + for (_, (uri, defs, refs, tags, models, text)) in scanned { if let Some(text) = text { self.virtual_files.insert(uri.clone(), text); } @@ -699,6 +1034,22 @@ impl Backend { } } } + let apply = apply_started.elapsed(); + self.client + .log_message( + MessageType::INFO, + format!( + "asset index scan complete (discovery {} ms, cache load {} ms, parse {} ms, cache write {} ms, apply {} ms, {} hits, {} misses)", + stats.discovery.as_millis(), + stats.cache_load.as_millis(), + stats.parse.as_millis(), + stats.cache_write.as_millis(), + apply.as_millis(), + stats.hits, + stats.misses, + ), + ) + .await; self.end_scan_progress(progress_token, ini_total, model_total) .await; } @@ -734,7 +1085,12 @@ impl Backend { } /// Forward one `done/total` update to the client's progress UI. - async fn report_scan_progress(&self, token: &Option, done: usize, total: usize) { + async fn report_scan_progress( + &self, + token: &Option, + done: usize, + total: usize, + ) { let Some(token) = token else { return }; self.client .send_notification::(ProgressParams { @@ -954,6 +1310,13 @@ impl LanguageServer for Backend { ..Default::default() }, )), + execute_command_provider: Some(ExecuteCommandOptions { + commands: vec![ + CLEAR_INDEX_CACHE_COMMAND.into(), + REBUILD_INDEX_CACHE_COMMAND.into(), + ], + work_done_progress_options: Default::default(), + }), ..Default::default() }, }) @@ -976,11 +1339,12 @@ impl LanguageServer for Backend { .unwrap_or_default(); (self.base_indexed_count.load(Ordering::Relaxed), models) }; + let cache_hits = self.last_scan_cache_hits.load(Ordering::Relaxed); self.client .log_message( MessageType::INFO, format!( - "zerosyntax language server ready ({ini} base INI files, {models} W3D models indexed)" + "zerosyntax language server ready ({ini} base INI files, {models} W3D models indexed, {cache_hits} files from cache)" ), ) .await; @@ -990,6 +1354,60 @@ impl LanguageServer for Backend { Ok(()) } + async fn execute_command( + &self, + params: ExecuteCommandParams, + ) -> Result> { + if params.command != CLEAR_INDEX_CACHE_COMMAND + && params.command != REBUILD_INDEX_CACHE_COMMAND + { + return Ok(None); + } + let path = self.current_index_cache_path(); + let cleared = match remove_index_cache(&path) { + Ok(cleared) => cleared, + Err(error) => { + tracing::warn!(%error, path = %path.display(), "could not clear asset index cache"); + return Err(tower_lsp::jsonrpc::Error::internal_error()); + } + }; + + if params.command == REBUILD_INDEX_CACHE_COMMAND { + self.scan_finished.store(false, Ordering::Relaxed); + self.base_indexed_count.store(0, Ordering::Relaxed); + self.last_scan_cache_hits.store(0, Ordering::Relaxed); + self.virtual_files.clear(); + if let Ok(mut index) = self.index.write() { + *index = WorkspaceIndex::new(); + } + for mut doc in self.docs.iter_mut() { + doc.diag_cache = DiagnosticsCache::new(); + } + self.scan_workspace().await; + let open: Vec = self.docs.iter().map(|doc| doc.key().clone()).collect(); + for uri in open { + self.refresh(&uri).await; + } + let message = "ZeroSyntax index cache rebuilt."; + self.client.show_message(MessageType::INFO, message).await; + return Ok(Some(serde_json::json!({ + "rebuilt": true, + "message": message + }))); + } + + let message = if cleared { + "ZeroSyntax index cache cleared. Restart the language server to rebuild it." + } else { + "ZeroSyntax index cache is already clear." + }; + self.client.show_message(MessageType::INFO, message).await; + Ok(Some(serde_json::json!({ + "cleared": cleared, + "message": message + }))) + } + async fn did_open(&self, params: DidOpenTextDocumentParams) { let uri = canonical_uri(params.text_document.uri); let text: Arc = params.text_document.text.into(); @@ -1673,6 +2091,62 @@ fn origin_copy_fixes( mod tests { use super::*; + fn test_dir(label: &str) -> PathBuf { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let dir = std::env::temp_dir().join(format!( + "zerosyntax-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn w3d_with_member(member: &str) -> Vec { + let mut pivot = vec![0; 60]; + let name = member.as_bytes(); + pivot[..name.len().min(16)].copy_from_slice(&name[..name.len().min(16)]); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&0x0000_0102u32.to_le_bytes()); + bytes.extend_from_slice(&(pivot.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&pivot); + bytes + } + + fn write_big(path: &Path, entries: &[(&str, Vec)]) { + let directory_len: usize = entries.iter().map(|(name, _)| 8 + name.len() + 1).sum(); + let mut offset = 0x10 + directory_len; + let archive_size = offset + entries.iter().map(|(_, bytes)| bytes.len()).sum::(); + let mut out = Vec::with_capacity(archive_size); + out.extend_from_slice(b"BIGF"); + out.extend_from_slice(&(archive_size as u32).to_be_bytes()); + out.extend_from_slice(&(entries.len() as u32).to_be_bytes()); + out.extend_from_slice(&0u32.to_be_bytes()); + for (name, bytes) in entries { + out.extend_from_slice(&(offset as u32).to_be_bytes()); + out.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); + out.extend_from_slice(name.as_bytes()); + out.push(0); + offset += bytes.len(); + } + for (_, bytes) in entries { + out.extend_from_slice(bytes); + } + std::fs::write(path, out).unwrap(); + } + + fn cached_scan( + analyzer: &Analyzer, + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], + cache: &Path, + workers: usize, + ) -> (Vec<(bool, ScanEntry)>, ScanStats) { + let paths = collect_scan_plan(workspace_roots, base_roots); + scan_with_cache(analyzer, &paths, cache, workers, &mut |_, _| {}) + } + #[test] fn canonical_uri_pass_through_non_file() { let u = Url::parse("untitled:///buffer").unwrap(); @@ -1686,6 +2160,166 @@ mod tests { assert!(!is_map_layer_file("C:/Data/INI/Object.ini")); } + #[test] + fn removes_only_the_requested_index_cache() { + let dir = test_dir("clear-cache"); + let current = dir.join("current.json"); + let other = dir.join("other.json"); + std::fs::write(¤t, b"cache").unwrap(); + std::fs::write(&other, b"cache").unwrap(); + + assert!(remove_index_cache(¤t).unwrap()); + assert!(!current.exists()); + assert!(other.exists()); + assert!(!remove_index_cache(¤t).unwrap()); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn cache_reuses_complete_scan_and_invalidates_changed_loose_file() { + let dir = test_dir("cache-loose"); + let cache = dir.join("cache.json"); + std::fs::write(dir.join("Object.ini"), "Object CachedTank\nEnd\n").unwrap(); + let model = dir.join("Tank.w3d"); + std::fs::write(&model, w3d_with_member("Muzzle01")).unwrap(); + let analyzer = Analyzer::embedded(); + let roots = std::slice::from_ref(&dir); + + let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((cold_stats.hits, cold_stats.misses), (0, 2)); + let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((warm_stats.hits, warm_stats.misses), (2, 0)); + assert_eq!(cold, warm); + + let mut changed = w3d_with_member("Muzzle02"); + changed.push(0); // size change guarantees invalidation on coarse-mtime filesystems + std::fs::write(&model, changed).unwrap(); + let (_, changed_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((changed_stats.hits, changed_stats.misses), (1, 1)); + + let mut incompatible = load_index_cache(&cache, schema_hash()); + incompatible.schema_hash ^= 1; + write_index_cache(&cache, &incompatible).unwrap(); + let (_, schema_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((schema_stats.hits, schema_stats.misses), (0, 2)); + + std::fs::write(&cache, b"not json").unwrap(); + let (_, corrupt_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((corrupt_stats.hits, corrupt_stats.misses), (0, 2)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn cache_invalidates_changed_big_archive_as_one_file() { + let dir = test_dir("cache-big"); + let archive = dir.join("Models.big"); + let cache = dir.join("cache.json"); + write_big( + &archive, + &[ + ("Data/INI/Object.ini", b"Object BigTank\nEnd\n".to_vec()), + ("Art/Tank.w3d", w3d_with_member("Muzzle01")), + ], + ); + let analyzer = Analyzer::embedded(); + let roots = std::slice::from_ref(&archive); + + let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((cold_stats.hits, cold_stats.misses), (0, 1)); + assert_eq!(cold.len(), 2); + let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((warm_stats.hits, warm_stats.misses), (1, 0)); + assert_eq!(cold, warm); + + write_big( + &archive, + &[ + ( + "Data/INI/Object.ini", + b"Object ChangedBigTank\nEnd\n".to_vec(), + ), + ("Art/Tank.w3d", w3d_with_member("Muzzle02")), + ], + ); + let (_, changed_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((changed_stats.hits, changed_stats.misses), (0, 1)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn parallel_scan_preserves_order_and_overlapping_roots_are_deduped() { + let dir = test_dir("parallel"); + for i in 0..32 { + std::fs::write( + dir.join(format!("Model{i:02}.w3d")), + w3d_with_member(&format!("Bone{i:02}")), + ) + .unwrap(); + } + let roots = std::slice::from_ref(&dir); + let plan = collect_scan_plan(roots, roots); + assert_eq!(plan.len(), 32); + assert!(plan.iter().all(|path| path.is_base)); + + let analyzer = Analyzer::embedded(); + let (serial, _) = cached_scan(&analyzer, &[], roots, &dir.join("serial.json"), 1); + let (parallel, _) = cached_scan(&analyzer, &[], roots, &dir.join("parallel.json"), 4); + assert_eq!(serial, parallel); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + #[ignore = "3,000-file synthetic timing harness; run with --ignored --nocapture"] + fn synthetic_3000_asset_cold_and_warm_timings() { + let dir = test_dir("benchmark"); + for i in 0..3000 { + std::fs::write( + dir.join(format!("Model{i:04}.w3d")), + w3d_with_member(&format!("Bone{:02}", i % 100)), + ) + .unwrap(); + } + let analyzer = Analyzer::embedded(); + let mut serial_times = Vec::new(); + let mut parallel_times = Vec::new(); + let mut expected = None; + let roots = std::slice::from_ref(&dir); + for run in 0..3 { + let started = Instant::now(); + let (out, _) = cached_scan( + &analyzer, + &[], + roots, + &dir.join(format!("serial-{run}.json")), + 1, + ); + serial_times.push(started.elapsed()); + expected.get_or_insert(out); + + let started = Instant::now(); + let (out, _) = cached_scan( + &analyzer, + &[], + roots, + &dir.join(format!("parallel-{run}.json")), + 4, + ); + parallel_times.push(started.elapsed()); + assert_eq!(expected.as_ref(), Some(&out)); + } + serial_times.sort_unstable(); + parallel_times.sort_unstable(); + let warm_started = Instant::now(); + let (_, warm_stats) = cached_scan(&analyzer, &[], roots, &dir.join("parallel-2.json"), 4); + let warm = warm_started.elapsed(); + assert_eq!((warm_stats.hits, warm_stats.misses), (3000, 0)); + eprintln!( + "3,000 W3Ds: serial median {:?}, parallel median {:?}, warm {:?}", + serial_times[1], parallel_times[1], warm + ); + let _ = std::fs::remove_dir_all(dir); + } + #[test] fn scans_ini_from_big_archive() { let dir = std::env::temp_dir(); @@ -1736,7 +2370,7 @@ mod tests { std::fs::write(dir.join("Good.w3d"), bytes).unwrap(); let analyzer = Analyzer::embedded(); - let scanned = scan_roots(&analyzer, &[dir.clone()]); + let scanned = scan_roots(&analyzer, std::slice::from_ref(&dir)); let _ = std::fs::remove_dir_all(&dir); let mut idx = WorkspaceIndex::new(); diff --git a/crates/server/tests/e2e.py b/crates/server/tests/e2e.py index daa11cc..937f45e 100644 --- a/crates/server/tests/e2e.py +++ b/crates/server/tests/e2e.py @@ -101,6 +101,9 @@ def wait_for(pred, what, timeout=15.0): caps = init["result"]["capabilities"] assert "completionProvider" in caps, "missing completionProvider" assert "semanticTokensProvider" in caps, "missing semanticTokensProvider" + commands = caps.get("executeCommandProvider", {}).get("commands", []) + assert "zerosyntax.clearIndexCache" in commands, "missing generic clear-cache command" + assert "zerosyntax.rebuildIndexCache" in commands, "missing generic rebuild-cache command" sync = caps.get("textDocumentSync") assert sync == 2, f"expected INCREMENTAL sync (2), got {sync!r}" # We offered no positionEncodings, so the server must stay on the baseline. @@ -411,6 +414,39 @@ def pos_off(text, pos): # ASCII docs: utf-16 char == byte offset assert new_texts == ["TREADS"], new_texts print("OK: code actions offer did-you-mean + missing End quickfixes") + # The cache command is a standard workspace/executeCommand request, so + # generic LSP editors can invoke the same command as the VS Code wrapper. + send({"jsonrpc": "2.0", "id": 25, "method": "workspace/executeCommand", + "params": {"command": "zerosyntax.clearIndexCache", "arguments": []}}) + clear = wait_for(lambda m: m.get("id") == 25 and "result" in m, + "clear index cache") + assert clear and "message" in clear["result"], clear + print("OK: generic workspace command clears the current index cache") + + # Rebuilding must rescan new files, refresh the live index, and recreate + # the persistent cache without restarting the process. + (workspace / "Rebuilt.INI").write_text("MappedImage RebuiltImage\nEnd\n") + send({"jsonrpc": "2.0", "id": 26, "method": "workspace/executeCommand", + "params": {"command": "zerosyntax.rebuildIndexCache", "arguments": []}}) + rebuilt = wait_for(lambda m: m.get("id") == 26 and "result" in m, + "rebuild index cache") + assert rebuilt and rebuilt["result"].get("rebuilt") is True, rebuilt + send({"jsonrpc": "2.0", "id": 27, "method": "textDocument/completion", + "params": {"textDocument": {"uri": scan_uri}, + "position": {"line": 1, "character": 16}}}) + rebuilt_comp = wait_for(lambda m: m.get("id") == 27 and "result" in m, + "rebuilt completion") + rebuilt_items = rebuilt_comp["result"] + if isinstance(rebuilt_items, dict): + rebuilt_items = rebuilt_items.get("items", []) + assert "RebuiltImage" in [i["label"] for i in rebuilt_items], rebuilt_items[:10] + send({"jsonrpc": "2.0", "id": 28, "method": "workspace/executeCommand", + "params": {"command": "zerosyntax.clearIndexCache", "arguments": []}}) + recreated = wait_for(lambda m: m.get("id") == 28 and "result" in m, + "clear rebuilt index cache") + assert recreated and recreated["result"].get("cleared") is True, recreated + print("OK: rebuild command refreshes the live index and recreates the cache") + # 9) shutdown send({"jsonrpc": "2.0", "id": 99, "method": "shutdown", "params": None}) wait_for(lambda m: m.get("id") == 99, "shutdown") diff --git a/editors/vscode/package.json b/editors/vscode/package.json index d1bc0fd..c28a056 100644 --- a/editors/vscode/package.json +++ b/editors/vscode/package.json @@ -18,9 +18,21 @@ ], "main": "./out/extension.js", "activationEvents": [ - "onLanguage:generals-ini" + "onLanguage:generals-ini", + "onCommand:zerosyntax.clearIndexCache", + "onCommand:zerosyntax.rebuildIndexCache" ], "contributes": { + "commands": [ + { + "command": "zerosyntax.clearIndexCache", + "title": "ZeroSyntax: Clear Index Cache" + }, + { + "command": "zerosyntax.rebuildIndexCache", + "title": "ZeroSyntax: Rebuild Index Cache" + } + ], "languages": [ { "id": "generals-ini", diff --git a/editors/vscode/src/extension.ts b/editors/vscode/src/extension.ts index 41c2c67..7d9571d 100644 --- a/editors/vscode/src/extension.ts +++ b/editors/vscode/src/extension.ts @@ -11,7 +11,7 @@ import { let client: LanguageClient | undefined; let baseIniRootsHintShown = false; -export function activate(context: vscode.ExtensionContext) { +export async function activate(context: vscode.ExtensionContext) { const serverPath = resolveServerPath(context); if (!serverPath) { vscode.window.showErrorMessage( @@ -57,7 +57,7 @@ export function activate(context: vscode.ExtensionContext) { ) ); - client.start(); + await client.start(); context.subscriptions.push({ dispose: () => { void client?.stop(); From b7dd806c0f25210c613b3c85a0f8a3e853cc35ba Mon Sep 17 00:00:00 2001 From: ViTeXFTW Date: Sun, 12 Jul 2026 21:54:26 +0200 Subject: [PATCH 2/4] refactor: extract asset index module --- crates/server/src/asset_index.rs | 1029 +++++++++++++++++++++++++ crates/server/src/backend.rs | 1197 ++---------------------------- crates/server/src/main.rs | 1 + 3 files changed, 1078 insertions(+), 1149 deletions(-) create mode 100644 crates/server/src/asset_index.rs diff --git a/crates/server/src/asset_index.rs b/crates/server/src/asset_index.rs new file mode 100644 index 0000000..d2eff1b --- /dev/null +++ b/crates/server/src/asset_index.rs @@ -0,0 +1,1029 @@ +//! Asset ingestion for workspace and configured game roots. +//! +//! Owns discovery, INI/W3D/BIG parsing, persistent cache compatibility, and +//! bounded parallel scanning. The LSP backend supplies roots and progress, +//! then applies the ordered result to its live workspace index. + +use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; +use std::hash::{Hash, Hasher}; +use std::io::{Read, Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Arc; +use std::time::{Duration, Instant, UNIX_EPOCH}; + +use serde::{Deserialize, Serialize}; +use tower_lsp::lsp_types::Url; +use zerosyntax_analysis::index::{ + definitions_in, module_tags_in, references_in, Definition, ModelAsset, ReferenceSite, +}; +use zerosyntax_analysis::Analyzer; + +const INDEX_CACHE_VERSION: u32 = 1; +const MAX_SCAN_WORKERS: usize = 4; +const MAX_W3D_CHUNK_DEPTH: usize = 16; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ScanEntry { + pub(crate) file: String, + pub(crate) definitions: Vec, + pub(crate) references: Vec, + pub(crate) tags: Vec<(String, String)>, + pub(crate) models: Vec, + pub(crate) text: Option>, + pub(crate) is_base: bool, +} + +#[derive(Debug, Default)] +pub(crate) struct ScanStats { + pub(crate) discovery: Duration, + pub(crate) cache_load: Duration, + pub(crate) parse: Duration, + pub(crate) cache_write: Duration, + pub(crate) hits: usize, + pub(crate) misses: usize, +} + +#[derive(Debug, Default)] +pub(crate) struct ScanResult { + pub(crate) entries: Vec, + pub(crate) stats: ScanStats, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct CachedScanEntry { + file: String, + definitions: Vec, + references: Vec, + tags: Vec<(String, String)>, + models: Vec, + text: Option, +} + +impl CachedScanEntry { + fn from_scan(entry: &ScanEntry) -> Self { + Self { + file: entry.file.clone(), + definitions: entry.definitions.clone(), + references: entry.references.clone(), + tags: entry.tags.clone(), + models: entry.models.clone(), + text: entry.text.as_deref().map(str::to_string), + } + } + + fn into_scan(self, is_base: bool) -> ScanEntry { + ScanEntry { + file: self.file, + definitions: self.definitions, + references: self.references, + tags: self.tags, + models: self.models, + text: self.text.map(Arc::from), + is_base, + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct FileFingerprint { + len: u64, + modified_secs: u64, + modified_nanos: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +struct CachedFile { + fingerprint: FileFingerprint, + entries: Vec, +} + +#[derive(Debug, Serialize, Deserialize)] +struct IndexCache { + version: u32, + schema_hash: u64, + files: HashMap, +} + +impl IndexCache { + fn empty(schema_hash: u64) -> Self { + Self { + version: INDEX_CACHE_VERSION, + schema_hash, + files: HashMap::new(), + } + } +} + +#[derive(Debug, Clone)] +struct ScanPath { + path: PathBuf, + key: String, + is_base: bool, + fingerprint: FileFingerprint, +} + +struct BigEntry { + name: String, + offset: u64, + size: usize, +} + +fn read_lossy(path: &Path) -> Option { + std::fs::read(path) + .ok() + .map(|bytes| String::from_utf8_lossy(&bytes).into_owned()) +} + +pub(crate) fn load_sibling_str_keys(ini_url: &Url) -> Vec { + let path = match ini_url.to_file_path() { + Ok(path) => path, + Err(_) => return Vec::new(), + }; + if let Some(text) = read_lossy(&path.with_extension("str")) { + return parse_str_keys(&text); + } + read_lossy(&path.with_extension("STR")) + .map(|text| parse_str_keys(&text)) + .unwrap_or_default() +} + +fn parse_str_keys(content: &str) -> Vec { + let mut keys = Vec::new(); + let mut expect_key = true; + for line in content.lines() { + let trimmed = line.trim(); + if trimmed.is_empty() || trimmed.starts_with(';') { + continue; + } + if trimmed.eq_ignore_ascii_case("END") { + expect_key = true; + } else if expect_key { + keys.push(trimmed.to_string()); + expect_key = false; + } + } + keys +} + +fn schema_hash() -> u64 { + let mut hasher = DefaultHasher::new(); + zerosyntax_schema::EMBEDDED_SCHEMA_JSON.hash(&mut hasher); + hasher.finish() +} + +fn path_key(path: &Path) -> String { + let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); + let key = path.to_string_lossy().replace('\\', "/"); + if cfg!(windows) { + key.to_ascii_lowercase() + } else { + key + } +} + +fn file_fingerprint(path: &Path) -> Option { + let metadata = std::fs::metadata(path).ok()?; + let modified = metadata + .modified() + .ok() + .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) + .unwrap_or_default(); + Some(FileFingerprint { + len: metadata.len(), + modified_secs: modified.as_secs(), + modified_nanos: modified.subsec_nanos(), + }) +} + +fn cache_dir() -> PathBuf { + #[cfg(windows)] + if let Some(path) = std::env::var_os("LOCALAPPDATA") { + return PathBuf::from(path).join("zerosyntax"); + } + #[cfg(not(windows))] + { + if let Some(path) = std::env::var_os("XDG_CACHE_HOME") { + return PathBuf::from(path).join("zerosyntax"); + } + if let Some(path) = std::env::var_os("HOME") { + return PathBuf::from(path).join(".cache/zerosyntax"); + } + } + std::env::temp_dir().join("zerosyntax") +} + +fn cache_path(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> PathBuf { + let mut roots: Vec<_> = workspace_roots + .iter() + .map(|root| format!("workspace:{}", path_key(root))) + .chain( + base_roots + .iter() + .map(|root| format!("base:{}", path_key(root))), + ) + .collect(); + roots.sort_unstable(); + let mut hasher = DefaultHasher::new(); + roots.hash(&mut hasher); + cache_dir().join(format!( + "index-v{INDEX_CACHE_VERSION}-{:016x}.json", + hasher.finish() + )) +} + +fn load_index_cache(path: &Path, expected_schema_hash: u64) -> IndexCache { + let Some(bytes) = std::fs::read(path).ok() else { + return IndexCache::empty(expected_schema_hash); + }; + let Ok(cache) = serde_json::from_slice::(&bytes) else { + return IndexCache::empty(expected_schema_hash); + }; + if cache.version != INDEX_CACHE_VERSION || cache.schema_hash != expected_schema_hash { + return IndexCache::empty(expected_schema_hash); + } + cache +} + +fn write_index_cache(path: &Path, cache: &IndexCache) -> std::io::Result<()> { + let Some(parent) = path.parent() else { + return Ok(()); + }; + std::fs::create_dir_all(parent)?; + let bytes = serde_json::to_vec(cache).map_err(std::io::Error::other)?; + std::fs::write(path, bytes) +} + +pub(crate) fn clear_cache( + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], +) -> std::io::Result { + match std::fs::remove_file(cache_path(workspace_roots, base_roots)) { + Ok(()) => Ok(true), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), + Err(error) => Err(error), + } +} + +fn read_u32_be(reader: &mut R) -> std::io::Result { + let mut buf = [0u8; 4]; + reader.read_exact(&mut buf)?; + Ok(u32::from_be_bytes(buf)) +} + +fn read_c_string(reader: &mut R) -> std::io::Result { + let mut buf = Vec::new(); + let mut byte = [0u8; 1]; + loop { + reader.read_exact(&mut byte)?; + if byte[0] == 0 { + break; + } + buf.push(byte[0]); + } + Ok(String::from_utf8_lossy(&buf).into_owned()) +} + +fn big_entries(file: &mut std::fs::File) -> std::io::Result> { + let mut magic = [0u8; 4]; + file.read_exact(&mut magic)?; + if &magic != b"BIGF" { + return Ok(Vec::new()); + } + let _archive_size = read_u32_be(&mut *file)?; + let count = read_u32_be(&mut *file)?; + file.seek(SeekFrom::Start(0x10))?; + let mut entries = Vec::with_capacity(count as usize); + for _ in 0..count { + entries.push(BigEntry { + offset: read_u32_be(&mut *file)? as u64, + size: read_u32_be(&mut *file)? as usize, + name: read_c_string(&mut *file)?.replace('\\', "/"), + }); + } + Ok(entries) +} + +fn read_big_entry_bytes(file: &mut std::fs::File, entry: &BigEntry) -> Option> { + file.seek(SeekFrom::Start(entry.offset)).ok()?; + let mut bytes = vec![0; entry.size]; + file.read_exact(&mut bytes).ok()?; + Some(bytes) +} + +fn big_uri(path: &Path, entry: &str) -> String { + let archive = path.to_string_lossy().replace('\\', "/"); + let mut uri = Url::parse("big:///").expect("static big URI is valid"); + uri.set_path(&format!("{archive}!/{entry}")); + uri.to_string() +} + +fn file_stem_str(path: &str) -> String { + let file_name = path.rsplit(['/', '\\']).next().unwrap_or(path); + file_name + .rsplit_once('.') + .map(|(stem, _)| stem) + .unwrap_or(file_name) + .to_string() +} + +fn parse_w3d_models(bytes: &[u8], fallback_name: &str) -> Vec { + let mut names = Vec::new(); + let mut members = Vec::new(); + if !fallback_name.is_empty() { + names.push(fallback_name.to_string()); + } + walk_w3d_chunks(bytes, 0, bytes.len(), 0, &mut |kind, payload| match kind { + 0x0000_001F if payload.len() >= 40 => { + push_name(&mut members, read_fixed_name(&payload[8..24])); + push_name(&mut names, read_fixed_name(&payload[24..40])); + } + 0x0000_0101 | 0x0000_0501 | 0x0000_0601 if payload.len() >= 20 => { + push_name(&mut names, read_fixed_name(&payload[4..20])); + } + 0x0000_0102 => { + for pivot in payload.chunks_exact(60) { + push_name(&mut members, read_fixed_name(&pivot[..16])); + } + } + 0x0000_0701 if payload.len() >= 40 => { + push_name(&mut names, read_fixed_name(&payload[8..24])); + push_name(&mut names, read_fixed_name(&payload[24..40])); + } + 0x0000_0704 if payload.len() >= 36 => { + push_name(&mut members, read_fixed_name(&payload[4..36])); + } + 0x0000_0740 if payload.len() >= 40 => { + push_name(&mut members, read_fixed_name(&payload[8..40])); + } + 0x0000_0750 if payload.len() >= 48 => { + push_name(&mut members, read_fixed_name(&payload[16..48])) + } + _ => {} + }); + dedup_case_insensitive(&mut names); + dedup_case_insensitive(&mut members); + names + .into_iter() + .filter(|name| !name.is_empty()) + .map(|name| ModelAsset { + name, + members: members.clone(), + }) + .collect() +} + +fn walk_w3d_chunks( + bytes: &[u8], + mut pos: usize, + end: usize, + depth: usize, + f: &mut impl FnMut(u32, &[u8]), +) { + if depth > MAX_W3D_CHUNK_DEPTH { + return; + } + while pos + 8 <= end && pos + 8 <= bytes.len() { + let kind = u32::from_le_bytes(bytes[pos..pos + 4].try_into().unwrap()); + let size_raw = u32::from_le_bytes(bytes[pos + 4..pos + 8].try_into().unwrap()); + let has_children = (size_raw & 0x8000_0000) != 0 || is_w3d_container(kind); + let size = (size_raw & 0x7fff_ffff) as usize; + let payload_start = pos + 8; + let Some(payload_end) = payload_start.checked_add(size) else { + break; + }; + if payload_end > end || payload_end > bytes.len() { + break; + } + let payload = &bytes[payload_start..payload_end]; + f(kind, payload); + if has_children { + walk_w3d_chunks(bytes, payload_start, payload_end, depth + 1, f); + } + pos = payload_end; + } +} + +fn is_w3d_container(kind: u32) -> bool { + matches!( + kind, + 0x0000_0000 + | 0x0000_0100 + | 0x0000_0500 + | 0x0000_0600 + | 0x0000_0700 + | 0x0000_0702 + | 0x0000_0705 + ) +} + +fn read_fixed_name(bytes: &[u8]) -> &str { + let end = bytes + .iter() + .position(|byte| *byte == 0) + .unwrap_or(bytes.len()); + std::str::from_utf8(&bytes[..end]).unwrap_or("").trim() +} + +fn push_name(out: &mut Vec, name: &str) { + if name.is_empty() { + return; + } + out.push(name.to_string()); + if let Some((_, short)) = name.rsplit_once('.') { + if !short.is_empty() { + out.push(short.to_string()); + } + } +} + +fn dedup_case_insensitive(values: &mut Vec) { + let mut seen = HashSet::new(); + values.retain(|value| seen.insert(value.to_ascii_lowercase())); +} + +fn scan_big(analyzer: &Analyzer, path: &Path, is_base: bool) -> Vec { + let mut out = Vec::new(); + let Ok(mut file) = std::fs::File::open(path) else { + return out; + }; + let Ok(entries) = big_entries(&mut file) else { + return out; + }; + for entry in entries { + let uri = big_uri(path, &entry.name); + if entry.name.ends_with(".ini") || entry.name.ends_with(".INI") { + let Some(bytes) = read_big_entry_bytes(&mut file, &entry) else { + continue; + }; + let text = String::from_utf8_lossy(&bytes).into_owned(); + let parse = analyzer.parse(&text); + out.push(ScanEntry { + definitions: definitions_in(analyzer, &parse, &uri), + references: references_in(analyzer, &parse), + tags: module_tags_in(analyzer, &parse), + file: uri, + models: Vec::new(), + text: Some(Arc::from(text)), + is_base, + }); + } else if entry.name.ends_with(".w3d") || entry.name.ends_with(".W3D") { + let Some(bytes) = read_big_entry_bytes(&mut file, &entry) else { + continue; + }; + let models = parse_w3d_models(&bytes, &file_stem_str(&entry.name)); + if !models.is_empty() { + out.push(ScanEntry { + file: uri, + definitions: Vec::new(), + references: Vec::new(), + tags: Vec::new(), + models, + text: None, + is_base, + }); + } + } + } + out +} + +fn collect_scan_paths(roots: &[PathBuf]) -> Vec { + let mut out = Vec::new(); + for root in roots { + if root + .extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| extension.eq_ignore_ascii_case("big")) + { + out.push(root.clone()); + continue; + } + for entry in walkdir::WalkDir::new(root) + .into_iter() + .filter_map(Result::ok) + { + let path = entry.path(); + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or(""); + if extension.eq_ignore_ascii_case("big") + || extension.eq_ignore_ascii_case("ini") + || extension.eq_ignore_ascii_case("w3d") + { + out.push(path.to_path_buf()); + } + } + } + out +} + +fn collect_scan_plan(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> Vec { + let mut seen = HashSet::new(); + let mut out = Vec::new(); + for (roots, is_base) in [(base_roots, true), (workspace_roots, false)] { + for path in collect_scan_paths(roots) { + let key = path_key(&path); + let Some(fingerprint) = file_fingerprint(&path) else { + continue; + }; + if seen.insert(key.clone()) { + out.push(ScanPath { + path, + key, + is_base, + fingerprint, + }); + } + } + } + out +} + +fn scan_path(analyzer: &Analyzer, path: &Path, is_base: bool) -> Vec { + let extension = path + .extension() + .and_then(|value| value.to_str()) + .unwrap_or(""); + if extension.eq_ignore_ascii_case("big") { + return scan_big(analyzer, path, is_base); + } + if extension.eq_ignore_ascii_case("ini") { + if let (Some(text), Ok(uri)) = (read_lossy(path), Url::from_file_path(path)) { + let parse = analyzer.parse(&text); + return vec![ScanEntry { + definitions: definitions_in(analyzer, &parse, uri.as_str()), + references: references_in(analyzer, &parse), + tags: module_tags_in(analyzer, &parse), + file: uri.to_string(), + models: Vec::new(), + text: None, + is_base, + }]; + } + } else if extension.eq_ignore_ascii_case("w3d") { + if let (Ok(bytes), Ok(uri)) = (std::fs::read(path), Url::from_file_path(path)) { + let stem = path + .file_stem() + .and_then(|value| value.to_str()) + .unwrap_or_default(); + let models = parse_w3d_models(&bytes, stem); + if !models.is_empty() { + return vec![ScanEntry { + file: uri.to_string(), + definitions: Vec::new(), + references: Vec::new(), + tags: Vec::new(), + models, + text: None, + is_base, + }]; + } + } + } + Vec::new() +} + +pub(crate) fn scan( + analyzer: &Analyzer, + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], + progress: &mut impl FnMut(usize, usize), +) -> ScanResult { + let discovery_started = Instant::now(); + let paths = collect_scan_plan(workspace_roots, base_roots); + let discovery = discovery_started.elapsed(); + let workers = std::thread::available_parallelism() + .map(usize::from) + .unwrap_or(1) + .min(MAX_SCAN_WORKERS); + let (entries, mut stats) = scan_with_cache( + analyzer, + &paths, + &cache_path(workspace_roots, base_roots), + workers, + progress, + ); + stats.discovery = discovery; + ScanResult { entries, stats } +} + +fn scan_with_cache( + analyzer: &Analyzer, + paths: &[ScanPath], + cache_path: &Path, + workers: usize, + progress: &mut impl FnMut(usize, usize), +) -> (Vec, ScanStats) { + let expected_schema_hash = schema_hash(); + let load_started = Instant::now(); + let mut old_cache = load_index_cache(cache_path, expected_schema_hash); + let cache_load = load_started.elapsed(); + let old_file_count = old_cache.files.len(); + let mut results: Vec>> = (0..paths.len()).map(|_| None).collect(); + let mut misses = Vec::new(); + let mut hits = 0; + let mut done = 0; + + for (index, path) in paths.iter().enumerate() { + match old_cache.files.remove(&path.key) { + Some(cached) if cached.fingerprint == path.fingerprint => { + results[index] = Some( + cached + .entries + .into_iter() + .map(|entry| entry.into_scan(path.is_base)) + .collect(), + ); + hits += 1; + done += 1; + progress(done, paths.len()); + } + _ => misses.push(index), + } + } + + let parse_started = Instant::now(); + if !misses.is_empty() { + let worker_count = workers.max(1).min(misses.len()); + let next = AtomicUsize::new(0); + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + for _ in 0..worker_count { + let tx = tx.clone(); + let misses = &misses; + let next = &next; + scope.spawn(move || loop { + let work = next.fetch_add(1, Ordering::Relaxed); + let Some(&index) = misses.get(work) else { + break; + }; + let path = &paths[index]; + if tx + .send((index, scan_path(analyzer, &path.path, path.is_base))) + .is_err() + { + break; + } + }); + } + drop(tx); + for (index, entries) in rx { + results[index] = Some(entries); + done += 1; + progress(done, paths.len()); + } + }); + } + let parse = parse_started.elapsed(); + + let mut cache = IndexCache::empty(expected_schema_hash); + let mut scanned = Vec::new(); + for (path, entries) in paths.iter().zip(results) { + let entries = entries.unwrap_or_default(); + cache.files.insert( + path.key.clone(), + CachedFile { + fingerprint: path.fingerprint.clone(), + entries: entries.iter().map(CachedScanEntry::from_scan).collect(), + }, + ); + scanned.extend(entries); + } + + let write_started = Instant::now(); + if (!misses.is_empty() || old_file_count != paths.len()) + && write_index_cache(cache_path, &cache).is_err() + { + tracing::warn!(path = %cache_path.display(), "could not write asset index cache"); + } + let cache_write = write_started.elapsed(); + ( + scanned, + ScanStats { + cache_load, + parse, + cache_write, + hits, + misses: misses.len(), + ..ScanStats::default() + }, + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use zerosyntax_analysis::{completion, WorkspaceIndex}; + + fn test_dir(label: &str) -> PathBuf { + static NEXT: AtomicUsize = AtomicUsize::new(0); + let dir = std::env::temp_dir().join(format!( + "zerosyntax-{label}-{}-{}", + std::process::id(), + NEXT.fetch_add(1, Ordering::Relaxed) + )); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir + } + + fn w3d_with_member(member: &str) -> Vec { + let mut pivot = vec![0; 60]; + let name = member.as_bytes(); + pivot[..name.len().min(16)].copy_from_slice(&name[..name.len().min(16)]); + let mut bytes = Vec::new(); + bytes.extend_from_slice(&0x0000_0102u32.to_le_bytes()); + bytes.extend_from_slice(&(pivot.len() as u32).to_le_bytes()); + bytes.extend_from_slice(&pivot); + bytes + } + + fn fixed(name: &str) -> [u8; N] { + let mut out = [0; N]; + let bytes = name.as_bytes(); + out[..bytes.len().min(N)].copy_from_slice(&bytes[..bytes.len().min(N)]); + out + } + + fn chunk(kind: u32, payload: Vec) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice(&kind.to_le_bytes()); + out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); + out.extend_from_slice(&payload); + out + } + + fn write_big(path: &Path, entries: &[(&str, Vec)]) { + let directory_len: usize = entries.iter().map(|(name, _)| 8 + name.len() + 1).sum(); + let mut offset = 0x10 + directory_len; + let archive_size = offset + entries.iter().map(|(_, bytes)| bytes.len()).sum::(); + let mut out = Vec::with_capacity(archive_size); + out.extend_from_slice(b"BIGF"); + out.extend_from_slice(&(archive_size as u32).to_be_bytes()); + out.extend_from_slice(&(entries.len() as u32).to_be_bytes()); + out.extend_from_slice(&0u32.to_be_bytes()); + for (name, bytes) in entries { + out.extend_from_slice(&(offset as u32).to_be_bytes()); + out.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); + out.extend_from_slice(name.as_bytes()); + out.push(0); + offset += bytes.len(); + } + for (_, bytes) in entries { + out.extend_from_slice(bytes); + } + std::fs::write(path, out).unwrap(); + } + + fn cached_scan( + analyzer: &Analyzer, + workspace_roots: &[PathBuf], + base_roots: &[PathBuf], + cache: &Path, + workers: usize, + ) -> (Vec, ScanStats) { + let paths = collect_scan_plan(workspace_roots, base_roots); + scan_with_cache(analyzer, &paths, cache, workers, &mut |_, _| {}) + } + + #[test] + fn removes_only_the_requested_index_cache() { + let dir = test_dir("clear-cache"); + std::fs::write(dir.join("Object.ini"), "Object CachedTank\nEnd\n").unwrap(); + let roots = std::slice::from_ref(&dir); + let _ = scan(&Analyzer::embedded(), &[], roots, &mut |_, _| {}); + assert!(clear_cache(&[], roots).unwrap()); + assert!(!clear_cache(&[], roots).unwrap()); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn cache_reuses_complete_scan_and_invalidates_changed_loose_file() { + let dir = test_dir("cache-loose"); + let cache = dir.join("cache.json"); + std::fs::write(dir.join("Object.ini"), "Object CachedTank\nEnd\n").unwrap(); + let model = dir.join("Tank.w3d"); + std::fs::write(&model, w3d_with_member("Muzzle01")).unwrap(); + let analyzer = Analyzer::embedded(); + let roots = std::slice::from_ref(&dir); + let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((cold_stats.hits, cold_stats.misses), (0, 2)); + let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((warm_stats.hits, warm_stats.misses), (2, 0)); + assert_eq!(cold, warm); + let mut changed = w3d_with_member("Muzzle02"); + changed.push(0); + std::fs::write(&model, changed).unwrap(); + let (_, changed_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((changed_stats.hits, changed_stats.misses), (1, 1)); + + let mut incompatible = load_index_cache(&cache, schema_hash()); + incompatible.schema_hash ^= 1; + write_index_cache(&cache, &incompatible).unwrap(); + let (_, schema_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((schema_stats.hits, schema_stats.misses), (0, 2)); + + std::fs::write(&cache, b"not json").unwrap(); + let (_, corrupt_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((corrupt_stats.hits, corrupt_stats.misses), (0, 2)); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn cache_invalidates_changed_big_archive_as_one_file() { + let dir = test_dir("cache-big"); + let archive = dir.join("Models.big"); + let cache = dir.join("cache.json"); + write_big( + &archive, + &[ + ("Data/INI/Object.ini", b"Object BigTank\nEnd\n".to_vec()), + ("Art/Tank.w3d", w3d_with_member("Muzzle01")), + ], + ); + let analyzer = Analyzer::embedded(); + let roots = std::slice::from_ref(&archive); + let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); + assert_eq!((cold_stats.hits, cold_stats.misses), (0, 1)); + assert_eq!(cold.len(), 2); + let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); + assert_eq!((warm_stats.hits, warm_stats.misses), (1, 0)); + assert_eq!(cold, warm); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn parallel_scan_preserves_order_and_overlapping_roots_are_deduped() { + let dir = test_dir("parallel"); + for index in 0..32 { + std::fs::write( + dir.join(format!("Model{index:02}.w3d")), + w3d_with_member(&format!("Bone{index:02}")), + ) + .unwrap(); + } + let roots = std::slice::from_ref(&dir); + let plan = collect_scan_plan(roots, roots); + assert_eq!(plan.len(), 32); + assert!(plan.iter().all(|path| path.is_base)); + let analyzer = Analyzer::embedded(); + let (serial, _) = cached_scan(&analyzer, &[], roots, &dir.join("serial.json"), 1); + let (parallel, _) = cached_scan(&analyzer, &[], roots, &dir.join("parallel.json"), 4); + assert_eq!(serial, parallel); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + #[ignore = "3,000-file synthetic timing harness; run with --ignored --nocapture"] + fn synthetic_3000_asset_cold_and_warm_timings() { + let dir = test_dir("benchmark"); + for index in 0..3000 { + std::fs::write( + dir.join(format!("Model{index:04}.w3d")), + w3d_with_member(&format!("Bone{:02}", index % 100)), + ) + .unwrap(); + } + let analyzer = Analyzer::embedded(); + let roots = std::slice::from_ref(&dir); + let mut serial_times = Vec::new(); + let mut parallel_times = Vec::new(); + let mut expected = None; + for run in 0..3 { + let started = Instant::now(); + let (out, _) = cached_scan( + &analyzer, + &[], + roots, + &dir.join(format!("serial-{run}.json")), + 1, + ); + serial_times.push(started.elapsed()); + expected.get_or_insert(out); + let started = Instant::now(); + let (out, _) = cached_scan( + &analyzer, + &[], + roots, + &dir.join(format!("parallel-{run}.json")), + 4, + ); + parallel_times.push(started.elapsed()); + assert_eq!(expected.as_ref(), Some(&out)); + } + serial_times.sort_unstable(); + parallel_times.sort_unstable(); + let warm_started = Instant::now(); + let (_, warm_stats) = cached_scan(&analyzer, &[], roots, &dir.join("parallel-2.json"), 4); + assert_eq!((warm_stats.hits, warm_stats.misses), (3000, 0)); + eprintln!( + "3,000 W3Ds: serial median {:?}, parallel median {:?}, warm {:?}", + serial_times[1], + parallel_times[1], + warm_started.elapsed() + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn scans_ini_from_big_archive() { + let dir = test_dir("big"); + let path = dir.join("test.big"); + write_big( + &path, + &[( + "Data/INI/Test.ini", + b"Object BigArchiveObject\nEnd\n".to_vec(), + )], + ); + let scanned = scan_big(&Analyzer::embedded(), &path, true); + assert_eq!(scanned.len(), 1); + assert!(scanned[0] + .definitions + .iter() + .any(|definition| definition.name == "BigArchiveObject")); + assert_eq!( + scanned[0].text.as_deref(), + Some("Object BigArchiveObject\nEnd\n") + ); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn w3d_root_scan_powers_model_and_bone_completions() { + let dir = test_dir("w3d"); + std::fs::write(dir.join("Good.w3d"), w3d_with_member("Tire01")).unwrap(); + let analyzer = Analyzer::embedded(); + let result = scan(&analyzer, &[], std::slice::from_ref(&dir), &mut |_, _| {}); + let mut index = WorkspaceIndex::new(); + for entry in result.entries { + index.set_file(&entry.file, entry.definitions); + index.set_file_refs(&entry.file, entry.references); + index.set_file_tags(&entry.file, entry.tags); + index.set_file_models(&entry.file, entry.models); + } + let source = "Object Tank\n Draw = W3DTankDraw ModuleTag_01\n DefaultConditionState\n Model = Good\n HideSubObject = \n End\n End\nEnd\n"; + let parse = analyzer.parse(source); + let offset = source.find("HideSubObject = ").unwrap() + "HideSubObject = ".len(); + let labels: Vec<_> = + completion::complete(&analyzer, &parse, offset as u32, Some(&index), None) + .into_iter() + .map(|completion| completion.label) + .collect(); + assert!(labels.contains(&"Tire".to_string())); + let _ = std::fs::remove_dir_all(dir); + } + + #[test] + fn parses_w3d_model_names_and_members() { + let mut hlod = Vec::new(); + hlod.extend_from_slice(&1u32.to_le_bytes()); + hlod.extend_from_slice(&1u32.to_le_bytes()); + hlod.extend_from_slice(&fixed::<16>("Good")); + hlod.extend_from_slice(&fixed::<16>("Good")); + let mut pivot = Vec::new(); + pivot.extend_from_slice(&fixed::<16>("Tire01")); + pivot.resize(60, 0); + let mut sub = Vec::new(); + sub.extend_from_slice(&0u32.to_le_bytes()); + sub.extend_from_slice(&fixed::<32>("Good.Cargo01")); + let mut bytes = Vec::new(); + bytes.extend(chunk(0x0000_0701, hlod)); + bytes.extend(chunk(0x0000_0102, pivot)); + bytes.extend(chunk(0x0000_0704, sub)); + let models = parse_w3d_models(&bytes, "Fallback"); + let good = models.iter().find(|model| model.name == "Good").unwrap(); + assert!(good.members.iter().any(|member| member == "Tire01")); + assert!(good.members.iter().any(|member| member == "Cargo01")); + } + + #[test] + fn parses_w3d_aggregate_and_emitter_names() { + let mut header = Vec::new(); + header.extend_from_slice(&1u32.to_le_bytes()); + header.extend_from_slice(&fixed::<16>("Aggro")); + let mut bytes = chunk(0x0000_0600, chunk(0x0000_0601, header)); + let mut header = Vec::new(); + header.extend_from_slice(&1u32.to_le_bytes()); + header.extend_from_slice(&fixed::<16>("Smoke")); + bytes.extend(chunk(0x0000_0500, chunk(0x0000_0501, header))); + let models = parse_w3d_models(&bytes, ""); + assert!(models.iter().any(|model| model.name == "Aggro")); + assert!(models.iter().any(|model| model.name == "Smoke")); + } + + #[test] + fn w3d_chunk_walker_survives_hostile_deep_nesting() { + let total = 64 * 1024; + let mut bytes = Vec::with_capacity(total); + let mut remaining = total; + while remaining >= 8 { + bytes.extend_from_slice(&0x0000_0700u32.to_le_bytes()); + bytes.extend_from_slice(&((remaining - 8) as u32).to_le_bytes()); + remaining -= 8; + } + let _ = parse_w3d_models(&bytes, "Fallback"); + } +} diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index 386440d..df5bfcb 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -6,28 +6,24 @@ //! `didChange` deltas are applied to the rope and the document is re-parsed //! once per change batch; read-only requests reuse the cached parse. -use std::collections::{hash_map::DefaultHasher, HashMap, HashSet}; -use std::hash::{Hash, Hasher}; -use std::io::{Read, Seek, SeekFrom}; -use std::path::{Path, PathBuf}; +use std::collections::HashSet; +use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock, RwLock}; -use std::time::{Duration, Instant, UNIX_EPOCH}; +use std::time::Instant; use dashmap::DashMap; use ropey::Rope; -use serde::{Deserialize, Serialize}; +use serde::Deserialize; use tower_lsp::lsp_types::*; use tower_lsp::{jsonrpc::Result, Client, LanguageServer}; use zerosyntax_analysis::diagnostics::DiagnosticsCache; -use zerosyntax_analysis::index::{ - definitions_in, module_tags_in, references_in, Definition, ModelAsset, ReferenceSite, - WorkspaceIndex, -}; +use zerosyntax_analysis::index::{definitions_in, module_tags_in, references_in, WorkspaceIndex}; use zerosyntax_analysis::nav::{definition_at, hover_at, reference_at, HoverInfo}; use zerosyntax_analysis::{actions, completion, diagnostics, format, outline, semantic, Analyzer}; use zerosyntax_syntax::{Edit, Parse}; +use crate::asset_index; use crate::convert::{self, PositionEnc}; /// An open document: its text (as both a rope for position math and a string @@ -87,217 +83,14 @@ pub struct Backend { semantic_result_id: std::sync::atomic::AtomicU64, } -type ScanEntry = ( - String, - Vec, - Vec, - Vec<(String, String)>, - Vec, - Option>, -); - -const INDEX_CACHE_VERSION: u32 = 1; -const MAX_SCAN_WORKERS: usize = 4; const CLEAR_INDEX_CACHE_COMMAND: &str = "zerosyntax.clearIndexCache"; const REBUILD_INDEX_CACHE_COMMAND: &str = "zerosyntax.rebuildIndexCache"; -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -struct CachedScanEntry { - file: String, - definitions: Vec, - references: Vec, - tags: Vec<(String, String)>, - models: Vec, - text: Option, -} - -impl CachedScanEntry { - fn from_scan(entry: &ScanEntry) -> Self { - Self { - file: entry.0.clone(), - definitions: entry.1.clone(), - references: entry.2.clone(), - tags: entry.3.clone(), - models: entry.4.clone(), - text: entry.5.as_deref().map(str::to_string), - } - } - - fn into_scan(self) -> ScanEntry { - ( - self.file, - self.definitions, - self.references, - self.tags, - self.models, - self.text.map(Arc::from), - ) - } -} - -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -struct FileFingerprint { - len: u64, - modified_secs: u64, - modified_nanos: u32, -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -struct CachedFile { - fingerprint: FileFingerprint, - entries: Vec, -} - -#[derive(Debug, Serialize, Deserialize)] -struct IndexCache { - version: u32, - schema_hash: u64, - files: HashMap, -} - -impl IndexCache { - fn empty(schema_hash: u64) -> Self { - Self { - version: INDEX_CACHE_VERSION, - schema_hash, - files: HashMap::new(), - } - } -} - -#[derive(Debug, Clone)] -struct ScanPath { - path: PathBuf, - key: String, - is_base: bool, - fingerprint: FileFingerprint, -} - -#[derive(Debug, Default)] -struct ScanStats { - discovery: Duration, - cache_load: Duration, - parse: Duration, - cache_write: Duration, - hits: usize, - misses: usize, -} - #[derive(Deserialize)] pub struct VirtualFileParams { uri: String, } -struct BigEntry { - name: String, - offset: u64, - size: usize, -} - -/// Read a file leniently: real INIs predate UTF-8 (Windows-1252 comments), so -/// a strict `read_to_string` would silently drop them from the index. -fn read_lossy(path: &Path) -> Option { - std::fs::read(path) - .ok() - .map(|b| String::from_utf8_lossy(&b).into_owned()) -} - -fn schema_hash() -> u64 { - let mut hasher = DefaultHasher::new(); - zerosyntax_schema::EMBEDDED_SCHEMA_JSON.hash(&mut hasher); - hasher.finish() -} - -fn path_key(path: &Path) -> String { - let path = std::fs::canonicalize(path).unwrap_or_else(|_| path.to_path_buf()); - let key = path.to_string_lossy().replace('\\', "/"); - if cfg!(windows) { - key.to_ascii_lowercase() - } else { - key - } -} - -fn file_fingerprint(path: &Path) -> Option { - let metadata = std::fs::metadata(path).ok()?; - let modified = metadata - .modified() - .ok() - .and_then(|time| time.duration_since(UNIX_EPOCH).ok()) - .unwrap_or_default(); - Some(FileFingerprint { - len: metadata.len(), - modified_secs: modified.as_secs(), - modified_nanos: modified.subsec_nanos(), - }) -} - -fn cache_dir() -> PathBuf { - #[cfg(windows)] - if let Some(path) = std::env::var_os("LOCALAPPDATA") { - return PathBuf::from(path).join("zerosyntax"); - } - #[cfg(not(windows))] - { - if let Some(path) = std::env::var_os("XDG_CACHE_HOME") { - return PathBuf::from(path).join("zerosyntax"); - } - if let Some(path) = std::env::var_os("HOME") { - return PathBuf::from(path).join(".cache/zerosyntax"); - } - } - std::env::temp_dir().join("zerosyntax") -} - -fn index_cache_path(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> PathBuf { - let mut roots: Vec<_> = workspace_roots - .iter() - .map(|root| format!("workspace:{}", path_key(root))) - .chain( - base_roots - .iter() - .map(|root| format!("base:{}", path_key(root))), - ) - .collect(); - roots.sort_unstable(); - let mut hasher = DefaultHasher::new(); - roots.hash(&mut hasher); - cache_dir().join(format!( - "index-v{INDEX_CACHE_VERSION}-{:016x}.json", - hasher.finish() - )) -} - -fn load_index_cache(path: &Path, expected_schema_hash: u64) -> IndexCache { - let Some(bytes) = std::fs::read(path).ok() else { - return IndexCache::empty(expected_schema_hash); - }; - let Ok(cache) = serde_json::from_slice::(&bytes) else { - return IndexCache::empty(expected_schema_hash); - }; - if cache.version != INDEX_CACHE_VERSION || cache.schema_hash != expected_schema_hash { - return IndexCache::empty(expected_schema_hash); - } - cache -} - -fn write_index_cache(path: &Path, cache: &IndexCache) -> std::io::Result<()> { - let Some(parent) = path.parent() else { - return Ok(()); - }; - std::fs::create_dir_all(parent)?; - let bytes = serde_json::to_vec(cache).map_err(std::io::Error::other)?; - std::fs::write(path, bytes) -} - -fn remove_index_cache(path: &Path) -> std::io::Result { - match std::fs::remove_file(path) { - Ok(()) => Ok(true), - Err(error) if error.kind() == std::io::ErrorKind::NotFound => Ok(false), - Err(error) => Err(error), - } -} - /// Normalise a client-supplied URI to the form [`Url::from_file_path`] produces. /// On Windows, VS Code sends `file:///c%3A/…` (percent-encoded colon, lowercase /// drive letter) while `from_file_path` produces `file:///C:/…`. The mismatch @@ -322,485 +115,6 @@ fn is_map_layer_file(file: &str) -> bool { }) } -/// Parse string table key names from a `.str` file (the Generals INI string format). -/// Each block starts with a bare key name, followed by `LANG: "text"` lines, and `END`. -fn parse_str_keys(content: &str) -> Vec { - let mut keys = Vec::new(); - let mut expect_key = true; - for line in content.lines() { - let trimmed = line.trim(); - if trimmed.is_empty() || trimmed.starts_with(';') { - continue; - } - if trimmed.eq_ignore_ascii_case("END") { - expect_key = true; - } else if expect_key { - keys.push(trimmed.to_string()); - expect_key = false; - } - // else: inside a block (language entries) — skip - } - keys -} - -/// Read the sibling `.str` file for `ini_url` (same directory, same basename) -/// and return its string table keys, or an empty vec if not found. -fn load_sibling_str_keys(ini_url: &Url) -> Vec { - let path = match ini_url.to_file_path() { - Ok(p) => p, - Err(_) => return Vec::new(), - }; - // map.ini → map.str (same stem, .str extension) - let str_path = path.with_extension("str"); - if let Some(text) = read_lossy(&str_path) { - return parse_str_keys(&text); - } - // Also try the uppercase variant (game data ships with mixed casing). - let str_upper = path.with_extension("STR"); - read_lossy(&str_upper) - .map(|t| parse_str_keys(&t)) - .unwrap_or_default() -} - -fn read_u32_be(reader: &mut R) -> std::io::Result { - let mut buf = [0u8; 4]; - reader.read_exact(&mut buf)?; - Ok(u32::from_be_bytes(buf)) -} - -fn read_c_string(reader: &mut R) -> std::io::Result { - let mut buf = Vec::new(); - let mut byte = [0u8; 1]; - loop { - reader.read_exact(&mut byte)?; - if byte[0] == 0 { - break; - } - buf.push(byte[0]); - } - Ok(String::from_utf8_lossy(&buf).into_owned()) -} - -fn big_entries(file: &mut std::fs::File) -> std::io::Result> { - let mut magic = [0u8; 4]; - file.read_exact(&mut magic)?; - if &magic != b"BIGF" { - return Ok(Vec::new()); - } - - let _archive_size = read_u32_be(&mut *file)?; - let count = read_u32_be(&mut *file)?; - file.seek(SeekFrom::Start(0x10))?; - - let mut entries = Vec::with_capacity(count as usize); - for _ in 0..count { - let offset = read_u32_be(&mut *file)? as u64; - let size = read_u32_be(&mut *file)? as usize; - let name = read_c_string(&mut *file)?.replace('\\', "/"); - entries.push(BigEntry { name, offset, size }); - } - Ok(entries) -} - -fn read_big_entry_bytes(file: &mut std::fs::File, entry: &BigEntry) -> Option> { - file.seek(SeekFrom::Start(entry.offset)).ok()?; - let mut bytes = vec![0; entry.size]; - file.read_exact(&mut bytes).ok()?; - Some(bytes) -} - -fn read_big_entry(file: &mut std::fs::File, entry: &BigEntry) -> Option { - let bytes = read_big_entry_bytes(file, entry)?; - Some(String::from_utf8_lossy(&bytes).into_owned()) -} - -fn big_uri(path: &Path, entry: &str) -> String { - let archive = path.to_string_lossy().replace('\\', "/"); - let mut uri = Url::parse("big:///").expect("static big URI is valid"); - uri.set_path(&format!("{archive}!/{entry}")); - uri.to_string() -} - -fn file_stem_str(path: &str) -> String { - let file_name = path.rsplit(['/', '\\']).next().unwrap_or(path); - file_name - .rsplit_once('.') - .map(|(stem, _)| stem) - .unwrap_or(file_name) - .to_string() -} - -fn parse_w3d_models(bytes: &[u8], fallback_name: &str) -> Vec { - let mut names = Vec::new(); - let mut members = Vec::new(); - if !fallback_name.is_empty() { - names.push(fallback_name.to_string()); - } - walk_w3d_chunks(bytes, 0, bytes.len(), 0, &mut |kind, payload| match kind { - 0x0000_001F if payload.len() >= 40 => { - push_name(&mut members, read_fixed_name(&payload[8..24])); - push_name(&mut names, read_fixed_name(&payload[24..40])); - } - // HIERARCHY_HEADER, EMITTER_HEADER, AGGREGATE_HEADER: Version + Name[16]. - 0x0000_0101 | 0x0000_0501 | 0x0000_0601 if payload.len() >= 20 => { - push_name(&mut names, read_fixed_name(&payload[4..20])); - } - 0x0000_0102 => { - for pivot in payload.chunks_exact(60) { - push_name(&mut members, read_fixed_name(&pivot[..16])); - } - } - 0x0000_0701 if payload.len() >= 40 => { - push_name(&mut names, read_fixed_name(&payload[8..24])); - push_name(&mut names, read_fixed_name(&payload[24..40])); - } - 0x0000_0704 if payload.len() >= 36 => { - push_name(&mut members, read_fixed_name(&payload[4..36])); - } - 0x0000_0740 if payload.len() >= 40 => { - push_name(&mut members, read_fixed_name(&payload[8..40])); - } - 0x0000_0750 if payload.len() >= 48 => { - push_name(&mut members, read_fixed_name(&payload[16..48])) - } - _ => {} - }); - dedup_case_insensitive(&mut names); - dedup_case_insensitive(&mut members); - names - .into_iter() - .filter(|name| !name.is_empty()) - .map(|name| ModelAsset { - name, - members: members.clone(), - }) - .collect() -} - -/// Real W3D files nest at most a handful of levels; the cap only guards -/// against corrupt or hostile files driving unbounded recursion. -const MAX_W3D_CHUNK_DEPTH: usize = 16; - -fn walk_w3d_chunks( - bytes: &[u8], - mut pos: usize, - end: usize, - depth: usize, - f: &mut impl FnMut(u32, &[u8]), -) { - if depth > MAX_W3D_CHUNK_DEPTH { - return; - } - while pos + 8 <= end && pos + 8 <= bytes.len() { - let kind = u32::from_le_bytes(bytes[pos..pos + 4].try_into().unwrap()); - let size_raw = u32::from_le_bytes(bytes[pos + 4..pos + 8].try_into().unwrap()); - let has_children = (size_raw & 0x8000_0000) != 0 || is_w3d_container(kind); - let size = (size_raw & 0x7fff_ffff) as usize; - let payload_start = pos + 8; - let Some(payload_end) = payload_start.checked_add(size) else { - break; - }; - if payload_end > end || payload_end > bytes.len() { - break; - } - let payload = &bytes[payload_start..payload_end]; - f(kind, payload); - if has_children { - walk_w3d_chunks(bytes, payload_start, payload_end, depth + 1, f); - } - pos = payload_end; - } -} - -fn is_w3d_container(kind: u32) -> bool { - // MESH, HIERARCHY, EMITTER, AGGREGATE, HLOD, HLOD_LOD_ARRAY, - // HLOD_AGGREGATE_ARRAY (w3d_file.h; the MSB size flag is the primary - // signal, these are fallbacks for files that omit it). - matches!( - kind, - 0x0000_0000 - | 0x0000_0100 - | 0x0000_0500 - | 0x0000_0600 - | 0x0000_0700 - | 0x0000_0702 - | 0x0000_0705 - ) -} - -fn read_fixed_name(bytes: &[u8]) -> &str { - let end = bytes.iter().position(|b| *b == 0).unwrap_or(bytes.len()); - std::str::from_utf8(&bytes[..end]).unwrap_or("").trim() -} - -fn push_name(out: &mut Vec, name: &str) { - if name.is_empty() { - return; - } - out.push(name.to_string()); - if let Some((_, short)) = name.rsplit_once('.') { - if !short.is_empty() { - out.push(short.to_string()); - } - } -} - -fn dedup_case_insensitive(values: &mut Vec) { - let mut seen = std::collections::HashSet::new(); - values.retain(|value| seen.insert(value.to_ascii_lowercase())); -} - -fn scan_big(analyzer: &Analyzer, path: &Path) -> Vec { - let mut out = Vec::new(); - let Ok(mut file) = std::fs::File::open(path) else { - return out; - }; - let Ok(entries) = big_entries(&mut file) else { - return out; - }; - for entry in entries { - let uri = big_uri(path, &entry.name); - if entry.name.ends_with(".ini") || entry.name.ends_with(".INI") { - let Some(text) = read_big_entry(&mut file, &entry) else { - continue; - }; - let parse = analyzer.parse(&text); - let defs = definitions_in(analyzer, &parse, &uri); - let refs = references_in(analyzer, &parse); - let tags = module_tags_in(analyzer, &parse); - out.push((uri, defs, refs, tags, Vec::new(), Some(Arc::from(text)))); - } else if entry.name.ends_with(".w3d") || entry.name.ends_with(".W3D") { - let Some(bytes) = read_big_entry_bytes(&mut file, &entry) else { - continue; - }; - let stem = file_stem_str(&entry.name); - let models = parse_w3d_models(&bytes, &stem); - if !models.is_empty() { - out.push((uri, Vec::new(), Vec::new(), Vec::new(), models, None)); - } - } - } - out -} - -/// Walk `roots` and index `.ini` files plus `.w3d` model assets in one call. -/// Production code goes through `collect_scan_plan` + `scan_with_cache` so the -/// scan can report progress; this convenience wrapper serves the tests. -#[cfg(test)] -fn scan_roots(analyzer: &Analyzer, roots: &[PathBuf]) -> Vec { - scan_files(analyzer, &collect_scan_paths(roots), &mut |_, _| {}) -} - -/// Phase 1 of the scan: a cheap walk collecting every indexable file -/// (`.ini` / `.big` / `.w3d`), so phase 2 can report `done/total` progress. -fn collect_scan_paths(roots: &[PathBuf]) -> Vec { - let mut out = Vec::new(); - for root in roots { - if root - .extension() - .and_then(|e| e.to_str()) - .is_some_and(|e| e.eq_ignore_ascii_case("big")) - { - out.push(root.clone()); - continue; - } - for entry in walkdir::WalkDir::new(root) - .into_iter() - .filter_map(|e| e.ok()) - { - let path = entry.path(); - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); - if ext.eq_ignore_ascii_case("big") - || ext.eq_ignore_ascii_case("ini") - || ext.eq_ignore_ascii_case("w3d") - { - out.push(path.to_path_buf()); - } - } - } - out -} - -fn collect_scan_plan(workspace_roots: &[PathBuf], base_roots: &[PathBuf]) -> Vec { - let mut seen = HashSet::new(); - let mut out = Vec::new(); - // Base paths come first to preserve the existing base-then-workspace apply - // order. A path present in both groups belongs to the base group. - for (roots, is_base) in [(base_roots, true), (workspace_roots, false)] { - for path in collect_scan_paths(roots) { - let key = path_key(&path); - let Some(fingerprint) = file_fingerprint(&path) else { - continue; - }; - if seen.insert(key.clone()) { - out.push(ScanPath { - path, - key, - is_base, - fingerprint, - }); - } - } - } - out -} - -fn scan_path(analyzer: &Analyzer, path: &Path) -> Vec { - let ext = path.extension().and_then(|e| e.to_str()).unwrap_or(""); - if ext.eq_ignore_ascii_case("big") { - return scan_big(analyzer, path); - } - if ext.eq_ignore_ascii_case("ini") { - if let (Some(text), Ok(uri)) = (read_lossy(path), Url::from_file_path(path)) { - let parse = analyzer.parse(&text); - return vec![( - uri.to_string(), - definitions_in(analyzer, &parse, uri.as_str()), - references_in(analyzer, &parse), - module_tags_in(analyzer, &parse), - Vec::new(), - None, - )]; - } - } else if ext.eq_ignore_ascii_case("w3d") { - if let (Ok(bytes), Ok(uri)) = (std::fs::read(path), Url::from_file_path(path)) { - let stem = path - .file_stem() - .and_then(|s| s.to_str()) - .unwrap_or_default(); - let models = parse_w3d_models(&bytes, stem); - if !models.is_empty() { - return vec![( - uri.to_string(), - Vec::new(), - Vec::new(), - Vec::new(), - models, - None, - )]; - } - } - } - Vec::new() -} - -/// Phase 2 of the scan: parse/index each collected file, invoking -/// `progress(done, total)` after each one (a `.big` archive counts as one -/// unit of work regardless of how many entries it holds). -#[cfg(test)] -fn scan_files( - analyzer: &Analyzer, - paths: &[PathBuf], - progress: &mut impl FnMut(usize, usize), -) -> Vec { - let mut out = Vec::new(); - for (i, path) in paths.iter().enumerate() { - out.extend(scan_path(analyzer, path)); - progress(i + 1, paths.len()); - } - out -} - -fn scan_with_cache( - analyzer: &Analyzer, - paths: &[ScanPath], - cache_path: &Path, - workers: usize, - progress: &mut impl FnMut(usize, usize), -) -> (Vec<(bool, ScanEntry)>, ScanStats) { - let expected_schema_hash = schema_hash(); - let load_started = Instant::now(); - let mut old_cache = load_index_cache(cache_path, expected_schema_hash); - let cache_load = load_started.elapsed(); - let old_file_count = old_cache.files.len(); - let mut results: Vec>> = (0..paths.len()).map(|_| None).collect(); - let mut misses = Vec::new(); - let mut hits = 0; - let mut done = 0; - - for (i, path) in paths.iter().enumerate() { - match old_cache.files.remove(&path.key) { - Some(cached) if cached.fingerprint == path.fingerprint => { - results[i] = Some( - cached - .entries - .into_iter() - .map(CachedScanEntry::into_scan) - .collect(), - ); - hits += 1; - done += 1; - progress(done, paths.len()); - } - _ => misses.push(i), - } - } - - let parse_started = Instant::now(); - if !misses.is_empty() { - let worker_count = workers.max(1).min(misses.len()); - let next = AtomicUsize::new(0); - let (tx, rx) = std::sync::mpsc::channel(); - std::thread::scope(|scope| { - for _ in 0..worker_count { - let tx = tx.clone(); - let misses = &misses; - let next = &next; - scope.spawn(move || loop { - let work = next.fetch_add(1, Ordering::Relaxed); - let Some(&index) = misses.get(work) else { - break; - }; - let entries = scan_path(analyzer, &paths[index].path); - if tx.send((index, entries)).is_err() { - break; - } - }); - } - drop(tx); - for (index, entries) in rx { - results[index] = Some(entries); - done += 1; - progress(done, paths.len()); - } - }); - } - let parse = parse_started.elapsed(); - - let mut cache = IndexCache::empty(expected_schema_hash); - let mut scanned = Vec::new(); - for (path, entries) in paths.iter().zip(results) { - let entries = entries.unwrap_or_default(); - cache.files.insert( - path.key.clone(), - CachedFile { - fingerprint: path.fingerprint.clone(), - entries: entries.iter().map(CachedScanEntry::from_scan).collect(), - }, - ); - scanned.extend(entries.into_iter().map(|entry| (path.is_base, entry))); - } - - let write_started = Instant::now(); - if !misses.is_empty() || old_file_count != paths.len() { - if let Err(error) = write_index_cache(cache_path, &cache) { - tracing::warn!(%error, path = %cache_path.display(), "could not write asset index cache"); - } - } - let cache_write = write_started.elapsed(); - ( - scanned, - ScanStats { - cache_load, - parse, - cache_write, - hits, - misses: misses.len(), - ..ScanStats::default() - }, - ) -} - impl Backend { pub fn new(client: Client) -> Self { Backend { @@ -837,20 +151,6 @@ impl Backend { .fetch_add(1, std::sync::atomic::Ordering::Relaxed) } - fn current_index_cache_path(&self) -> PathBuf { - let roots = self - .roots - .lock() - .map(|roots| roots.clone()) - .unwrap_or_default(); - let base_roots = self - .base_roots - .lock() - .map(|roots| roots.clone()) - .unwrap_or_default(); - index_cache_path(&roots, &base_roots) - } - /// Update the cross-file index from the document's cached parse, run /// diagnostics (via the per-block cache), and publish. The parse itself is /// maintained synchronously by `did_open`/`did_change`. @@ -874,7 +174,7 @@ impl Backend { let defs = definitions_in(&self.analyzer, &parse, uri.as_str()); let refs = references_in(&self.analyzer, &parse); let tags = module_tags_in(&self.analyzer, &parse); - let str_keys = load_sibling_str_keys(uri); + let str_keys = asset_index::load_sibling_str_keys(uri); if let Ok(mut idx) = self.index.write() { idx.set_file(uri.as_str(), defs); idx.set_file_refs(uri.as_str(), refs); @@ -964,14 +264,6 @@ impl Backend { // each update as a progress report while waiting for the results. let (tx, mut rx) = tokio::sync::mpsc::unbounded_channel::<(usize, usize)>(); let handle = tokio::task::spawn_blocking(move || { - let discovery_started = Instant::now(); - let paths = collect_scan_plan(&roots, &base_roots); - let discovery = discovery_started.elapsed(); - let cache_path = index_cache_path(&roots, &base_roots); - let workers = std::thread::available_parallelism() - .map(usize::from) - .unwrap_or(1) - .min(MAX_SCAN_WORKERS); let mut last_percent = u32::MAX; // Throttle to whole-percent changes (plus the final count) so a // 10k-file scan sends ~100 notifications, not 10k. @@ -982,33 +274,29 @@ impl Backend { let _ = tx.send((done, total)); } }; - let (scanned, mut stats) = - scan_with_cache(&analyzer, &paths, &cache_path, workers, &mut progress); - stats.discovery = discovery; - (scanned, stats) + asset_index::scan(&analyzer, &roots, &base_roots, &mut progress) }); while let Some((done, total)) = rx.recv().await { self.report_scan_progress(&progress_token, done, total) .await; } - let (scanned, stats) = handle.await.unwrap_or_default(); - let base_ini_count = scanned + let result = handle.await.unwrap_or_default(); + let base_ini_count = result + .entries .iter() - .filter(|(is_base, (_, _, _, _, models, _))| *is_base && models.is_empty()) + .filter(|entry| entry.is_base && entry.models.is_empty()) .count(); self.base_indexed_count .store(base_ini_count, Ordering::Relaxed); self.last_scan_cache_hits - .store(stats.hits, Ordering::Relaxed); + .store(result.stats.hits, Ordering::Relaxed); self.scan_finished.store(true, Ordering::Relaxed); - let ini_total = scanned + let ini_total = result + .entries .iter() - .filter(|(_, (_, _, _, _, models, _))| models.is_empty()) + .filter(|entry| entry.models.is_empty()) .count(); - let model_total: usize = scanned - .iter() - .map(|(_, (_, _, _, _, models, _))| models.len()) - .sum(); + let model_total: usize = result.entries.iter().map(|entry| entry.models.len()).sum(); // Don't overwrite index entries for already-open documents with stale // disk content; `initialized` calls `refresh` for each open doc right // after this returns, so they will populate the index from live text. @@ -1022,15 +310,15 @@ impl Backend { let Ok(mut idx) = self.index.write() else { return; }; - for (_, (uri, defs, refs, tags, models, text)) in scanned { - if let Some(text) = text { - self.virtual_files.insert(uri.clone(), text); + for entry in result.entries { + if let Some(text) = entry.text { + self.virtual_files.insert(entry.file.clone(), text); } - if !open.contains(&uri) { - idx.set_file(&uri, defs); - idx.set_file_refs(&uri, refs); - idx.set_file_tags(&uri, tags); - idx.set_file_models(&uri, models); + if !open.contains(&entry.file) { + idx.set_file(&entry.file, entry.definitions); + idx.set_file_refs(&entry.file, entry.references); + idx.set_file_tags(&entry.file, entry.tags); + idx.set_file_models(&entry.file, entry.models); } } } @@ -1040,13 +328,13 @@ impl Backend { MessageType::INFO, format!( "asset index scan complete (discovery {} ms, cache load {} ms, parse {} ms, cache write {} ms, apply {} ms, {} hits, {} misses)", - stats.discovery.as_millis(), - stats.cache_load.as_millis(), - stats.parse.as_millis(), - stats.cache_write.as_millis(), + result.stats.discovery.as_millis(), + result.stats.cache_load.as_millis(), + result.stats.parse.as_millis(), + result.stats.cache_write.as_millis(), apply.as_millis(), - stats.hits, - stats.misses, + result.stats.hits, + result.stats.misses, ), ) .await; @@ -1145,7 +433,9 @@ impl Backend { .map(|text| Rope::from_str(&text)); } let path = uri.to_file_path().ok()?; - read_lossy(&path).map(|s| Rope::from_str(&s)) + std::fs::read(path) + .ok() + .map(|bytes| Rope::from_str(&String::from_utf8_lossy(&bytes))) } pub async fn read_virtual_file(&self, params: VirtualFileParams) -> Result> { @@ -1363,11 +653,20 @@ impl LanguageServer for Backend { { return Ok(None); } - let path = self.current_index_cache_path(); - let cleared = match remove_index_cache(&path) { + let roots = self + .roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + let base_roots = self + .base_roots + .lock() + .map(|roots| roots.clone()) + .unwrap_or_default(); + let cleared = match asset_index::clear_cache(&roots, &base_roots) { Ok(cleared) => cleared, Err(error) => { - tracing::warn!(%error, path = %path.display(), "could not clear asset index cache"); + tracing::warn!(%error, "could not clear asset index cache"); return Err(tower_lsp::jsonrpc::Error::internal_error()); } }; @@ -2091,66 +1390,10 @@ fn origin_copy_fixes( mod tests { use super::*; - fn test_dir(label: &str) -> PathBuf { - static NEXT: AtomicUsize = AtomicUsize::new(0); - let dir = std::env::temp_dir().join(format!( - "zerosyntax-{label}-{}-{}", - std::process::id(), - NEXT.fetch_add(1, Ordering::Relaxed) - )); - let _ = std::fs::remove_dir_all(&dir); - std::fs::create_dir_all(&dir).unwrap(); - dir - } - - fn w3d_with_member(member: &str) -> Vec { - let mut pivot = vec![0; 60]; - let name = member.as_bytes(); - pivot[..name.len().min(16)].copy_from_slice(&name[..name.len().min(16)]); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&0x0000_0102u32.to_le_bytes()); - bytes.extend_from_slice(&(pivot.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&pivot); - bytes - } - - fn write_big(path: &Path, entries: &[(&str, Vec)]) { - let directory_len: usize = entries.iter().map(|(name, _)| 8 + name.len() + 1).sum(); - let mut offset = 0x10 + directory_len; - let archive_size = offset + entries.iter().map(|(_, bytes)| bytes.len()).sum::(); - let mut out = Vec::with_capacity(archive_size); - out.extend_from_slice(b"BIGF"); - out.extend_from_slice(&(archive_size as u32).to_be_bytes()); - out.extend_from_slice(&(entries.len() as u32).to_be_bytes()); - out.extend_from_slice(&0u32.to_be_bytes()); - for (name, bytes) in entries { - out.extend_from_slice(&(offset as u32).to_be_bytes()); - out.extend_from_slice(&(bytes.len() as u32).to_be_bytes()); - out.extend_from_slice(name.as_bytes()); - out.push(0); - offset += bytes.len(); - } - for (_, bytes) in entries { - out.extend_from_slice(bytes); - } - std::fs::write(path, out).unwrap(); - } - - fn cached_scan( - analyzer: &Analyzer, - workspace_roots: &[PathBuf], - base_roots: &[PathBuf], - cache: &Path, - workers: usize, - ) -> (Vec<(bool, ScanEntry)>, ScanStats) { - let paths = collect_scan_plan(workspace_roots, base_roots); - scan_with_cache(analyzer, &paths, cache, workers, &mut |_, _| {}) - } - #[test] fn canonical_uri_pass_through_non_file() { - let u = Url::parse("untitled:///buffer").unwrap(); - assert_eq!(canonical_uri(u.clone()), u); + let uri = Url::parse("untitled:///buffer").unwrap(); + assert_eq!(canonical_uri(uri.clone()), uri); } #[test] @@ -2160,353 +1403,9 @@ mod tests { assert!(!is_map_layer_file("C:/Data/INI/Object.ini")); } - #[test] - fn removes_only_the_requested_index_cache() { - let dir = test_dir("clear-cache"); - let current = dir.join("current.json"); - let other = dir.join("other.json"); - std::fs::write(¤t, b"cache").unwrap(); - std::fs::write(&other, b"cache").unwrap(); - - assert!(remove_index_cache(¤t).unwrap()); - assert!(!current.exists()); - assert!(other.exists()); - assert!(!remove_index_cache(¤t).unwrap()); - let _ = std::fs::remove_dir_all(dir); - } - - #[test] - fn cache_reuses_complete_scan_and_invalidates_changed_loose_file() { - let dir = test_dir("cache-loose"); - let cache = dir.join("cache.json"); - std::fs::write(dir.join("Object.ini"), "Object CachedTank\nEnd\n").unwrap(); - let model = dir.join("Tank.w3d"); - std::fs::write(&model, w3d_with_member("Muzzle01")).unwrap(); - let analyzer = Analyzer::embedded(); - let roots = std::slice::from_ref(&dir); - - let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); - assert_eq!((cold_stats.hits, cold_stats.misses), (0, 2)); - let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); - assert_eq!((warm_stats.hits, warm_stats.misses), (2, 0)); - assert_eq!(cold, warm); - - let mut changed = w3d_with_member("Muzzle02"); - changed.push(0); // size change guarantees invalidation on coarse-mtime filesystems - std::fs::write(&model, changed).unwrap(); - let (_, changed_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); - assert_eq!((changed_stats.hits, changed_stats.misses), (1, 1)); - - let mut incompatible = load_index_cache(&cache, schema_hash()); - incompatible.schema_hash ^= 1; - write_index_cache(&cache, &incompatible).unwrap(); - let (_, schema_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); - assert_eq!((schema_stats.hits, schema_stats.misses), (0, 2)); - - std::fs::write(&cache, b"not json").unwrap(); - let (_, corrupt_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); - assert_eq!((corrupt_stats.hits, corrupt_stats.misses), (0, 2)); - let _ = std::fs::remove_dir_all(dir); - } - - #[test] - fn cache_invalidates_changed_big_archive_as_one_file() { - let dir = test_dir("cache-big"); - let archive = dir.join("Models.big"); - let cache = dir.join("cache.json"); - write_big( - &archive, - &[ - ("Data/INI/Object.ini", b"Object BigTank\nEnd\n".to_vec()), - ("Art/Tank.w3d", w3d_with_member("Muzzle01")), - ], - ); - let analyzer = Analyzer::embedded(); - let roots = std::slice::from_ref(&archive); - - let (cold, cold_stats) = cached_scan(&analyzer, &[], roots, &cache, 1); - assert_eq!((cold_stats.hits, cold_stats.misses), (0, 1)); - assert_eq!(cold.len(), 2); - let (warm, warm_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); - assert_eq!((warm_stats.hits, warm_stats.misses), (1, 0)); - assert_eq!(cold, warm); - - write_big( - &archive, - &[ - ( - "Data/INI/Object.ini", - b"Object ChangedBigTank\nEnd\n".to_vec(), - ), - ("Art/Tank.w3d", w3d_with_member("Muzzle02")), - ], - ); - let (_, changed_stats) = cached_scan(&analyzer, &[], roots, &cache, 4); - assert_eq!((changed_stats.hits, changed_stats.misses), (0, 1)); - let _ = std::fs::remove_dir_all(dir); - } - - #[test] - fn parallel_scan_preserves_order_and_overlapping_roots_are_deduped() { - let dir = test_dir("parallel"); - for i in 0..32 { - std::fs::write( - dir.join(format!("Model{i:02}.w3d")), - w3d_with_member(&format!("Bone{i:02}")), - ) - .unwrap(); - } - let roots = std::slice::from_ref(&dir); - let plan = collect_scan_plan(roots, roots); - assert_eq!(plan.len(), 32); - assert!(plan.iter().all(|path| path.is_base)); - - let analyzer = Analyzer::embedded(); - let (serial, _) = cached_scan(&analyzer, &[], roots, &dir.join("serial.json"), 1); - let (parallel, _) = cached_scan(&analyzer, &[], roots, &dir.join("parallel.json"), 4); - assert_eq!(serial, parallel); - let _ = std::fs::remove_dir_all(dir); - } - - #[test] - #[ignore = "3,000-file synthetic timing harness; run with --ignored --nocapture"] - fn synthetic_3000_asset_cold_and_warm_timings() { - let dir = test_dir("benchmark"); - for i in 0..3000 { - std::fs::write( - dir.join(format!("Model{i:04}.w3d")), - w3d_with_member(&format!("Bone{:02}", i % 100)), - ) - .unwrap(); - } - let analyzer = Analyzer::embedded(); - let mut serial_times = Vec::new(); - let mut parallel_times = Vec::new(); - let mut expected = None; - let roots = std::slice::from_ref(&dir); - for run in 0..3 { - let started = Instant::now(); - let (out, _) = cached_scan( - &analyzer, - &[], - roots, - &dir.join(format!("serial-{run}.json")), - 1, - ); - serial_times.push(started.elapsed()); - expected.get_or_insert(out); - - let started = Instant::now(); - let (out, _) = cached_scan( - &analyzer, - &[], - roots, - &dir.join(format!("parallel-{run}.json")), - 4, - ); - parallel_times.push(started.elapsed()); - assert_eq!(expected.as_ref(), Some(&out)); - } - serial_times.sort_unstable(); - parallel_times.sort_unstable(); - let warm_started = Instant::now(); - let (_, warm_stats) = cached_scan(&analyzer, &[], roots, &dir.join("parallel-2.json"), 4); - let warm = warm_started.elapsed(); - assert_eq!((warm_stats.hits, warm_stats.misses), (3000, 0)); - eprintln!( - "3,000 W3Ds: serial median {:?}, parallel median {:?}, warm {:?}", - serial_times[1], parallel_times[1], warm - ); - let _ = std::fs::remove_dir_all(dir); - } - - #[test] - fn scans_ini_from_big_archive() { - let dir = std::env::temp_dir(); - let path = dir.join(format!("zerosyntax-test-{}.big", std::process::id())); - let entry_name = b"Data\\INI\\Test.ini\0"; - let ini = b"Object BigArchiveObject\nEnd\n"; - let data_offset = 0x10 + 8 + entry_name.len(); - let archive_size = data_offset + ini.len(); - - let mut bytes = Vec::new(); - bytes.extend_from_slice(b"BIGF"); - bytes.extend_from_slice(&(archive_size as u32).to_be_bytes()); - bytes.extend_from_slice(&1u32.to_be_bytes()); - bytes.extend_from_slice(&0u32.to_be_bytes()); - bytes.extend_from_slice(&(data_offset as u32).to_be_bytes()); - bytes.extend_from_slice(&(ini.len() as u32).to_be_bytes()); - bytes.extend_from_slice(entry_name); - bytes.extend_from_slice(ini); - std::fs::write(&path, bytes).unwrap(); - - let analyzer = Analyzer::embedded(); - let scanned = scan_big(&analyzer, &path); - let _ = std::fs::remove_file(&path); - - assert_eq!(scanned.len(), 1); - assert!(Url::parse(&scanned[0].0).is_ok()); - assert!(scanned[0].1.iter().any(|d| d.name == "BigArchiveObject")); - assert_eq!( - scanned[0].5.as_deref(), - Some("Object BigArchiveObject\nEnd\n") - ); - } - - #[test] - fn w3d_root_scan_powers_model_and_bone_completions() { - // End-to-end over the `baseIniRoots` path: a directory containing a - // loose .w3d file is scanned, indexed, and drives completions. - let mut pivots = Vec::new(); - pivots.extend_from_slice(b"Tire01"); - pivots.resize(60, 0); - let mut bytes = Vec::new(); - bytes.extend_from_slice(&0x0000_0102u32.to_le_bytes()); - bytes.extend_from_slice(&(pivots.len() as u32).to_le_bytes()); - bytes.extend_from_slice(&pivots); - - let dir = std::env::temp_dir().join(format!("zerosyntax-w3d-{}", std::process::id())); - std::fs::create_dir_all(&dir).unwrap(); - std::fs::write(dir.join("Good.w3d"), bytes).unwrap(); - - let analyzer = Analyzer::embedded(); - let scanned = scan_roots(&analyzer, std::slice::from_ref(&dir)); - let _ = std::fs::remove_dir_all(&dir); - - let mut idx = WorkspaceIndex::new(); - for (uri, defs, refs, tags, models, _) in scanned { - idx.set_file(&uri, defs); - idx.set_file_refs(&uri, refs); - idx.set_file_tags(&uri, tags); - idx.set_file_models(&uri, models); - } - assert!(idx.is_model_asset("Good"), "model name from file stem"); - - let src = "\ -Object Tank - Draw = W3DTankDraw ModuleTag_01 - DefaultConditionState - Model = - HideSubObject = - End - End -End -"; - let parse = analyzer.parse(src); - let labels_at = |offset: usize| -> Vec { - completion::complete(&analyzer, &parse, offset as u32, Some(&idx), None) - .into_iter() - .map(|c| c.label) - .collect() - }; - let model_offset = src.find("Model = ").unwrap() + "Model = ".len(); - assert!(labels_at(model_offset).contains(&"Good".to_string())); - - let src = src.replace("Model = ", "Model = Good"); - let parse = analyzer.parse(&src); - let bone_offset = src.find("HideSubObject = ").unwrap() + "HideSubObject = ".len(); - let labels: Vec = - completion::complete(&analyzer, &parse, bone_offset as u32, Some(&idx), None) - .into_iter() - .map(|c| c.label) - .collect(); - assert!(labels.contains(&"Tire".to_string()), "{labels:?}"); - } - - #[test] - fn parses_w3d_model_names_and_members() { - fn fixed(name: &str) -> [u8; N] { - let mut out = [0; N]; - let bytes = name.as_bytes(); - out[..bytes.len().min(N)].copy_from_slice(&bytes[..bytes.len().min(N)]); - out - } - fn chunk(kind: u32, payload: Vec) -> Vec { - let mut out = Vec::new(); - out.extend_from_slice(&kind.to_le_bytes()); - out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); - out.extend_from_slice(&payload); - out - } - - let mut hlod = Vec::new(); - hlod.extend_from_slice(&1u32.to_le_bytes()); - hlod.extend_from_slice(&1u32.to_le_bytes()); - hlod.extend_from_slice(&fixed::<16>("Good")); - hlod.extend_from_slice(&fixed::<16>("Good")); - - let mut pivot = Vec::new(); - pivot.extend_from_slice(&fixed::<16>("Tire01")); - pivot.resize(60, 0); - - let mut sub = Vec::new(); - sub.extend_from_slice(&0u32.to_le_bytes()); - sub.extend_from_slice(&fixed::<32>("Good.Cargo01")); - - let mut bytes = Vec::new(); - bytes.extend(chunk(0x0000_0701, hlod)); - bytes.extend(chunk(0x0000_0102, pivot)); - bytes.extend(chunk(0x0000_0704, sub)); - - let models = parse_w3d_models(&bytes, "Fallback"); - let good = models.iter().find(|m| m.name == "Good").unwrap(); - assert!(good.members.iter().any(|m| m == "Tire01"), "{good:?}"); - assert!(good.members.iter().any(|m| m == "Cargo01"), "{good:?}"); - } - - #[test] - fn parses_w3d_aggregate_and_emitter_names() { - fn fixed(name: &str) -> [u8; N] { - let mut out = [0; N]; - let bytes = name.as_bytes(); - out[..bytes.len().min(N)].copy_from_slice(&bytes[..bytes.len().min(N)]); - out - } - fn chunk(kind: u32, payload: Vec) -> Vec { - let mut out = Vec::new(); - out.extend_from_slice(&kind.to_le_bytes()); - out.extend_from_slice(&(payload.len() as u32).to_le_bytes()); - out.extend_from_slice(&payload); - out - } - // Version + Name[16] header, wrapped in its container chunk. - let mut header = Vec::new(); - header.extend_from_slice(&1u32.to_le_bytes()); - header.extend_from_slice(&fixed::<16>("Aggro")); - let mut bytes = chunk(0x0000_0600, chunk(0x0000_0601, header)); - - let mut header = Vec::new(); - header.extend_from_slice(&1u32.to_le_bytes()); - header.extend_from_slice(&fixed::<16>("Smoke")); - bytes.extend(chunk(0x0000_0500, chunk(0x0000_0501, header))); - - let models = parse_w3d_models(&bytes, ""); - assert!(models.iter().any(|m| m.name == "Aggro"), "{models:?}"); - assert!(models.iter().any(|m| m.name == "Smoke"), "{models:?}"); - } - - #[test] - fn w3d_chunk_walker_survives_hostile_deep_nesting() { - // A self-nesting container at every level: each 8-byte header claims - // the rest of the file as payload. Without a depth cap this recursed - // once per 8 bytes and could overflow the stack on a large file. - let total: usize = 64 * 1024; - let mut bytes = Vec::with_capacity(total); - let mut remaining = total; - while remaining >= 8 { - bytes.extend_from_slice(&0x0000_0700u32.to_le_bytes()); - bytes.extend_from_slice(&((remaining - 8) as u32).to_le_bytes()); - remaining -= 8; - } - // Must terminate without smashing the stack; content is garbage. - let _ = parse_w3d_models(&bytes, "Fallback"); - } - #[test] #[cfg(windows)] fn canonical_uri_normalises_windows_file_uri() { - // VS Code on Windows sends percent-encoded colon + lowercase drive - // letter; `from_file_path` produces uppercase drive + no encoding. let client = Url::parse("file:///c%3A/CodeProjects/mod/Object.ini").unwrap(); let expected = Url::from_file_path(r"C:\CodeProjects\mod\Object.ini").unwrap(); assert_eq!(canonical_uri(client), expected); diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 1e5e174..6ee87b8 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -1,6 +1,7 @@ //! `zerosyntax-lsp`: the IDE-agnostic language server for C&C Generals: Zero Hour //! INI files. Speaks LSP over stdio. +mod asset_index; mod backend; mod convert; From 9eb7b375dd461d8b063298f2e11feb4d96c9c8b6 Mon Sep 17 00:00:00 2001 From: ViTeXFTW Date: Sun, 12 Jul 2026 21:58:24 +0200 Subject: [PATCH 3/4] refactor: encapsulate server documents --- crates/server/src/backend.rs | 272 ++++++---------------- crates/server/src/document.rs | 421 ++++++++++++++++++++++++++++++++++ crates/server/src/main.rs | 1 + 3 files changed, 493 insertions(+), 201 deletions(-) create mode 100644 crates/server/src/document.rs diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index df5bfcb..2c67025 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -1,55 +1,30 @@ //! The tower-lsp language server backend. //! -//! Holds open documents as [`DocumentState`]s — a `ropey` rope plus the cached -//! parse for that text — alongside a shared [`Analyzer`] (the embedded schema) -//! and a workspace symbol [`WorkspaceIndex`]. Document sync is INCREMENTAL: -//! `didChange` deltas are applied to the rope and the document is re-parsed -//! once per change batch; read-only requests reuse the cached parse. +//! Adapts LSP requests to the shared analysis engine, document store, and asset +//! index. -use std::collections::HashSet; use std::path::PathBuf; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex, OnceLock, RwLock}; use std::time::Instant; -use dashmap::DashMap; use ropey::Rope; use serde::Deserialize; use tower_lsp::lsp_types::*; use tower_lsp::{jsonrpc::Result, Client, LanguageServer}; -use zerosyntax_analysis::diagnostics::DiagnosticsCache; use zerosyntax_analysis::index::{definitions_in, module_tags_in, references_in, WorkspaceIndex}; use zerosyntax_analysis::nav::{definition_at, hover_at, reference_at, HoverInfo}; use zerosyntax_analysis::{actions, completion, diagnostics, format, outline, semantic, Analyzer}; -use zerosyntax_syntax::{Edit, Parse}; +use zerosyntax_syntax::Parse; use crate::asset_index; use crate::convert::{self, PositionEnc}; - -/// An open document: its text (as both a rope for position math and a string -/// for the parser) and the parse of that exact text. `did_open`/`did_change` -/// are the only places a new parse is produced for an open document; -/// `did_change` reparses incrementally by splicing at block boundaries. -struct DocumentState { - rope: Rope, - /// The same text as `rope`; the source the cached `parse` was built from. - text: Arc, - parse: Arc, - version: i32, - /// Per-block diagnostics, reused across edits for unchanged blocks. - diag_cache: DiagnosticsCache, - /// The last `semanticTokens/full` response (result id + encoded data), - /// kept so `full/delta` can answer with a splice instead of the world. - last_semantic: Option<(u64, Vec)>, -} +use crate::document::DocumentStore; pub struct Backend { client: Client, analyzer: Arc, - /// Open documents, keyed by URI. - docs: DashMap, - /// Read-only documents synthesized from configured `.big` archives. - virtual_files: DashMap>, + documents: DocumentStore, index: RwLock, /// Workspace roots, captured at `initialize` and scanned in `initialized`. roots: Mutex>, @@ -120,8 +95,7 @@ impl Backend { Backend { client, analyzer: Arc::new(Analyzer::embedded()), - docs: DashMap::new(), - virtual_files: DashMap::new(), + documents: DocumentStore::new(), index: RwLock::new(WorkspaceIndex::new()), roots: Mutex::new(Vec::new()), encoding: OnceLock::new(), @@ -157,23 +131,17 @@ impl Backend { async fn refresh(&self, uri: &Url) { // Take the cache out so diagnostics run without holding the doc entry // (avoids lock-order entanglement with the index RwLock). - let Some((rope, parse, version, mut cache)) = self.docs.get_mut(uri).map(|mut d| { - ( - d.rope.clone(), - d.parse.clone(), - d.version, - std::mem::take(&mut d.diag_cache), - ) - }) else { + let Some(mut diagnostic) = self.documents.checkout_diagnostics(uri) else { return; }; + let document = &diagnostic.document; // `set_file` bumps the index generation only when definition *names* // changed, so ordinary keystrokes keep diagnostics caches warm. // Reference sites never bump it. - let defs = definitions_in(&self.analyzer, &parse, uri.as_str()); - let refs = references_in(&self.analyzer, &parse); - let tags = module_tags_in(&self.analyzer, &parse); + let defs = definitions_in(&self.analyzer, &document.parse, uri.as_str()); + let refs = references_in(&self.analyzer, &document.parse); + let tags = module_tags_in(&self.analyzer, &document.parse); let str_keys = asset_index::load_sibling_str_keys(uri); if let Ok(mut idx) = self.index.write() { idx.set_file(uri.as_str(), defs); @@ -187,30 +155,27 @@ impl Backend { let idx = self.index.read().ok(); let diags = diagnostics::diagnose_with_cache( &self.analyzer, - &parse, + &document.parse, idx.as_deref(), Some(uri.as_str()), - &mut cache, + &mut diagnostic.cache, ); diags .iter() - .map(|d| convert::to_lsp_diagnostic(&rope, d, enc)) + .map(|d| convert::to_lsp_diagnostic(&document.rope, d, enc)) .collect() }; // Hand the warmed cache back unless a newer change superseded us (the // newer change runs its own refresh against its own parse). + if !self + .documents + .restore_diagnostics(uri, document.version, diagnostic.cache) { - let Some(mut entry) = self.docs.get_mut(uri) else { - return; - }; - if entry.version != version { - return; - } - entry.diag_cache = cache; + return; } self.client - .publish_diagnostics(uri.clone(), lsp_diags, Some(version)) + .publish_diagnostics(uri.clone(), lsp_diags, Some(document.version)) .await; self.maybe_warn_missing_base_roots(uri).await; } @@ -300,20 +265,24 @@ impl Backend { // Don't overwrite index entries for already-open documents with stale // disk content; `initialized` calls `refresh` for each open doc right // after this returns, so they will populate the index from live text. - let open: HashSet = self - .docs + let open = self.documents.open_uri_strings(); + let apply_started = Instant::now(); + let virtual_files: Vec<_> = result + .entries .iter() - .map(|e| e.key().as_str().to_string()) + .filter_map(|entry| { + entry + .text + .as_ref() + .map(|text| (entry.file.clone(), Arc::clone(text))) + }) .collect(); - let apply_started = Instant::now(); + self.documents.replace_virtual_files(virtual_files); { let Ok(mut idx) = self.index.write() else { return; }; for entry in result.entries { - if let Some(text) = entry.text { - self.virtual_files.insert(entry.file.clone(), text); - } if !open.contains(&entry.file) { idx.set_file(&entry.file, entry.definitions); idx.set_file_refs(&entry.file, entry.references); @@ -415,34 +384,19 @@ impl Backend { /// The cached state for an open document (rope + parse), if any. fn doc(&self, uri: &Url) -> Option<(Rope, Arc)> { - self.docs - .get(uri) - .map(|d| (d.rope.clone(), d.parse.clone())) + self.documents + .snapshot(uri) + .map(|document| (document.rope, document.parse)) } /// Resolve a URI's text to a rope, preferring open documents and falling /// back to disk (for go-to-definition into unopened files). fn rope_for(&self, uri: &Url) -> Option { - if let Some(doc) = self.docs.get(uri) { - return Some(doc.rope.clone()); - } - if uri.scheme() == "big" { - return self - .virtual_files - .get(uri.as_str()) - .map(|text| Rope::from_str(&text)); - } - let path = uri.to_file_path().ok()?; - std::fs::read(path) - .ok() - .map(|bytes| Rope::from_str(&String::from_utf8_lossy(&bytes))) + self.documents.rope_for(uri) } pub async fn read_virtual_file(&self, params: VirtualFileParams) -> Result> { - Ok(self - .virtual_files - .get(¶ms.uri) - .map(|text| text.to_string())) + Ok(self.documents.read_virtual_file(¶ms.uri)) } /// The (kind, name, span) under the cursor — a reference-typed value token @@ -617,8 +571,7 @@ impl LanguageServer for Backend { // Re-publish diagnostics for any already-open docs now that the index // is populated (so cross-file references resolve). The cached parse is // still valid — only the index changed. - let open: Vec = self.docs.iter().map(|e| e.key().clone()).collect(); - for uri in open { + for uri in self.documents.open_uris() { self.refresh(&uri).await; } let (ini, models) = { @@ -675,16 +628,13 @@ impl LanguageServer for Backend { self.scan_finished.store(false, Ordering::Relaxed); self.base_indexed_count.store(0, Ordering::Relaxed); self.last_scan_cache_hits.store(0, Ordering::Relaxed); - self.virtual_files.clear(); + self.documents.clear_virtual_files(); if let Ok(mut index) = self.index.write() { *index = WorkspaceIndex::new(); } - for mut doc in self.docs.iter_mut() { - doc.diag_cache = DiagnosticsCache::new(); - } + self.documents.clear_diagnostic_caches(); self.scan_workspace().await; - let open: Vec = self.docs.iter().map(|doc| doc.key().clone()).collect(); - for uri in open { + for uri in self.documents.open_uris() { self.refresh(&uri).await; } let message = "ZeroSyntax index cache rebuilt."; @@ -709,20 +659,12 @@ impl LanguageServer for Backend { async fn did_open(&self, params: DidOpenTextDocumentParams) { let uri = canonical_uri(params.text_document.uri); - let text: Arc = params.text_document.text.into(); - let rope = Rope::from_str(&text); - let parse = Arc::new(self.analyzer.parse(&text)); let version = params.text_document.version; - self.docs.insert( + self.documents.open( + &self.analyzer, uri.clone(), - DocumentState { - rope, - text, - parse, - version, - diag_cache: DiagnosticsCache::new(), - last_semantic: None, - }, + params.text_document.text, + version, ); self.refresh(&uri).await; } @@ -731,69 +673,12 @@ impl LanguageServer for Backend { let uri = canonical_uri(params.text_document.uri); let version = params.text_document.version; let enc = self.enc(); + if self + .documents + .change(&self.analyzer, &uri, params.content_changes, version, enc) + .is_none() { - let Some(mut entry) = self.docs.get_mut(&uri) else { - return; - }; - let entry = entry.value_mut(); - // Bulk batches (e.g. format-on-save applying coalesced reindent - // edits) take a different path: incremental reparse needs the full - // old/new text per edit, so running it per change is - // O(changes × file). Apply every delta to the rope first, then - // rebuild the text and parse once — one full parse beats a pile - // of incremental ones. Triggers on many changes or on a - // multi-change batch with a large total payload (a few coalesced - // format edits can each carry kilobytes). - const BULK_CHANGE_THRESHOLD: usize = 8; - const BULK_TEXT_BYTES: usize = 32 * 1024; - let bulk = params.content_changes.len() > BULK_CHANGE_THRESHOLD - || (params.content_changes.len() > 1 - && params - .content_changes - .iter() - .map(|c| c.text.len()) - .sum::() - > BULK_TEXT_BYTES); - if bulk && params.content_changes.iter().all(|c| c.range.is_some()) { - for change in params.content_changes { - convert::apply_change(&mut entry.rope, change.range, &change.text, enc); - } - entry.text = entry.rope.to_string().into(); - entry.parse = Arc::new(self.analyzer.parse(&entry.text)); - entry.version = version; - } else { - // Each change applies to the text produced by the previous - // one. The parse is kept in lockstep via incremental reparse, - // so the cost per keystroke is the edited block, not the - // whole file. - for change in params.content_changes { - match change.range { - Some(range) => { - let start = convert::position_to_offset(&entry.rope, range.start, enc); - let old_end = convert::position_to_offset(&entry.rope, range.end, enc); - convert::apply_change(&mut entry.rope, Some(range), &change.text, enc); - let new_text: Arc = entry.rope.to_string().into(); - let edit = Edit { - start: start as usize, - old_end: old_end as usize, - new_len: change.text.len(), - }; - let (parse, _strategy) = - self.analyzer - .reparse(&entry.parse, &entry.text, &new_text, edit); - entry.parse = Arc::new(parse); - entry.text = new_text; - } - None => { - // Full-document replacement. - entry.rope = Rope::from_str(&change.text); - entry.text = change.text.into(); - entry.parse = Arc::new(self.analyzer.parse(&entry.text)); - } - } - } - entry.version = version; - } + return; } self.refresh(&uri).await; } @@ -801,7 +686,8 @@ impl LanguageServer for Backend { async fn did_close(&self, params: DidCloseTextDocumentParams) { // Keep the file's symbols in the index (it still exists on disk); just // drop the in-memory buffer. - self.docs.remove(&canonical_uri(params.text_document.uri)); + self.documents + .close(&canonical_uri(params.text_document.uri)); } async fn completion(&self, params: CompletionParams) -> Result> { @@ -837,9 +723,8 @@ impl LanguageServer for Backend { let tokens = semantic::semantic_tokens(&self.analyzer, &parse); let data = convert::to_lsp_semantic_tokens(&rope, &tokens, self.enc()); let id = self.next_semantic_id(); - if let Some(mut doc) = self.docs.get_mut(&uri) { - doc.last_semantic = Some((id, data.clone())); - } + self.documents + .replace_semantic_history(&uri, (id, data.clone())); Ok(Some(SemanticTokensResult::Tokens(SemanticTokens { result_id: Some(id.to_string()), data, @@ -858,9 +743,8 @@ impl LanguageServer for Backend { let data = convert::to_lsp_semantic_tokens(&rope, &tokens, self.enc()); let id = self.next_semantic_id(); let previous = self - .docs - .get_mut(&uri) - .and_then(|mut doc| doc.last_semantic.replace((id, data.clone()))); + .documents + .replace_semantic_history(&uri, (id, data.clone())); // Only splice against the exact result the client says it holds; // anything else (stale id, no history) falls back to a full response. match previous { @@ -888,11 +772,7 @@ impl LanguageServer for Backend { let uri = canonical_uri(params.text_document.uri); // `text` is kept in lockstep with `rope` by did_open/did_change, so no // rope-to-string rebuild is needed here. - let Some((rope, text, parse)) = self - .docs - .get(&uri) - .map(|d| (d.rope.clone(), d.text.clone(), d.parse.clone())) - else { + let Some(document) = self.documents.snapshot(&uri) else { return Ok(None); }; let indent = if params.options.insert_spaces { @@ -901,10 +781,10 @@ impl LanguageServer for Backend { "\t".to_string() }; let enc = self.enc(); - let edits = format::format_edits(&parse, &text, &indent) + let edits = format::format_edits(&document.parse, &document.text, &indent) .into_iter() .map(|e| TextEdit { - range: convert::span_to_range(&rope, e.span, enc), + range: convert::span_to_range(&document.rope, e.span, enc), new_text: e.new_text, }) .collect(); @@ -916,36 +796,29 @@ impl LanguageServer for Backend { // Take the diagnostics cache out without holding the DashMap entry // across the index lock (avoids lock-order deadlock with the index RwLock). - let Some((rope, parse, text, version, mut cache)) = self.docs.get_mut(&uri).map(|mut d| { - ( - d.rope.clone(), - d.parse.clone(), - Arc::clone(&d.text), - d.version, - std::mem::take(&mut d.diag_cache), - ) - }) else { + let Some(mut diagnostic) = self.documents.checkout_diagnostics(&uri) else { return Ok(None); }; + let document = &diagnostic.document; let enc = self.enc(); - let start = convert::position_to_offset(&rope, params.range.start, enc); - let end = convert::position_to_offset(&rope, params.range.end, enc); + let start = convert::position_to_offset(&document.rope, params.range.start, enc); + let end = convert::position_to_offset(&document.rope, params.range.end, enc); let range_span = zerosyntax_analysis::Span::new(start, end); let fixes = { let idx = self.index.read().ok(); let diags = diagnostics::diagnose_with_cache( &self.analyzer, - &parse, + &document.parse, idx.as_deref(), Some(uri.as_str()), - &mut cache, + &mut diagnostic.cache, ); let mut f = actions::fixes( &self.analyzer, - &parse, - &text, + &document.parse, + &document.text, range_span, &diags, idx.as_deref(), @@ -954,29 +827,26 @@ impl LanguageServer for Backend { if let Some(idx) = idx.as_deref() { f.extend(origin_copy_fixes( &self.analyzer, - &parse, - &text, + &document.parse, + &document.text, range_span, &diags, idx, - |base_uri| self.rope_for(base_uri).map(|rope| rope.to_string()), + |base_uri| self.documents.source_text(base_uri.as_str()), )); } f }; // Hand the warmed cache back unless a newer change superseded us. - if let Some(mut entry) = self.docs.get_mut(&uri) { - if entry.version == version { - entry.diag_cache = cache; - } - } + self.documents + .restore_diagnostics(&uri, document.version, diagnostic.cache); let response: CodeActionResponse = fixes .into_iter() .map(|f| { let edit = TextEdit { - range: convert::span_to_range(&rope, f.span, enc), + range: convert::span_to_range(&document.rope, f.span, enc), new_text: f.new_text, }; CodeActionOrCommand::CodeAction(CodeAction { diff --git a/crates/server/src/document.rs b/crates/server/src/document.rs new file mode 100644 index 0000000..0c7d962 --- /dev/null +++ b/crates/server/src/document.rs @@ -0,0 +1,421 @@ +//! Open-document state and source lookup for the language server. + +use std::sync::Arc; + +use dashmap::DashMap; +use ropey::Rope; +use tower_lsp::lsp_types::{SemanticToken, TextDocumentContentChangeEvent, Url}; +use zerosyntax_analysis::diagnostics::DiagnosticsCache; +use zerosyntax_analysis::Analyzer; +use zerosyntax_syntax::{Edit, Parse}; + +use crate::convert::{self, PositionEnc}; + +const BULK_CHANGE_THRESHOLD: usize = 8; +const BULK_TEXT_BYTES: usize = 32 * 1024; + +/// An immutable, internally consistent view of an open document. +#[derive(Clone)] +pub(crate) struct DocumentSnapshot { + pub(crate) rope: Rope, + pub(crate) text: Arc, + pub(crate) parse: Arc, + pub(crate) version: i32, +} + +/// A snapshot plus exclusive ownership of its diagnostics cache. +pub(crate) struct DiagnosticSnapshot { + pub(crate) document: DocumentSnapshot, + pub(crate) cache: DiagnosticsCache, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ChangeStrategy { + Incremental, + Full, + Bulk, +} + +struct DocumentState { + rope: Rope, + text: Arc, + parse: Arc, + version: i32, + diag_cache: DiagnosticsCache, + last_semantic: Option<(u64, Vec)>, +} + +impl DocumentState { + fn snapshot(&self) -> DocumentSnapshot { + DocumentSnapshot { + rope: self.rope.clone(), + text: Arc::clone(&self.text), + parse: Arc::clone(&self.parse), + version: self.version, + } + } +} + +/// Owns live editor buffers and read-only files synthesized from BIG archives. +pub(crate) struct DocumentStore { + open: DashMap, + virtual_files: DashMap>, +} + +impl DocumentStore { + pub(crate) fn new() -> Self { + Self { + open: DashMap::new(), + virtual_files: DashMap::new(), + } + } + + pub(crate) fn open(&self, analyzer: &Analyzer, uri: Url, text: String, version: i32) { + let text: Arc = text.into(); + let state = DocumentState { + rope: Rope::from_str(&text), + parse: Arc::new(analyzer.parse(&text)), + text, + version, + diag_cache: DiagnosticsCache::new(), + last_semantic: None, + }; + self.open.insert(uri, state); + } + + pub(crate) fn change( + &self, + analyzer: &Analyzer, + uri: &Url, + changes: Vec, + version: i32, + encoding: PositionEnc, + ) -> Option { + let mut state = self.open.get_mut(uri)?; + let bulk = changes.len() > BULK_CHANGE_THRESHOLD + || (changes.len() > 1 + && changes + .iter() + .map(|change| change.text.len()) + .sum::() + > BULK_TEXT_BYTES); + + if bulk && changes.iter().all(|change| change.range.is_some()) { + for change in changes { + convert::apply_change(&mut state.rope, change.range, &change.text, encoding); + } + state.text = state.rope.to_string().into(); + state.parse = Arc::new(analyzer.parse(&state.text)); + state.version = version; + return Some(ChangeStrategy::Bulk); + } + + let mut strategy = ChangeStrategy::Incremental; + for change in changes { + match change.range { + Some(range) => { + let start = convert::position_to_offset(&state.rope, range.start, encoding); + let old_end = convert::position_to_offset(&state.rope, range.end, encoding); + convert::apply_change(&mut state.rope, Some(range), &change.text, encoding); + let new_text: Arc = state.rope.to_string().into(); + let edit = Edit { + start: start as usize, + old_end: old_end as usize, + new_len: change.text.len(), + }; + let (parse, _) = analyzer.reparse(&state.parse, &state.text, &new_text, edit); + state.parse = Arc::new(parse); + state.text = new_text; + } + None => { + state.rope = Rope::from_str(&change.text); + state.text = change.text.into(); + state.parse = Arc::new(analyzer.parse(&state.text)); + strategy = ChangeStrategy::Full; + } + } + } + state.version = version; + Some(strategy) + } + + pub(crate) fn close(&self, uri: &Url) { + self.open.remove(uri); + } + + pub(crate) fn snapshot(&self, uri: &Url) -> Option { + self.open.get(uri).map(|state| state.snapshot()) + } + + pub(crate) fn checkout_diagnostics(&self, uri: &Url) -> Option { + self.open.get_mut(uri).map(|mut state| DiagnosticSnapshot { + document: state.snapshot(), + cache: std::mem::take(&mut state.diag_cache), + }) + } + + /// Restore a warmed cache only when the document version is still current. + pub(crate) fn restore_diagnostics( + &self, + uri: &Url, + version: i32, + cache: DiagnosticsCache, + ) -> bool { + let Some(mut state) = self.open.get_mut(uri) else { + return false; + }; + if state.version != version { + return false; + } + state.diag_cache = cache; + true + } + + pub(crate) fn clear_diagnostic_caches(&self) { + for mut state in self.open.iter_mut() { + state.diag_cache = DiagnosticsCache::new(); + } + } + + pub(crate) fn replace_semantic_history( + &self, + uri: &Url, + history: (u64, Vec), + ) -> Option<(u64, Vec)> { + self.open + .get_mut(uri) + .and_then(|mut state| state.last_semantic.replace(history)) + } + + pub(crate) fn open_uris(&self) -> Vec { + self.open.iter().map(|state| state.key().clone()).collect() + } + + pub(crate) fn open_uri_strings(&self) -> std::collections::HashSet { + self.open + .iter() + .map(|state| state.key().as_str().to_string()) + .collect() + } + + pub(crate) fn replace_virtual_files( + &self, + files: impl IntoIterator)>, + ) { + self.virtual_files.clear(); + for (uri, text) in files { + self.virtual_files.insert(uri, text); + } + } + + pub(crate) fn clear_virtual_files(&self) { + self.virtual_files.clear(); + } + + pub(crate) fn read_virtual_file(&self, uri: &str) -> Option { + self.virtual_files.get(uri).map(|text| text.to_string()) + } + + /// Resolve source with live editor buffers taking precedence over virtual + /// BIG entries, which in turn take precedence over disk. + pub(crate) fn rope_for(&self, uri: &Url) -> Option { + if let Some(document) = self.open.get(uri) { + return Some(document.rope.clone()); + } + if let Some(text) = self.virtual_files.get(uri.as_str()) { + return Some(Rope::from_str(&text)); + } + let path = uri.to_file_path().ok()?; + std::fs::read(path) + .ok() + .map(|bytes| Rope::from_str(&String::from_utf8_lossy(&bytes))) + } + + pub(crate) fn source_text(&self, uri: &str) -> Option { + self.rope_for(&Url::parse(uri).ok()?) + .map(|rope| rope.to_string()) + } +} + +#[cfg(test)] +mod tests { + use std::fs; + use std::time::{SystemTime, UNIX_EPOCH}; + + use tower_lsp::lsp_types::{Position, Range}; + + use super::*; + + fn uri(name: &str) -> Url { + Url::parse(&format!("file:///C:/zerosyntax-tests/{name}")).unwrap() + } + + fn ranged(start: Position, end: Position, text: &str) -> TextDocumentContentChangeEvent { + TextDocumentContentChangeEvent { + range: Some(Range { start, end }), + range_length: None, + text: text.into(), + } + } + + #[test] + fn ranged_incremental_change_keeps_snapshot_synchronized() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri("incremental.ini"); + store.open(&analyzer, uri.clone(), "Object Old\nEnd\n".into(), 1); + + let strategy = store.change( + &analyzer, + &uri, + vec![ranged(Position::new(0, 7), Position::new(0, 10), "New")], + 2, + PositionEnc::Utf16, + ); + + assert_eq!(strategy, Some(ChangeStrategy::Incremental)); + let snapshot = store.snapshot(&uri).unwrap(); + assert_eq!(&*snapshot.text, "Object New\nEnd\n"); + assert_eq!(snapshot.rope.to_string(), &*snapshot.text); + assert_eq!(snapshot.parse.syntax().text().to_string(), &*snapshot.text); + assert_eq!(snapshot.version, 2); + } + + #[test] + fn full_replacement_rebuilds_all_representations() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri("full.ini"); + store.open(&analyzer, uri.clone(), "Object Old\nEnd\n".into(), 1); + let strategy = store.change( + &analyzer, + &uri, + vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "Object New\nEnd\n".into(), + }], + 2, + PositionEnc::Utf16, + ); + assert_eq!(strategy, Some(ChangeStrategy::Full)); + let snapshot = store.snapshot(&uri).unwrap(); + assert_eq!(snapshot.rope.to_string(), &*snapshot.text); + assert_eq!(snapshot.parse.syntax().text().to_string(), &*snapshot.text); + } + + #[test] + fn many_ranged_changes_use_bulk_fallback() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri("bulk.ini"); + store.open(&analyzer, uri.clone(), "123456789\n".into(), 1); + let changes = (0..9) + .map(|column| ranged(Position::new(0, column), Position::new(0, column + 1), "x")) + .collect(); + assert_eq!( + store.change(&analyzer, &uri, changes, 2, PositionEnc::Utf16), + Some(ChangeStrategy::Bulk) + ); + let snapshot = store.snapshot(&uri).unwrap(); + assert_eq!(snapshot.rope.to_string(), &*snapshot.text); + assert_eq!(snapshot.parse.syntax().text().to_string(), &*snapshot.text); + } + + #[test] + fn crlf_changes_honor_utf8_and_utf16_positions() { + for (encoding, character) in [(PositionEnc::Utf8, 4), (PositionEnc::Utf16, 2)] { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri(if encoding == PositionEnc::Utf8 { + "utf8.ini" + } else { + "utf16.ini" + }); + store.open(&analyzer, uri.clone(), "😀x\r\nEnd\r\n".into(), 1); + store.change( + &analyzer, + &uri, + vec![ranged( + Position::new(0, character), + Position::new(0, character + 1), + "y", + )], + 2, + encoding, + ); + assert_eq!(&*store.snapshot(&uri).unwrap().text, "😀y\r\nEnd\r\n"); + } + } + + #[test] + fn rejects_stale_diagnostic_cache_restoration() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri("diagnostics.ini"); + store.open(&analyzer, uri.clone(), "Object Old\nEnd\n".into(), 1); + let checked_out = store.checkout_diagnostics(&uri).unwrap(); + store.change( + &analyzer, + &uri, + vec![TextDocumentContentChangeEvent { + range: None, + range_length: None, + text: "Object New\nEnd\n".into(), + }], + 2, + PositionEnc::Utf16, + ); + assert!(!store.restore_diagnostics(&uri, checked_out.document.version, checked_out.cache)); + } + + #[test] + fn semantic_history_is_replaced() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let uri = uri("semantic.ini"); + store.open(&analyzer, uri.clone(), String::new(), 1); + assert!(store + .replace_semantic_history(&uri, (1, Vec::new())) + .is_none()); + assert_eq!( + store.replace_semantic_history(&uri, (2, Vec::new())), + Some((1, Vec::new())) + ); + } + + #[test] + fn source_precedence_is_open_then_virtual_then_disk() { + let analyzer = Analyzer::embedded(); + let store = DocumentStore::new(); + let stamp = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_nanos(); + let path = std::env::temp_dir().join(format!("zerosyntax-document-{stamp}.ini")); + fs::write(&path, "disk").unwrap(); + let disk_uri = Url::from_file_path(&path).unwrap(); + assert_eq!( + store.source_text(disk_uri.as_str()).as_deref(), + Some("disk") + ); + store.open(&analyzer, disk_uri.clone(), "open".into(), 1); + assert_eq!( + store.source_text(disk_uri.as_str()).as_deref(), + Some("open") + ); + + let big_uri = Url::parse("big:///archive.big!/entry.ini").unwrap(); + store.replace_virtual_files([(big_uri.to_string(), Arc::::from("virtual"))]); + assert_eq!( + store.source_text(big_uri.as_str()).as_deref(), + Some("virtual") + ); + store.open(&analyzer, big_uri.clone(), "open-big".into(), 1); + assert_eq!( + store.source_text(big_uri.as_str()).as_deref(), + Some("open-big") + ); + fs::remove_file(path).unwrap(); + } +} diff --git a/crates/server/src/main.rs b/crates/server/src/main.rs index 6ee87b8..06eb8e5 100644 --- a/crates/server/src/main.rs +++ b/crates/server/src/main.rs @@ -4,6 +4,7 @@ mod asset_index; mod backend; mod convert; +mod document; use backend::Backend; use tower_lsp::{LspService, Server}; From 660f2a8bb8132e15c077c928f1611031da473517 Mon Sep 17 00:00:00 2001 From: ViTeXFTW Date: Sun, 12 Jul 2026 22:01:17 +0200 Subject: [PATCH 4/4] refactor: move origin copy fixes to analysis --- crates/analysis/src/actions.rs | 207 ++++++++++++++++++++++++++++++++- crates/analysis/tests/spec.rs | 2 +- crates/server/src/backend.rs | 125 +------------------- 3 files changed, 205 insertions(+), 129 deletions(-) diff --git a/crates/analysis/src/actions.rs b/crates/analysis/src/actions.rs index 22ee1b1..8564257 100644 --- a/crates/analysis/src/actions.rs +++ b/crates/analysis/src/actions.rs @@ -4,7 +4,7 @@ //! unreachable sets, scaffold stub definitions, suppress diagnostics in-file). use zerosyntax_schema::{RefKind, ValueType}; -use zerosyntax_syntax::ast::{Field, Module}; +use zerosyntax_syntax::ast::{Block, Field, Module}; use zerosyntax_syntax::{Parse, SyntaxErrorKind, SyntaxKind, SyntaxNode, SyntaxToken}; use crate::diagnostics::{pragma_rest, pragma_words, Diagnostic, Severity}; @@ -22,7 +22,8 @@ pub struct Fix { /// All fixes applicable within `range` (the editor's selection/cursor line). /// `diags` should be the suppression-filtered diagnostics for the document /// (from `diagnose_with_cache`); `index` is the workspace index for cross-file -/// checks (e.g. stub creation). +/// checks (e.g. stub creation). `load_source` resolves indexed file URIs when +/// a fix needs source from another file. pub fn fixes( analyzer: &Analyzer, parse: &Parse, @@ -30,11 +31,127 @@ pub fn fixes( range: Span, diags: &[Diagnostic], index: Option<&WorkspaceIndex>, + load_source: impl Fn(&str) -> Option, ) -> Vec { let mut out = Vec::new(); insert_missing_ends(parse, text, range, &mut out); suggest_members(analyzer, parse, range, &mut out); diagnostic_fixes(analyzer, parse, text, range, diags, index, &mut out); + if let Some(index) = index { + out.extend(origin_copy_fixes( + analyzer, + parse, + text, + range, + diags, + index, + load_source, + )); + } + out +} + +/// Offer to append the complete base definition for a map-layer override. +fn origin_copy_fixes( + analyzer: &Analyzer, + parse: &Parse, + text: &str, + range: Span, + diags: &[Diagnostic], + index: &WorkspaceIndex, + load_source: impl Fn(&str) -> Option, +) -> Vec { + use std::collections::HashSet; + + let mut out = Vec::new(); + let mut seen = HashSet::new(); + for diagnostic in diags { + if diagnostic.code != "overrides" + || diagnostic.span.end < range.start + || diagnostic.span.start > range.end + { + continue; + } + let block_node = parse + .syntax() + .children() + .filter(|node| node.kind() == SyntaxKind::BLOCK) + .find(|node| { + let node_range = node.text_range(); + u32::from(node_range.start()) <= diagnostic.span.start + && diagnostic.span.end <= u32::from(node_range.end()) + }); + let Some(block_node) = block_node else { + continue; + }; + let block = Block(block_node); + let Some(keyword) = block.keyword() else { + continue; + }; + let Some(kind) = analyzer + .block(keyword.text()) + .and_then(|schema_block| schema_block.defines) + else { + continue; + }; + let Some(name) = block.name().map(|token| token.text().to_string()) else { + continue; + }; + if !seen.insert(name.to_ascii_lowercase()) { + continue; + } + + let base_location = index.locations(kind, &name).iter().find(|location| { + !location + .file + .rsplit(['/', '\\']) + .next() + .is_some_and(|file| { + file.eq_ignore_ascii_case("map.ini") || file.eq_ignore_ascii_case("solo.ini") + }) + }); + let Some(base_location) = base_location else { + continue; + }; + let Some(base_text) = load_source(&base_location.file) else { + continue; + }; + let base_parse = analyzer.parse(&base_text); + let base_block = base_parse + .syntax() + .children() + .filter(|node| node.kind() == SyntaxKind::BLOCK) + .find(|node| { + Block(node.clone()) + .name() + .is_some_and(|token| token.text().eq_ignore_ascii_case(&name)) + }); + let Some(base_block) = base_block else { + continue; + }; + let block_range = base_block.text_range(); + let block_text = + &base_text[usize::from(block_range.start())..usize::from(block_range.end())]; + let base_short = base_location + .file + .rsplit(['/', '\\']) + .next() + .filter(|name| !name.is_empty()) + .unwrap_or("base"); + let lead = if text.is_empty() || text.ends_with('\n') { + "" + } else { + "\n" + }; + let at = text.len() as u32; + out.push(Fix { + title: format!("Insert reference copy of `{name}` from {base_short}"), + span: Span::new(at, at), + new_text: format!( + "{lead}\n; === Reference copy of {name} from {base_short} ===\n; Remove the fields and modules you don't need.\n{block_text}\n" + ), + }); + } out } @@ -483,6 +600,7 @@ mod tests { Span::new(0, src.len() as u32), &diags, Some(&idx), + |_| None, ) } @@ -491,7 +609,15 @@ mod tests { let a = Analyzer::embedded(); let src = "Object Tank\n MaxHealth = 1\n"; let parse = a.parse(src); - let fx = fixes(&a, &parse, src, Span::new(0, src.len() as u32), &[], None); + let fx = fixes( + &a, + &parse, + src, + Span::new(0, src.len() as u32), + &[], + None, + |_| None, + ); assert_eq!(fx.len(), 1, "{fx:?}"); assert!(fx[0].title.contains("`End`") && fx[0].title.contains("Object")); assert_eq!(fx[0].new_text, "End\n"); @@ -504,7 +630,15 @@ mod tests { // `Appearance` is enum locomotor_appearance; TREDS ≈ TREADS. let src = "Locomotor L\n Appearance = TREDS\nEnd\n"; let parse = a.parse(src); - let fx = fixes(&a, &parse, src, Span::new(0, src.len() as u32), &[], None); + let fx = fixes( + &a, + &parse, + src, + Span::new(0, src.len() as u32), + &[], + None, + |_| None, + ); assert!(fx.iter().any(|f| f.new_text == "TREADS"), "{fx:?}"); // Bitflag with prefix op: the `+` is preserved in the replacement. @@ -517,6 +651,7 @@ mod tests { Span::new(0, src2.len() as u32), &[], None, + |_| None, ); assert!(fx2.iter().any(|f| f.new_text == "+EXPLODED"), "{fx2:?}"); } @@ -526,7 +661,16 @@ mod tests { let a = Analyzer::embedded(); let src = "Locomotor L\n Appearance = TREADS\nEnd\n"; let parse = a.parse(src); - assert!(fixes(&a, &parse, src, Span::new(0, src.len() as u32), &[], None).is_empty()); + assert!(fixes( + &a, + &parse, + src, + Span::new(0, src.len() as u32), + &[], + None, + |_| None, + ) + .is_empty()); } // ── unreachable-set fixes ──────────────────────────────────────────────── @@ -869,6 +1013,55 @@ mod tests { assert_eq!(suppress_count, 1, "duplicate suppress actions: {fx_dup:?}"); } + #[test] + fn reference_copy_uses_source_loader_and_requires_source() { + let analyzer = Analyzer::embedded(); + let base_uri = "file:///game/Data/INI/Object.ini"; + let map_uri = "file:///game/Maps/Test/map.ini"; + let base = "Object Tank\n MaxHealth = 500\nEnd\n"; + let source = "Object Tank\n MaxHealth = 750\nEnd\n"; + let parse = analyzer.parse(source); + let mut index = WorkspaceIndex::new(); + index.set_file( + base_uri, + definitions_in(&analyzer, &analyzer.parse(base), base_uri), + ); + index.set_file(map_uri, definitions_in(&analyzer, &parse, map_uri)); + let diags = diagnose(&analyzer, &parse, Some(&index), Some(map_uri)); + assert!(diags + .iter() + .any(|diagnostic| diagnostic.code == "overrides")); + + let loaded = fixes( + &analyzer, + &parse, + source, + Span::new(0, source.len() as u32), + &diags, + Some(&index), + |uri| (uri == base_uri).then(|| base.to_string()), + ); + let reference = loaded + .iter() + .find(|fix| fix.title.contains("Insert reference copy")) + .expect("reference-copy fix should be supplied by the source loader"); + assert!(reference.title.contains("Object.ini")); + assert!(reference.new_text.contains("MaxHealth = 500")); + + let unavailable = fixes( + &analyzer, + &parse, + source, + Span::new(0, source.len() as u32), + &diags, + Some(&index), + |_| None, + ); + assert!(!unavailable + .iter() + .any(|fix| fix.title.contains("Insert reference copy"))); + } + #[test] fn fixes_ignore_diags_outside_range() { // The diagnostic is at EOF; the range covers only the first byte. @@ -880,7 +1073,9 @@ mod tests { idx.set_file("test.ini", definitions_in(&a, &parse, "test.ini")); let diags = diagnose(&a, &parse, Some(&idx), Some("test.ini")); // Range = first byte only (nowhere near the unresolved-reference token). - let fx = fixes(&a, &parse, src, Span::new(0, 1), &diags, Some(&idx)); + let fx = fixes(&a, &parse, src, Span::new(0, 1), &diags, Some(&idx), |_| { + None + }); assert!( !fx.iter() .any(|f| f.title.contains("Suppress") || f.title.contains("Create stub")), diff --git a/crates/analysis/tests/spec.rs b/crates/analysis/tests/spec.rs index ba96947..237a594 100644 --- a/crates/analysis/tests/spec.rs +++ b/crates/analysis/tests/spec.rs @@ -307,7 +307,7 @@ fn check_action( )); }; let range = Span::new(off, off + spec.on.len() as u32); - let fixes = actions::fixes(analyzer, parse, src, range, diags, Some(index)); + let fixes = actions::fixes(analyzer, parse, src, range, diags, Some(index), |_| None); let titles: Vec<&str> = fixes.iter().map(|f| f.title.as_str()).collect(); let missing: Vec<&String> = spec diff --git a/crates/server/src/backend.rs b/crates/server/src/backend.rs index 2c67025..6aa3bfb 100644 --- a/crates/server/src/backend.rs +++ b/crates/server/src/backend.rs @@ -815,27 +815,15 @@ impl LanguageServer for Backend { Some(uri.as_str()), &mut diagnostic.cache, ); - let mut f = actions::fixes( + actions::fixes( &self.analyzer, &document.parse, &document.text, range_span, &diags, idx.as_deref(), - ); - // Origin-copy fix: requires file I/O, so computed here in the server. - if let Some(idx) = idx.as_deref() { - f.extend(origin_copy_fixes( - &self.analyzer, - &document.parse, - &document.text, - range_span, - &diags, - idx, - |base_uri| self.documents.source_text(base_uri.as_str()), - )); - } - f + |source_uri| self.documents.source_text(source_uri), + ) }; // Hand the warmed cache back unless a newer change superseded us. @@ -1149,113 +1137,6 @@ impl LanguageServer for Backend { } } -/// Build "Insert reference copy of " fixes for any `overrides` diagnostic -/// intersecting `range`. Reads the base definition file via `read_file` and -/// extracts the full Object block text to insert at the end of the current file. -fn origin_copy_fixes( - analyzer: &Analyzer, - parse: &zerosyntax_syntax::Parse, - text: &str, - range: zerosyntax_analysis::Span, - diags: &[zerosyntax_analysis::diagnostics::Diagnostic], - index: &WorkspaceIndex, - read_file: impl Fn(&Url) -> Option, -) -> Vec { - use zerosyntax_analysis::index::Location; - use zerosyntax_syntax::ast::Block; - use zerosyntax_syntax::SyntaxKind; - - let mut out = Vec::new(); - let mut seen: std::collections::HashSet = std::collections::HashSet::new(); - - for d in diags { - if d.code != "overrides" { - continue; - } - if d.span.end < range.start || d.span.start > range.end { - continue; - } - // Find the top-level BLOCK whose name token's span contains the diagnostic. - let block_node = parse - .syntax() - .children() - .filter(|n| n.kind() == SyntaxKind::BLOCK) - .find(|n| { - let nr = n.text_range(); - u32::from(nr.start()) <= d.span.start && d.span.end <= u32::from(nr.end()) - }); - let Some(block_node) = block_node else { - continue; - }; - let block = Block(block_node.clone()); - let Some(kw) = block.keyword() else { continue }; - let Some(schema_block) = analyzer.block(kw.text()) else { - continue; - }; - let Some(kind) = schema_block.defines else { - continue; - }; - let Some(name_tok) = block.name() else { - continue; - }; - let name = name_tok.text().to_string(); - if !seen.insert(name.to_ascii_lowercase()) { - continue; - } - // Find a base-game (non-override-layer) definition location. - let base_loc: Option<&Location> = index.locations(kind, &name).iter().find(|l| { - !l.file.rsplit(['/', '\\']).next().is_some_and(|f| { - f.eq_ignore_ascii_case("map.ini") || f.eq_ignore_ascii_case("solo.ini") - }) - }); - let Some(base_loc) = base_loc else { continue }; - let base_url = match Url::parse(&base_loc.file) { - Ok(u) => u, - Err(_) => continue, - }; - let Some(base_text) = read_file(&base_url) else { - continue; - }; - // Re-parse the base file and extract the block at the name token. - let base_parse = analyzer.parse(&base_text); - let base_block_text: Option = base_parse - .syntax() - .children() - .filter(|n| n.kind() == SyntaxKind::BLOCK) - .find(|n| { - Block(n.clone()) - .name() - .map(|t| t.text().eq_ignore_ascii_case(&name)) - .unwrap_or(false) - }) - .map(|n| { - let r = n.text_range(); - base_text[usize::from(r.start())..usize::from(r.end())].to_string() - }); - let Some(block_text) = base_block_text else { - continue; - }; - let base_short = base_url - .path_segments() - .and_then(|mut s| s.next_back()) - .unwrap_or("base"); - let lead = if text.is_empty() || text.ends_with('\n') { - "" - } else { - "\n" - }; - let at = text.len() as u32; - out.push(actions::Fix { - title: format!("Insert reference copy of `{name}` from {base_short}"), - span: zerosyntax_analysis::Span::new(at, at), - new_text: format!( - "{lead}\n; === Reference copy of {name} from {base_short} ===\n; Remove the fields and modules you don't need.\n{block_text}\n" - ), - }); - } - out -} - #[cfg(test)] mod tests { use super::*;