diff --git a/Cargo.lock b/Cargo.lock index 3266696ae80d..5e0972f2e358 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4471,6 +4471,7 @@ dependencies = [ "rspack_error", "rspack_fs", "rspack_hash", + "rspack_napi", "rspack_paths", "rspack_sources", "rspack_util", @@ -4574,6 +4575,7 @@ dependencies = [ "napi", "oneshot", "rspack_error", + "rustc-hash", "serde_json", "tokio", ] diff --git a/crates/node_binding/napi-binding.d.ts b/crates/node_binding/napi-binding.d.ts index 2285e7815e3a..625d43b23a10 100644 --- a/crates/node_binding/napi-binding.d.ts +++ b/crates/node_binding/napi-binding.d.ts @@ -420,6 +420,37 @@ export declare class JsLoaderCache { store(loaderIndex: number, output: JsLoaderCacheEntry): Promise } +/** + * Owns the boxed native context while JavaScript executes. Returning the class + * moves the context back to Rust and leaves the cached JavaScript instance empty. + * The same instance is reattached on the next entry for this native lifetime. + */ +export declare class JsLoaderContext { + /** + * Snapshot the mutable execution state in one crossing. Output remains lazy: + * leaving it absent on writeback preserves native content and source maps. + */ + get state(): JsLoaderContextState + get resource(): string + get _module(): Module + /** Content may be empty in the pitching stage. */ + get content(): string | Buffer | null + get additionalData(): any + get sourceMap(): Buffer | null + get loaderItems(): Array + get __internal__loaderCache(): JsLoaderCache | undefined + /** + * Commit the JavaScript wrapper's state in one crossing. An absent output + * preserves native content and source maps when pitching produces no output. + */ + set state(result: JsLoaderContextState) + /** + * Return unexpected JavaScript failures together with the owned context, + * even when reading or converting the state object itself failed. + */ + set __internal__error(error: RspackError) +} + export declare class JsModuleGraph { getModule(dependency: Dependency): Module | null getResolvedModule(dependency: Dependency): Module | null @@ -972,23 +1003,18 @@ export interface JsLoaderCacheEntry { parseMeta: Record } -export interface JsLoaderContext { - loaderContextState?: object | undefined - resource: string - _module: Module - hot: Readonly - /** Content maybe empty in pitching stage */ - content: string | Buffer | null - additionalData?: any - __internal__parseMeta: Record - sourceMap?: Buffer +export interface JsLoaderContextState { cacheable: boolean dependencies: JsLoaderDependencies - loaderItems: Array + hot: boolean + /** The native scheduler owns phase transitions; writeback does not change it. */ + loaderState: JsLoaderState + loaderItemStates: Array loaderIndex: number - loaderState: Readonly - __internal__error?: RspackError - __internal__loaderCache?: JsLoaderCache | undefined + /** JavaScript additions, merged into the native typed parse metadata. */ + parseMeta: Record + output?: JsLoaderOutput + error?: RspackError } export interface JsLoaderDependencies { @@ -998,6 +1024,18 @@ export interface JsLoaderDependencies { buildDependencies: Array } +/** + * The before-loaders hook borrows the native context and exchanges only owned + * snapshots. It neither moves the Box nor materializes source content/maps. + */ +export interface JsLoaderHookContext { + identity: JsLoaderContext + state: JsLoaderContextState + resource: string + _module: Module + loaderItems?: Array +} + export interface JsLoaderItem { loader: string type: string @@ -1008,6 +1046,26 @@ export interface JsLoaderItem { noPitch: boolean } +export interface JsLoaderItemState { + data: any + normalExecuted: boolean + pitchExecuted: boolean + noPitch: boolean +} + +/** Immutable loader metadata, materialized once for the JavaScript facade. */ +export interface JsLoaderMetadata { + loader: string + type: string + cache: boolean +} + +export interface JsLoaderOutput { + content: string | Buffer | null + sourceMap?: Buffer + additionalData?: any +} + export declare enum JsLoaderState { Pitching = 'Pitching', Normal = 'Normal' @@ -3335,7 +3393,7 @@ export interface RegisterJsTaps { registerCompilationAfterProcessAssetsTaps: (stages: Array) => Array<{ function: ((arg: JsCompilation) => void); stage: number; }> registerCompilationSealTaps: (stages: Array) => Array<{ function: (() => void); stage: number; }> registerCompilationAfterSealTaps: (stages: Array) => Array<{ function: (() => Promise); stage: number; }> - registerNormalModuleLoaderTaps: (stages: Array) => Array<{ function: ((arg: JsLoaderContext) => JsLoaderContext); stage: number; }> + registerNormalModuleLoaderTaps: (stages: Array) => Array<{ function: ((arg: JsLoaderHookContext) => JsLoaderContextState); stage: number; }> registerNormalModuleFactoryBeforeResolveTaps: (stages: Array) => Array<{ function: ((arg: JsResolveData) => Promise<[boolean | undefined, JsResolveData]>); stage: number; }> registerNormalModuleFactoryFactorizeTaps: (stages: Array) => Array<{ function: ((arg: JsResolveData) => Promise); stage: number; }> registerNormalModuleFactoryResolveTaps: (stages: Array) => Array<{ function: ((arg: JsResolveData) => Promise); stage: number; }> diff --git a/crates/node_binding/rspack.wasi-browser.js b/crates/node_binding/rspack.wasi-browser.js index 789b1c70518a..d5c5c18064d9 100644 --- a/crates/node_binding/rspack.wasi-browser.js +++ b/crates/node_binding/rspack.wasi-browser.js @@ -96,6 +96,7 @@ export const JsContextModuleFactoryBeforeResolveData = __napiModule.exports.JsCo export const JsCoordinator = __napiModule.exports.JsCoordinator export const JsDependencies = __napiModule.exports.JsDependencies export const JsEntries = __napiModule.exports.JsEntries +export const JsLoaderContext = __napiModule.exports.JsLoaderContext export const JsExportsInfo = __napiModule.exports.JsExportsInfo export const JsModuleGraph = __napiModule.exports.JsModuleGraph export const JsResolver = __napiModule.exports.JsResolver diff --git a/crates/node_binding/rspack.wasi.cjs b/crates/node_binding/rspack.wasi.cjs index f95783d1dfac..5658ab2cd089 100644 --- a/crates/node_binding/rspack.wasi.cjs +++ b/crates/node_binding/rspack.wasi.cjs @@ -136,6 +136,7 @@ module.exports.JsContextModuleFactoryBeforeResolveData = __napiModule.exports.Js module.exports.JsCoordinator = __napiModule.exports.JsCoordinator module.exports.JsDependencies = __napiModule.exports.JsDependencies module.exports.JsEntries = __napiModule.exports.JsEntries +module.exports.JsLoaderContext = __napiModule.exports.JsLoaderContext module.exports.JsExportsInfo = __napiModule.exports.JsExportsInfo module.exports.JsModuleGraph = __napiModule.exports.JsModuleGraph module.exports.JsResolver = __napiModule.exports.JsResolver diff --git a/crates/rspack_binding_api/src/modules/normal_module.rs b/crates/rspack_binding_api/src/modules/normal_module.rs index 75bd9cd85681..4d29294aa273 100644 --- a/crates/rspack_binding_api/src/modules/normal_module.rs +++ b/crates/rspack_binding_api/src/modules/normal_module.rs @@ -48,7 +48,7 @@ impl NormalModule { module .loaders() .iter() - .map(JsLoaderItem::from) + .map(|resolved| JsLoaderItem::from(&resolved.loader)) .collect::>(), )? }); diff --git a/crates/rspack_binding_api/src/plugins/interceptor.rs b/crates/rspack_binding_api/src/plugins/interceptor.rs index 96681719da50..0c799190f951 100644 --- a/crates/rspack_binding_api/src/plugins/interceptor.rs +++ b/crates/rspack_binding_api/src/plugins/interceptor.rs @@ -101,7 +101,10 @@ use crate::{ JsCreateData, JsNormalModuleFactoryCreateModuleArgs, JsResolveData, JsResolveForSchemeArgs, JsResolveForSchemeOutput, }, - plugins::js_loader::{JsLoaderContext, merge_loader_context}, + plugins::js_loader::{ + JsLoaderContextState, JsLoaderHookContext, + context::{JsLoaderHookContextObject, check_loader_error}, + }, rsdoctor::{ JsRsdoctorAssetPatch, JsRsdoctorChunkGraph, JsRsdoctorModuleGraph, JsRsdoctorModuleIdsPatch, JsRsdoctorModuleSourcesPatch, @@ -626,7 +629,7 @@ pub struct RegisterJsTaps { )] pub register_compilation_after_seal_taps: RegisterFunction, #[napi( - ts_type = "(stages: Array) => Array<{ function: ((arg: JsLoaderContext) => JsLoaderContext); stage: number; }>" + ts_type = "(stages: Array) => Array<{ function: ((arg: JsLoaderHookContext) => JsLoaderContextState); stage: number; }>" )] pub register_normal_module_loader_taps: RegisterFunction, #[napi( @@ -942,7 +945,7 @@ define_register!( /* NormalModule Hooks */ define_register!( RegisterNormalModuleLoaderTaps, - tap = NormalModuleLoaderTap @ NormalModuleLoaderHook, + tap = NormalModuleLoaderTap @ NormalModuleLoaderHook, cache = true, kind = RegisterJsTapKind::NormalModuleLoader, skip = true, @@ -1705,11 +1708,14 @@ impl CompilationAfterSeal for CompilationAfterSealTap { #[async_trait] impl NormalModuleLoader for NormalModuleLoaderTap { async fn run(&self, context: &mut LoaderContext) -> rspack_error::Result<()> { - let data = self + let state = self .function - .call_with_sync(JsLoaderContext::try_from(&mut *context)?) + .call_with_sync(JsLoaderHookContextObject(JsLoaderHookContext::new(context))) .await?; - merge_loader_context(context, data) + let (error, _) = state + .apply(context) + .map_err(|error| rspack_error::error!("{error}"))?; + check_loader_error(error) } fn stage(&self) -> i32 { diff --git a/crates/rspack_binding_api/src/plugins/js_loader/context.rs b/crates/rspack_binding_api/src/plugins/js_loader/context.rs index ccc2a13dfc50..ba247c93d30d 100644 --- a/crates/rspack_binding_api/src/plugins/js_loader/context.rs +++ b/crates/rspack_binding_api/src/plugins/js_loader/context.rs @@ -3,10 +3,11 @@ use std::{ptr::NonNull, sync::Arc}; use napi::bindgen_prelude::*; use napi_derive::napi; use rspack_collections::Identifiable; -use rspack_core::{Content, LoaderContext, LoaderDependencies, Module, RunnerContext}; -use rspack_error::ToStringResultToRspackResultExt; -use rspack_loader_runner::State as LoaderState; -use rspack_napi::threadsafe_js_value_ref::ThreadsafeJsValueRef; +use rspack_core::{AdditionalData, Content, LoaderContext, LoaderDependencies, RunnerContext}; +use rspack_loader_runner::{LoaderContextLifetime, State as LoaderState}; +use rspack_napi::{ + LifecycleId, ThreadLocalReference, threadsafe_js_value_ref::ThreadsafeJsValueRef, +}; use rustc_hash::FxHashMap as HashMap; use super::cache::JsLoaderCacheObject; @@ -29,20 +30,20 @@ pub struct JsLoaderItem { pub no_pitch: bool, } -impl From<&rspack_loader_runner::LoaderItem> for JsLoaderItem { - fn from(value: &rspack_loader_runner::LoaderItem) -> Self { - JsLoaderItem { - loader: value.request().to_string(), - r#type: value.r#type().to_string(), - cache: value.cache(), - - data: value.data().clone(), - normal_executed: value.normal_executed(), - pitch_executed: value.pitch_executed(), +/// Immutable loader metadata, materialized once for the JavaScript facade. +#[napi(object)] +pub struct JsLoaderMetadata { + pub loader: String, + pub r#type: String, + pub cache: bool, +} - no_pitch: false, - } - } +#[napi(object)] +pub struct JsLoaderItemState { + pub data: serde_json::Value, + pub normal_executed: bool, + pub pitch_executed: bool, + pub no_pitch: bool, } impl From<&Arc>> for JsLoaderItem @@ -169,98 +170,383 @@ impl From for LoaderDependencies { } } -#[napi(object)] +/// Owns the boxed native context while JavaScript executes. Returning the class +/// moves the context back to Rust and leaves the cached JavaScript instance empty. +/// The same instance is reattached on the next entry for this native lifetime. +#[napi] pub struct JsLoaderContext { - #[napi(ts_type = "object | undefined")] - pub loader_context_state: Option>>, - pub resource: String, - #[napi(js_name = "_module", ts_type = "Module")] - pub module: ModuleObject, - #[napi(ts_type = "Readonly")] - pub hot: bool, + pub(crate) context: Option>>, + pub(crate) error: Option, + pub(crate) loaders_without_pitch: Vec, +} + +impl JsLoaderContext { + fn empty() -> Self { + Self { + context: None, + error: None, + loaders_without_pitch: Vec::new(), + } + } + + fn unavailable() -> napi::Error { + napi::Error::from_reason( + "Loader context is no longer available after the JavaScript loader runner has finished", + ) + } + + fn with_context( + &self, + f: impl FnOnce(&LoaderContext) -> napi::Result, + ) -> napi::Result { + f(self.context.as_deref().ok_or_else(Self::unavailable)?) + } + + pub(crate) fn take_error(&mut self) -> rspack_error::Result<()> { + check_loader_error(self.error.take()) + } +} + +/// Transport wrapper: only JS-thread conversion accesses the reference cache. +pub struct JsLoaderContextObject(pub Box>); + +impl ToNapiValue for JsLoaderContextObject { + unsafe fn to_napi_value(env: sys::napi_env, value: Self) -> napi::Result { + let env = Env::from_raw(env); + let mut instance = + ThreadLocalReference::::get_or_insert_with( + &env, + value.0.lifecycle.id(), + JsLoaderContext::empty, + )?; + if instance.context.is_some() { + return Err(napi::Error::from_reason( + "Loader context is already executing in JavaScript", + )); + } + instance.context = Some(value.0); + Ok(instance.value) + } +} + +/// Hooks identify the same cached class without transferring the native Box. +pub struct JsLoaderContextIdentity(LifecycleId); + +impl ToNapiValue for JsLoaderContextIdentity { + unsafe fn to_napi_value(env: sys::napi_env, value: Self) -> napi::Result { + let env = Env::from_raw(env); + Ok( + ThreadLocalReference::::get_or_insert_with( + &env, + value.0, + JsLoaderContext::empty, + )? + .value, + ) + } +} + +impl FromNapiValue for JsLoaderContext { + unsafe fn from_napi_value(env: sys::napi_env, value: sys::napi_value) -> napi::Result { + // Conversion runs on the JavaScript thread. Take the Box before sending the + // returned value to Rust; later access through the JS class sees None. + let mut instance = unsafe { ClassInstance::::from_napi_value(env, value)? }; + Ok(Self { + context: Some(instance.context.take().ok_or_else(Self::unavailable)?), + error: instance.error.take(), + loaders_without_pitch: std::mem::take(&mut instance.loaders_without_pitch), + }) + } +} + +#[napi] +impl JsLoaderContext { + /// Snapshot the mutable execution state in one crossing. Output remains lazy: + /// leaving it absent on writeback preserves native content and source maps. + #[napi(getter)] + pub fn state(&self) -> napi::Result { + self.with_context(|cx| Ok(JsLoaderContextState::from_context(cx))) + } + + #[napi(getter)] + pub fn resource(&self) -> napi::Result { + self.with_context(|cx| Ok(cx.resource().to_owned())) + } + + #[napi(getter, js_name = "_module", ts_return_type = "Module")] + pub fn module(&self) -> napi::Result { + self.with_context(|cx| { + Ok(ModuleObject::with_ptr( + NonNull::from(cx.context.module.as_ref() as &dyn rspack_core::Module), + cx.context.compiler_id, + )) + }) + } + + /// Content may be empty in the pitching stage. + #[napi(getter)] + pub fn content(&self) -> napi::Result> { + self.with_context(|cx| { + Ok(match cx.content() { + Some(Content::String(content)) => Either3::A(content.clone()), + Some(Content::Buffer(content)) => Either3::B(content.clone().into()), + None => Either3::C(Null), + }) + }) + } + + #[napi(getter, ts_return_type = "any")] + pub fn additional_data(&self) -> napi::Result>>> { + self.with_context(|cx| { + Ok( + cx.additional_data() + .and_then(|data| data.get::>()) + .cloned(), + ) + }) + } + + #[napi(getter)] + pub fn source_map(&self) -> napi::Result> { + self.with_context(|cx| Ok(cx.source_map().map(|map| map.to_json().into_bytes().into()))) + } + + #[napi(getter)] + pub fn loader_items(&self) -> napi::Result> { + self.with_context(|cx| { + Ok( + cx.loader_items() + .iter() + .map(|item| JsLoaderMetadata { + loader: item.request().to_string(), + r#type: item.r#type().to_string(), + cache: item.cache(), + }) + .collect(), + ) + }) + } + + #[napi( + getter, + js_name = "__internal__loaderCache", + ts_return_type = "JsLoaderCache | undefined" + )] + pub fn loader_cache(&self) -> napi::Result> { + self.with_context(|cx| { + Ok( + cx.loader_items() + .iter() + .any(|loader| loader.cache()) + .then(|| { + JsLoaderCacheObject::new( + cx.context.loader_cache.clone(), + cx.context.file_system_info.clone(), + cx.context.module.identifier().to_string(), + cx.loader_items() + .iter() + .map(|loader| loader.cache_options().cloned().unwrap_or_default()) + .collect(), + ) + }), + ) + }) + } + /// Commit the JavaScript wrapper's state in one crossing. An absent output + /// preserves native content and source maps when pitching produces no output. + #[napi(setter)] + pub fn set_state(&mut self, result: JsLoaderContextState) -> napi::Result<()> { + let cx = self.context.as_deref_mut().ok_or_else(Self::unavailable)?; + let (error, loaders_without_pitch) = result.apply(cx)?; + self.error = error; + self.loaders_without_pitch.extend(loaders_without_pitch); + Ok(()) + } - /// Content maybe empty in pitching stage + /// Return unexpected JavaScript failures together with the owned context, + /// even when reading or converting the state object itself failed. + #[napi(setter, js_name = "__internal__error")] + pub fn set_error(&mut self, error: RspackError) -> napi::Result<()> { + self.context.as_ref().ok_or_else(Self::unavailable)?; + self.error = Some(error); + Ok(()) + } +} + +#[napi(object)] +pub struct JsLoaderOutput { pub content: Either3, + pub source_map: Option, #[napi(ts_type = "any")] pub additional_data: Option>>, - #[napi(js_name = "__internal__parseMeta")] - pub parse_meta: HashMap, - pub source_map: Option, +} + +#[napi(object)] +pub struct JsLoaderContextState { pub cacheable: bool, pub dependencies: JsLoaderDependencies, - - pub loader_items: Vec, - pub loader_index: i32, - #[napi(ts_type = "Readonly")] + pub hot: bool, + /// The native scheduler owns phase transitions; writeback does not change it. pub loader_state: JsLoaderState, - #[napi(js_name = "__internal__error")] + pub loader_item_states: Vec, + pub loader_index: i32, + /// JavaScript additions, merged into the native typed parse metadata. + pub parse_meta: HashMap, + pub output: Option, pub error: Option, - #[napi( - js_name = "__internal__loaderCache", - ts_type = "JsLoaderCache | undefined" - )] - pub loader_cache: Option, } -impl TryFrom<&mut LoaderContext> for JsLoaderContext { - type Error = rspack_error::Error; +impl JsLoaderContextState { + pub(crate) fn from_context(cx: &LoaderContext) -> Self { + Self { + hot: cx.hot, + loader_state: cx.state().into(), + loader_item_states: cx + .loader_item_states + .iter() + .map(|state| JsLoaderItemState { + data: state.data().clone(), + normal_executed: state.normal_executed(), + pitch_executed: state.pitch_executed(), + no_pitch: false, + }) + .collect(), + loader_index: cx.loader_index, + cacheable: cx.cacheable, + dependencies: cx.dependencies().as_ref().into(), + parse_meta: HashMap::default(), + output: None, + error: None, + } + } - fn try_from( - cx: &mut rspack_core::LoaderContext, - ) -> std::result::Result { - let module = &cx.context.module; + pub(crate) fn apply( + self, + cx: &mut LoaderContext, + ) -> napi::Result<(Option, Vec)> { + cx.hot = self.hot; + cx.cacheable = self.cacheable; + cx.replace_dependencies(self.dependencies.into()); + if self.error.is_some() { + return Ok((self.error, Vec::new())); + } + if let Some(output) = self.output { + let source_map = output + .source_map + .map(|buffer| rspack_core::rspack_sources::SourceMap::from_bytes(buffer.into())) + .transpose() + .map_err(|error| napi::Error::from_reason(error.to_string()))?; + let content = match output.content { + Either3::C(_) => None, + Either3::B(buffer) => Some(Content::from(Vec::::from(buffer))), + Either3::A(string) => Some(Content::from(string)), + }; + let additional_data = output.additional_data.map(|value| { + let mut data = AdditionalData::default(); + data.insert(value); + data + }); + cx.__finish_with((content, source_map, additional_data)); + } + let mut loaders_without_pitch = Vec::new(); + let pitching = cx.state() == LoaderState::Pitching; + for (index, item) in self + .loader_item_states + .into_iter() + .take(cx.loader_item_states.len()) + .enumerate() + { + if item.no_pitch && pitching { + loaders_without_pitch.push(cx.loader_items()[index].path().to_string()); + } + let loader = &mut cx.loader_item_states[index]; + if item.normal_executed { + loader.set_normal_executed(); + loader.set_finish_called(); + } + if item.pitch_executed { + loader.set_pitch_executed(); + } + loader.set_data(item.data); + } + cx.loader_index = self.loader_index; + cx.parse_meta.extend( + self + .parse_meta + .into_iter() + .map(|(key, value)| (key, Box::new(value) as _)), + ); + Ok((None, loaders_without_pitch)) + } +} - #[allow(clippy::unwrap_used)] - Ok(JsLoaderContext { - loader_context_state: cx - .context - .loader_context_data - .get::>() - .cloned(), - resource: cx.resource_data.resource().to_owned(), +/// The before-loaders hook borrows the native context and exchanges only owned +/// snapshots. It neither moves the Box nor materializes source content/maps. +#[napi(object, object_from_js = false)] +pub struct JsLoaderHookContext { + #[napi(ts_type = "JsLoaderContext")] + pub identity: JsLoaderContextIdentity, + pub state: JsLoaderContextState, + pub resource: String, + #[napi(js_name = "_module", ts_type = "Module")] + pub module: ModuleObject, + pub loader_items: Option>, +} + +pub struct JsLoaderHookContextObject(pub JsLoaderHookContext); + +impl ToNapiValue for JsLoaderHookContextObject { + unsafe fn to_napi_value(env: sys::napi_env, mut value: Self) -> napi::Result { + let env_wrapper = Env::from_raw(env); + let mut created = false; + ThreadLocalReference::::get_or_insert_with( + &env_wrapper, + value.0.identity.0, + || { + created = true; + JsLoaderContext::empty() + }, + )?; + // Only the first hook needs to materialize loader metadata for the facade. + if !created { + value.0.loader_items = None; + } + unsafe { ToNapiValue::to_napi_value(env, value.0) } + } +} + +impl JsLoaderHookContext { + pub(crate) fn new(cx: &LoaderContext) -> Self { + let state = JsLoaderContextState::from_context(cx); + let loader_items = Some({ + cx.loader_items() + .iter() + .map(|item| JsLoaderMetadata { + loader: item.request().to_string(), + r#type: item.r#type().to_string(), + cache: item.cache(), + }) + .collect() + }); + Self { + identity: JsLoaderContextIdentity(cx.lifecycle.id()), + state, + resource: cx.resource().to_owned(), module: ModuleObject::with_ptr( - NonNull::new(module.as_ref() as *const dyn Module as *mut dyn Module).unwrap(), + NonNull::from(cx.context.module.as_ref() as &dyn rspack_core::Module), cx.context.compiler_id, ), - hot: cx.hot, - content: match cx.content() { - Some(Content::String(content)) => Either3::A(content.clone()), - Some(Content::Buffer(content)) => Either3::B(content.clone().into()), - None => Either3::C(Null), - }, - // Since js side only set parse meta, and can't read it, so we can use Default here to only bring the - // set values from js side to rust side. - parse_meta: Default::default(), - additional_data: cx - .additional_data() - .and_then(|data| data.get::>()) - .cloned(), - source_map: cx - .source_map() - .map(|v| v.to_json()) - .map(|v| v.into_bytes().into()), - cacheable: cx.cacheable, - dependencies: cx.dependencies().as_ref().into(), + loader_items, + } + } +} - loader_items: cx.loader_items.iter().map(Into::into).collect(), - loader_index: cx.loader_index, - loader_state: cx.state().into(), - error: None, - loader_cache: cx - .loader_items - .iter() - .any(|loader| loader.cache()) - .then(|| { - JsLoaderCacheObject::new( - cx.context.loader_cache.clone(), - cx.context.file_system_info.clone(), - module.identifier().to_string(), - cx.loader_items - .iter() - .map(|loader| loader.cache_options().cloned().unwrap_or_default()) - .collect(), - ) - }), - }) +pub(crate) fn check_loader_error(error: Option) -> rspack_error::Result<()> { + if let Some(error) = error { + if let Some(diagnostic) = error.rust_diagnostic.as_ref() { + return Err(diagnostic.error.clone()); + } + return Err(error.with_parent_error_name("ModuleBuildError").into()); } + Ok(()) } diff --git a/crates/rspack_binding_api/src/plugins/js_loader/mod.rs b/crates/rspack_binding_api/src/plugins/js_loader/mod.rs index e3dba3aaec48..ba3ba61a2996 100644 --- a/crates/rspack_binding_api/src/plugins/js_loader/mod.rs +++ b/crates/rspack_binding_api/src/plugins/js_loader/mod.rs @@ -1,5 +1,5 @@ mod cache; -mod context; +pub(crate) mod context; mod resolver; mod scheduler; @@ -11,7 +11,10 @@ use std::{ }; pub use cache::{JsLoaderCache, JsLoaderCacheEntry}; -pub use context::{JsLoaderContext, JsLoaderDependencies, JsLoaderItem}; +use context::JsLoaderContextObject; +pub use context::{ + JsLoaderContext, JsLoaderContextState, JsLoaderDependencies, JsLoaderHookContext, JsLoaderItem, +}; use napi::{ bindgen_prelude::*, sys::{napi_call_threadsafe_function, napi_threadsafe_function}, @@ -24,15 +27,14 @@ use rspack_core::{ use rspack_error::Result; use rspack_hook::{plugin, plugin_hook}; use rustc_hash::FxHashSet; -pub(crate) use scheduler::merge_loader_context; use tokio::sync::{OnceCell, RwLock}; use crate::{COMPILER_REFERENCES, error::RspackResultToNapiResultExt}; pub type JsLoaderRunner = ThreadsafeFunction< - JsLoaderContext, + JsLoaderContextObject, Promise, - JsLoaderContext, + JsLoaderContextObject, Status, false, true, @@ -65,9 +67,11 @@ extern "C" fn napi_js_callback( Object::from_napi_value(env, napi_value)? }; let run_loader = compiler_object - .get_named_property::>>("_runLoader")?; + .get_named_property::>>( + "_runLoader", + )?; let ts_fn: JsLoaderRunner = run_loader - .build_threadsafe_function::() + .build_threadsafe_function::() .weak::() .callee_handled::() .max_queue_size::<0>() @@ -209,10 +213,8 @@ impl Plugin for JsLoaderRspackPlugin { .resolve_loader .tap(resolver::resolve_loader::new(self)); - ctx - .normal_module_hooks - .loader_yield - .tap(scheduler::loader_yield::new(self)); + ctx.normal_module_hooks.javascript_loader_runner = + Some(Arc::new(Self::from_inner(self.inner()))); // TODO: tap compiler done hook will be better. ctx.compiler_hooks.emit.tap(done::new(self)); diff --git a/crates/rspack_binding_api/src/plugins/js_loader/scheduler.rs b/crates/rspack_binding_api/src/plugins/js_loader/scheduler.rs index 49710d577234..577116294b99 100644 --- a/crates/rspack_binding_api/src/plugins/js_loader/scheduler.rs +++ b/crates/rspack_binding_api/src/plugins/js_loader/scheduler.rs @@ -1,10 +1,9 @@ -use napi::bindgen_prelude::{Either3, JsValuesTupleIntoVec}; -use rspack_core::{AdditionalData, LoaderContext, NormalModuleLoaderStartYielding, RunnerContext}; +use napi::bindgen_prelude::JsValuesTupleIntoVec; +use rspack_core::{LoaderContext, RunnerContext}; use rspack_error::{Result, ToStringResultToRspackResultExt}; -use rspack_hook::plugin_hook; use rspack_loader_runner::State as LoaderState; -use super::{JsLoaderContext, JsLoaderRspackPlugin, JsLoaderRspackPluginInner}; +use super::{JsLoaderRspackPlugin, context::JsLoaderContextObject}; impl JsLoaderRspackPlugin { async fn update_loaders_without_pitch(&self, list: Vec) { @@ -15,129 +14,59 @@ impl JsLoaderRspackPlugin { } } -#[plugin_hook(NormalModuleLoaderStartYielding for JsLoaderRspackPlugin,tracing=false)] -pub(crate) async fn loader_yield( - &self, - loader_context: &mut LoaderContext, -) -> Result<()> { - // Keep pitch capability discovery on the JS side of the runtime boundary. - // A loader known not to have a pitch function does not need a JS callback. - if loader_context.state() == LoaderState::Pitching - && self - .loaders_without_pitch - .read() - .await - .contains(loader_context.current_loader().path().as_str()) - { - loader_context.current_loader().set_pitch_executed(); - loader_context.loader_index += 1; - return Ok(()); - } - - let runner = self.runner.lock().expect("should get lock").clone(); - let runner = runner - .get_or_try_init(|| async { - #[allow(clippy::unwrap_used)] - let compiler_id = self.compiler_id.get().unwrap(); - self.runner_getter.call(compiler_id).await - }) - .await - .to_rspack_result()?; +#[async_trait::async_trait] +impl rspack_loader_runner::LoaderRunner for JsLoaderRspackPlugin { + type Context = RunnerContext; - let new_cx = runner - .call_async(loader_context.try_into()?) - .await - .to_rspack_result()? - .await - .to_rspack_result()?; - - if loader_context.state() == LoaderState::Pitching { - let list = collect_loaders_without_pitch(loader_context, &new_cx); - if !list.is_empty() { - self.update_loaders_without_pitch(list).await; + async fn run( + &self, + mut cx: Box>, + ) -> (Box>, Result<()>) { + // Keep pitch capability discovery on the JS side of the runtime boundary. + // A loader known not to have a pitch function does not need a JS callback. + if cx.state() == LoaderState::Pitching + && self + .loaders_without_pitch + .read() + .await + .contains(cx.current_loader().path().as_str()) + { + cx.set_current_loader_pitch_executed(); + cx.loader_index += 1; + return (cx, Ok(())); } - } - - merge_loader_context(loader_context, new_cx)?; - - Ok(()) -} - -pub(crate) fn merge_loader_context( - to: &mut LoaderContext, - mut from: JsLoaderContext, -) -> Result<()> { - if let Some(state) = from.loader_context_state.take() { - to.context.loader_context_data.insert(state); - } - to.cacheable = from.cacheable; - to.replace_dependencies(from.dependencies.into()); - if let Some(error) = from.error { - if let Some(diagnostic) = error.rust_diagnostic.as_ref() { - return Err(diagnostic.error.clone()); - } - return Err(error.with_parent_error_name("ModuleBuildError").into()); - } - - let content = match from.content { - Either3::A(content) => Some(rspack_core::Content::String(content)), - Either3::B(content) => Some(rspack_core::Content::Buffer(content.into())), - Either3::C(_) => None, - }; - let source_map = from - .source_map - .map(|buffer| rspack_core::rspack_sources::SourceMap::from_bytes(buffer.into())) - .transpose() - .to_rspack_result()?; - let additional_data = from.additional_data.take().map(|data| { - let mut additional = AdditionalData::default(); - additional.insert(data); - additional - }); - to.__finish_with((content, source_map, additional_data)); - - // update loader status - let to_state = to.state(); - to.loader_items = to - .loader_items - .drain(..) - .zip(from.loader_items.drain(..)) - .map(|(mut to, from)| { - if from.normal_executed { - to.set_normal_executed() - } - if from.pitch_executed { - to.set_pitch_executed() - } - to.set_data(from.data); - // The loader hook also merges a snapshot, before any loader has run. - if to_state != LoaderState::Init { - to.set_finish_called(); - } - to - }) - .collect(); - to.loader_index = from.loader_index; - to.parse_meta.extend( - from - .parse_meta - .into_iter() - .map(|(k, v)| (k, Box::new(v) as _)), - ); - - Ok(()) -} + let runner = self.runner.lock().expect("should get lock").clone(); + let runner = match runner + .get_or_try_init(|| async { + #[allow(clippy::unwrap_used)] + let compiler_id = self.compiler_id.get().unwrap(); + self.runner_getter.call(compiler_id).await + }) + .await + .to_rspack_result() + { + Ok(runner) => runner, + Err(error) => return (cx, Err(error)), + }; -fn collect_loaders_without_pitch( - ctx: &LoaderContext, - js_ctx: &JsLoaderContext, -) -> Vec { - let mut list = Vec::new(); - for (js_loader_item, loader_item) in js_ctx.loader_items.iter().zip(ctx.loader_items.iter()) { - if js_loader_item.no_pitch { - list.push(loader_item.path().to_string()); + // Once transferred, the JS boundary must return the class even when a loader + // throws. Transport failures cannot recover an allocation already sent to JS. + let mut js_context = runner + .call_async(JsLoaderContextObject(cx)) + .await + .expect("JavaScript loader call must return the owned context") + .await + .expect("JavaScript loader promise must return the owned context"); + let cx = js_context + .context + .take() + .expect("JavaScript returned the loader context"); + if !js_context.loaders_without_pitch.is_empty() { + self + .update_loaders_without_pitch(std::mem::take(&mut js_context.loaders_without_pitch)) + .await; } + (cx, js_context.take_error()) } - list } diff --git a/crates/rspack_core/Cargo.toml b/crates/rspack_core/Cargo.toml index 617c55a3eef5..40394e5b7162 100644 --- a/crates/rspack_core/Cargo.toml +++ b/crates/rspack_core/Cargo.toml @@ -96,4 +96,4 @@ workspace = true codspeed = ["rspack_parallel/codspeed"] debug_tool = ["rspack_util/debug_tool"] default = [] -napi = ["dep:napi", "dep:rspack_napi", "dep:rkyv"] +napi = ["dep:napi", "dep:rspack_napi", "dep:rkyv", "rspack_loader_runner/napi"] diff --git a/crates/rspack_core/src/diagnostics.rs b/crates/rspack_core/src/diagnostics.rs index b8c411b637ea..9c5f3f196eb0 100644 --- a/crates/rspack_core/src/diagnostics.rs +++ b/crates/rspack_core/src/diagnostics.rs @@ -1,7 +1,7 @@ use itertools::Itertools; use rspack_error::{Diagnostic, Error, Label, dim}; -use crate::{BoxLoader, DependencyRange}; +use crate::{DependencyRange, ResolvedLoader}; ///////////////////// Module Factory ///////////////////// @@ -96,7 +96,7 @@ impl From for Error { } impl ModuleParseError { - pub fn new(source: Error, loaders: &[BoxLoader]) -> Self { + pub fn new(source: Error, loaders: &[ResolvedLoader]) -> Self { let mut help = String::new(); let mut title = "Module parse failed:"; if source.is_error() { @@ -106,7 +106,7 @@ impl ModuleParseError { let s = loaders .iter() .map(|l| { - let l = l.identifier().to_string(); + let l = l.loader.identifier().to_string(); format!("\n * {l}") }) .join(""); @@ -129,7 +129,7 @@ impl ModuleParseError { /// then, map it to diagnostics pub fn map_box_diagnostics_to_module_parse_diagnostics( diagnostic: Vec, - loaders: &[BoxLoader], + loaders: &[ResolvedLoader], ) -> Vec { diagnostic .into_iter() diff --git a/crates/rspack_core/src/loader/loader_runner.rs b/crates/rspack_core/src/loader/loader_runner.rs index a2c6e2733b40..e631eca745d1 100644 --- a/crates/rspack_core/src/loader/loader_runner.rs +++ b/crates/rspack_core/src/loader/loader_runner.rs @@ -1,15 +1,15 @@ use std::sync::Arc; use rspack_fs::ReadableFileSystem; +use rspack_loader_runner::LoaderRunnerContext; pub use rspack_loader_runner::{ Content, Loader, LoaderContext, LoaderDependencies, LoaderExecutionKind, LoaderRunnerOptions, - run_loaders, }; use rspack_util::source_map::SourceMapKind; use crate::{ - AdditionalData, CacheFacade, CompilationId, CompilerId, CompilerOptions, FileSystemInfo, - NormalModule, ResolverFactory, + CacheFacade, CompilationId, CompilerId, CompilerOptions, FileSystemInfo, NormalModule, + ResolverFactory, }; #[derive(Debug)] @@ -23,8 +23,14 @@ pub struct RunnerContext { pub resolver_factory: Arc, pub module: Box, pub source_map_kind: SourceMapKind, - /// Binding state shared by hooks and loaders for this module build only. - pub loader_context_data: AdditionalData, } +impl LoaderRunnerContext for RunnerContext { + fn loaders(&self) -> &Loaders { + &self.module.loaders + } +} + +pub type Loaders = rspack_loader_runner::Loaders; +pub type ResolvedLoader = rspack_loader_runner::ResolvedLoader; pub type BoxLoader = Arc Loader>; diff --git a/crates/rspack_core/src/loader/rspack_loader.rs b/crates/rspack_core/src/loader/rspack_loader.rs index a43d8b76bcde..eacacd4135bb 100644 --- a/crates/rspack_core/src/loader/rspack_loader.rs +++ b/crates/rspack_core/src/loader/rspack_loader.rs @@ -98,13 +98,24 @@ impl LoaderRunnerPlugin for RspackLoaderRunnerPlugin { Ok(None) } - async fn start_yielding(&self, context: &mut LoaderContext) -> Result<()> { - self + async fn start_yielding( + &self, + context: Box>, + ) -> (Box>, Result<()>) { + if let Some(runner) = &self .plugin_driver .normal_module_hooks - .loader_yield - .call(context) - .await + .javascript_loader_runner + { + runner.run(context).await + } else { + ( + context, + Err(rspack_error::error!( + "JavaScript loader runner is not registered" + )), + ) + } } async fn run_normal_loader( @@ -118,12 +129,12 @@ impl LoaderRunnerPlugin for RspackLoaderRunnerPlugin { LoaderCacheAction::Disabled }; if matches!(cache_action, LoaderCacheAction::Hit) { - context.current_loader().set_finish_called(); + context.set_current_loader_finish_called(); return Ok(()); } loader.run(context).await?; - if !context.current_loader().finish_called() { + if !context.current_loader_state().finish_called() { context.finish_with_empty(); } if let LoaderCacheAction::Miss(state) = cache_action { diff --git a/crates/rspack_core/src/normal_module.rs b/crates/rspack_core/src/normal_module.rs index d26eb0c2a3b0..2a4d62a4151c 100644 --- a/crates/rspack_core/src/normal_module.rs +++ b/crates/rspack_core/src/normal_module.rs @@ -16,9 +16,7 @@ use rspack_error::{Diagnosable, Diagnostic, Result, error}; use rspack_fs::ReadableFileSystem; use rspack_hash::{RspackHash, RspackHashDigest, RspackHasher}; use rspack_hook::define_hook; -use rspack_loader_runner::{ - AdditionalData, Content, LoaderContext, LoaderRunnerOptions, ResourceData, run_loaders, -}; +use rspack_loader_runner::{AdditionalData, Content, LoaderContext, ResourceData, run_loaders}; use rspack_sources::{ BoxSource, CachedSource, OriginalSource, RawBufferSource, RawStringSource, SourceExt, SourceMap, SourceMapSource, WithoutOriginalOptions, @@ -28,13 +26,13 @@ use serde_json::json; use tracing::{Instrument, info_span}; use crate::{ - BoxLoader, BoxModule, BuildContext, BuildInfo, BuildMeta, ChunkGraph, - CodeGenerationResultBuilder, Compilation, ConnectionState, Context, DependenciesBlock, - DependenciesBlockData, DependencyCodeGenerationRef, DependencyId, FactoryMeta, FactoryMetaStore, - FreezeLock, GenerateContext, GeneratorOptions, ImportPhase, LibIdentOptions, Module, + BoxModule, BuildContext, BuildInfo, BuildMeta, ChunkGraph, CodeGenerationResultBuilder, + Compilation, ConnectionState, Context, DependenciesBlock, DependenciesBlockData, + DependencyCodeGenerationRef, DependencyId, FactoryMeta, FactoryMetaStore, FreezeLock, + GenerateContext, GeneratorOptions, ImportPhase, LibIdentOptions, Loaders, Module, ModuleCodeGenerationContext, ModuleGraph, ModuleGraphCacheArtifact, ModuleIdentifier, ModuleLayer, ModuleType, NeedBuildContext, OptimizationBailoutItem, OutputOptions, ParseContext, - ParseResult, ParserAndGenerator, ParserOptions, Resolve, ResolvedModuleOptions, + ParseResult, ParserAndGenerator, ParserOptions, Resolve, ResolvedLoader, ResolvedModuleOptions, RspackLoaderRunnerPlugin, RunnerContext, RuntimeGlobals, RuntimeSpec, SideEffectsStateArtifact, SnapshotValidationResult, SourceType, cache::SnapshotStrategyOptions, @@ -82,7 +80,6 @@ impl ModuleIssuer { define_hook!(NormalModuleReadResource: SeriesBail(resource_data: &ResourceData, fs: &Arc) -> Content,tracing=false); define_hook!(NormalModuleLoader: Series(loader_context: &mut LoaderContext),tracing=false); -define_hook!(NormalModuleLoaderStartYielding: Series(loader_context: &mut LoaderContext),tracing=false); define_hook!(NormalModuleBeforeLoaders: Series(module: &mut NormalModule),tracing=false); define_hook!(NormalModuleAdditionalData: Series(additional_data: &mut Option<&mut AdditionalData>),tracing=false); @@ -90,7 +87,8 @@ define_hook!(NormalModuleAdditionalData: Series(additional_data: &mut Option<&mu pub struct NormalModuleHooks { pub read_resource: NormalModuleReadResourceHook, pub loader: NormalModuleLoaderHook, - pub loader_yield: NormalModuleLoaderStartYieldingHook, + pub javascript_loader_runner: + Option>>, pub before_loaders: NormalModuleBeforeLoadersHook, pub additional_data: NormalModuleAdditionalDataHook, } @@ -119,8 +117,7 @@ pub struct NormalModule { resource_data: Arc, /// Loaders for the module #[debug(skip)] - loaders: Vec, - loader_options: Option>, + pub(crate) loaders: Loaders, /// Resolve options derived from [Rule.resolve] resolve_options: Option>, @@ -193,8 +190,7 @@ impl NormalModule { match_resource: Option, resource_data: Arc, resolve_options: Option>, - loaders: Vec, - loader_options: Option>, + loaders: Loaders, context: Option, extract_source_map: Option, import_phase: ImportPhase, @@ -219,7 +215,6 @@ impl NormalModule { resource_data, resolve_options, loaders, - loader_options, debug_id: DEBUG_ID.fetch_add(1, Ordering::Relaxed), extract_source_map, @@ -269,8 +264,8 @@ impl NormalModule { &self.raw_request } - pub fn loaders(&self) -> &[BoxLoader] { - &self.loaders + pub fn loaders(&self) -> &[ResolvedLoader] { + self.loaders.loaders() } pub fn parser_and_generator(&self) -> &dyn ParserAndGenerator { @@ -461,7 +456,7 @@ impl Module for NormalModule { perfetto.process_name = format!("Rspack Build Detail"), module.resource = self.resource_resolved_data().resource(), module.identifier = self.identifier().as_str(), - module.loaders = ?self.loaders.iter().map(|l| l.identifier().as_str()).collect::>()) + module.loaders = ?self.loaders.loaders().iter().map(|l| l.loader.identifier().as_str()).collect::>()) )] async fn build( mut self: Box, @@ -500,8 +495,6 @@ impl Module for NormalModule { let resolver_factory = build_context.resolver_factory.clone(); let fs = build_context.fs.clone(); let (mut loader_result, err) = run_loaders( - self.loaders.clone(), - self.loader_options.clone(), self.resource_data.clone(), Some(plugin.clone()), RunnerContext { @@ -513,14 +506,12 @@ impl Module for NormalModule { file_system_info: build_context.file_system_info.clone(), resolver_factory, source_map_kind: self.source_map_kind, - loader_context_data: Default::default(), module: self, }, fs, ) .instrument(info_span!("NormalModule:run_loaders",)) .await; - drop(loader_result.context.loader_context_data); self = loader_result.context.module; if let Some(err) = err { @@ -617,7 +608,7 @@ impl Module for NormalModule { module_user_request: &self.user_request, module_match_resource: self.match_resource.as_ref(), module_source_map_kind: self.source_map_kind, - loaders: &self.loaders, + loaders: self.loaders.loaders(), resource_data: &self.resource_data, compiler_options: &build_context.compiler_options, additional_data: loader_result.additional_data, diff --git a/crates/rspack_core/src/normal_module_factory.rs b/crates/rspack_core/src/normal_module_factory.rs index 19e66c133148..286f10953730 100644 --- a/crates/rspack_core/src/normal_module_factory.rs +++ b/crates/rspack_core/src/normal_module_factory.rs @@ -15,13 +15,13 @@ use crate::{ AssetInlineGeneratorOptions, AssetResourceGeneratorOptions, BoxLoader, BoxModule, CompilerOptions, Context, CssAutoOrModuleParserOptions, CssModuleGeneratorOptions, CssModuleParserOptions, Dependency, DependencyCategory, DependencyType, FactoryMeta, FuncUseCtx, - GeneratorOptions, MatchContext, ModuleExt, ModuleFactory, ModuleFactoryCreateData, + GeneratorOptions, Loaders, MatchContext, ModuleExt, ModuleFactory, ModuleFactoryCreateData, ModuleFactoryResult, ModuleIdentifier, ModuleLayer, ModuleRuleEffect, ModuleRuleEnforce, ModuleRuleUse, ModuleRuleUseLoader, ModuleType, NormalModule, ParserAndGenerator, ParserOptions, ParserOptionsMap, RawModule, Resolve, ResolveArgs, ResolveOptionsWithDependencyType, - ResolveResult, ResolvedModuleOptions, ResolvedModuleOptionsCacheKey, Resolver, ResolverFactory, - ResourceData, ResourceParsedData, RunnerContext, RuntimeGlobals, SharedPluginDriver, - diagnostics::EmptyDependency, module_rules_matcher, parse_resource, resolve, + ResolveResult, ResolvedLoader, ResolvedModuleOptions, ResolvedModuleOptionsCacheKey, Resolver, + ResolverFactory, ResourceData, ResourceParsedData, RunnerContext, RuntimeGlobals, + SharedPluginDriver, diagnostics::EmptyDependency, module_rules_matcher, parse_resource, resolve, stringify_loaders_and_resource, }; @@ -1101,24 +1101,7 @@ module.exports = "data:,"; } else { resource_data.resource().to_owned() }; - let has_cached_loader = resolved_loaders - .iter() - .any(|resolved| resolved.options.cache); - let (loaders, loader_options) = if has_cached_loader { - let (loaders, loader_options) = resolved_loaders - .into_iter() - .map(|resolved| (resolved.loader, resolved.options)) - .unzip(); - (loaders, Some(loader_options)) - } else { - ( - resolved_loaders - .into_iter() - .map(|resolved| resolved.loader) - .collect(), - None, - ) - }; + let loaders = Loaders::new(resolved_loaders); let resolved_module_type = self.calculate_module_type(match_module_type, &matched_module_rules); let resolved_module_layer = @@ -1200,7 +1183,6 @@ module.exports = "data:,"; resource_resolve_data, resolved_resolve_options, loaders, - loader_options, create_data.context.clone().map(|x| x.into()), resolved_extract_source_map, dependency_phase, @@ -1369,20 +1351,6 @@ async fn resolve_each( .ok_or_else(|| error!("Unable to resolve loader {}", l.loader)) } -struct ResolvedLoader { - loader: BoxLoader, - options: LoaderRunnerOptions, -} - -impl ResolvedLoader { - fn uncached(loader: BoxLoader) -> Self { - Self { - loader, - options: LoaderRunnerOptions::default(), - } - } -} - async fn resolve_each_with_options( plugin_driver: &SharedPluginDriver, options: &CompilerOptions, diff --git a/crates/rspack_core/src/parser_and_generator.rs b/crates/rspack_core/src/parser_and_generator.rs index 96d0a21b5052..b9bbf57ba62f 100644 --- a/crates/rspack_core/src/parser_and_generator.rs +++ b/crates/rspack_core/src/parser_and_generator.rs @@ -14,11 +14,11 @@ use rspack_util::{ext::AsAny, source_map::SourceMapKind}; use rustc_hash::{FxHashMap, FxHashSet}; use crate::{ - AsyncDependenciesBlock, BoxDependency, BoxLoader, BuildInfo, BuildMeta, ChunkGraph, - CodeGenerationData, Compilation, CompilerOptions, ConcatenationScope, Context, - DependencyCodeGenerationRef, DependencyId, DependencyLocation, DependencyRange, - EvaluatedInlinableValue, FactoryMeta, GeneratorOptions, Module, ModuleCodeTemplate, ModuleGraph, - ModuleIdentifier, ModuleLayer, ModuleType, NormalModule, ParserOptions, RuntimeSpec, SourceType, + AsyncDependenciesBlock, BoxDependency, BuildInfo, BuildMeta, ChunkGraph, CodeGenerationData, + Compilation, CompilerOptions, ConcatenationScope, Context, DependencyCodeGenerationRef, + DependencyId, DependencyLocation, DependencyRange, EvaluatedInlinableValue, FactoryMeta, + GeneratorOptions, Module, ModuleCodeTemplate, ModuleGraph, ModuleIdentifier, ModuleLayer, + ModuleType, NormalModule, ParserOptions, ResolvedLoader, RuntimeSpec, SourceType, }; #[derive(Debug)] @@ -34,7 +34,7 @@ pub struct ParseContext<'a> { pub module_source_map_kind: SourceMapKind, pub module_match_resource: Option<&'a ResourceData>, #[debug(skip)] - pub loaders: &'a [BoxLoader], + pub loaders: &'a [ResolvedLoader], pub resource_data: &'a ResourceData, pub compiler_options: &'a CompilerOptions, pub additional_data: Option, diff --git a/crates/rspack_loader_runner/Cargo.toml b/crates/rspack_loader_runner/Cargo.toml index 2da218c9b27d..5f7e0764fe53 100644 --- a/crates/rspack_loader_runner/Cargo.toml +++ b/crates/rspack_loader_runner/Cargo.toml @@ -5,6 +5,11 @@ license = "MIT" name = "rspack_loader_runner" repository = "https://github.com/web-infra-dev/rspack" version.workspace = true + +[features] +napi = ["dep:rspack_napi"] +test-loader = [] + [dependencies] anymap = { workspace = true } async-trait = { workspace = true } @@ -19,6 +24,7 @@ rspack_collections = { workspace = true } rspack_error = { workspace = true } rspack_fs = { workspace = true } rspack_hash = { workspace = true } +rspack_napi = { workspace = true, optional = true } rspack_paths = { workspace = true, features = ["cacheable"] } rspack_sources = { workspace = true } rspack_util = { workspace = true } diff --git a/crates/rspack_loader_runner/src/context.rs b/crates/rspack_loader_runner/src/context.rs index 296b07e941c6..f9fa3a13f689 100644 --- a/crates/rspack_loader_runner/src/context.rs +++ b/crates/rspack_loader_runner/src/context.rs @@ -6,9 +6,11 @@ use rspack_error::Diagnostic; use rspack_paths::{InternedPath, InternedPathSet, Utf8Path}; use rspack_sources::SourceMap; +#[cfg(feature = "test-loader")] +use crate::loader::LoaderItemList; use crate::{ - AdditionalData, Content, LoaderItem, LoaderRunnerPlugin, ParseMeta, ResourceData, - loader::LoaderItemList, + AdditionalData, Content, LoaderItem, LoaderItemState, LoaderRunnerPlugin, Loaders, ParseMeta, + ResourceData, }; #[derive(Clone, Copy, PartialEq, Eq, Debug)] @@ -51,8 +53,17 @@ impl LoaderDependencies { } } +pub trait LoaderRunnerContext: Send + Sized { + fn loaders(&self) -> &Loaders; +} + +#[cfg(feature = "napi")] +pub enum LoaderContextLifetime {} + #[derive(Debug)] pub struct LoaderContext { + #[cfg(feature = "napi")] + pub lifecycle: rspack_napi::LifecycleGuard, pub hot: bool, pub resource_data: Arc, #[debug(skip)] @@ -77,7 +88,7 @@ pub struct LoaderContext { /// Loader States pub(crate) state: State, pub loader_index: i32, - pub loader_items: Vec>, + pub loader_item_states: Vec, #[debug(skip)] pub plugin: Option>>, } @@ -281,21 +292,62 @@ impl LoaderContext { self.added_dependencies.context.clear(); self.added_dependencies.missing.clear(); } +} + +impl LoaderContext { + #[inline] + pub fn loader_items(&self) -> &[LoaderItem] { + self.context.loaders().loader_items() + } + #[cfg(feature = "test-loader")] pub fn remaining_request(&self) -> LoaderItemList<'_, Context> { - if self.loader_index >= self.loader_items.len() as i32 - 1 { + if self.loader_index >= self.loader_items().len() as i32 - 1 { return Default::default(); } - LoaderItemList(&self.loader_items[self.loader_index as usize + 1..]) + LoaderItemList(&self.loader_items()[self.loader_index as usize + 1..]) } + #[cfg(feature = "test-loader")] pub fn previous_request(&self) -> LoaderItemList<'_, Context> { - LoaderItemList(&self.loader_items[..self.loader_index as usize]) + LoaderItemList(&self.loader_items()[..self.loader_index as usize]) } #[inline] pub fn current_loader(&self) -> &LoaderItem { - &self.loader_items[self.loader_index as usize] + &self.loader_items()[self.loader_index as usize] + } +} + +impl LoaderContext { + #[inline] + pub fn loader_item_state(&self, index: usize) -> &LoaderItemState { + &self.loader_item_states[index] + } + + #[inline] + pub fn loader_item_state_mut(&mut self, index: usize) -> &mut LoaderItemState { + &mut self.loader_item_states[index] + } + + pub fn current_loader_state(&self) -> &LoaderItemState { + self.loader_item_state(self.loader_index as usize) + } + + pub fn current_loader_state_mut(&mut self) -> &mut LoaderItemState { + self.loader_item_state_mut(self.loader_index as usize) + } + + pub fn set_current_loader_pitch_executed(&mut self) { + self.current_loader_state_mut().set_pitch_executed(); + } + + pub fn set_current_loader_normal_executed(&mut self) { + self.current_loader_state_mut().set_normal_executed(); + } + + pub fn set_current_loader_finish_called(&mut self) { + self.current_loader_state_mut().set_finish_called(); } /// Emit a diagnostic, it can be a `warning` or `error`. @@ -361,14 +413,14 @@ impl LoaderContext { pub fn finish_with(&mut self, patch: impl Into) { self.__finish_with(patch); - self.current_loader().set_finish_called(); + self.set_current_loader_finish_called(); } pub fn finish_with_empty(&mut self) { self.content = None; self.source_map = None; self.additional_data = None; - self.current_loader().set_finish_called(); + self.set_current_loader_finish_called(); } #[inline] diff --git a/crates/rspack_loader_runner/src/lib.rs b/crates/rspack_loader_runner/src/lib.rs index ed93eb6a4c06..4385a4b727ed 100644 --- a/crates/rspack_loader_runner/src/lib.rs +++ b/crates/rspack_loader_runner/src/lib.rs @@ -8,17 +8,20 @@ mod plugin; mod runner; mod scheme; +pub use cache::LoaderRunnerOptions; pub use content::{ AdditionalData, Content, DescriptionData, ParseMeta, ParseMetaValue, ResourceData, }; -pub use context::{LoaderContext, LoaderDependencies, State}; +#[cfg(feature = "napi")] +pub use context::LoaderContextLifetime; +pub use context::{LoaderContext, LoaderDependencies, LoaderRunnerContext, State}; pub use loader::{ - DisplayWithSuffix, Loader, LoaderExecutionKind, LoaderItem, ResourceParsedData, parse_resource, + DisplayWithSuffix, Loader, LoaderExecutionKind, LoaderItem, LoaderItemState, ResourceParsedData, + parse_resource, }; -pub use plugin::LoaderRunnerPlugin; +pub use plugin::{LoaderRunner, LoaderRunnerPlugin}; pub use rspack_collections::{Identifiable, Identifier}; -pub use runner::{LoaderResult, run_loaders}; +pub use runner::{LoaderResult, Loaders, ResolvedLoader, run_loaders}; pub use scheme::{Scheme, get_scheme}; pub const BUILTIN_LOADER_PREFIX: &str = "builtin:"; -pub use cache::LoaderRunnerOptions; diff --git a/crates/rspack_loader_runner/src/loader.rs b/crates/rspack_loader_runner/src/loader.rs index ad2b3c60339f..2cb99571f182 100644 --- a/crates/rspack_loader_runner/src/loader.rs +++ b/crates/rspack_loader_runner/src/loader.rs @@ -1,11 +1,6 @@ -use std::{ - fmt::Display, - ops::Deref, - sync::{ - Arc, - atomic::{AtomicBool, Ordering}, - }, -}; +#[cfg(feature = "test-loader")] +use std::ops::Deref; +use std::{fmt::Display, sync::Arc}; use async_trait::async_trait; use derive_more::Debug; @@ -41,21 +36,19 @@ pub struct LoaderItem { /// Fragment of a loader, starts with `#`. #[allow(dead_code)] fragment: Option, - /// Data shared between pitching and normal - data: serde_json::Value, r#type: String, cache_options: Option>, execution_kind: LoaderExecutionKind, - pitch_executed: AtomicBool, - normal_executed: AtomicBool, +} + +#[derive(Debug, Default)] +pub struct LoaderItemState { + /// Data shared between pitching and normal. + data: serde_json::Value, + pitch_executed: bool, + normal_executed: bool, /// Whether loader was called with [LoaderContext::finish_with]. - /// - /// Indicates that the loader has finished its work, - /// otherwise loader runner will reset [`LoaderContext::content`], [`LoaderContext::source_map`], [`LoaderContext::additional_data`]. - /// - /// This flag is used to align with webpack's behavior: - /// If nothing is modified in the loader, the loader will reset the content, source map, and additional data. - finish_called: AtomicBool, + finish_called: bool, } impl LoaderItem { @@ -121,7 +114,9 @@ impl LoaderItem { pub fn cache_options(&self) -> Option<&LoaderRunnerOptions> { self.cache_options.as_deref() } +} +impl LoaderItemState { #[inline] pub fn data(&self) -> &serde_json::Value { &self.data @@ -136,36 +131,36 @@ impl LoaderItem { #[inline] #[doc(hidden)] pub fn pitch_executed(&self) -> bool { - self.pitch_executed.load(Ordering::Relaxed) + self.pitch_executed } #[inline] pub fn normal_executed(&self) -> bool { - self.normal_executed.load(Ordering::Relaxed) + self.normal_executed } #[inline] #[doc(hidden)] pub fn finish_called(&self) -> bool { - self.finish_called.load(Ordering::Relaxed) + self.finish_called } #[inline] #[doc(hidden)] - pub fn set_pitch_executed(&self) { - self.pitch_executed.store(true, Ordering::Relaxed) + pub fn set_pitch_executed(&mut self) { + self.pitch_executed = true; } #[inline] #[doc(hidden)] - pub fn set_normal_executed(&self) { - self.normal_executed.store(true, Ordering::Relaxed) + pub fn set_normal_executed(&mut self) { + self.normal_executed = true; } #[inline] #[doc(hidden)] - pub fn set_finish_called(&self) { - self.finish_called.store(true, Ordering::Relaxed) + pub fn set_finish_called(&mut self) { + self.finish_called = true; } } @@ -175,9 +170,11 @@ impl Display for LoaderItem { } } +#[cfg(feature = "test-loader")] #[derive(Debug)] pub struct LoaderItemList<'a, Context: Send>(pub &'a [LoaderItem]); +#[cfg(feature = "test-loader")] impl Deref for LoaderItemList<'_, Context> { type Target = [LoaderItem]; @@ -186,6 +183,7 @@ impl Deref for LoaderItemList<'_, Context> { } } +#[cfg(feature = "test-loader")] impl Default for LoaderItemList<'_, Context> { fn default() -> Self { Self(&[]) @@ -202,8 +200,10 @@ pub trait DisplayWithSuffix: Display { } } +#[cfg(feature = "test-loader")] impl DisplayWithSuffix for LoaderItemList<'_, Context> {} impl DisplayWithSuffix for LoaderItem {} +#[cfg(feature = "test-loader")] impl Display for LoaderItemList<'_, Context> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { let s = self @@ -229,7 +229,7 @@ where async fn run(&self, loader_context: &mut LoaderContext) -> Result<()> { // If loader does not implement normal stage, // it should inherit the result from the previous loader. - loader_context.current_loader().set_finish_called(); + loader_context.set_current_loader_finish_called(); Ok(()) } @@ -255,6 +255,7 @@ where } } +#[cfg(test)] impl From>> for LoaderItem { fn from(loader: Arc>) -> Self { Self::new(loader, LoaderRunnerOptions::default()) @@ -279,13 +280,9 @@ impl LoaderItem { path, query, fragment, - data: serde_json::Value::Null, r#type: ty, cache_options, execution_kind, - pitch_executed: AtomicBool::new(false), - normal_executed: AtomicBool::new(false), - finish_called: AtomicBool::new(false), }; } let ident = loader.identifier(); @@ -300,13 +297,9 @@ impl LoaderItem { path, query, fragment, - data: serde_json::Value::Null, r#type: String::default(), cache_options, execution_kind, - pitch_executed: AtomicBool::new(false), - normal_executed: AtomicBool::new(false), - finish_called: AtomicBool::new(false), } } } diff --git a/crates/rspack_loader_runner/src/plugin.rs b/crates/rspack_loader_runner/src/plugin.rs index d4bb43972002..a880a0e26266 100644 --- a/crates/rspack_loader_runner/src/plugin.rs +++ b/crates/rspack_loader_runner/src/plugin.rs @@ -6,13 +6,13 @@ use rspack_paths::InternedPathSet; use rspack_sources::SourceMap; use crate::{ - Loader, LoaderContext, + Loader, LoaderContext, LoaderRunnerContext, content::{Content, ResourceData}, }; #[async_trait::async_trait] pub trait LoaderRunnerPlugin: Send + Sync { - type Context: Send; + type Context: LoaderRunnerContext; fn name(&self) -> &'static str { "unknown" @@ -22,8 +22,13 @@ pub trait LoaderRunnerPlugin: Send + Sync { Ok(()) } - async fn start_yielding(&self, _context: &mut LoaderContext) -> Result<()> { - Ok(()) + /// Transfer the same allocation to the foreign runner and back, including on + /// loader errors. Native hooks and loaders only borrow the context. + async fn start_yielding( + &self, + context: Box>, + ) -> (Box>, Result<()>) { + (context, Ok(())) } async fn run_normal_loader( @@ -32,7 +37,7 @@ pub trait LoaderRunnerPlugin: Send + Sync { loader: Arc>, ) -> Result<()> { loader.run(context).await?; - if !context.current_loader().finish_called() { + if !context.current_loader_state().finish_called() { context.finish_with_empty(); } Ok(()) @@ -44,3 +49,14 @@ pub trait LoaderRunnerPlugin: Send + Sync { fs: Arc, ) -> Result>, InternedPathSet)>>; } + +/// A foreign loader runner owns the context for the duration of its invocation. +#[async_trait::async_trait] +pub trait LoaderRunner: std::fmt::Debug + Send + Sync { + type Context: LoaderRunnerContext; + + async fn run( + &self, + context: Box>, + ) -> (Box>, Result<()>); +} diff --git a/crates/rspack_loader_runner/src/runner.rs b/crates/rspack_loader_runner/src/runner.rs index 7f1fa128f686..83d14e06b443 100644 --- a/crates/rspack_loader_runner/src/runner.rs +++ b/crates/rspack_loader_runner/src/runner.rs @@ -1,5 +1,7 @@ -use std::{fmt::Debug, sync::Arc}; +use std::sync::{Arc, OnceLock}; +use derive_more::Debug; +use rspack_cacheable::{cacheable, with::Skip}; use rspack_error::{Diagnostic, Error, Result, error}; use rspack_fs::ReadableFileSystem; use rspack_paths::Utf8PathBuf; @@ -9,28 +11,76 @@ use tracing::{Instrument, info_span}; use crate::{ LoaderExecutionKind, LoaderRunnerOptions, ParseMeta, content::{AdditionalData, Content, ResourceData}, - context::{LoaderContext, LoaderDependencies, State}, - loader::{Loader, LoaderItem}, + context::{LoaderContext, LoaderDependencies, LoaderRunnerContext, State}, + loader::{Loader, LoaderItem, LoaderItemState}, plugin::LoaderRunnerPlugin, }; -impl LoaderContext { - async fn start_yielding(&mut self) -> Result { - if self.current_loader().execution_kind() == LoaderExecutionKind::JavaScript - && let Some(plugin) = &self.plugin - { - plugin.clone().start_yielding(self).await?; - return Ok(true); +#[cacheable] +#[derive(Debug)] +pub struct ResolvedLoader { + #[debug("{}", loader.identifier())] + pub loader: Arc>, + pub options: LoaderRunnerOptions, +} + +impl ResolvedLoader { + #[inline] + pub fn uncached(loader: Arc>) -> Self { + Self { + loader, + options: LoaderRunnerOptions::default(), + } + } +} + +#[cacheable] +#[derive(Debug)] +pub struct Loaders { + #[debug(skip)] + loaders: Vec>, + #[cacheable(with=Skip)] + loader_items: OnceLock>>, +} + +impl Loaders { + #[inline] + pub fn new(loaders: Vec>) -> Self { + Self { + loaders, + loader_items: OnceLock::new(), } - Ok(false) } + + #[inline] + pub fn loaders(&self) -> &[ResolvedLoader] { + &self.loaders + } + + pub(crate) fn loader_items(&self) -> &[LoaderItem] { + self.loader_items.get_or_init(|| { + self + .loaders + .iter() + .map(|resolved| LoaderItem::new(resolved.loader.clone(), resolved.options.clone())) + .collect() + }) + } +} + +fn yielding_plugin( + context: &LoaderContext, +) -> Option>> { + (context.current_loader().execution_kind() == LoaderExecutionKind::JavaScript) + .then(|| context.plugin.clone()) + .flatten() } #[tracing::instrument("LoaderRunner:process_resource", skip_all, fields(resource = loader_context.resource_data.resource()) )] -async fn process_resource( +async fn process_resource( loader_context: &mut LoaderContext, fs: Arc, ) -> Result<()> { @@ -63,8 +113,7 @@ You may need an additional plugin to handle "{scheme}:" URIs."# )) } -fn create_loader_context( - loader_items: Vec>, +fn create_loader_context( resource_data: Arc, plugin: Option>>, context: Context, @@ -76,7 +125,13 @@ fn create_loader_context( dependencies.file.insert(resource_path.into()); } + let loader_items = context.loaders().loader_items(); + let loader_item_states = (0..loader_items.len()) + .map(|_| LoaderItemState::default()) + .collect(); LoaderContext { + #[cfg(feature = "napi")] + lifecycle: Default::default(), hot: false, cacheable: true, parse_meta: Default::default(), @@ -84,12 +139,12 @@ fn create_loader_context( added_dependencies: Default::default(), removed_dependencies: Default::default(), content: None, - context, source_map: None, + context, additional_data: None, state: State::Init, loader_index: 0, - loader_items, + loader_item_states, plugin, resource_data, diagnostics: vec![], @@ -97,39 +152,31 @@ fn create_loader_context( } #[tracing::instrument("LoaderRunner:run_loaders", skip_all, level = "trace")] -pub async fn run_loaders( - loaders: Vec>>, - loader_options: Option>, +pub async fn run_loaders( resource_data: Arc, plugin: Option>>, context: Context, fs: Arc, ) -> (LoaderResult, Option) { - let loaders = if let Some(loader_options) = loader_options { - assert_eq!( - loaders.len(), - loader_options.len(), - "loader options must stay aligned with loaders" - ); - loaders - .into_iter() - .zip(loader_options) - .map(|(loader, options)| LoaderItem::new(loader, options)) - .collect::>>() - } else { - loaders.into_iter().map(LoaderItem::from).collect() - }; - let mut cx = create_loader_context(loaders, resource_data, plugin, context); - let result = run_loaders_impl(&mut cx, fs).await; - (LoaderResult::new(cx), result.err()) + let cx = Box::new(create_loader_context(resource_data, plugin, context)); + let (cx, result) = run_loaders_impl(cx, fs).await; + (LoaderResult::new(*cx), result.err()) } -async fn run_loaders_impl( - cx: &mut LoaderContext, +async fn run_loaders_impl( + mut cx: Box>, fs: Arc, -) -> Result<()> { +) -> (Box>, Result<()>) { + macro_rules! try_with_context { + ($result:expr) => { + match $result { + Ok(value) => value, + Err(error) => return (cx, Err(error)), + } + }; + } if let Some(plugin) = cx.plugin.clone() { - plugin.before_all(cx).await?; + try_with_context!(plugin.before_all(&mut cx).await); } let resource = cx.resource().to_owned(); let resource = resource.as_str(); @@ -139,12 +186,15 @@ async fn run_loaders_impl( cx.state.transition(State::Pitching); } State::Pitching => { - if cx.loader_index >= cx.loader_items.len() as i32 { + if cx.loader_index >= cx.loader_items().len() as i32 { cx.state.transition(State::ProcessResource); continue; } let span = info_span!("run_loader:pitch:yield_to_js", resource); - if cx.start_yielding().instrument(span).await? { + if let Some(plugin) = yielding_plugin(&cx) { + let result; + (cx, result) = plugin.start_yielding(cx).instrument(span).await; + try_with_context!(result); if cx.content.is_some() { cx.state.transition(State::Normal); cx.loader_index -= 1; @@ -152,18 +202,18 @@ async fn run_loaders_impl( continue; } - if cx.current_loader().pitch_executed() { + if cx.current_loader_state().pitch_executed() { cx.loader_index += 1; continue; } - cx.current_loader().set_pitch_executed(); + cx.set_current_loader_pitch_executed(); let loader = cx.current_loader().loader().clone(); let span = info_span!("run_loader:pitch", resource); cx.reset_dependency_changes(); - let result = loader.pitch(cx).instrument(span).await; + let result = loader.pitch(&mut cx).instrument(span).await; cx.merge_dependency_changes(); - result?; + try_with_context!(result); if cx.content.is_some() { cx.state.transition(State::Normal); cx.loader_index -= 1; @@ -171,8 +221,8 @@ async fn run_loaders_impl( } State::ProcessResource => { let span = info_span!("run_loader:process_resource", resource); - process_resource(cx, fs.clone()).instrument(span).await?; - cx.loader_index = cx.loader_items.len() as i32 - 1; + try_with_context!(process_resource(&mut cx, fs.clone()).instrument(span).await); + cx.loader_index = cx.loader_items().len() as i32 - 1; cx.state.transition(State::Normal); } State::Normal => { @@ -181,30 +231,36 @@ async fn run_loaders_impl( continue; } - if cx.loader_index == 0 && cx.current_loader().normal_executed() { + if cx.loader_index == 0 && cx.current_loader_state().normal_executed() { cx.state.transition(State::Finished); continue; } let span = info_span!("run_loader:yield_to_js", resource); - if cx.start_yielding().instrument(span).await? { + if let Some(plugin) = yielding_plugin(&cx) { + let result; + (cx, result) = plugin.start_yielding(cx).instrument(span).await; + try_with_context!(result); continue; } - if cx.current_loader().normal_executed() { + if cx.current_loader_state().normal_executed() { cx.loader_index -= 1; continue; } - cx.current_loader().set_normal_executed(); + cx.set_current_loader_normal_executed(); let loader = cx.current_loader().loader().clone(); let span = info_span!("run_loader:normal", resource); cx.reset_dependency_changes(); let result = if let Some(plugin) = cx.plugin.clone() { - plugin.run_normal_loader(cx, loader).instrument(span).await + plugin + .run_normal_loader(&mut cx, loader) + .instrument(span) + .await } else { - let result = loader.run(cx).instrument(span).await; - if result.is_ok() && !cx.current_loader().finish_called() { + let result = loader.run(&mut cx).instrument(span).await; + if result.is_ok() && !cx.current_loader_state().finish_called() { // If nothing is returned from this loader, // we set everything to [None] and move to the next loader. // This mocks the behavior of webpack loader-runner. @@ -213,24 +269,27 @@ async fn run_loaders_impl( result }; cx.merge_dependency_changes(); - result?; + try_with_context!(result); } State::Finished => break, } } if cx.content.is_none() { - if !cx.loader_items.is_empty() { - let loader = cx.loader_items[0].to_string(); - return Err(error!( - "Final loader({loader}) didn't return a Buffer or String" - )); + if !cx.loader_items().is_empty() { + let loader = cx.loader_items()[0].to_string(); + return ( + cx, + Err(error!( + "Final loader({loader}) didn't return a Buffer or String" + )), + ); } else { panic!("content should be available"); } } - Ok(()) + (cx, Ok(())) } #[derive(Debug)] @@ -246,8 +305,16 @@ pub struct LoaderResult { pub current_loader: Option, } -impl LoaderResult { +impl LoaderResult { pub fn new(loader_context: LoaderContext) -> Self { + let current_loader = (loader_context.loader_index >= 0) + .then(|| { + loader_context + .loader_items() + .get(loader_context.loader_index as usize) + }) + .flatten() + .map(|loader| loader.path().to_path_buf()); LoaderResult { context: loader_context.context, cacheable: loader_context.cacheable, @@ -259,14 +326,7 @@ impl LoaderResult { source_map: loader_context.source_map, additional_data: loader_context.additional_data, parse_meta: loader_context.parse_meta, - current_loader: (loader_context.loader_index >= 0) - .then(|| { - loader_context - .loader_items - .get(loader_context.loader_index as usize) - }) - .flatten() - .map(|loader| loader.path().to_path_buf()), + current_loader, } } } @@ -282,14 +342,24 @@ mod test { use rspack_paths::InternedPathSet; use rspack_sources::SourceMap; - use super::{Loader, LoaderContext, ResourceData, run_loaders}; + use super::{ + Loader, LoaderContext, LoaderRunnerContext, Loaders, ResolvedLoader, ResourceData, run_loaders, + }; use crate::{AdditionalData, content::Content, plugin::LoaderRunnerPlugin}; + struct TestContext(Loaders); + + impl LoaderRunnerContext for TestContext { + fn loaders(&self) -> &Loaders { + &self.0 + } + } + struct TestContentPlugin; #[async_trait::async_trait] impl LoaderRunnerPlugin for TestContentPlugin { - type Context = (); + type Context = TestContext; fn name(&self) -> &'static str { "test-content" @@ -319,12 +389,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Pitching { + impl Loader for Pitching { fn identifier(&self) -> Identifier { "/rspack/pitching-loader1".into() } - async fn pitch(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn pitch(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch1".to_string())); Ok(()) } @@ -335,12 +405,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Pitching2 { + impl Loader for Pitching2 { fn identifier(&self) -> Identifier { "/rspack/pitching-loader2".into() } - async fn pitch(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn pitch(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch2".to_string())); Ok(()) } @@ -351,12 +421,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal { + impl Loader for Normal { fn identifier(&self) -> Identifier { "/rspack/normal-loader1".into() } - async fn run(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("normal1".to_string())); Ok(()) } @@ -367,12 +437,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal2 { + impl Loader for Normal2 { fn identifier(&self) -> Identifier { "/rspack/normal-loader2".into() } - async fn run(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("normal2".to_string())); Ok(()) } @@ -383,17 +453,17 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for PitchNormalBase { + impl Loader for PitchNormalBase { fn identifier(&self) -> Identifier { "/rspack/pitch-normal-base-loader".into() } - async fn run(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-base-normal".to_string())); Ok(()) } - async fn pitch(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn pitch(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-base-pitch".to_string())); Ok(()) } @@ -404,17 +474,17 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for PitchNormal { + impl Loader for PitchNormal { fn identifier(&self) -> Identifier { "/rspack/pitch-normal-loader".into() } - async fn run(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-normal".to_string())); Ok(()) } - async fn pitch(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn pitch(&self, loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-pitch".to_string())); loader_context.content = Some(Content::Buffer(vec![])); Ok(()) @@ -426,27 +496,27 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for PitchNormal2 { + impl Loader for PitchNormal2 { fn identifier(&self) -> Identifier { "/rspack/pitch-normal-2-loader".into() } - async fn run(&self, _loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, _loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-normal-2".to_string())); Ok(()) } - async fn pitch(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn pitch(&self, loader_context: &mut LoaderContext) -> Result<()> { IDENTS.with(|i| i.borrow_mut().push("pitch-normal-pitch-2".to_string())); loader_context.content = Some(Content::Buffer(vec![])); Ok(()) } } - let c1 = Arc::new(Normal) as Arc>; - let c2 = Arc::new(Normal2) as Arc>; - let p1 = Arc::new(Pitching) as Arc>; - let p2 = Arc::new(Pitching2) as Arc>; + let c1 = ResolvedLoader::uncached(Arc::new(Normal)); + let c2 = ResolvedLoader::uncached(Arc::new(Normal2)); + let p1 = ResolvedLoader::uncached(Arc::new(Pitching)); + let p2 = ResolvedLoader::uncached(Arc::new(Pitching2)); let rs = Arc::new(ResourceData::new_with_resource( "/rspack/main.js?abc=123#efg".to_owned(), @@ -455,12 +525,10 @@ mod test { // Ignore error: Final loader didn't return a Buffer or String assert!( run_loaders( - vec![p1, p2, c1, c2], - None, rs.clone(), Some(Arc::new(TestContentPlugin)), - (), - Arc::new(NativeFileSystem::new(false)) + TestContext(Loaders::new(vec![p1, p2, c1, c2])), + Arc::new(NativeFileSystem::new(false)), ) .await .1 @@ -469,19 +537,17 @@ mod test { IDENTS.with(|i| assert_eq!(*i.borrow(), &["pitch1", "pitch2", "normal2", "normal1"])); IDENTS.with(|i| i.borrow_mut().clear()); - let p1 = Arc::new(PitchNormalBase) as Arc>; - let p2 = Arc::new(PitchNormal) as Arc>; - let p3 = Arc::new(PitchNormal2) as Arc>; + let p1 = ResolvedLoader::uncached(Arc::new(PitchNormalBase)); + let p2 = ResolvedLoader::uncached(Arc::new(PitchNormal)); + let p3 = ResolvedLoader::uncached(Arc::new(PitchNormal2)); // Ignore error: Final loader didn't return a Buffer or String assert!( run_loaders( - vec![p1, p2, p3], - None, rs.clone(), Some(Arc::new(TestContentPlugin)), - (), - Arc::new(NativeFileSystem::new(false)) + TestContext(Loaders::new(vec![p1, p2, p3])), + Arc::new(NativeFileSystem::new(false)), ) .await .1 @@ -508,12 +574,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal { + impl Loader for Normal { fn identifier(&self) -> Identifier { "/rspack/normal-loader1".into() } - async fn run(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, loader_context: &mut LoaderContext) -> Result<()> { let data = loader_context .additional_data .as_ref() @@ -531,12 +597,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal2 { + impl Loader for Normal2 { fn identifier(&self) -> Identifier { "/rspack/normal-loader2".into() } - async fn run(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, loader_context: &mut LoaderContext) -> Result<()> { let mut additional_data: AdditionalData = Default::default(); additional_data.insert("additional-data"); loader_context.finish_with((String::new(), None, Some(additional_data))); @@ -550,11 +616,12 @@ mod test { assert!( run_loaders( - vec![Arc::new(Normal) as Arc, Arc::new(Normal2)], - None, rs, Some(Arc::new(TestContentPlugin)), - (), + TestContext(Loaders::new(vec![ + ResolvedLoader::uncached(Arc::new(Normal)), + ResolvedLoader::uncached(Arc::new(Normal2)), + ])), Arc::new(NativeFileSystem::new(false)), ) .await @@ -570,12 +637,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal { + impl Loader for Normal { fn identifier(&self) -> Identifier { "/rspack/normal-loader1".into() } - async fn run(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, loader_context: &mut LoaderContext) -> Result<()> { assert!(loader_context.content.is_some()); // Does not call `LoaderContext::finish_with` Ok(()) @@ -591,12 +658,12 @@ mod test { #[cacheable_dyn] #[async_trait::async_trait] - impl Loader<()> for Normal2 { + impl Loader for Normal2 { fn identifier(&self) -> Identifier { "/rspack/normal-loader2".into() } - async fn run(&self, loader_context: &mut LoaderContext<()>) -> Result<()> { + async fn run(&self, loader_context: &mut LoaderContext) -> Result<()> { let (content, source_map, additional_data) = loader_context.take_all(); assert!(content.is_none()); assert!(source_map.is_none()); @@ -608,12 +675,13 @@ mod test { // Ignore error: Final loader didn't return a Buffer or String assert!( run_loaders( - vec![Arc::new(Normal2), Arc::new(Normal)], - None, rs, Some(Arc::new(TestContentPlugin)), - (), - Arc::new(NativeFileSystem::new(false)) + TestContext(Loaders::new(vec![ + ResolvedLoader::uncached(Arc::new(Normal2)), + ResolvedLoader::uncached(Arc::new(Normal)), + ])), + Arc::new(NativeFileSystem::new(false)), ) .await .1 diff --git a/crates/rspack_loader_testing/Cargo.toml b/crates/rspack_loader_testing/Cargo.toml index 3716ea250622..b0bfb8a4713f 100644 --- a/crates/rspack_loader_testing/Cargo.toml +++ b/crates/rspack_loader_testing/Cargo.toml @@ -16,7 +16,7 @@ async-trait = { workspace = true } rspack_cacheable = { workspace = true } rspack_core = { workspace = true } rspack_error = { workspace = true } -rspack_loader_runner = { workspace = true } +rspack_loader_runner = { workspace = true, features = ["test-loader"] } serde_json = { workspace = true } [lints] diff --git a/crates/rspack_napi/Cargo.toml b/crates/rspack_napi/Cargo.toml index 8ffca1b36891..71c92d69d4a4 100644 --- a/crates/rspack_napi/Cargo.toml +++ b/crates/rspack_napi/Cargo.toml @@ -11,6 +11,7 @@ version.workspace = true napi = { workspace = true, features = ["serde-json", "anyhow", "napi9", "compat-mode"] } oneshot = { workspace = true } rspack_error = { workspace = true } +rustc-hash = { workspace = true } serde_json = { workspace = true } tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "time"] } diff --git a/crates/rspack_napi/src/lib.rs b/crates/rspack_napi/src/lib.rs index adac6a2ccb2d..e7531d7413ba 100644 --- a/crates/rspack_napi/src/lib.rs +++ b/crates/rspack_napi/src/lib.rs @@ -2,8 +2,13 @@ mod ext; mod js_values; +mod lifecycle; +mod thread_local_reference; mod utils; +pub use lifecycle::{LifecycleGuard, LifecycleId}; +pub use thread_local_reference::ThreadLocalReference; + mod errors; pub use errors::NapiErrorToRspackErrorExt; diff --git a/crates/rspack_napi/src/lifecycle.rs b/crates/rspack_napi/src/lifecycle.rs new file mode 100644 index 000000000000..2bf45b326fcc --- /dev/null +++ b/crates/rspack_napi/src/lifecycle.rs @@ -0,0 +1,81 @@ +use std::{ + any::TypeId, + fmt, + hash::{Hash, Hasher}, + marker::PhantomData, + sync::atomic::{AtomicU64, Ordering}, +}; + +static NEXT_ID: AtomicU64 = AtomicU64::new(1); + +/// A process-unique identity, typed independently of the owner's generic parameters. +pub struct LifecycleId { + pub(crate) value: u64, + marker: PhantomData T>, +} + +impl Copy for LifecycleId {} +impl Clone for LifecycleId { + fn clone(&self) -> Self { + *self + } +} +impl PartialEq for LifecycleId { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } +} +impl Eq for LifecycleId {} +impl Hash for LifecycleId { + fn hash(&self, state: &mut H) { + self.value.hash(state); + } +} +impl fmt::Debug for LifecycleId { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.value.fmt(f) + } +} + +/// Owns an identity without owning or borrowing `T`. Moving the guard preserves +/// its identity; dropping it releases associated JS references on their threads. +/// This guard deliberately cannot be cloned. +pub struct LifecycleGuard { + id: LifecycleId, +} + +impl LifecycleGuard { + pub fn new() -> Self { + let value = NEXT_ID + .fetch_update(Ordering::Relaxed, Ordering::Relaxed, |id| id.checked_add(1)) + .expect("Lifecycle IDs exhausted"); + Self { + id: LifecycleId { + value, + marker: PhantomData, + }, + } + } + + pub fn id(&self) -> LifecycleId { + self.id + } +} + +impl Default for LifecycleGuard { + fn default() -> Self { + Self::new() + } +} + +impl fmt::Debug for LifecycleGuard { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.id.fmt(f) + } +} + +impl Drop for LifecycleGuard { + fn drop(&mut self) { + crate::thread_local_reference::notify_drop(TypeId::of::(), self.id.value); + } +} diff --git a/crates/rspack_napi/src/thread_local_reference.rs b/crates/rspack_napi/src/thread_local_reference.rs new file mode 100644 index 000000000000..dce69d1601fe --- /dev/null +++ b/crates/rspack_napi/src/thread_local_reference.rs @@ -0,0 +1,250 @@ +use std::{ + any::TypeId, + cell::RefCell, + ffi::c_void, + marker::PhantomData, + ptr, + sync::{Arc, LazyLock, Mutex}, +}; + +use napi::{ + Env, Result, + bindgen_prelude::{ClassInstance, FromNapiValue, JavaScriptClassExt}, + check_status, sys, +}; +use rustc_hash::FxHashMap as HashMap; + +use crate::LifecycleId; + +type DropEvent = (TypeId, u64); + +// Only queues and TSFN handles cross threads. All references remain in ENVS. +static DISPATCHERS: LazyLock>>> = + LazyLock::new(Default::default); + +thread_local! { + static ENVS: RefCell> = RefCell::default(); +} + +struct References { + lifetime_type: TypeId, + values: HashMap, +} + +struct EnvReferences { + types: HashMap, +} + +#[derive(Default)] +struct DispatchState { + // Serialized with enqueue and environment shutdown; never dereferenced in Rust. + tsfn: Option, + pending: Vec, + scheduled: bool, +} + +#[derive(Default)] +struct Dispatcher(Mutex); + +impl Dispatcher { + fn enqueue(self: &Arc, event: DropEvent) { + let mut state = self.0.lock().expect("reference cleanup queue lock"); + let Some(tsfn) = state.tsfn else { return }; + state.pending.push(event); + if state.scheduled { + return; + } + state.scheduled = true; + let data = Box::into_raw(Box::new(self.clone())); + // An unlimited, nonblocking queue also works when a guard drops on the JS + // thread. Batch notifications until this callback drains them. + let status = unsafe { + sys::napi_call_threadsafe_function( + tsfn as sys::napi_threadsafe_function, + data.cast(), + sys::ThreadsafeFunctionCallMode::nonblocking, + ) + }; + if status != sys::Status::napi_ok { + // Closing environments may reject notifications; their cleanup hook + // releases every reference. No queued allocation may escape on failure. + unsafe { drop(Box::from_raw(data)) }; + state.pending.clear(); + state.scheduled = false; + } + } +} + +pub(crate) fn notify_drop(lifetime_type: TypeId, id: u64) { + for dispatcher in DISPATCHERS + .lock() + .expect("reference dispatcher lock") + .values() + { + dispatcher.enqueue((lifetime_type, id)); + } +} + +extern "C" fn cleanup_callback( + env: sys::napi_env, + _callback: sys::napi_value, + _context: *mut c_void, + data: *mut c_void, +) { + // Node also drains queued data with a null env during shutdown. + let dispatcher = unsafe { Box::>::from_raw(data.cast()) }; + let events = { + let mut state = dispatcher.0.lock().expect("reference cleanup queue lock"); + state.scheduled = false; + std::mem::take(&mut state.pending) + }; + if env.is_null() { + return; + } + ENVS.with_borrow_mut(|envs| { + if let Some(cache) = envs.get_mut(&(env as usize)) { + for (lifetime_type, id) in events { + for references in cache.types.values_mut() { + if references.lifetime_type == lifetime_type + && let Some(reference) = references.values.remove(&id) + { + unsafe { sys::napi_delete_reference(env, reference) }; + } + } + } + } + }); +} + +fn initialize(env: &Env) -> Result<()> { + let key = env.raw() as usize; + if ENVS.with_borrow(|envs| envs.contains_key(&key)) { + return Ok(()); + } + let dispatcher = Arc::new(Dispatcher::default()); + let mut name = ptr::null_mut(); + check_status!(unsafe { + sys::napi_create_string_utf8(env.raw(), c"lifecycle_cleanup".as_ptr(), 17, &mut name) + })?; + let mut tsfn = ptr::null_mut(); + check_status!(unsafe { + sys::napi_create_threadsafe_function( + env.raw(), + ptr::null_mut(), + ptr::null_mut(), + name, + 0, + 1, + ptr::null_mut(), + None, + ptr::null_mut(), + Some(cleanup_callback), + &mut tsfn, + ) + })?; + let setup = (|| { + check_status!(unsafe { sys::napi_unref_threadsafe_function(env.raw(), tsfn) })?; + env.add_env_cleanup_hook(key, |key| { + if let Some(dispatcher) = DISPATCHERS + .lock() + .expect("reference dispatcher lock") + .remove(&key) + { + let mut state = dispatcher.0.lock().expect("reference cleanup queue lock"); + if let Some(tsfn) = state.tsfn.take() { + unsafe { + sys::napi_release_threadsafe_function( + tsfn as sys::napi_threadsafe_function, + sys::ThreadsafeFunctionReleaseMode::release, + ) + }; + } + state.pending.clear(); + } + // Remove the cache before deleting references, including for environments + // with no subsequent loader invocation (hook failure, cancellation, exit). + let cache = ENVS.with_borrow_mut(|envs| envs.remove(&key)); + if let Some(cache) = cache { + for references in cache.types.into_values() { + for reference in references.values.into_values() { + unsafe { sys::napi_delete_reference(key as sys::napi_env, reference) }; + } + } + } + })?; + Ok(()) + })(); + if let Err(error) = setup { + unsafe { + sys::napi_release_threadsafe_function(tsfn, sys::ThreadsafeFunctionReleaseMode::release) + }; + return Err(error); + } + dispatcher + .0 + .lock() + .expect("reference cleanup queue lock") + .tsfn = Some(tsfn as usize); + ENVS.with_borrow_mut(|envs| { + envs.insert( + key, + EnvReferences { + types: HashMap::default(), + }, + ); + }); + DISPATCHERS + .lock() + .expect("reference dispatcher lock") + .insert(key, dispatcher); + Ok(()) +} + +/// Reuses a JS class for a native lifetime on the current JS thread and env. +/// References are released asynchronously when the guard drops, or at env exit. +/// `T` identifies the native lifetime; `J` identifies its JS representation. +pub struct ThreadLocalReference(PhantomData (T, J)>); + +impl ThreadLocalReference { + /// The caller must keep the corresponding guard alive throughout this call. + /// Returned instances follow the class's own native access-window policy. + pub fn get_or_insert_with<'env>( + env: &'env Env, + id: LifecycleId, + create: impl FnOnce() -> J, + ) -> Result> { + initialize(env)?; + let key = env.raw() as usize; + let cached = ENVS.with_borrow(|envs| { + envs + .get(&key) + .and_then(|cache| cache.types.get(&TypeId::of::<(T, J)>())) + .and_then(|references| references.values.get(&id.value).copied()) + }); + if let Some(reference) = cached { + let mut value = ptr::null_mut(); + check_status!(unsafe { sys::napi_get_reference_value(env.raw(), reference, &mut value) })?; + return unsafe { ClassInstance::from_napi_value(env.raw(), value) }; + } + // Do not hold a RefCell borrow while constructing a native-backed class. + let instance = create().into_instance(env)?; + let mut reference = ptr::null_mut(); + check_status!(unsafe { + sys::napi_create_reference(env.raw(), instance.value, 1, &mut reference) + })?; + ENVS.with_borrow_mut(|envs| { + envs + .get_mut(&key) + .expect("initialized reference cache") + .types + .entry(TypeId::of::<(T, J)>()) + .or_insert_with(|| References { + lifetime_type: TypeId::of::(), + values: HashMap::default(), + }) + .values + .insert(id.value, reference); + }); + Ok(instance) + } +} diff --git a/packages/rspack/src/Compiler.ts b/packages/rspack/src/Compiler.ts index 593dc184842c..1d56712701ce 100644 --- a/packages/rspack/src/Compiler.ts +++ b/packages/rspack/src/Compiler.ts @@ -49,6 +49,7 @@ import type { FileSystemInfoEntry } from './FileSystemInfo'; import type { rspack } from './index'; import Cache from './lib/Cache'; import CacheFacade from './lib/CacheFacade'; +import { runWithLoaderContext } from './loader-runner/context'; import { Logger, type LogTypeEnum } from './logging/Logger'; import { NormalModuleFactory } from './NormalModuleFactory'; import { ResolverFactory } from './ResolverFactory'; @@ -976,7 +977,20 @@ class Compiler { this.#instance = new instanceBinding.JsCompiler( this.compilerPath, this.#rawOptions, - this.#builtinPlugins, + this.#builtinPlugins.map((plugin) => + plugin.name === instanceBinding.BuiltinPluginName.JsLoaderRspackPlugin + ? { + ...plugin, + options: (context: binding.JsLoaderContext) => + runWithLoaderContext( + context, + plugin.options as ( + context: binding.JsLoaderContext, + ) => Promise, + ), + } + : plugin, + ), this.#registers, ThreadsafeOutputNodeFS.__to_binding(this.outputFileSystem!), this.intermediateFileSystem diff --git a/packages/rspack/src/loader-runner/cache.ts b/packages/rspack/src/loader-runner/cache.ts index fcc70c62f773..4a7cfe33ef31 100644 --- a/packages/rspack/src/loader-runner/cache.ts +++ b/packages/rspack/src/loader-runner/cache.ts @@ -1,4 +1,4 @@ -import type { JsLoaderContext } from '@rspack/binding'; +import type { LoaderContextState } from './context'; import { isNil } from '../util'; import { @@ -32,10 +32,13 @@ type LoaderCacheApi = { export class LoaderCache { readonly #api: LoaderCacheApi; - readonly #context: JsLoaderContext; + readonly #context: LoaderContextState; readonly #dependencies: LoaderDependenciesState; - constructor(context: JsLoaderContext, dependencies: LoaderDependenciesState) { + constructor( + context: LoaderContextState, + dependencies: LoaderDependenciesState, + ) { this.#context = context; this.#api = (context as any).__internal__loaderCache as LoaderCacheApi; this.#dependencies = dependencies; @@ -47,7 +50,7 @@ export class LoaderCache { additionalData: unknown, ): Promise { const context = this.#context; - const loader = context.loaderItems[loaderIndex]; + const loader = context.state.loaderItemStates[loaderIndex]; if ( !context.cacheable || !loader || diff --git a/packages/rspack/src/loader-runner/context.ts b/packages/rspack/src/loader-runner/context.ts new file mode 100644 index 000000000000..c6887bc4451a --- /dev/null +++ b/packages/rspack/src/loader-runner/context.ts @@ -0,0 +1,158 @@ +import type { + JsLoaderContext, + JsLoaderHookContext, + JsLoaderOutput, + RspackError, +} from '@rspack/binding'; + +const UNREAD = Symbol('unread'); + +/** Reused wrapper with one owned state snapshot and read cache per entry. */ +export class LoaderContextState { + native: JsLoaderContext | JsLoaderHookContext; + state: JsLoaderContext['state']; + #module?: JsLoaderContext['_module']; + #resource?: string; + // Reset on every native entry. Cache missing values too. + #loaderCache: JsLoaderContext['__internal__loaderCache'] | typeof UNREAD = + UNREAD; + #content: JsLoaderContext['content'] | typeof UNREAD = UNREAD; + #sourceMap: JsLoaderOutput['sourceMap'] | typeof UNREAD = UNREAD; + #additionalData: JsLoaderOutput['additionalData'] | typeof UNREAD = UNREAD; + + constructor(native: JsLoaderContext | JsLoaderHookContext) { + this.native = native; + this.state = native.state; + for (const item of this.state.loaderItemStates) item.data ??= {}; + } + + get identity(): JsLoaderContext { + return 'identity' in this.native ? this.native.identity : this.native; + } + + enter(native: JsLoaderContext | JsLoaderHookContext) { + this.native = native; + this.state = native.state; + for (const item of this.state.loaderItemStates) item.data ??= {}; + this.#module = undefined; + this.#resource = undefined; + this.#loaderCache = UNREAD; + this.#content = UNREAD; + this.#sourceMap = UNREAD; + this.#additionalData = UNREAD; + } + + get loaderState() { + return this.state.loaderState; + } + get loaderIndex() { + return this.state.loaderIndex; + } + set loaderIndex(value: number) { + this.state.loaderIndex = value; + } + get cacheable() { + return this.state.cacheable; + } + set cacheable(value: boolean) { + this.state.cacheable = value; + } + get dependencies() { + return this.state.dependencies; + } + get __internal__parseMeta() { + return this.state.parseMeta; + } + set __internal__error(value: RspackError) { + this.state.error = value; + } + get __internal__loaderCache() { + if (this.#loaderCache === UNREAD) { + this.#loaderCache = + '__internal__loaderCache' in this.native + ? this.native.__internal__loaderCache + : undefined; + } + return this.#loaderCache; + } + get resource() { + return (this.#resource ??= this.native.resource); + } + get hot() { + return this.state.hot; + } + set hot(value: boolean) { + this.state.hot = value; + } + get _module() { + return (this.#module ??= this.native._module); + } + get content() { + if (this.state.output) return this.state.output.content; + if (this.#content === UNREAD) { + this.#content = 'content' in this.native ? this.native.content : null; + } + return this.#content; + } + get sourceMap() { + if (this.state.output) return this.state.output.sourceMap; + if (this.#sourceMap === UNREAD) { + this.#sourceMap = + 'sourceMap' in this.native + ? (this.native.sourceMap ?? undefined) + : undefined; + } + return this.#sourceMap; + } + get additionalData() { + if (this.state.output) return this.state.output.additionalData; + if (this.#additionalData === UNREAD) { + this.#additionalData = + 'additionalData' in this.native + ? (this.native.additionalData ?? undefined) + : undefined; + } + return this.#additionalData; + } + + finish(output: JsLoaderOutput) { + this.state.output = output; + } + + commit() { + this.native.state = this.state; + } +} + +/** Preserve ownership when a runner fails before it can commit its local state. */ +export function setLoaderContextError( + context: JsLoaderContext, + error: unknown, +) { + context.__internal__error = toLoaderContextError(error); +} + +export function toLoaderContextError(error: unknown): RspackError { + if (typeof error !== 'object' || error === null) { + const wrapped = new Error( + `(Emitted value instead of an instance of Error) ${String(error)}`, + ); + wrapped.name = 'NonErrorEmittedError'; + return wrapped; + } else { + return error as RspackError; + } +} + +/** The native boundary always receives its class back, including on rejection. */ +export async function runWithLoaderContext( + context: JsLoaderContext, + run: (context: JsLoaderContext) => unknown, +): Promise { + try { + await run(context); + } catch (error) { + setLoaderContextError(context, error); + } + return context; +} diff --git a/packages/rspack/src/loader-runner/dependencies.ts b/packages/rspack/src/loader-runner/dependencies.ts index 061224191a38..96b7fac61fe3 100644 --- a/packages/rspack/src/loader-runner/dependencies.ts +++ b/packages/rspack/src/loader-runner/dependencies.ts @@ -1,6 +1,6 @@ import type { JsLoaderContext } from '@rspack/binding'; -export type LoaderDependencies = JsLoaderContext['dependencies']; +export type LoaderDependencies = JsLoaderContext['state']['dependencies']; const DEPENDENCY_KEYS = [ 'fileDependencies', diff --git a/packages/rspack/src/loader-runner/index.ts b/packages/rspack/src/loader-runner/index.ts index 66fe66e803b8..626e7bfdd267 100644 --- a/packages/rspack/src/loader-runner/index.ts +++ b/packages/rspack/src/loader-runner/index.ts @@ -11,7 +11,9 @@ import querystring from 'node:querystring'; import { formatDiagnostic, type JsLoaderContext, - type JsLoaderItem, + type JsLoaderHookContext, + type JsLoaderMetadata, + type JsLoaderItemState, JsLoaderState, JsRspackSeverity, } from '@rspack/binding'; @@ -23,6 +25,7 @@ import { } from 'webpack-sources'; import { commitCustomFieldsToRust } from '../BuildInfo'; +import { LoaderContextState } from './context'; import type { Compiler } from '../Compiler'; import { BUILTIN_LOADER_PREFIX, @@ -89,9 +92,15 @@ export class LoaderObject { /** * @internal This field is rspack internal. Do not edit. */ - loaderItem: JsLoaderItem; + readonly loaderItem: JsLoaderMetadata; + readonly #getState: () => JsLoaderItemState; - constructor(loaderItem: JsLoaderItem, compiler: Compiler) { + constructor( + loaderItem: JsLoaderMetadata, + getState: () => JsLoaderItemState, + compiler: Compiler, + ) { + this.#getState = getState; const splittedRequest = parseResourceWithoutFragment(loaderItem.loader); this.path = splittedRequest.path; this.fragment = ''; @@ -140,11 +149,14 @@ export class LoaderObject { ) as LoaderObject['parallel']) : false; this.loaderItem = loaderItem; - this.loaderItem.data = this.loaderItem.data ?? {}; + } + + get state() { + return this.#getState(); } get pitchExecuted() { - return this.loaderItem.pitchExecuted; + return this.state.pitchExecuted; } set pitchExecuted(value: boolean) { @@ -152,11 +164,11 @@ export class LoaderObject { throw new Error('pitchExecuted should be true'); } - this.loaderItem.pitchExecuted = true; + this.state.pitchExecuted = true; } get normalExecuted() { - return this.loaderItem.normalExecuted; + return this.state.normalExecuted; } set normalExecuted(value: boolean) { @@ -164,30 +176,19 @@ export class LoaderObject { throw new Error('normalExecuted should be true'); } - this.loaderItem.normalExecuted = true; + this.state.normalExecuted = true; } set noPitch(value: boolean) { if (!value) { throw new Error('noPitch should be true'); } - this.loaderItem.noPitch = true; + this.state.noPitch = true; } shouldYield() { return this.request.startsWith(BUILTIN_LOADER_PREFIX); } - - static __from_binding( - loaderItem: JsLoaderItem, - compiler: Compiler, - ): LoaderObject { - return new this(loaderItem, compiler); - } - - static __to_binding(loader: LoaderObject): JsLoaderItem { - return loader.loaderItem; - } } class JsSourceMap { @@ -228,22 +229,37 @@ function getCurrentLoader( return null; } -interface LoaderContextState { +interface SharedLoaderContextState { + context: LoaderContextState; loaderContext: LoaderContext; update( - context: JsLoaderContext, + context: LoaderContextState, dependencies: LoaderDependenciesState, traceData?: Pick, ): void; } +const loaderContexts = new WeakMap(); + +export function getLoaderContextState( + native: JsLoaderContext | JsLoaderHookContext, +): LoaderContextState { + const identity = 'identity' in native ? native.identity : native; + const shared = loaderContexts.get(identity); + if (shared) { + shared.context.enter(native); + return shared.context; + } + return new LoaderContextState(native); +} + export function createLoaderContext( compiler: Compiler, - context: JsLoaderContext, + context: LoaderContextState, dependencies: LoaderDependenciesState, traceData?: Pick, ): LoaderContext { - const state = context.loaderContextState as LoaderContextState | undefined; + const state = loaderContexts.get(context.identity); if (state) { state.update(context, dependencies, traceData); return state.loaderContext; @@ -260,11 +276,22 @@ export function createLoaderContext( /// Construct `loaderContext` const loaderContext = {} as LoaderContext; - loaderContext.loaders = context.loaderItems.map((item) => { - return LoaderObject.__from_binding(item, compiler); - }); + loaderContext.loaders = context.native.loaderItems!.map( + (item, index) => + new LoaderObject( + item, + () => context.state.loaderItemStates[index], + compiler, + ), + ); - loaderContext.hot = context.hot; + Object.defineProperty(loaderContext, 'hot', { + enumerable: true, + get: () => context.hot, + set: (hot: boolean) => { + context.hot = hot; + }, + }); loaderContext.context = contextDirectory; loaderContext.resourcePath = resourcePath!; loaderContext.resourceQuery = resourceQuery!; @@ -705,9 +732,9 @@ export function createLoaderContext( }); Object.defineProperty(loaderContext, 'data', { enumerable: true, - get: () => loaderContext.loaders[loaderContext.loaderIndex].loaderItem.data, + get: () => loaderContext.loaders[loaderContext.loaderIndex].state.data, set: (data) => - (loaderContext.loaders[loaderContext.loaderIndex].loaderItem.data = data), + (loaderContext.loaders[loaderContext.loaderIndex].state.data = data), }); /// Rspack private @@ -715,30 +742,27 @@ export function createLoaderContext( context.__internal__parseMeta[key] = value; }; - // Rust retains this state only for the current run_loaders invocation. Update - // the captured snapshot on every entry so hook-installed closures use the - // current loader index, dependencies and module pointer across native loaders. - context.loaderContextState = { + // The native lifetime cache keeps the key alive across hooks and loaders. + // WeakMap entries disappear once Rust releases that class and JS lets it go. + loaderContexts.set(context.identity, { + context, loaderContext, update(nextContext, nextDependencies, nextTraceData) { context = nextContext; dependencies = nextDependencies; traceData = nextTraceData; - loaderContext.hot = context.hot; loaderContext._module = context._module; - loaderContext.loaders = context.loaderItems.map((item) => - LoaderObject.__from_binding(item, compiler), - ); }, - } satisfies LoaderContextState; + }); return loaderContext; } export async function runLoaders( compiler: Compiler, - context: JsLoaderContext, + nativeContext: JsLoaderContext, ): Promise { + const context = getLoaderContextState(nativeContext); const loaderState = context.loaderState; const pitch = loaderState === JsLoaderState.Pitching; @@ -802,6 +826,7 @@ export async function runLoaders( } return { ...item, + state: item.state, options, pitch: undefined, normal: undefined, @@ -960,16 +985,15 @@ export async function runLoaders( } case RequestType.UpdateLoaderObjects: { const updates = args[0]; - loaderContext.loaders = loaderContext.loaders.map((item, index) => { + loaderContext.loaders.forEach((item, index) => { const update = updates[index]; - item.loaderItem.data = update.data; + item.state.data = update.data; if (update.pitchExecuted) { item.pitchExecuted = true; } if (update.normalExecuted) { item.normalExecuted = true; } - return item; }); break; } @@ -1123,7 +1147,7 @@ export async function runLoaders( args = await isomorphoicRun(fn, [ loaderContext.remainingRequest, loaderContext.previousRequest, - currentLoaderObject.loaderItem.data, + currentLoaderObject.state.data, ]); } finally { dependencies.mergeChanges(); @@ -1133,13 +1157,15 @@ export async function runLoaders( if (hasArg) { const [content, sourceMap, additionalData] = args; - context.content = isNil(content) - ? null - : typeof content === 'string' - ? content - : toBuffer(content); - context.sourceMap = serializeObject(sourceMap); - context.additionalData = additionalData || undefined; + context.finish({ + content: isNil(content) + ? null + : typeof content === 'string' + ? content + : toBuffer(content), + sourceMap: serializeObject(sourceMap), + additionalData: additionalData || undefined, + }); break; } } @@ -1149,7 +1175,7 @@ export async function runLoaders( case JsLoaderState.Normal: { let content: Parameters[0] | null | undefined = context.content; - const rawSourceMap = context.sourceMap; + let outputChanged = false; let sourceMap: string | object | undefined; let sourceMapParsed = false; let additionalData = context.additionalData; @@ -1180,6 +1206,7 @@ export async function runLoaders( if (cached) { currentLoaderObject.normalExecuted = true; content = cached.content; + outputChanged = true; sourceMap = JsSourceMap.__from_binding(cached.sourceMap); sourceMapParsed = true; loaderContext.loaderIndex--; @@ -1197,7 +1224,7 @@ export async function runLoaders( // Parse source map lazily only when a JavaScript loader consumes it. if (!sourceMapParsed) { - sourceMap = JsSourceMap.__from_binding(rawSourceMap); + sourceMap = JsSourceMap.__from_binding(context.sourceMap); sourceMapParsed = true; } @@ -1207,6 +1234,7 @@ export async function runLoaders( additionalData, ]); + outputChanged = true; if (cached === null) { await loaderCache?.store( loaderContext.loaderIndex, @@ -1220,26 +1248,23 @@ export async function runLoaders( } } - context.content = isNil(content) - ? null - : typeof content === 'string' - ? content - : toBuffer(content); - context.sourceMap = sourceMapParsed - ? JsSourceMap.__to_binding(sourceMap) - : rawSourceMap; - context.additionalData = additionalData || undefined; + if (outputChanged) { + context.finish({ + content: isNil(content) + ? null + : typeof content === 'string' + ? content + : toBuffer(content), + sourceMap: JsSourceMap.__to_binding(sourceMap), + additionalData: additionalData || undefined, + }); + } break; } default: throw new Error(`Unexpected loader runner state: ${loaderState}`); } - - // update loader state - context.loaderItems = loaderContext.loaders.map((item) => - LoaderObject.__to_binding(item), - ); } catch (e) { if (typeof e !== 'object' || e === null) { const error = new Error( @@ -1263,5 +1288,6 @@ export async function runLoaders( commitCustomFieldsToRust(context._module.buildInfo); } - return context; + context.commit(); + return nativeContext; } diff --git a/packages/rspack/src/loader-runner/worker.ts b/packages/rspack/src/loader-runner/worker.ts index d0f9539d769c..a60b830f0ef5 100644 --- a/packages/rspack/src/loader-runner/worker.ts +++ b/packages/rspack/src/loader-runner/worker.ts @@ -484,9 +484,9 @@ async function loaderImpl( Object.defineProperty(loaderContext, 'data', { enumerable: true, - get: () => loaderContext.loaders[loaderContext.loaderIndex].loaderItem.data, + get: () => loaderContext.loaders[loaderContext.loaderIndex].state.data, set: (value) => { - loaderContext.loaders[loaderContext.loaderIndex].loaderItem.data = value; + loaderContext.loaders[loaderContext.loaderIndex].state.data = value; }, }); @@ -523,7 +523,7 @@ async function loaderImpl( (await runSyncOrAsync(fn, loaderContext, [ loaderContext.remainingRequest, loaderContext.previousRequest, - currentLoaderObject.loaderItem.data, + currentLoaderObject.state.data, ])) || []; const hasArg = args.some((value) => value !== undefined); @@ -604,7 +604,7 @@ async function loaderImpl( RequestType.UpdateLoaderObjects, loaderContext.loaders.map((item) => { return { - data: item.loaderItem.data, + data: item.state.data, normalExecuted: item.normalExecuted, pitchExecuted: item.pitchExecuted, }; diff --git a/packages/rspack/src/taps/normalModule.ts b/packages/rspack/src/taps/normalModule.ts index 26dbac7c0ec4..d45c6ffa51e1 100644 --- a/packages/rspack/src/taps/normalModule.ts +++ b/packages/rspack/src/taps/normalModule.ts @@ -1,6 +1,7 @@ import binding from '@rspack/binding'; import { commitCustomFieldsToRust } from '../BuildInfo'; -import { createLoaderContext, LoaderObject } from '../loader-runner'; +import { createLoaderContext, getLoaderContextState } from '../loader-runner'; +import { toLoaderContextError } from '../loader-runner/context'; import { LoaderDependenciesState } from '../loader-runner/dependencies'; import { NormalModule } from '../NormalModule'; import type { CreatePartialRegisters } from './types'; @@ -14,23 +15,25 @@ export const createNormalModuleHooksRegisters: CreatePartialRegisters< NormalModule.getCompilationHooks( getCompiler().__internal__get_compilation()!, ).loader, - (queried) => (context: binding.JsLoaderContext) => { - const compiler = getCompiler(); - const dependencies = new LoaderDependenciesState(context.dependencies); - const loaderContext = createLoaderContext( - compiler, - context, - dependencies, - ); - queried.call(loaderContext, loaderContext._module); - dependencies.mergeChanges(); - context.loaderItems = loaderContext.loaders.map( - LoaderObject.__to_binding, - ); - if (compiler.options.cache) { - commitCustomFieldsToRust(context._module.buildInfo); + (queried) => (nativeContext: binding.JsLoaderHookContext) => { + try { + const context = getLoaderContextState(nativeContext); + const compiler = getCompiler(); + const dependencies = new LoaderDependenciesState(context.dependencies); + const loaderContext = createLoaderContext( + compiler, + context, + dependencies, + ); + queried.call(loaderContext, loaderContext._module); + dependencies.mergeChanges(); + if (compiler.options.cache) { + commitCustomFieldsToRust(context._module.buildInfo); + } + } catch (error) { + nativeContext.state.error = toLoaderContextError(error); } - return context; + return nativeContext.state; }, ), }); diff --git a/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/gc-check-loader-context.cjs b/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/gc-check-loader-context.cjs new file mode 100644 index 000000000000..ebd2bd77cb81 --- /dev/null +++ b/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/gc-check-loader-context.cjs @@ -0,0 +1,99 @@ +const assert = require('node:assert/strict'); +const { Worker, isMainThread } = require('node:worker_threads'); +const rspack = require('@rspack/core'); +const { createFsFromVolume, Volume } = require('memfs'); +const { closeCompiler, createGCTracker, forceGC, runCompiler } = require('./helpers.cjs'); + +async function exercise() { + for (const mode of ['normal', 'hook-error', 'runner-error', 'builtin-only', 'no-loaders']) { + const tracker = createGCTracker(); + const loader = require.resolve('./lifecycle-loader.cjs'); + const use = mode === 'no-loaders' ? [] : mode === 'builtin-only' + ? ['builtin:swc-loader'] : [loader, 'builtin:swc-loader', loader]; + let build = 0; + let nativeCalls = 0; + let native; + let facade; + const compiler = rspack({ + context: __dirname, + mode: 'development', + cache: false, + experiments: { incremental: false }, + entry: () => `./entry.js?build=${build}`, + output: { path: '/', filename: 'bundle.js' }, + module: { rules: [{ test: /entry\.js$/, use }] }, + plugins: [{ + apply(compiler) { + compiler.hooks.beforeRun.tap('LoaderLifecycle', () => { + build++; + nativeCalls = 0; + native = undefined; + facade = undefined; + }); + compiler.hooks.compilation.tap('LoaderLifecycle', compilation => { + const hook = compiler.webpack.NormalModule.getCompilationHooks(compilation).loader; + hook.tap({ name: 'LoaderLifecycle', stage: -10 }, context => { + tracker.track(context, `${build}:facade`); + facade = new WeakRef(context); + context.fromHook = 'preserved'; + context.hookSelf = () => context; + if (mode === 'hook-error') throw new Error('lifecycle hook error'); + }); + hook.tap({ name: 'LoaderLifecycle', stage: 10 }, context => { + assert.equal(context, facade.deref()); + }); + }); + const plugin = compiler.__internal__builtinPlugins.find(p => p.name === 'JsLoaderRspackPlugin'); + const run = plugin.options; + plugin.options = async context => { + nativeCalls++; + if (native) assert.equal(context, native.deref()); + else { + native = new WeakRef(context); + tracker.track(context, `${build}:native`); + } + // Both native and facade identities must survive GC between entries. + await forceGC(2); + assert(facade.deref()); + if (mode === 'runner-error') throw new Error('lifecycle runner error'); + return run(context); + }; + }, + }], + }); + compiler.outputFileSystem = createFsFromVolume(new Volume()); + try { + for (let i = 0; i < 2; i++) { + const stats = await runCompiler(compiler); + assert.equal(stats.hasErrors(), mode.endsWith('error')); + if (mode === 'normal') assert(nativeCalls > 1, `build ${build}: ${nativeCalls} native calls`); + // Collect before another build or compiler.close: guard destruction + // must actively release the final batch, including hook-only failures. + await tracker.waitForCollection(`${build}:facade`); + if (native) await tracker.waitForCollection(`${build}:native`); + } + } finally { + await closeCompiler(compiler); + } + } +} + +async function main() { + if (!isMainThread) return exercise(); + // Independent N-API environments must never reuse each other's references. + const worker = new Worker(__filename); + await Promise.all([ + exercise(), + new Promise((resolve, reject) => { + worker.on('error', reject); + worker.on('exit', code => code === 0 ? resolve() : reject(new Error(`worker exited ${code}`))); + }), + ]); + // A worker's environment cleanup must not disable the surviving environment. + await exercise(); +} + +main().catch(error => { + console.error(error); + process.exitCode = 1; +}); diff --git a/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/lifecycle-loader.cjs b/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/lifecycle-loader.cjs new file mode 100644 index 000000000000..3c1a8ebfce45 --- /dev/null +++ b/tests/rspack-test/compilerCases/fixtures/tsfn-lifecycle/lifecycle-loader.cjs @@ -0,0 +1,12 @@ +const assert = require('node:assert/strict'); + +module.exports = function (source) { + assert.equal(this.fromHook, 'preserved'); + assert.equal(this.hookSelf(), this); + return source; +}; + +module.exports.pitch = function () { + assert.equal(this.fromHook, 'preserved'); + assert.equal(this.hookSelf(), this); +}; diff --git a/tests/rspack-test/compilerCases/tsfn-lifecycle.js b/tests/rspack-test/compilerCases/tsfn-lifecycle.js index 71e7a3991587..e7a5e5036873 100644 --- a/tests/rspack-test/compilerCases/tsfn-lifecycle.js +++ b/tests/rspack-test/compilerCases/tsfn-lifecycle.js @@ -37,6 +37,12 @@ function runChild(script) { /** @type {import('@rspack/test-tools').TCompilerCaseConfig[]} */ module.exports = [ + { + description: "should reuse loader contexts and release references when native lifetimes end", + async build() { + await runChild(path.join(__dirname, "fixtures", "tsfn-lifecycle", "gc-check-loader-context.cjs")); + }, + }, { description: "should garbage collect hook closures that capture both compilation and compiler", diff --git a/tests/rspack-test/configCases/loader/boxed-context-rejection/errors.js b/tests/rspack-test/configCases/loader/boxed-context-rejection/errors.js new file mode 100644 index 000000000000..f7d63150de84 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-context-rejection/errors.js @@ -0,0 +1 @@ +module.exports = [/Rejected loader invocation/]; diff --git a/tests/rspack-test/configCases/loader/boxed-context-rejection/index.js b/tests/rspack-test/configCases/loader/boxed-context-rejection/index.js new file mode 100644 index 000000000000..05eccb912901 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-context-rejection/index.js @@ -0,0 +1 @@ +import './input'; diff --git a/tests/rspack-test/configCases/loader/boxed-context-rejection/input.js b/tests/rspack-test/configCases/loader/boxed-context-rejection/input.js new file mode 100644 index 000000000000..aef22247d752 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-context-rejection/input.js @@ -0,0 +1 @@ +export default 1; diff --git a/tests/rspack-test/configCases/loader/boxed-context-rejection/loader.js b/tests/rspack-test/configCases/loader/boxed-context-rejection/loader.js new file mode 100644 index 000000000000..57caa9e538ca --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-context-rejection/loader.js @@ -0,0 +1 @@ +module.exports = content => content; diff --git a/tests/rspack-test/configCases/loader/boxed-context-rejection/rspack.config.js b/tests/rspack-test/configCases/loader/boxed-context-rejection/rspack.config.js new file mode 100644 index 000000000000..51e659b4ef84 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-context-rejection/rspack.config.js @@ -0,0 +1,23 @@ +module.exports = { + module: { rules: [{ test: /input\.js$/, use: './loader.js' }] }, + plugins: [ + { + apply(compiler) { + let retained; + compiler.hooks.beforeRun.tap('RejectLoaderContext', () => { + const plugin = compiler.__internal__builtinPlugins.find( + (plugin) => plugin.name === 'JsLoaderRspackPlugin', + ); + plugin.options = async (context) => { + retained = context; + throw new Error('Rejected loader invocation'); + }; + }); + compiler.hooks.afterCompile.tap('RejectLoaderContext', () => { + expect(retained).toBeDefined(); + expect(() => retained.resource).toThrow('no longer available'); + }); + }, + }, + ], +}; diff --git a/tests/rspack-test/configCases/loader/boxed-hook-rejection/errors.js b/tests/rspack-test/configCases/loader/boxed-hook-rejection/errors.js new file mode 100644 index 000000000000..05757b0bf5f4 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-hook-rejection/errors.js @@ -0,0 +1 @@ +module.exports = [/Rejected loader hook/]; diff --git a/tests/rspack-test/configCases/loader/boxed-hook-rejection/index.js b/tests/rspack-test/configCases/loader/boxed-hook-rejection/index.js new file mode 100644 index 000000000000..05eccb912901 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-hook-rejection/index.js @@ -0,0 +1 @@ +import './input'; diff --git a/tests/rspack-test/configCases/loader/boxed-hook-rejection/input.js b/tests/rspack-test/configCases/loader/boxed-hook-rejection/input.js new file mode 100644 index 000000000000..aef22247d752 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-hook-rejection/input.js @@ -0,0 +1 @@ +export default 1; diff --git a/tests/rspack-test/configCases/loader/boxed-hook-rejection/rspack.config.js b/tests/rspack-test/configCases/loader/boxed-hook-rejection/rspack.config.js new file mode 100644 index 000000000000..71c3510a0270 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-hook-rejection/rspack.config.js @@ -0,0 +1,26 @@ +const path = require('node:path'); + +module.exports = { + plugins: [ + { + apply(compiler) { + let calls = 0; + compiler.hooks.compilation.tap('RejectLoaderHook', (compilation) => { + compiler.webpack.NormalModule.getCompilationHooks( + compilation, + ).loader.tap('RejectLoaderHook', (context) => { + if (context.resourcePath === path.resolve(__dirname, 'input.js')) { + calls++; + throw new Error('Rejected loader hook'); + } + }); + }); + compiler.hooks.afterCompile.tap('RejectLoaderHook', () => { + // Hooks only borrow the native context. A failed hook must report + // the module error while leaving the runner able to finish normally. + expect(calls).toBe(1); + }); + }, + }, + ], +}; diff --git a/tests/rspack-test/configCases/loader/boxed-source-roundtrip/index.js b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/index.js new file mode 100644 index 000000000000..3b3904099628 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/index.js @@ -0,0 +1,7 @@ +import normal from './input.js?normal'; +import pitched from './input.js?pitch'; + +it('preserves binary bytes, maps and additional data across native loaders', () => { + expect(normal).toEqual({ hex: '00fffe800a', value: 42 }); + expect(pitched).toEqual(normal); +}); diff --git a/tests/rspack-test/configCases/loader/boxed-source-roundtrip/input.js b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/input.js new file mode 100644 index 000000000000..15112f96a873 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/input.js @@ -0,0 +1 @@ +throw new Error("The producer replaces this source"); diff --git a/tests/rspack-test/configCases/loader/boxed-source-roundtrip/produce.js b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/produce.js new file mode 100644 index 000000000000..a64f630e0691 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/produce.js @@ -0,0 +1,17 @@ +function produce() { + this.cacheable(false); + this.addDependency(__filename); + this.__internal__setParseMeta('boxed-source', 'producer'); + this.callback(null, Buffer.from([0, 255, 254, 128, 10]), { + version: 3, + names: [], + sources: ['original.js'], + sourcesContent: ['original'], + mappings: 'AAAA', + }, { value: () => 42 }); +} + +module.exports = produce; +module.exports.pitch = function () { + if (this.resourceQuery === '?pitch') return produce.call(this); +}; diff --git a/tests/rspack-test/configCases/loader/boxed-source-roundtrip/rspack.config.js b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/rspack.config.js new file mode 100644 index 000000000000..9413fab244ad --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/rspack.config.js @@ -0,0 +1,151 @@ +const path = require('node:path'); + +module.exports = { + devtool: 'source-map', + module: { + rules: [ + { + test: /input\.js$/, + use: ['./verify.js', 'builtin:test-passthrough-loader', './produce.js'], + }, + ], + }, + plugins: [ + { + apply(compiler) { + const contexts = []; + const cachedKeys = [ + 'content', + 'sourceMap', + 'additionalData', + 'resource', + '_module', + '__internal__loaderCache', + ]; + const identities = new Map(); + const seen = new WeakSet(); + compiler.hooks.beforeRun.tap('BoxedSourceRoundtrip', () => { + const plugin = compiler.__internal__builtinPlugins.find( + (plugin) => plugin.name === 'JsLoaderRspackPlugin', + ); + const run = plugin.options; + plugin.options = async (context) => { + const prototype = Object.getPrototypeOf(context); + expect(prototype.constructor.name).toBe('JsLoaderContext'); + const reused = seen.has(context); + seen.add(context); + const resource = Object.getOwnPropertyDescriptor( + prototype, + 'resource', + ).get.call(context); + if (identities.has(resource)) + expect(context).toBe(identities.get(resource)); + else identities.set(resource, context); + let reads = 0; + let commits = 0; + const getterReads = {}; + for (const key of cachedKeys) { + const descriptor = Object.getOwnPropertyDescriptor( + prototype, + key, + ); + Object.defineProperty(context, key, { + configurable: true, + get() { + getterReads[key] = (getterReads[key] ?? 0) + 1; + if (key === 'content' || key === 'sourceMap') reads++; + return descriptor.get.call(context); + }, + }); + } + let stateReads = 0; + let metadataReads = 0; + let snapshot; + const state = Object.getOwnPropertyDescriptor(prototype, 'state'); + const metadata = Object.getOwnPropertyDescriptor( + prototype, + 'loaderItems', + ); + Object.defineProperty(context, 'loaderItems', { + configurable: true, + get() { + metadataReads++; + const items = metadata.get.call(context); + expect(items.every((item) => !('data' in item))).toBe(true); + return items; + }, + }); + Object.defineProperty(context, 'state', { + configurable: true, + get() { + stateReads++; + snapshot = state.get.call(context); + expect(Object.getPrototypeOf(snapshot)).toBe(Object.prototype); + expect( + snapshot.loaderItemStates.every( + (item) => !('loader' in item), + ), + ).toBe(true); + expect('loaderContextState' in snapshot).toBe(false); + if ( + snapshot.loaderState === 'Normal' && + snapshot.loaderIndex === 0 + ) { + expect(snapshot.cacheable).toBe(false); + } + return snapshot; + }, + set(value) { + commits++; + expect(value).toBe(snapshot); + state.set.call(context, value); + }, + }); + expect(await run(context)).toBe(context); + expect(stateReads).toBe(1); + expect(commits).toBe(1); + expect(metadataReads).toBe(reused ? 0 : 1); + for (const count of Object.values(getterReads)) { + expect(count).toBe(1); + } + // Restore native accessors before reusing the same class or checking + // revoked access with snapshots from earlier entries. + for (const key of [...cachedKeys, 'state', 'loaderItems']) + delete context[key]; + contexts.push({ context, snapshot }); + if (snapshot.loaderState === 'Pitching') expect(reads).toBe(0); + }; + }); + compiler.hooks.afterCompile.tap( + 'BoxedSourceRoundtrip', + (compilation) => { + expect(contexts.length).toBeGreaterThan(0); + expect(contexts.length).toBeGreaterThan(identities.size); + for (const { context, snapshot } of contexts) { + expect(() => context.content).toThrow('no longer available'); + expect(() => context._module).toThrow('no longer available'); + expect(() => context.state).toThrow('no longer available'); + // Owned state remains readable after the native class is revoked. + expect(Array.isArray(snapshot.loaderItemStates)).toBe(true); + expect(() => { + context.state = snapshot; + }).toThrow('no longer available'); + expect(() => { + context.__internal__error = new Error('late write'); + }).toThrow('no longer available'); + } + const modules = [...compilation.modules].filter((module) => + module.resource?.includes('input.js'), + ); + expect(modules).toHaveLength(2); + expect( + compilation.fileDependencies.has( + path.join(__dirname, 'produce.js'), + ), + ).toBe(true); + }, + ); + }, + }, + ], +}; diff --git a/tests/rspack-test/configCases/loader/boxed-source-roundtrip/verify.js b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/verify.js new file mode 100644 index 000000000000..1963e5713063 --- /dev/null +++ b/tests/rspack-test/configCases/loader/boxed-source-roundtrip/verify.js @@ -0,0 +1,10 @@ +module.exports = function (content, sourceMap, additionalData) { + expect(Buffer.isBuffer(content)).toBe(true); + expect(content.equals(Buffer.from([0, 255, 254, 128, 10]))).toBe(true); + expect(sourceMap.sources).toEqual(['original.js']); + expect(sourceMap.mappings).toBe('AAAA'); + expect(this.getDependencies()).toContain(require.resolve('./produce.js')); + this.__internal__setParseMeta('boxed-source', 'verified'); + return `module.exports = ${JSON.stringify({ hex: content.toString('hex'), value: additionalData.value() })}`; +}; +module.exports.raw = true; diff --git a/tests/rspack-test/configCases/loader/hook-before-loaders/loader.js b/tests/rspack-test/configCases/loader/hook-before-loaders/loader.js index 79c6f7edb482..697f78148aed 100644 --- a/tests/rspack-test/configCases/loader/hook-before-loaders/loader.js +++ b/tests/rspack-test/configCases/loader/hook-before-loaders/loader.js @@ -2,6 +2,10 @@ const assert = require('node:assert/strict'); const path = require('node:path'); function checkHookContext(context) { + assert.equal(context.loaders, context.hookLoaders); + context.loaders.forEach((loader, index) => { + assert.equal(loader, context.hookLoaderObjects[index]); + }); assert.equal(context.hookValue, 'from loader hook'); assert.equal(context.hookContext, context); assert.equal(context[Symbol.for('loader-hook-value')].value, 42); @@ -13,13 +17,19 @@ function checkHookContext(context) { module.exports = function (source) { checkHookContext(this); - assert.equal(this.hot, true); - this._compiler.loaderHookEvents.push('normal:' + this.getOptions().name); + assert.equal(this.hot, false); + const name = this.getOptions().name; + assert.equal(this.data.name, name); + assert(this.hookLoaderObjects.every(loader => loader.pitchExecuted)); + assert(this.hookLoaderObjects.slice(this.loaderIndex).every(loader => loader.normalExecuted)); + this._compiler.loaderHookEvents.push('normal:' + name); return source; }; module.exports.pitch = function () { checkHookContext(this); - assert.equal(this.hot, true); + assert.equal(this.hot, false); + this.data.name = this.getOptions().name; + assert(this.hookLoaderObjects.slice(0, this.loaderIndex + 1).every(loader => loader.pitchExecuted)); this._compiler.loaderHookEvents.push('pitch:' + this.getOptions().name); }; diff --git a/tests/rspack-test/configCases/loader/hook-before-loaders/rspack.config.js b/tests/rspack-test/configCases/loader/hook-before-loaders/rspack.config.js index fbac486f5bcf..500181a378e7 100644 --- a/tests/rspack-test/configCases/loader/hook-before-loaders/rspack.config.js +++ b/tests/rspack-test/configCases/loader/hook-before-loaders/rspack.config.js @@ -41,6 +41,8 @@ module.exports = [ // The native HMR tap runs at stage 0. if (stage === -10) { assert.equal(context.hot, false); + context.hookLoaders = context.loaders; + context.hookLoaderObjects = [...context.loaders]; context.hookValue = 'from loader hook'; context[Symbol.for('loader-hook-value')] = { value: 42 }; Object.defineProperty(context, 'hookContext', { @@ -49,6 +51,10 @@ module.exports = [ const addDependency = context.addDependency; context.addHookDependency = () => addDependency(dependency); } else { + assert.equal(context.loaders, context.hookLoaders); + context.loaders.forEach((loader, index) => { + assert.equal(loader, context.hookLoaderObjects[index]); + }); assert.equal(context.hookValue, 'from loader hook'); assert.equal(context.hookContext, context); assert.equal( @@ -58,6 +64,8 @@ module.exports = [ } if (stage === Infinity) { assert.equal(context.hot, true); + // JS writes must survive the native boundary too. + context.hot = false; if (name === 'no-loaders') { context.emitError(new Error('error from loader hook')); context.emitWarning(new Error('warning from loader hook')); diff --git a/website/docs/en/api/javascript-api/architecture.mdx b/website/docs/en/api/javascript-api/architecture.mdx index 9ad13bfdfe04..f1f21bcf21ba 100644 --- a/website/docs/en/api/javascript-api/architecture.mdx +++ b/website/docs/en/api/javascript-api/architecture.mdx @@ -112,6 +112,32 @@ When equivalent functionality is available, prefer an Rspack builtin loader. For Builtin loaders run on the Rust side, reducing JavaScript queueing and cross-language data conversion while making better use of Rspack's parallelism. +### Loader source ownership + +The native loader runner keeps its `LoaderContext` in a `Box`. Throughout the loader chain, content is stored as `Content::String` or `Content::Buffer`, with the source map stored separately. This preserves the input type for each loader. `NormalModule` constructs the final source after the loader chain finishes. + +When execution reaches JavaScript, a native `JsLoaderContext` class directly owns an `Option>` for that invocation. Its getters materialize content and source maps only when the JavaScript runner reads them. The JavaScript wrapper caches each getter result, including missing values, for that entry. The same wrapper resets its read cache on the next entry; output produced in JavaScript takes precedence over cached input. + +The JavaScript loader-context wrapper reads one plain `JsLoaderContextState` object (`#[napi(object)]`), accesses its mutable fields through JavaScript getters and setters, and passes the same state object back in one call before returning. Loader metadata is read once when the wrapper is created; per-loader data and execution flags travel in the state object. Reused loader objects read the latest state after each native loader. The state carries an optional output only when JavaScript produces one. A pitch that produces no output leaves native content and its source map unchanged. + +Each native loader context contains a typed `LifecycleGuard` from `rspack_napi`. The guard allocates a process-unique, incrementing ID. `ThreadLocalReference` caches the corresponding `JsLoaderContext` by this ID on its JavaScript thread, separately for each N-API environment. Hooks use the same identity without moving the Box. A JavaScript `WeakMap` associates the class with its wrapper and webpack-compatible `this` object, so neither needs to travel in the mutable state object. + +When the native context is consumed into its loader result, its guard drops and queues reference cleanup on each registered JavaScript environment. Cleanup notifications are batched and wake the JavaScript thread even if no more loaders run. Environment shutdown also releases cached references. Removing a reference permits garbage collection; a class retained by user code still follows the native access-window checks. + +Before loaders run, Rust loader hooks borrow `&mut LoaderContext`. A JavaScript loader hook receives a `JsLoaderHookContext` snapshot and returns only its state; the native runner keeps ownership of the Box throughout. Only the JavaScript loader runner takes and returns the Box, together with the loader result. Hook-installed properties and closures survive later builtin and JavaScript loaders through a shared JavaScript wrapper, whose captured execution state is refreshed on each entry. + +```text +Native loader: String/Buffer + separate source map + ↓ move boxed LoaderContext +JsLoaderContext: state snapshot + lazy content/source-map getters + ↓ +JavaScript loader-context wrapper: getters/setters access state + ↓ pass the whole state object back +Native runner: recover boxed LoaderContext and continue +``` + +JavaScript returns the same class instance on success or with a captured error. The return-value conversion takes its boxed context on the JavaScript thread before moving it back to the native runner. The instance is left with `None`, revoking native access until the next loader-runner entry reattaches the Box, without a shared ownership slot or per-access locking. This is an internal bridge; loaders continue to use the webpack-compatible `this` context, string/Buffer content, source-map objects, and callbacks. + ## Efficient usage ### Read only the assets you need diff --git a/website/docs/zh/api/javascript-api/architecture.mdx b/website/docs/zh/api/javascript-api/architecture.mdx index 4d3151df5ee4..0631d767c139 100644 --- a/website/docs/zh/api/javascript-api/architecture.mdx +++ b/website/docs/zh/api/javascript-api/architecture.mdx @@ -110,6 +110,32 @@ Rspack 可以同时处理多个文件，但 JavaScript loader 需要在 JavaScri Builtin loader 可以在 Rust 侧执行，减少 JavaScript 排队和跨语言数据转换，并更好地利用 Rspack 的并行能力。 +### Loader 源码的所有权 + +原生 loader runner 用 `Box` 持有 `LoaderContext`。loader 链内分别保存 `Content::String` 或 `Content::Buffer` 内容以及 source map，保留每个 loader 接收到的内容类型。`NormalModule` 在 loader 链执行结束后构造最终 Source。 + +执行切换到 JavaScript 时，原生 `JsLoaderContext` class 在本次调用期间直接持有 `Option>`。只有 JavaScript runner 读取对应 getter 时，才转换内容和 source map。JavaScript 封装在本次调用中缓存各 getter 的结果，包括空值；下次进入 JavaScript 时，同一封装会重置读取缓存。JavaScript 产生的输出优先于缓存的输入。 + +JavaScript 侧的 loader-context 封装读取一个普通的 `JsLoaderContextState` 对象（`#[napi(object)]`），通过 JavaScript getter/setter 访问其中的可变字段，并在返回前将同一个 state 对象整体传回。loader 元数据只在创建封装时读取一次，各 loader 的 data 和执行标记随 state 往返。复用的 loader 对象会在每次内置 loader 执行后读取最新 state。只有 JavaScript 产生输出时，state 才携带可选的 output。pitch 没有产生输出时，会保留原生内容和 source map。 + +每个原生 loader context 持有 `rspack_napi` 提供的带类型标记的 `LifecycleGuard`，由 guard 分配进程内唯一的自增 ID。`ThreadLocalReference` 在所属 JavaScript 线程按 ID 缓存对应的 `JsLoaderContext`，并隔离不同 N-API 环境。hook 使用同一个身份关联实例，无需移动 Box。JavaScript `WeakMap` 将 class 与封装和兼容 webpack 的 `this` 对象关联，因此它们不再随可变 state 对象往返。 + +原生 context 被消耗并转成 loader result 时，guard 析构，向已注册的 JavaScript 环境发送引用清理通知。通知会批量处理，并主动唤醒 JavaScript 线程，即使不再有 loader 调用也能清理。环境退出时也会释放缓存引用。移除引用使对象可以被垃圾回收；用户代码保留的 class 仍受原生访问窗口检查约束。 + +loader 执行前，Rust hook 借用 `&mut LoaderContext`。JavaScript loader hook 接收 `JsLoaderHookContext` 快照，只返回 state；Box 的所有权始终留在原生 runner。只有 JavaScript loader runner 会接收并归还 Box，同时返回 loader 的执行结果。hook 安装的属性和闭包通过共享的 JavaScript 封装跨内置及 JavaScript loader 保留，每次进入 JavaScript 时都会更新封装捕获的执行状态。 + +```text +原生 loader：String/Buffer + 独立 source map + ↓ move 装箱的 LoaderContext +JsLoaderContext：state 快照 + 按需 content/source-map getter + ↓ +JavaScript loader-context 封装：getter/setter 访问 state + ↓ 整体传回 state 对象 +原生 runner：收回装箱的 LoaderContext 并继续执行 +``` + +JavaScript 无论成功还是捕获错误，都会返回同一个 class 实例。返回值转换在 JavaScript 线程取走装箱的上下文，再将其 move 回原生 runner。实例中留下 `None`，原生访问随即失效，直到下次 loader runner 调用重新放入 Box；不需要共享所有权槽位，也不需要为每次访问加锁。这是内部桥接机制；loader 仍然使用兼容 webpack 的 `this` 上下文、字符串或 Buffer 内容、source-map 对象以及回调。 + ## 高效用法 ### 只读取需要的资源