diff --git a/dependencies.py b/dependencies.py index 114cfd145..d41bb9567 100755 --- a/dependencies.py +++ b/dependencies.py @@ -367,8 +367,10 @@ def on_install_confirmed(settings: Settings) -> None: "kokkos", settings.kokkos_version, settings.kokkos_backend, - settings.kokkos_arch.strip().lower(), ) + if settings.kokkos_arch.strip().lower(): + kokkos_modfile_file = os.path.join(kokkos_modfile_file, settings.kokkos_arch.strip().lower()) + os.makedirs(os.path.dirname(kokkos_modfile_file), exist_ok=True) if os.path.exists(kokkos_modfile_file) and not settings.overwrite: raise FileExistsError( diff --git a/input.example.toml b/input.example.toml index 8d6f25117..974e556b3 100644 --- a/input.example.toml +++ b/input.example.toml @@ -7,7 +7,7 @@ # Simulation engine to use # @required # @type: string - # @enum: "SRPIC", "GRPIC" + # @enum: "SRPIC", "GRPIC", "HYBRID" engine = "" # Max runtime in physical (code) units # @required @@ -224,6 +224,82 @@ # @from: `scales.larmor0` # @value: `1 / larmor0` +# Electron-fluid closure for the hybrid Ohm's law +# @note: Only read by the `HYBRID` engine (ignored by `SRPIC`/`GRPIC`) +# @note: In `team_policy=ON` builds the fused ion push+moment deposit runs +# tiled (TeamPolicy + SLM scratch); for it to actually hit the fast +# path, keep `particles.spatial_sorting_interval = 1` for ion species +# (see the note there) +[hybrid] + # Adiabatic index of the massless electron fluid + # @type: float [> 0.0] + # @default: 1.66667 (= 5/3) + # @note: `1.0` corresponds to an isothermal closure + gamma_ad = "" + # Electron temperature `T_e` in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: `0.0` corresponds to cold electrons + theta0 = "" + # Vacuum threshold for the hybrid Ohm's law, in units of `n0` + # @type: float [> 0.0] + # @default: 1e-3 + # @note: Above it `E` follows the usual `[...]/N`; below it `E` ramps continuously + # to zero so plasma-free cells keep `B` frozen instead of amplifying the + # right-hand side by `1/dens_min` (statistically-empty or unfilled ghost + # cells, a fraction `e^-ppc` for finite `ppc`) + dens_min = "" + # Cap on the local whistler Courant the Ohm's-law Hall term may imply + # @type: float + # @default: 0.5 + # @note: The implied whistler speed `v_w ~ |B|/N` can outrun `dx/dt` at shock + # overshoots and magnetic cavities; the Hall term is scaled so its + # whistler Courant never exceeds this. Identity in resolvable cells + # @note: `<= 0.0` disables the limiter + hall_lim = "" + # Vacuum resistivity, in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: `> 0.0` makes the sub-`dens_min` vacuum resistive: `E` crossfades to + # `eta * curl B` below the threshold, so vacuum regions diffuse toward a + # curl-free field instead of freezing. Needed when fast plasma sweeps + # along a vacuum boundary, where a hard `E = 0` mask pumps `B` at `~ u/dx` + # @note: Internally clamped to the explicit-diffusion stability limit; any value is safe + # @note: `0.0` disables it (hard `E = 0` vacuum) + resist_vac = "" + # Hyper-resistivity coefficient `eta_H` (adds `-eta_H grad^4 B` to the field advance) + # @type: float [>= 0.0] + # @default: 0.0 + # @note: Damps the few-cell band the moments filter leaves without kinetic + # feedback, while leaving physical scales (`k dx < 0.3`) essentially + # untouched; size so `eta_H (pi/dx)^4` exceeds the drive + # @note: Internally clamped to the explicit `grad^4` stability limit; any value is safe + # @note: `0.0` disables it + resist_hyper = "" + # Pegasus-style sub-cycled magnetic-field advance + # @type: bool + # @default: true + # @note: Integrates the whistler-stiff Ohm/Faraday loop with adaptive SSP-RK3 + # sub-steps at its own CFL; `false` uses the legacy 3-push scheme + subcycle = "" + # Target whistler Courant per field sub-step + # @type: float [> 0.0] + # @default: 0.5 + # @note: Only read when `subcycle = true` + subcycle_courant = "" + # Cap on the number of field sub-steps per advance + # @type: int [>= 1] + # @default: 64 + # @note: Beyond the cap the per-cell Hall limiter (`hall_lim`) takes over + subcycle_max = "" + # Characteristic flow speed for the hybrid CFL, in physical (code) units + # @type: float [>= 0.0] + # @default: 0.0 + # @note: Added to the Alfven + whistler signal speeds in the Pegasus CFL + # `dt <= CFL * min(dx0 / v_max, 2*pi/Omega)` [Kunz, Stone & Bai (2014), §3.5] + # @note: `0.0` lets the Alfven + whistler speeds set `dt`; raise it for super-whistler flows + v_max = "" + [radiation] [radiation.drag] [radiation.drag.synchrotron] @@ -436,6 +512,15 @@ # @type: uint # @default: 0 # @note: Set to 0 to disable spatial sorting + # @note: In `team_policy=ON` builds the tiled deposits (currents, and the + # HYBRID fused push+moment deposit) stay correct for any interval, + # but their per-tile SLM scratch is only *used* by particles that + # are still within `team_policy_drift` (CMake, default 1) cells of + # their sorted tile: with an interval > team_policy_drift + 1 most + # particles take the slower global escape valve and the tiled + # kernels perform like the flat ones. Keep this at 1 for HYBRID ion + # species (each push moves particles), or build with + # `team_policy_drift` = interval. spatial_sorting_interval = "" # @inferred: diff --git a/pgens/hybrid/hybrid.toml b/pgens/hybrid/hybrid.toml new file mode 100644 index 000000000..e670e3a8e --- /dev/null +++ b/pgens/hybrid/hybrid.toml @@ -0,0 +1,50 @@ +[simulation] + name = "hybrid" + engine = "hybrid" + runtime = 1.0 + +[grid] + resolution = [512, 256] + extent = [[0.0, 2.0], [0.0, 1.0]] + + [grid.metric] + metric = "minkowski" + + [grid.boundaries] + fields = [["PERIODIC"], ["PERIODIC"]] + particles = [["PERIODIC"], ["PERIODIC"]] + +[scales] + larmor0 = 1.0 + skindepth0 = 1.0 + +[algorithms] + [algorithms.timestep] + CFL = 0.5 + +[particles] + ppc0 = 10.0 + + [[particles.species]] + label = "ions" + mass = 1.0 + charge = 1.0 + maxnpart = 1e5 + +[setup] + +[output] + interval_time = 0.1 + + [output.fields] + quantities = ["N_1", "B"] + + [output.particles] + species = [1] + stride = 10 + +[checkpoint] + keep = 0 + +[diagnostics] + colored_stdout = true diff --git a/pgens/hybrid/pgen.hpp b/pgens/hybrid/pgen.hpp new file mode 100644 index 000000000..e3359fefa --- /dev/null +++ b/pgens/hybrid/pgen.hpp @@ -0,0 +1,30 @@ +#ifndef PROBLEM_GENERATOR_H +#define PROBLEM_GENERATOR_H + +#include "enums.h" +#include "global.h" + +#include "traits/pgen.h" + +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" + +namespace user { + + template + struct PGen { + static constexpr auto D { M::Dim }; + + static constexpr auto engines = + ::traits::pgen::compatible_with {}; + static constexpr auto metrics = + ::traits::pgen::compatible_with {}; + static constexpr auto dimensions = + ::traits::pgen::compatible_with {}; + + PGen(const SimulationParams&, const Metadomain&) {} + }; + +} // namespace user + +#endif diff --git a/pgens/shock/pgen.hpp b/pgens/shock/pgen.hpp index 7a7aaac21..4553b7d37 100644 --- a/pgens/shock/pgen.hpp +++ b/pgens/shock/pgen.hpp @@ -259,7 +259,7 @@ namespace user { domain.fields.em, init_flds, domain.mesh.metric }); - metadomain.CommunicateFields(domain, Comm::E | Comm::B); + metadomain.CommunicateFields(domain, Comm::EM); /* tag particles inside the injection zone as dead diff --git a/pgens/streaming/pgen.hpp b/pgens/streaming/pgen.hpp index 6983d249d..470cc780f 100644 --- a/pgens/streaming/pgen.hpp +++ b/pgens/streaming/pgen.hpp @@ -54,7 +54,8 @@ namespace user { struct PGen { static constexpr auto D { M::Dim }; // compatibility traits for the problem generator - static constexpr auto engines = ::traits::pgen::compatible_with {}; + static constexpr auto engines = + ::traits::pgen::compatible_with {}; static constexpr auto metrics = ::traits::pgen::compatible_with {}; static constexpr auto dimensions = @@ -87,13 +88,13 @@ namespace user { raise::ErrorIf(nspec % 2 != 0, "Number of species must be even for this setup", HERE); - for (auto n = 0u; n < nspec; n += 2) { - raise::ErrorIf( - global_domain.species_params()[n].charge() != - -global_domain.species_params()[n + 1].charge(), - "Charges of i-th and i+1-th species must be opposite for this setup", - HERE); - } + // for (auto n = 0u; n < nspec; n += 2) { + // raise::ErrorIf( + // global_domain.species_params()[n].charge() != + // -global_domain.species_params()[n + 1].charge(), + // "Charges of i-th and i+1-th species must be opposite for this + // setup", HERE); + // } for (auto* specs : { &drifts_in_x, &drifts_in_y, &drifts_in_z, &temperatures }) { if (specs->empty()) { @@ -126,13 +127,29 @@ namespace user { const auto drift_2 = prmvec_t { drifts_in_x[n + 1], drifts_in_y[n + 1], drifts_in_z[n + 1] }; - arch::InjectUniformMaxwellians( + const auto edist1 = arch::energy_dist::MaxwellianNonRel { + domain.random_pool(), + temperatures[n], + drift_1 + }; + const auto edist2 = arch::energy_dist::MaxwellianNonRel { + domain.random_pool(), + temperatures[n + 1], + drift_2 + }; + arch::InjectUniform( params, domain, - densities[n / 2], - { temperatures[n], temperatures[n + 1] }, - { n + 1, n + 2 }, - { drift_1, drift_2 }); + { 1u, 2u }, + { edist1, edist2 }, + ONE); + // arch::InjectUniformMaxwellians( + // params, + // domain, + // densities[n / 2], + // { temperatures[n], temperatures[n + 1] }, + // { n + 1, n + 2 }, + // { drift_1, drift_2 }); } } }; diff --git a/src/archetypes/energy_dist.h b/src/archetypes/energy_dist.h index 11fe04369..7af8dfd03 100644 --- a/src/archetypes/energy_dist.h +++ b/src/archetypes/energy_dist.h @@ -76,6 +76,38 @@ namespace arch::energy_dist { random_number_pool_t pool; }; + Inline void NonRelMaxwellian(vec_t& v, + real_t temp, + const random_number_pool_t& pool) { + auto rand_gen = pool.get_state(); + // Juttner-Synge distribution using the Box-Muller method - non-relativistic + real_t randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + real_t randX2 = static_cast(constant::TWO_PI) * + Random(rand_gen); + v[0] = randX1 * math::cos(randX2) * math::sqrt(temp); + + randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + randX2 = static_cast(constant::TWO_PI) * Random(rand_gen); + v[1] = randX1 * math::cos(randX2) * math::sqrt(temp); + + randX1 = Random(rand_gen); + while (cmp::AlmostZero(randX1)) { + randX1 = Random(rand_gen); + } + randX1 = math::sqrt(-TWO * math::log(randX1)); + randX2 = static_cast(constant::TWO_PI) * Random(rand_gen); + v[2] = randX1 * math::cos(randX2) * math::sqrt(temp); + pool.free_state(rand_gen); + } + Inline void JuttnerSinge(vec_t& v, real_t temp, const random_number_pool_t& pool) { @@ -291,6 +323,48 @@ namespace arch::energy_dist { short drift_dir { 0 }; }; + template + struct MaxwellianNonRel { + MaxwellianNonRel(random_number_pool_t& pool, + real_t temperature, + const std::vector& drift_vel = { ZERO, ZERO, ZERO }) + : pool { pool } + , temperature { temperature } { + raise::ErrorIf(drift_vel.size() != 3, + "Maxwellian: Drift velocity must be a 3D vector", + HERE); + raise::ErrorIf(temperature < ZERO, + "Maxwellian: Temperature must be non-negative", + HERE); + if (not cmp::AlmostZero_host(NORM(drift_vel[0], drift_vel[1], drift_vel[2]))) { + drift_vel_x1 = drift_vel[0]; + drift_vel_x2 = drift_vel[1]; + drift_vel_x3 = drift_vel[2]; + } + } + + Inline void operator()(const coord_t&, vec_t& v) const { + + if (cmp::AlmostZero(temperature)) { + v[0] = ZERO; + v[1] = ZERO; + v[2] = ZERO; + } else { + NonRelMaxwellian(v, temperature, pool); + } + v[0] += drift_vel_x1; + v[1] += drift_vel_x2; + v[2] += drift_vel_x3; + } + + private: + random_number_pool_t pool; + + const real_t temperature; + + real_t drift_vel_x1 { ZERO }, drift_vel_x2 { ZERO }, drift_vel_x3 { ZERO }; + }; + } // namespace arch::energy_dist #endif // ARCHETYPES_ENERGY_DIST_HPP diff --git a/src/archetypes/field_setter.h b/src/archetypes/field_setter.h index 1e3137d01..0758c3221 100644 --- a/src/archetypes/field_setter.h +++ b/src/archetypes/field_setter.h @@ -27,6 +27,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/numeric.h" @@ -64,7 +65,7 @@ namespace arch { if constexpr (D == Dim::_1D) { const auto i1_ = COORD(i1); coord_t x_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { if constexpr (HasEx1) { metric.template convert({ i1_ + HALF }, x_Phys); EM(i1, em::ex1) = metric.template transform<1, Idx::T, Idx::U>( @@ -114,7 +115,7 @@ namespace arch { const auto i1_ = COORD(i1); const auto i2_ = COORD(i2); // srpic - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { coord_t x_Phys { ZERO }; if constexpr (HasEx1) { metric.template convert({ i1_ + HALF, i2_ }, x_Phys); @@ -153,7 +154,7 @@ namespace arch { { i1_ + HALF, i2_ + HALF }, finit.bx3(x_Phys)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::UserFieldsInContravariantBasis) { // grpic if constexpr (HasDx1 && HasDx2 && HasDx3) { const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; @@ -203,7 +204,7 @@ namespace arch { const auto i2_ = COORD(i2); const auto i3_ = COORD(i3); coord_t x_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { // srpic if constexpr (HasEx1) { metric.template convert({ i1_ + HALF, i2_, i3_ }, @@ -250,7 +251,7 @@ namespace arch { { i1_ + HALF, i2_ + HALF, i3_ }, finit.bx3(x_Phys)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::UserFieldsInContravariantBasis) { // grpic const real_t x1_0 { metric.template convert<1, Crd::Cd, Crd::Ph>(i1_) }; const real_t x1_H { metric.template convert<1, Crd::Cd, Crd::Ph>( @@ -295,6 +296,9 @@ namespace arch { template FS> struct CustomSetEMFields_kernel { + static_assert(::traits::engine::UserFieldsInTetradBasis, + "CustomSetEMFields_kernel is only implemented for SimEngines " + "with user-defined fields in tetrad basis"); static constexpr auto D = M::Dim; static constexpr auto HasEx1 = ::traits::fieldsetter::HasConditionalEx1; static constexpr auto HasEx2 = ::traits::fieldsetter::HasConditionalEx2; diff --git a/src/archetypes/moving_window.h b/src/archetypes/moving_window.h index b8dc74fa2..d1aebd54f 100644 --- a/src/archetypes/moving_window.h +++ b/src/archetypes/moving_window.h @@ -310,7 +310,7 @@ namespace arch { } // synch ghost zones after moving the window - metadomain.CommunicateFields(domain, Comm::E | Comm::B); + metadomain.CommunicateFields(domain, Comm::EM); // communicate particles after moving metadomain.CommunicateParticles(domain); diff --git a/src/archetypes/particle_injector.h b/src/archetypes/particle_injector.h index 415da851e..e9f358270 100644 --- a/src/archetypes/particle_injector.h +++ b/src/archetypes/particle_injector.h @@ -6,6 +6,8 @@ * - arch::ComputeNumInject<> -> tuple, array_t> * - arch::AtmosphereDensityProfile<> * - arch::InjectUniform<> -> void + * - arch::BalancedLattice -> std::array + * - arch::InjectLattice<> -> void * - arch::InjectGlobally<> -> void * - arch::InjectNonUniform<> -> void * @namespaces: @@ -34,6 +36,7 @@ #include #endif +#include #include #include #include @@ -262,6 +265,285 @@ namespace arch { } } + /** + * @brief Injects uniform number density of a single species everywhere in the domain + * @note for engines with a lone species (e.g. HYBRID ions), where the + * charge-neutral pair injector does not apply + * @param params Simulation parameters + * @param domain Domain object + * @param species Species index (1-based) + * @param energy_dist Energy distribution object + * @param number_density Number density (in units of n0) + * @param use_weights Use weights + * @param box Region to inject the particles in global coords + * @tparam S Simulation engine type + * @tparam M Metric type + * @tparam ED Energy distribution type + */ + template ED> + inline void InjectUniform(const SimulationParams& params, + Domain& domain, + spidx_t species, + const ED& energy_dist, + real_t number_density, + bool use_weights = false, + const boundaries_t& box = {}) { + raise::ErrorIf((M::CoordType != Coord::Cartesian) && (not use_weights), + "Weights must be used for non-Cartesian coordinates", + HERE); + raise::ErrorIf((M::CoordType == Coord::Cartesian) && use_weights, + "Weights should not be used for Cartesian coordinates", + HERE); + raise::ErrorIf(params.template get("particles.use_weights") != use_weights, + "Weights must be enabled from the input file to use them in " + "the injector", + HERE); + + { + boundaries_t nonempty_box; + for (auto d { 0u }; d < M::Dim; ++d) { + if (d < box.size()) { + nonempty_box.emplace_back(box[d].first, box[d].second); + } else { + nonempty_box.push_back(Range::All); + } + } + const auto result = ComputeNumInject(params, domain, number_density, nonempty_box); + if (not std::get<0>(result)) { + return; + } + const auto nparticles = std::get<1>(result); + const auto xi_min = std::get<2>(result); + const auto xi_max = std::get<3>(result); + + Kokkos::parallel_for("InjectUniform", + nparticles, + kernel::UniformInjectorSingle_kernel( + domain.species[species - 1], + domain.index(), + domain.mesh.metric, + xi_min, + xi_max, + energy_dist, + ONE / params.template get("scales.V0"), + domain.random_pool())); + domain.species[species - 1].set_npart( + domain.species[species - 1].npart() + nparticles); + domain.species[species - 1].set_counter( + domain.species[species - 1].counter() + nparticles); + } + } + + /** + * @brief Factorizes `ppc` into `dim` per-direction lattice counts, as close + * to each other as the divisors allow (e.g. 512 in 2D -> 16 x 32) + * @param ppc Number of lattice sites per cell + * @param dim Number of directions to distribute them over + * @return Array of 3 counts (unused directions are 1); their product is `ppc` + */ + inline auto BalancedLattice(npart_t ppc, dim_t dim) -> std::array { + std::array nppd { 1, 1, 1 }; + auto rest = ppc; + for (auto d { 0u }; d + 1u < static_cast(dim); ++d) { + // largest n <= rest^(1/k) that divides rest, k = remaining directions + const auto k = static_cast(dim) - d; + npart_t n { 1 }; + while (true) { + npart_t pw { 1 }; + for (auto i { 0u }; i < k; ++i) { + pw *= (n + 1); + } + if (pw > rest) { + break; + } + ++n; + } + while (n > 1 and (rest % n) != 0) { + --n; + } + nppd[d] = n; + rest /= n; + } + nppd[static_cast(dim) - 1u] = rest; + return nppd; + } + + /** + * @brief Injects a uniform number density on a deterministic sub-cell lattice, + * i.e. exactly `ppc` particles per cell at fixed + * positions, instead of at random positions as `InjectUniform` + * @note Drop-in replacement for `InjectUniform` with the same arguments and the + * same density convention (`nparticles = ppc0 * number_density / 2` per + * cell, see `ComputeNumInject`). + * @note The per-cell count is factorized into per-direction counts + * (`BalancedLattice`), so any integer ppc works; it need not be a perfect + * square/cube. The injection region must be cell-aligned, which the + * default (whole-domain) box always is. + * @param params Simulation parameters + * @param domain Domain object + * @param species Species index + * @param energy_dist Energy distribution object + * @param number_density Number density (in units of n0) + * @param use_weights Use weights + * @param box Region to inject the particles in global coords + * @tparam S Simulation engine type + * @tparam M Metric type + * @tparam ED Energy distribution type + */ + template ED> + inline void InjectLattice(const SimulationParams& params, + Domain& domain, + spidx_t species, + const ED& energy_dist, + real_t number_density, + bool use_weights = false, + const boundaries_t& box = {}) { + raise::ErrorIf((M::CoordType != Coord::Cartesian) && (not use_weights), + "Weights must be used for non-Cartesian coordinates", + HERE); + raise::ErrorIf((M::CoordType == Coord::Cartesian) && use_weights, + "Weights should not be used for Cartesian coordinates", + HERE); + raise::ErrorIf(params.template get("particles.use_weights") != use_weights, + "Weights must be enabled from the input file to use them in " + "the injector", + HERE); + + constexpr auto dim { static_cast(M::Dim) }; + // # of lattice sites per cell, same convention as ComputeNumInject + const auto ppc_real = params.template get("particles.ppc0") * + number_density * HALF; + const auto ppc_cell = static_cast(ppc_real + HALF); + raise::ErrorIf(ppc_cell == 0, + "InjectLattice: ppc0 * number_density / 2 rounds to zero", + HERE); + raise::ErrorIf( + math::abs(ppc_real - static_cast(ppc_cell)) > + static_cast(1e-4), + "InjectLattice: ppc0 * number_density / 2 = " + std::to_string(ppc_real) + + " must be an integer (particles per cell)", + HERE); + // sub-cell lattice, as isotropic as the divisors of ppc_cell allow + const auto nppd = BalancedLattice(ppc_cell, dim); + if (nppd[static_cast(dim) - 1u] > + 4u * nppd[0]) { // e.g. a prime ppc -> 1 x ppc + raise::Warning( + "InjectLattice: " + std::to_string(ppc_cell) + + " particles per cell factorize into a lopsided sub-cell " + "lattice; pick a ppc0 with a more balanced factorization", + HERE); + } + + boundaries_t nonempty_box; + for (auto d { 0u }; d < M::Dim; ++d) { + if (d < box.size()) { + nonempty_box.emplace_back(box[d].first, box[d].second); + } else { + nonempty_box.push_back(Range::All); + } + } + const auto region = DeduceRegion(domain, nonempty_box); + if (not std::get<0>(region)) { + return; + } + auto xi_min_h = Kokkos::create_mirror_view(std::get<1>(region)); + auto xi_max_h = Kokkos::create_mirror_view(std::get<2>(region)); + Kokkos::deep_copy(xi_min_h, std::get<1>(region)); + Kokkos::deep_copy(xi_max_h, std::get<2>(region)); + + // the lattice is defined per cell, so the region has to be cell-aligned + constexpr auto tol { static_cast(1e-4) }; + std::array offsets { 0, 0, 0 }; + std::array ncells { 1, 1, 1 }; + npart_t nparticles { ppc_cell }; + for (auto d { 0u }; d < M::Dim; ++d) { + raise::ErrorIf(xi_min_h(d) < -tol, + "InjectLattice: negative injection region", + HERE); + offsets[d] = static_cast(xi_min_h(d) + HALF); + ncells[d] = static_cast(xi_max_h(d) - xi_min_h(d) + HALF); + const auto dlo = xi_min_h(d) - static_cast(offsets[d]); + const auto dhi = (xi_max_h(d) - xi_min_h(d)) - + static_cast(ncells[d]); + raise::ErrorIf(math::abs(dlo) > tol or math::abs(dhi) > tol, + "InjectLattice: the injection region must be aligned with " + "cell boundaries (direction " + + std::to_string(d) + ": [" + std::to_string(xi_min_h(d)) + + ", " + std::to_string(xi_max_h(d)) + "] in cell units)", + HERE); + nparticles *= ncells[d]; + } + if (nparticles == 0) { + return; + } + + Kokkos::parallel_for("InjectLattice", + nparticles, + kernel::LatticeInjector_kernel( + domain.species[species - 1], + domain.index(), + domain.mesh.metric, + offsets[0], + offsets[1], + offsets[2], + ncells[0], + ncells[1], + nppd[0], + nppd[1], + nppd[2], + energy_dist, + ONE / params.template get("scales.V0"))); + domain.species[species - 1].set_npart( + domain.species[species - 1].npart() + nparticles); + domain.species[species - 1].set_counter( + domain.species[species - 1].counter() + nparticles); + } + + /** + * @brief Charge-neutral pair variant of `InjectLattice` + * @note Both species land on the SAME lattice sites (the placement is + * deterministic), so the initial charge density is exactly zero. + * @param params Simulation parameters + * @param domain Domain object + * @param species Pair of species indices + * @param energy_dists Pair of energy distribution objects + * @param number_density Total number density (in units of n0) + * @param use_weights Use weights + * @param box Region to inject the particles in global coords + * @tparam S Simulation engine type + * @tparam M Metric type + * @tparam ED1 Energy distribution type for species 1 + * @tparam ED2 Energy distribution type for species 2 + */ + template ED1, EnrgDistClass ED2> + inline void InjectLattice(const SimulationParams& params, + Domain& domain, + const std::pair& species, + const std::pair& energy_dists, + real_t number_density, + bool use_weights = false, + const boundaries_t& box = {}) { + if (domain.species[species.first - 1].charge() + + domain.species[species.second - 1].charge() != + 0.0f) { + raise::Warning("Total charge of the injected species is non-zero", HERE); + } + InjectLattice(params, + domain, + species.first, + energy_dists.first, + number_density, + use_weights, + box); + InjectLattice(params, + domain, + species.second, + energy_dists.second, + number_density, + use_weights, + box); + } + /** * @brief Injects particles from a globally-defined map * @note very inefficient, should only be used for debug purposes diff --git a/src/archetypes/utils.h b/src/archetypes/utils.h index 1ae775309..7013d52e0 100644 --- a/src/archetypes/utils.h +++ b/src/archetypes/utils.h @@ -170,7 +170,7 @@ namespace arch { template inline void UpdateEMFields(Domain& domain, const F& fieldsetter) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Kokkos::deep_copy(domain.fields.bckp, domain.fields.em); Kokkos::parallel_for( "UpdateEMFields", @@ -181,7 +181,9 @@ namespace arch { fieldsetter)); // comm here } else { - raise::Error("Custom fieldsetter is only implemented for SRPIC", HERE); + raise::Error("Custom fieldsetter is only implemented for simengine which " + "defines user fields in tetrad basis", + HERE); } } diff --git a/src/engines/engine.hpp b/src/engines/engine.hpp index 161339a9d..eb0a41603 100644 --- a/src/engines/engine.hpp +++ b/src/engines/engine.hpp @@ -247,15 +247,32 @@ namespace ntt { void Engine::run() { init(); + // engine-specific timer set: PIC (SRPIC/GRPIC) and HYBRID exercise + // different sub-steps, so each gets only the timers it actually uses (no + // permanently-zero rows, no Moments leaking into PIC). + const auto timer_names = []() -> std::vector { + // shared bookkeeping timers driven from Engine::run (all engines): + // Custom (post-step), LoadBalance, and the ParticleSort/Output/Checkpoint + // "extras" that printAll() prints separately below the total. + if constexpr (S == SimEngine::HYBRID) { + return { + "FieldSolver", "ParticlePusher", // field solve + push (fused deposit) + "MomentFiltering", // binomial smoothing of N,V (+ its comm) + "FieldBoundaries", "Communications", "Custom", "LoadBalance", + "ParticleSort", "Output", "Checkpoint" + }; + } else { + return { + "FieldSolver", "CurrentFiltering", "CurrentDeposit", + "ParticlePusher", "FieldBoundaries", "ParticleBoundaries", + "Communications", "Injector", "Custom", + "LoadBalance", "ParticleSort", "Output", + "Checkpoint" + }; + } + }(); auto timers = timer::Timers { - { "FieldSolver", - "CurrentFiltering", "CurrentDeposit", - "ParticlePusher", "FieldBoundaries", - "ParticleBoundaries", "Communications", - "Injector", "Custom", - "LoadBalance", - "ParticleSort", "Output", - "Checkpoint" }, + timer_names, []() { Kokkos::fence(); }, diff --git a/src/engines/grpic/currents.h b/src/engines/grpic/currents.h index cb4032f54..1d5eede28 100644 --- a/src/engines/grpic/currents.h +++ b/src/engines/grpic/currents.h @@ -91,41 +91,12 @@ namespace ntt { dt, layout, species.npart() }; - const auto scratch = Kokkos::PerTeam( - decltype(deposit_kernel)::scratch_bytes()); - - // Team (work-group) size. The default (team_size_req == 0) leaves - // Kokkos::AUTO, which sizes the team from the backend occupancy - // heuristic. A positive `algorithms.deposit.team_policy_team_size` - // overrides it, clamped to the scratch/backend-feasible maximum so an - // over-large request cannot abort the launch (Kokkos errors when - // team_size > team_size_max). No portable subgroup rounding is applied; - // pick a multiple of the device subgroup width (printed per arch by - // ideal_tile_size.py) for the best occupancy. - Kokkos::TeamPolicy<> policy(static_cast(layout.ntiles_total), - Kokkos::AUTO); - policy.set_scratch_size(0, scratch); - if (team_size_req > 0) { - const int ts_max = policy.team_size_max(deposit_kernel, - Kokkos::ParallelForTag {}); - int ts = team_size_req; - if (ts > ts_max) { - raise::Warning( - fmt::format("algorithms.deposit.team_policy_team_size = %d exceeds " - "the tiled-deposit maximum %d on this backend; clamping " - "to %d", - team_size_req, - ts_max, - ts_max), - HERE); - ts = ts_max; - } - policy = Kokkos::TeamPolicy<>(static_cast(layout.ntiles_total), ts); - policy.set_scratch_size(0, scratch); - logger::Checkpoint( - fmt::format("Tiled deposit: explicit team size %d", ts), - HERE); - } + // Policy boilerplate (scratch sizing, optional explicit team size with + // clamping) is shared by all tiled scatters — see MakeTiledPolicy in + // kernels/tiled_scatter.hpp. + const auto policy = kernel::MakeTiledPolicy(deposit_kernel, + layout.ntiles_total, + team_size_req); Kokkos::parallel_for("CurrentsDepositTiled", policy, deposit_kernel); // Particles appended since the last sort (injection / MPI receive on a @@ -264,7 +235,7 @@ namespace ntt { domain.fields.buff, size, domain.mesh.flds_bc())); - metadomain.CommunicateFields(domain, Comm::J); // J0 + metadomain.CommunicateFields(domain, ::Comm::CUR0); // J0 } } diff --git a/src/engines/grpic/grpic.hpp b/src/engines/grpic/grpic.hpp index 3487d2867..4f2bdd82a 100644 --- a/src/engines/grpic/grpic.hpp +++ b/src/engines/grpic/grpic.hpp @@ -38,6 +38,16 @@ namespace ntt { + namespace grpic::Comm { + static constexpr auto E = ::Comm::AUX_012; + static constexpr auto H = ::Comm::AUX_345; + static constexpr auto D = ::Comm::EM_012; + static constexpr auto D0 = ::Comm::EM0_012; + static constexpr auto B = ::Comm::EM_345; + static constexpr auto B0 = ::Comm::EM0_345; + static constexpr auto J = ::Comm::CUR0; + } // namespace grpic::Comm + template class GRPICEngine : public Engine { using base_t = Engine; @@ -64,6 +74,7 @@ namespace ntt { ~GRPICEngine() override = default; void step_forward(timer::Timers& timers, domain_t& dom) override { + const auto fieldsolver_enabled = m_params.template get( "algorithms.fieldsolver.enable"); const auto deposit_enabled = m_params.template get( @@ -94,7 +105,8 @@ namespace ntt { * em0::D, em::D, em0::B, em::B <- boundary conditions */ m_metadomain.CommunicateFields(dom, - Comm::B | Comm::B0 | Comm::D | Comm::D0); + grpic::Comm::B | grpic::Comm::B0 | + grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -122,7 +134,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::H | grpic::Comm::E); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -144,7 +156,7 @@ namespace ntt { /** * em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -166,7 +178,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -186,7 +198,7 @@ namespace ntt { /** * aux::E, aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H | Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::H | grpic::Comm::E); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -209,7 +221,7 @@ namespace ntt { /** * em0::B, em::B <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -226,7 +238,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -243,7 +255,7 @@ namespace ntt { /** * aux::H <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -261,7 +273,7 @@ namespace ntt { /** * em0::D, em::D <- boundary conditions */ - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); grpic::FieldBoundaries(dom, m_metadomain.mesh(), m_pgen, @@ -336,7 +348,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::E); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -360,7 +372,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); timers.stop("Communications"); /** * em0::B, em::B <- boundary conditions @@ -384,7 +396,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -422,8 +434,8 @@ namespace ntt { timers.stop("CurrentDeposit"); timers.start("Communications"); - m_metadomain.SynchronizeFields(dom, Comm::J); - m_metadomain.CommunicateFields(dom, Comm::J); + m_metadomain.SynchronizeFields(dom, grpic::Comm::J); + m_metadomain.CommunicateFields(dom, grpic::Comm::J); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -464,7 +476,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E); + m_metadomain.CommunicateFields(dom, grpic::Comm::E); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -492,7 +504,7 @@ namespace ntt { * em0::B, em::B <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B | Comm::B0); + m_metadomain.CommunicateFields(dom, grpic::Comm::B | grpic::Comm::B0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, @@ -530,7 +542,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, @@ -551,7 +563,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::H); + m_metadomain.CommunicateFields(dom, grpic::Comm::H); timers.stop("Communications"); timers.start("FieldBoundaries"); /** @@ -601,7 +613,7 @@ namespace ntt { * em0::D, em::D <- boundary conditions */ timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::D | Comm::D0); + m_metadomain.CommunicateFields(dom, grpic::Comm::D | grpic::Comm::D0); timers.stop("Communications"); timers.start("FieldBoundaries"); grpic::FieldBoundaries(dom, diff --git a/src/engines/hybrid/fields_bcs.h b/src/engines/hybrid/fields_bcs.h new file mode 100644 index 000000000..f9a4d6ec4 --- /dev/null +++ b/src/engines/hybrid/fields_bcs.h @@ -0,0 +1,368 @@ +/** + * @file engines/hybrid/fields_bcs.h + * @brief Non-periodic (reflecting-wall / perfect-conductor) boundary + * conditions for the HYBRID engine, x1 walls only. + * @implements + * - ntt::hybrid::PerfectConductorFieldsIn<> -> void + * - ntt::hybrid::FieldBoundaries<> -> void (em: Ee and/or Bf) + * - ntt::hybrid::WallEPrime<> -> void (em0::345 = Ee'/Ee'') + * - ntt::hybrid::WallScratchB<> -> void (cur = Bf*, Bf**) + * - ntt::hybrid::WallBckpFill<> -> void (bckp = Ec'/Bc') + * - ntt::hybrid::MomentsWallBC<> -> void (aux = V, N) + * @namespaces: + * - ntt::hybrid:: + * + * PERIODIC boundaries are realized by the metadomain halo exchange + * (Metadomain::CommunicateFields) and are skipped here. + * + * The conductor wall (kernels/hybrid/wall_bcs.hpp) enforces E_tan = 0 on the + * wall plane; Faraday then freezes the wall-plane B_n automatically, which is + * the correct condition for an OBLIQUE background field. The wall-plane B_n is + * deliberately never overwritten (zeroing it is only valid for perpendicular + * fields and plants a div(B) layer at the wall). B ghosts get an even + * (zero-gradient) mirror so the Hall term sees no fake wall current. + * + * EVERY buffer whose wall ghosts (or wall-plane values) enter a stencil needs + * explicit treatment -- there is no halo exchange at a physical wall, so + * untreated ghosts keep their allocation value (zero) forever: + * - em (Ee/Bf) -> FieldBoundaries (BC::E and/or BC::B) + * - em0::345 (Ee'/Ee'') -> WallEPrime, before Faraday #2/#3 + * - cur (Bf*, Bf**) -> WallScratchB, before EMF #1/#2 (Hall stencil) + * - bckp (Ec'/Bc') -> WallBckpFill, before the particle gathers + * - aux (V/N) -> MomentsWallBC after each deposit: folds the + * ghost deposit tails back as the image-plasma contribution (V_x odd) and + * mirror-fills the ghosts for the EMF / filter stencils + * + * @note Hybrid is Cartesian Minkowski, so only the conductor wall is + * meaningful (no axis / GR-horizon boundaries). + * @note Only x1 walls are implemented (the hybrid shock geometry); a + * CONDUCTOR boundary in x2/x3 raises. + */ + +#ifndef ENGINES_HYBRID_FIELDS_BCS_H +#define ENGINES_HYBRID_FIELDS_BCS_H + +#include "enums.h" +#include "global.h" + +#include "arch/directions.h" +#include "arch/kokkos_aliases.h" +#include "utils/error.h" + +#include "metrics/minkowski.h" + +#include "framework/domain/domain.h" +#include "framework/domain/grid.h" +#include "kernels/hybrid/wall_bcs.hpp" + +#include + +#include + +namespace ntt { + namespace hybrid { + + namespace wall { + + /** + * @brief Range over the wall layer for x1-staggered (edge-E / face-B) + * kernels: i1 = 0 (wall plane) .. N_GHOSTS (ghost layers; the + * +x1 side has one node less), full transverse extent. + */ + template + auto StaggeredRange(Domain>& domain, + short sign) -> range_t { + std::vector xi_min, xi_max; + xi_min.push_back(0); + xi_max.push_back((sign < 0) ? (N_GHOSTS + 1) : N_GHOSTS); + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + for (auto d { 1u }; d < static_cast(D); ++d) { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(all_dirs[d])); + } + if constexpr (D == Dim::_1D) { + return CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (D == Dim::_2D) { + return CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else { + return CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } + } + + /** + * @brief Range over the wall layer for cell-centered kernels: + * i1 = ghost layer 0 .. N_GHOSTS; transverse extent is + * active-only when `active_transverse` (the moment fold; corner + * tails are already remapped by the transverse sync) and full + * otherwise. + */ + template + auto CellRange(Domain>& domain, + bool active_transverse) -> range_t { + std::vector xi_min, xi_max; + xi_min.push_back(0); + xi_max.push_back(N_GHOSTS); + const std::vector all_dirs { in::x1, in::x2, in::x3 }; + for (auto d { 1u }; d < static_cast(D); ++d) { + const auto dd = all_dirs[d]; + if (active_transverse) { + xi_min.push_back(domain.mesh.i_min(dd)); + xi_max.push_back(domain.mesh.i_max(dd)); + } else { + xi_min.push_back(0); + xi_max.push_back(domain.mesh.n_all(dd)); + } + } + if constexpr (D == Dim::_1D) { + return CreateRangePolicy({ xi_min[0] }, { xi_max[0] }); + } else if constexpr (D == Dim::_2D) { + return CreateRangePolicy({ xi_min[0], xi_min[1] }, + { xi_max[0], xi_max[1] }); + } else { + return CreateRangePolicy({ xi_min[0], xi_min[1], xi_min[2] }, + { xi_max[0], xi_max[1], xi_max[2] }); + } + } + + /** + * @brief Invoke `f(sign, i_edge)` for every x1 conductor wall this rank + * owns. Raises on non-x1 conductor walls (unimplemented). + */ + template + void ForEachX1Wall(Domain>& domain, + const Grid& global_grid, + F f) { + for (const auto& direction : dir::Directions::orth) { + const auto gbc = global_grid.flds_bc_in(direction); + if (gbc == FldsBC::PERIODIC) { + continue; // realized by the halo exchange + } else if (gbc == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) != FldsBC::CONDUCTOR) { + continue; // not an edge-owning rank + } + const auto dim = direction.get_dim(); + const auto sign = direction.get_sign(); + raise::ErrorIf(dim != in::x1, + "hybrid engine: CONDUCTOR walls are only " + "implemented in x1", + HERE); + const ncells_t i_edge = (sign > 0) ? domain.mesh.i_max(in::x1) + : domain.mesh.i_min(in::x1); + f(sign, i_edge); + } else { + raise::Error("hybrid engine: only PERIODIC and CONDUCTOR field " + "boundaries are implemented", + HERE); + } + } + } + + } // namespace wall + + /** + * @brief Apply the perfect-conductor wall condition on `em` along one + * orthogonal direction: BC::E -> Ee (comps 0..2): E_tan = 0 on the + * wall plane + mirror ghosts; BC::B -> Bf (comps 3..5): even mirror + * ghosts (the wall-plane B_n is left to Faraday, which freezes it + * because of E_tan = 0). + */ + template + void PerfectConductorFieldsIn( + const dir::direction_t& direction, + Domain>& domain, + BCTags tags) { + const auto sign = direction.get_sign(); + raise::ErrorIf(direction.get_dim() != in::x1, + "hybrid engine: CONDUCTOR walls are only implemented in x1", + HERE); + const ncells_t i_edge = (sign > 0) ? domain.mesh.i_max(in::x1) + : domain.mesh.i_min(in::x1); + const auto range = wall::StaggeredRange(domain, sign); + if (tags & BC::E) { + if (sign > 0) { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em, i_edge, 0)); + } else { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em, i_edge, 0)); + } + } + if (tags & BC::B) { + if (sign > 0) { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.em, + i_edge, + 3)); + } else { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.em, + i_edge, + 3)); + } + } + } + + /** + * @brief Apply non-periodic field boundaries to the evolved `em` fields. + * @param global_grid the global grid (for the configured boundary type) + * @param tags which fields to act on (BC::E, BC::B, or both) + */ + template + void FieldBoundaries(Domain>& domain, + const Grid& global_grid, + BCTags tags) { + for (const auto& direction : dir::Directions::orth) { + const auto gbc = global_grid.flds_bc_in(direction); + if (gbc == FldsBC::PERIODIC) { + continue; // realized by the halo exchange + } else if (gbc == FldsBC::CONDUCTOR) { + if (domain.mesh.flds_bc_in(direction) == FldsBC::CONDUCTOR) { + PerfectConductorFieldsIn(direction, domain, tags); + } + } else { + raise::Error("hybrid engine: only PERIODIC and CONDUCTOR field " + "boundaries are implemented", + HERE); + } + } + } + + /** + * @brief Conductor wall condition on the half-step edge-E (em0::345 = + * Ee'/Ee''). Call after EMF #1/#2, before the Faraday push that + * consumes it. + */ + template + void WallEPrime(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::StaggeredRange(domain, sign); + if (sign > 0) { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em0, i_edge, 3)); + } else { + Kokkos::parallel_for( + "WallEdgeE", + range, + kernel::hybrid::WallEdgeE_kernel(domain.fields.em0, i_edge, 3)); + } + }); + } + + /** + * @brief Even-mirror the wall ghosts of the predictor face-B scratch + * (cur = Bf*, Bf**). Call after each Faraday #1/#2 (+ halo + * exchange), before the EMF that reads it: the Hall stencil reads + * the wall ghosts, which no halo exchange fills (left at zero they + * fake a wall current sheet J ~ B/dx every substage). + */ + template + void WallScratchB(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::StaggeredRange(domain, sign); + if (sign > 0) { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.cur, + i_edge, + 0)); + } else { + Kokkos::parallel_for( + "WallFaceB", + range, + kernel::hybrid::WallFaceB_kernel(domain.fields.cur, + i_edge, + 0)); + } + }); + } + + /** + * @brief Conductor-sign mirror fill of the wall ghosts of the + * cell-centered gather fields (bckp: Ec' 0..2, Bc' 3..5). Call + * after each EMF #1/#2 (+ halo exchange), before the particle push + * that gathers from bckp. + */ + template + void WallBckpFill(Domain>& domain, + const Grid& global_grid) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + const auto range = wall::CellRange(domain, false); + if (sign > 0) { + Kokkos::parallel_for( + "WallBckp", + range, + kernel::hybrid::WallBckp_kernel(domain.fields.bckp, i_edge)); + } else { + Kokkos::parallel_for( + "WallBckp", + range, + kernel::hybrid::WallBckp_kernel(domain.fields.bckp, i_edge)); + } + }); + } + + /** + * @brief Image-plasma wall treatment of the deposited moments (aux). + * @param fold also fold the ghost deposit tails back into the active cells + * (V_x sign-flipped) -- exactly once per deposit, after + * SynchronizeFields(AUX) + CommunicateFields(AUX). With + * fold = false only the (idempotent) ghost mirror fill runs, e.g. + * to re-fill after a filter pass. + */ + template + void MomentsWallBC(Domain>& domain, + const Grid& global_grid, + bool fold) { + wall::ForEachX1Wall(domain, global_grid, [&](short sign, ncells_t i_edge) { + if (fold) { + const auto fold_range = wall::CellRange(domain, true); + if (sign > 0) { + Kokkos::parallel_for("WallMomentsFold", + fold_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } else { + Kokkos::parallel_for("WallMomentsFold", + fold_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } + } + const auto fill_range = wall::CellRange(domain, false); + if (sign > 0) { + Kokkos::parallel_for("WallMomentsFill", + fill_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } else { + Kokkos::parallel_for("WallMomentsFill", + fill_range, + kernel::hybrid::WallMoments_kernel( + domain.fields.aux, + i_edge)); + } + }); + } + + } // namespace hybrid +} // namespace ntt + +#endif // ENGINES_HYBRID_FIELDS_BCS_H diff --git a/src/engines/hybrid/fieldsolvers.h b/src/engines/hybrid/fieldsolvers.h new file mode 100644 index 000000000..88078d4c6 --- /dev/null +++ b/src/engines/hybrid/fieldsolvers.h @@ -0,0 +1,354 @@ +#ifndef ENGINES_HYBRID_FIELDSOLVERS_H +#define ENGINES_HYBRID_FIELDSOLVERS_H + +#include "enums.h" +#include "global.h" + +#include "utils/error.h" +#include "utils/param_container.h" + +#include "metrics/minkowski.h" + +#include "engines/hybrid/fields_bcs.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/EMF.hpp" +#include "kernels/hybrid/faraday.hpp" +#include "kernels/hybrid/subcycle.hpp" + +#include + +#if defined(MPI_ENABLED) + #include "arch/mpi_aliases.h" + + #include +#endif + +namespace ntt { + namespace hybrid { + + enum class faraday : uint8_t { + push1, + push2, + push3, + }; + + enum class emf : uint8_t { + push0, + push12, + }; + + template + void Faraday(Domain>& domain, + const prm::Parameters& engine_params, + const faraday& flag) { + const auto dt = engine_params.get("dt"); + const auto dx = domain.mesh.metric.get_dx(); + if (flag == faraday::push1) { + Kokkos::parallel_for( + "FaradayPush1", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em, + domain.fields.em, + domain.fields.cur, + 0, + 3, + 0, + dt, + dx)); + } else if (flag == faraday::push2) { + Kokkos::parallel_for( + "FaradayPush2", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.cur, + 3, + 3, + 0, + dt, + dx)); + } else if (flag == faraday::push3) { + Kokkos::parallel_for( + "FaradayPush3", + domain.mesh.rangeActiveCells(), + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.em, + domain.fields.em, + 3, + 3, + 3, + dt, + dx)); + } else { + raise::Error("Wrong option for `flag`", HERE); + } + } + + template + void EMF(Domain>& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + const emf& flag) { + const auto dt = engine_params.get("dt"); + const auto gamma_ad = params.get("hybrid.gamma_ad"); + const auto theta = params.get("hybrid.theta0"); + const auto d0 = params.get("scales.skindepth0"); + const auto rho0 = params.get("scales.larmor0"); + const auto dens_min = params.get("hybrid.dens_min"); + const auto hall_lim = params.get("hybrid.hall_lim"); + const auto res_vac = params.get("hybrid.resist_vac"); + const auto res_hyp = params.get("hybrid.resist_hyper"); + const auto res_uni = params.get("hybrid.resist"); + const auto dx = domain.mesh.metric.get_dx(); + if (flag == emf::push0) { + // clang-format off + Kokkos::parallel_for( + "EMFPush", + domain.mesh.rangeActiveCells(), + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em0, // Ee_in + domain.fields.cur, // Bf + domain.fields.bckp, // Ec + domain.fields.em, // Bfs + domain.fields.em, // Ee_out + domain.fields.em0, // Ec_out + domain.fields.bckp, // Bc_out + 0, 3, 3, // P, N, Ee_in + 0, 0, 3, // Bf, Ec, Bfs + 0, 0, 3, // Ee_out, Ec_out, Bc_out + dt, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, res_uni, dx)); + // clang-format on + } else { + // clang-format off + Kokkos::parallel_for( + "EMFPush", + domain.mesh.rangeActiveCells(), + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em, // Ee_in + domain.fields.em, // Bf + domain.fields.em0, // Ec + domain.fields.cur, // Bfs + domain.fields.em0, // Ee_out + domain.fields.bckp, // Ec_out + domain.fields.bckp, // Bc_out + 0, 3, 0, // P, N, Ee_in + 3, 0, 0, // Bf, Ec, Bfs + 3, 0, 3, // Ee_out, Ec_out, Bc_out + dt, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, res_uni, dx)); + // clang-format on + } + } + + /** + * @brief Sub-cycled (Pegasus-style) magnetic-field advance: integrates + * dB/dt = -curl E(B) over one full timestep with the Ohm's-law E + * refreshed from the (frozen) deposited moments every sub-step, so + * the whistler-stiff part of hybrid Ohm's law is integrated at its + * own CFL instead of the particle dt. + * + * Reads Bf^(n) from em::345 and the frozen moments from aux; leaves the + * advanced B (with valid ghosts) in `cur` -- the same contract as the + * single-Euler Faraday push #1/#2 it replaces. The third (accepting) + * Faraday push is redundant here: this advance already integrates the full + * interval, so the caller accepts `cur` as B^(n+1) (AcceptSubcycledB). + * + * Number of sub-steps m: adaptive from the GLOBAL max of the local whistler + * speed v_w = d0^2 pi |B| / (dx max(N, dens_min)), targeting a per-sub-step + * whistler Courant `hybrid.subcycle_courant`, capped at `hybrid.subcycle_max` + * (beyond the cap the per-cell Hall limiter, which here receives the + * SUB-step dt, takes over). The max is MPI-allreduced so every rank runs the + * same m and the halo exchanges inside the loop pair up. + * + * Each sub-step is SSP-RK3 (Shu-Osher): 2-stage schemes are marginally + * unstable for purely oscillatory (whistler) modes (|G| = 1 + (w dt)^4/8), + * while RK3 is damped up to w dt < sqrt(3). + * + * Scratch usage: cur = running B (in place), buff = sub-step base for the + * RK3 combinations, em0::345 = stage E. Combines run over the FULL extent so + * valid ghosts stay valid; E/B halo exchanges + wall conditions run after + * every stage. + */ + template + auto SubcycledFaraday(Metadomain>& metadomain, + Domain>& domain, + const prm::Parameters& engine_params, + const SimulationParams& params) -> int { + const auto dt = engine_params.get("dt"); + const auto dx = domain.mesh.metric.get_dx(); + const auto gamma_ad = params.get("hybrid.gamma_ad"); + const auto theta = params.get("hybrid.theta0"); + const auto d0 = params.get("scales.skindepth0"); + const auto rho0 = params.get("scales.larmor0"); + const auto dens_min = params.get("hybrid.dens_min"); + const auto hall_lim = params.get("hybrid.hall_lim"); + const auto res_vac = params.get("hybrid.resist_vac"); + const auto res_hyp = params.get("hybrid.resist_hyper"); + const auto res_uni = params.get("hybrid.resist"); + const auto courant = params.get("hybrid.subcycle_courant"); + auto m_max = params.get("hybrid.subcycle_max"); + raise::ErrorIf(courant <= ZERO, + "hybrid.subcycle_courant must be positive", + HERE); + if (m_max < 1) { + m_max = 1; + } + + // pick m from the global max whistler speed at the advance start + real_t vw_max = ZERO; + Kokkos::parallel_reduce("VwMax", + domain.mesh.rangeActiveCells(), + kernel::hybrid::VwMax_kernel(domain.fields.em, + domain.fields.aux, + 3, + 3, + d0, + dens_min, + dx), + Kokkos::Max(vw_max)); +#if defined(MPI_ENABLED) + MPI_Allreduce(MPI_IN_PLACE, + &vw_max, + 1, + mpi::get_type(), + MPI_MAX, + MPI_COMM_WORLD); +#endif + // clamp in double before the int cast: at runaway field amplitudes + // vw_max*dt/(courant*dx) exceeds INT_MAX, and casting that to int is UB + // (wraps negative), which the m < 1 -> m = 1 clamp would then turn into + // m = 1, disabling the subcycler exactly when it is needed. Comparing in + // double and taking the else-branch also routes a NaN/Inf vw_max to m_max. + const double m_want = std::ceil(static_cast(vw_max) * dt / + (courant * dx)); + int m; + if (m_want < static_cast(m_max)) { + m = static_cast(m_want); + if (m < 1) { + m = 1; + } + } else { + m = m_max; + } + const real_t dt_sub = dt / static_cast(m); + + const auto range_act = domain.mesh.rangeActiveCells(); + const auto range_all = domain.mesh.rangeAllCells(); + + // stage E from frozen moments and the current sub-stepped B (cur): + // raw edge-E only -> em0::345 (+ halo + wall condition). The Hall limiter + // inside receives the SUB-step dt. + auto emf_stage = [&]() { + // clang-format off + Kokkos::parallel_for( + "EMFSub", + range_act, + kernel::hybrid::EMF_kernel(domain.fields.aux, // P + domain.fields.aux, // N + domain.fields.em0, // Ee_in (unused) + domain.fields.cur, // Bf (unused) + domain.fields.em0, // Ec (unused) + domain.fields.cur, // Bfs = B_s + domain.fields.em0, // Ee_out + domain.fields.em0, // Ec_out (unused) + domain.fields.bckp, // Bc_out (unused) + 0, 3, 0, // P, N, Ee_in + 0, 0, 0, // Bf, Ec, Bfs + 3, 0, 0, // Ee_out, Ec_out, Bc_out + dt_sub, gamma_ad, theta, d0, rho0, dens_min, hall_lim, res_vac, res_hyp, res_uni, dx)); + // clang-format on + metadomain.CommunicateFields(domain, ::Comm::EM0_345); + WallEPrime(domain, metadomain.mesh()); + }; + // cur += dt_sub * curl(em0::345), in place (+ halo + wall condition) + auto faraday_stage = [&]() { + Kokkos::parallel_for( + "FaradaySub", + range_act, + kernel::hybrid::Faraday_kernel(domain.fields.em0, + domain.fields.cur, + domain.fields.cur, + 3, + 0, + 0, + dt_sub, + dx)); + metadomain.CommunicateFields(domain, ::Comm::CUR); + WallScratchB(domain, metadomain.mesh()); + }; + // cur = a*cur + b*buff over the full extent (ghosts of both are valid) + auto combine = [&](real_t a, real_t b) { + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.cur, + domain.fields.buff, + a, + b, + 0, + 0)); + }; + + // start from Bf^(n), ghosts included (valid: comm'd + wall'd at the end + // of the previous step) + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.cur, + domain.fields.em, + ZERO, + ONE, + 0, + 3)); + for (int s = 0; s < m; ++s) { + // sub-step base: buff <- cur + Kokkos::parallel_for( + "FieldCombine", + range_all, + kernel::hybrid::FieldCombine_kernel(domain.fields.buff, + domain.fields.cur, + ZERO, + ONE, + 0, + 0)); + // SSP-RK3 (Shu-Osher): u1 = u + dt L(u) + emf_stage(); + faraday_stage(); + // u2 = 3/4 u + 1/4 (u1 + dt L(u1)) + emf_stage(); + faraday_stage(); + combine(static_cast(0.25), static_cast(0.75)); + // u^(s+1) = 1/3 u + 2/3 (u2 + dt L(u2)) + emf_stage(); + faraday_stage(); + combine(static_cast(2.0 / 3.0), static_cast(1.0 / 3.0)); + } + return m; + } + + /** + * @brief Accept the sub-cycled advance as B^(n+1): em::345 <- cur (full + * extent -- the ghosts of cur are valid). Replaces Faraday push #3, + * which in sub-cycled mode would redundantly re-integrate the same + * interval with a stiff-unstable single Euler step. + */ + template + void AcceptSubcycledB(Domain>& domain) { + Kokkos::parallel_for( + "FieldCombine", + domain.mesh.rangeAllCells(), + kernel::hybrid::FieldCombine_kernel(domain.fields.em, + domain.fields.cur, + ZERO, + ONE, + 3, + 0)); + } + + } // namespace hybrid +} // namespace ntt + +#endif diff --git a/src/engines/hybrid/hybrid.hpp b/src/engines/hybrid/hybrid.hpp new file mode 100644 index 000000000..718c75d74 --- /dev/null +++ b/src/engines/hybrid/hybrid.hpp @@ -0,0 +1,334 @@ +#ifndef ENGINES_HYBRID_HYBRID_HPP +#define ENGINES_HYBRID_HYBRID_HPP + +#include "enums.h" + +#include "traits/metric.h" +#include "utils/timer.h" + +#include "engines/hybrid/fields_bcs.h" +#include "engines/hybrid/fieldsolvers.h" +#include "engines/hybrid/moments_filter.h" +#include "engines/hybrid/particle_pusher.h" + +#include "engines/engine.hpp" + +namespace ntt { + + template + class HYBRIDEngine : public Engine { + using base_t = Engine; + using pgen_t = user::PGen; + using domain_t = Domain; + // contents + using base_t::m_metadomain; + using base_t::m_params; + using base_t::m_pgen; + // methods + using base_t::init; + // variables + using base_t::dt; + using base_t::max_steps; + using base_t::runtime; + using base_t::step; + using base_t::time; + + public: + static constexpr auto S { SimEngine::HYBRID }; + + HYBRIDEngine(const SimulationParams& params) : base_t { params } {} + + ~HYBRIDEngine() override = default; + + void step_forward(timer::Timers& timers, domain_t& dom) override { + /** + * Initially: em::012 -- + * em::345 Bf^(n) + * + * em0::012 -- + * em0::345 -- + * + * aux::012 V^(n) (except step 0) + * aux::3 N^(n) (except step 0) + * + * bckp::012 -- + * bckp::345 -- + * + * cur::012 -- + * + * x_prtl at n + * u_prtl at n + */ + // Pegasus-style sub-cycled field advance (fieldsolvers.h::SubcycledFaraday): + // the whistler-stiff Ohm's-law/Faraday loop is integrated with adaptive + // SSP-RK3 sub-steps at its own CFL instead of single full-dt Euler pushes. + // false -> the legacy 3-push scheme. + const bool subcycle = m_params.template get("hybrid.subcycle"); + + if (step == 0) { + // fill Bf^(n) ghosts (periodic / MPI) so the field-solver stencils are valid + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_345); + timers.stop("Communications"); + // non-periodic (reflecting-wall) boundaries on the initial Bf, if any + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::B); + timers.stop("FieldBoundaries"); + + // compute N^(0) and V^(0) -> aux::012, aux::3 (deposit-only, no push) + hybrid::DepositMoments(dom, m_params); + + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, ::Comm::AUX); // additive remap of deposit tails + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for EMF reads + timers.stop("Communications"); + // reflecting wall: fold the wall-ghost deposit tails back (image plasma) + // + mirror-fill the wall ghosts for the EMF stencils + timers.start("FieldBoundaries"); + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + timers.stop("FieldBoundaries"); + // smooth N^(0),V^(0) (kills grid-scale shot noise before the field solve) + timers.start("MomentFiltering"); + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("MomentFiltering"); + } + + // EMF calculation #0 — the un-averaged field E^(n). + // Using: aux::012 [V^(n)], aux::3 [N^(n)], em::345 [Bf^(n)] + // + // Ee^(n) = EMF(N^(n), V^(n), Bf^(n)) -> em::012 + // Ec^(n) = EMF(N^(n), V^(n), Bc^(n)=interp Bf) -> em0::012 + // (these seed the trapezoidal average E' = 1/2(E* + E^(n)) in EMF #1/#2) + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push0); + timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_012 | ::Comm::EM0_012); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee^(n) before Faraday #1 + // (this is what freezes the wall-plane B_n for an oblique background field) + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::E); + timers.stop("FieldBoundaries"); + + // Faraday push #1 (predictor field advance) + // Using: em::012 [Ee^(n)], em::345 [Bf^(n)] + // + // legacy: Bf* = Bf^(n) + dt * curl Ee^(n) + // subcycled: Bf* = integrate dB/dt = -curl EMF(N^(n), V^(n), B) over dt + // (E refreshed every sub-step; comms + wall conditions inside, + // cur comes back with valid ghosts) + // + // Now: cur::012 <-- Bf* + timers.start("FieldSolver"); + if (subcycle) { + hybrid::SubcycledFaraday(m_metadomain, dom, this->engineParams(), m_params); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push1); + } + timers.stop("FieldSolver"); + if (not subcycle) { + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::CUR); // fill Bf* ghosts for EMF #1 + timers.stop("Communications"); + // reflecting wall: even-mirror the Bf* scratch ghosts (a physical wall + // has no halo exchange to fill them) so the Hall stencil sees no fake + // wall current + timers.start("FieldBoundaries"); + hybrid::WallScratchB(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + } + + // EMF calculation #1 + // Using: + // aux::012 [P^(n)] + // aux::3 [N^(n)] + // em::012 [Ee^(n)] + // em::345 [Bf^(n)] + // em0::012 [Ec^(n)] + // cur::012 [Bf*] + // + // Bc* = interpolate Bf* + // Bc^(n) = interpolate Bf^(n) + // Ee* = EMF(N^(n), P^(n), Bf*) + // Ec* = EMF(N^(n), P^(n), Bc*) + // + // Ee' = 0.5 * (Ee* + Ee^(n)) + // Ec' = 0.5 * (Ec* + Ec^(n)) + // Bc' = 0.5 * (Bc* + Bc^(n)) + // + // Now: + // em0::345 <-- Ee' + // bckp::012 <-- Ec' + // bckp::345 <-- Bc' + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push12); + timers.stop("FieldSolver"); + + // Particle push #1 (predictor) — Pegasus Fig. 2 steps 7+8. + // Using: bckp::012 [Ec'], bckp::345 [Bc']; transient (no store, no particle BCs). + // Pushes in registers and deposits predicted moments: aux::012 <-- V', aux::3 <-- N' + timers.start("Communications"); + // Ee' (em0::345) ghosts for Faraday push #2; Ec'/Bc' (bckp) ghosts for the gather + m_metadomain.CommunicateFields(dom, ::Comm::EM0_345 | ::Comm::Bckp); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee' (consumed by + // Faraday #2) + conductor-mirror the bckp wall ghosts for the gather + timers.start("FieldBoundaries"); + hybrid::WallEPrime(dom, m_metadomain.mesh()); + hybrid::WallBckpFill(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + timers.start("ParticlePusher"); + hybrid::ParticlePush(dom, this->engineParams(), m_params, /* corrector */ false); + timers.stop("ParticlePusher"); + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, + ::Comm::AUX); // additive remap of deposit tails + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for EMF #2 + timers.stop("Communications"); + // reflecting wall: image-plasma fold + ghost fill of the predicted moments + timers.start("FieldBoundaries"); + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + timers.stop("FieldBoundaries"); + // smooth predicted N',V' before EMF #2 + timers.start("MomentFiltering"); + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("MomentFiltering"); + + // Faraday push #2 (corrector field advance) + // Using: em::345 [Bf^(n)] + em0::345 [Ee'] (legacy) / aux [N', V'] (subcycled) + // + // legacy: Bf** = Bf^(n) + dt * curl Ee' + // subcycled: Bf** = integrate dB/dt = -curl EMF(N', V', B) over dt + // (E refreshed from the PREDICTED moments every sub-step; + // this advance IS the full-interval B^(n) -> B^(n+1) + // integration, accepted as Bf^(n+1) below) + // + // Now: cur::012 <-- Bf** + timers.start("FieldSolver"); + if (subcycle) { + hybrid::SubcycledFaraday(m_metadomain, dom, this->engineParams(), m_params); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push2); + } + timers.stop("FieldSolver"); + if (not subcycle) { + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::CUR); // fill Bf** ghosts for EMF #2 + timers.stop("Communications"); + // reflecting wall: even-mirror the Bf** scratch ghosts (see Faraday #1) + timers.start("FieldBoundaries"); + hybrid::WallScratchB(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + } + + // EMF calculation #2 + // Using: + // aux::012 [P'] + // aux::3 [N'] + // em::012 [Ee^(n)] + // em::345 [Bf^(n)] + // em0::012 [Ec^(n)] + // cur::012 [Bf**] + // + // Bc** = interpolate Bf** + // Bc^(n) = interpolate Bf^(n) + // Ee** = EMF(N', P', Bf**) + // Ec** = EMF(N', P', Bc**) + // + // Ee'' = 0.5 * (Ee** + Ee^(n)) + // Ec'' = 0.5 * (Ec** + Ec^(n)) + // Bc'' = 0.5 * (Bc** + Bc^(n)) + // + // Now: + // em0::345 <-- Ee'' + // bckp:012 <-- Ec'' + // bckp:345 <-- Bc'' + timers.start("FieldSolver"); + hybrid::EMF(dom, this->engineParams(), m_params, hybrid::emf::push12); + timers.stop("FieldSolver"); + timers.start("Communications"); + // Ee'' (em0::345) ghosts for Faraday push #3; Ec''/Bc'' (bckp) ghosts for the gather + m_metadomain.CommunicateFields(dom, ::Comm::EM0_345 | ::Comm::Bckp); + timers.stop("Communications"); + // reflecting wall: E_tan = 0 on the wall plane of Ee'' (consumed by + // Faraday #3) + conductor-mirror the bckp wall ghosts for the corrector + timers.start("FieldBoundaries"); + hybrid::WallEPrime(dom, m_metadomain.mesh()); + hybrid::WallBckpFill(dom, m_metadomain.mesh()); + timers.stop("FieldBoundaries"); + + // Faraday push #3 (accept B^(n+1)) + // Using: em0::345 [Ee''], em::345 [Bf^(n)] (legacy) / cur [Bf**] (subcycled) + // + // legacy: Bf^(n+1) = Bf^(n) + dt * curl Ee'' + // subcycled: Bf^(n+1) = Bf** -- the corrector advance above already + // integrated the full interval; a third full-dt Euler push + // would redundantly re-integrate it (stiff-unstable) + // + // Now: + // em::345 <-- Bf^(n+1) + timers.start("FieldSolver"); + if (subcycle) { + hybrid::AcceptSubcycledB(dom); + } else { + hybrid::Faraday(dom, this->engineParams(), hybrid::faraday::push3); + } + timers.stop("FieldSolver"); + timers.start("Communications"); + m_metadomain.CommunicateFields(dom, ::Comm::EM_345); // Bf^(n+1) ghosts for next step + timers.stop("Communications"); + // non-periodic (reflecting-wall) boundaries on Bf^(n+1), if any + timers.start("FieldBoundaries"); + hybrid::FieldBoundaries(dom, m_metadomain.mesh(), BC::B); + timers.stop("FieldBoundaries"); + + // Particle push #2 (corrector) — Pegasus Fig. 2 step 12. + // Using: bckp::012 [Ec''], bckp::345 [Bc'']; accepted (store-back + + // particle BCs). Pushes and deposits final moments: x_prtl at n+1, + // aux::012 <-- V^(n+1), aux::3 <-- N^(n+1) (bckp ghosts already filled + // after EMF #2; Faraday push #3 does not touch bckp) + timers.start("ParticlePusher"); + hybrid::ParticlePush(dom, this->engineParams(), m_params, /* corrector */ true); + timers.stop("ParticlePusher"); + + timers.start("Communications"); + m_metadomain.SynchronizeFields(dom, ::Comm::AUX); + m_metadomain.CommunicateFields(dom, ::Comm::AUX); // fill ghosts for next step's EMF + timers.stop("Communications"); + // reflecting wall: image-plasma fold + ghost fill of the final moments + timers.start("FieldBoundaries"); + hybrid::MomentsWallBC(dom, m_metadomain.mesh(), /* fold */ true); + timers.stop("FieldBoundaries"); + // smooth final N^(n+1),V^(n+1) for next step's EMF #0/#1 + timers.start("MomentFiltering"); + hybrid::MomentsFilter(m_metadomain, dom, m_params); + timers.stop("MomentFiltering"); + + timers.start("Communications"); + m_metadomain.CommunicateParticles(dom); + timers.stop("Communications"); + + /** + * Finally: em::012 -- + * em::345 Bf^(n+1) + * + * em0::012 -- + * em0::345 -- + * + * aux::012 V^(n+1) + * aux::3 N^(n+1) + * + * bckp::012 -- + * bckp::345 -- + * + * cur::012 -- + * + * x_prtl at n+1 + * u_prtl at n+1 + */ + } + }; +} // namespace ntt + +#endif // ENGINES_HYBRID_HYBRID_HPP diff --git a/src/engines/hybrid/moments_filter.h b/src/engines/hybrid/moments_filter.h new file mode 100644 index 000000000..2e2646878 --- /dev/null +++ b/src/engines/hybrid/moments_filter.h @@ -0,0 +1,62 @@ +#ifndef ENGINES_HYBRID_MOMENTS_FILTER_H +#define ENGINES_HYBRID_MOMENTS_FILTER_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/log.h" +#include "utils/param_container.h" + +#include "metrics/minkowski.h" + +#include "engines/hybrid/fields_bcs.h" +#include "framework/domain/domain.h" +#include "framework/domain/metadomain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/moments_filter.hpp" + +namespace ntt { + namespace hybrid { + + /** + * Binomially smooth the deposited moments (aux::012 = V, aux::3 = N) in + * place, `algorithms.current_filters` times. Without this, cold/fast beams + * inject grid-scale shot noise straight into the Ohm's-law E and drive a + * numerical instability (see kernels/hybrid/moments_filter.hpp). + * + * Must be called AFTER the deposit's additive ghost remap + * (SynchronizeFields(AUX)) and active->ghost copy (CommunicateFields(AUX)), + * since the stencil reads the i +/- 1 ghosts; each pass re-fills them. + * + * Uses `bckp` as the read buffer: it is dead scratch at every deposit point + * in the step (always overwritten by the following EMF before it is read). + */ + template + void MomentsFilter(Metadomain>& metadomain, + Domain>& domain, + const SimulationParams& params) { + const auto nfilter = params.template get( + "algorithms.current_filters"); + if (nfilter == 0) { + return; + } + logger::Checkpoint("Launching hybrid moments filtering kernels", HERE); + for (auto i { 0u }; i < nfilter; ++i) { + Kokkos::deep_copy(domain.fields.bckp, domain.fields.aux); + Kokkos::parallel_for( + "MomentsFilter", + domain.mesh.rangeActiveCells(), + kernel::hybrid::MomentsFilter_kernel(domain.fields.aux, + domain.fields.bckp)); + metadomain.CommunicateFields(domain, ::Comm::AUX); + // re-mirror the reflecting-wall ghosts (fill only; the deposit tails + // were already folded once, before the filter) + MomentsWallBC(domain, metadomain.mesh(), /* fold */ false); + } + } + + } // namespace hybrid +} // namespace ntt + +#endif // ENGINES_HYBRID_MOMENTS_FILTER_H diff --git a/src/engines/hybrid/particle_pusher.h b/src/engines/hybrid/particle_pusher.h new file mode 100644 index 000000000..f82619486 --- /dev/null +++ b/src/engines/hybrid/particle_pusher.h @@ -0,0 +1,272 @@ +/** + * @file engines/hybrid/particle_pusher.h + * @brief Particle push + fused ion-moment deposit drivers for the HYBRID engine + * @implements + * - ntt::hybrid::ParticlePush<> -> void + * - ntt::hybrid::DepositMoments<> -> void + * @namespaces: + * - ntt::hybrid:: + * + * The Pegasus step (Kunz, Stone & Bai 2014, Fig. 2) performs two ion pushes that + * BOTH start from the stored state x^(n), v^(n). The pusher kernel deposits the + * ion moments (N, V) in the SAME pass as the push, so the transient predictor can + * produce its predicted moments without ever writing the particle arrays — hence + * no save/restore of x^(n), v^(n) is needed: + * + * DepositMoments(dom, params); // step 0 only: seed aux with N^(0), V^(0) + * ...EMF #1... + * ParticlePush(dom, ep, params, false); // predictor: push (registers) + deposit N', V' + * ...EMF #2, Faraday... + * ParticlePush(dom, ep, params, true); // corrector: push + deposit + store x^(n+1) + * + * Each call rebuilds the `aux` ScatterView, zeroes aux, runs the kernel (which + * scatter-deposits), and contributes. The caller then remaps/fills aux ghosts: + * SynchronizeFields(dom, ::Comm::AUX) // additive ghost->active (Pegasus §3.6) + * CommunicateFields(dom, ::Comm::AUX) // copy active->ghost for the EMF reads + * and, before each push, fills the bckp (Ec/Bc) ghosts the gather reads: + * CommunicateFields(dom, ::Comm::Bckp) + * + * TEAM_POLICY: the fused deposit is launched through the generic + * `kernel::TiledScatter_kernel` harness instead of the flat ScatterView — + * one team per spatial tile, per-team SLM scratch of (T + 2*HALO)^D x 4, + * HALO = window + TEAM_POLICY_DRIFT — mirroring the tiled current deposit + * (engines/srpic/currents.h). Coverage is identical to the tiled currents + * launcher: flat fallback when the species has no tile layout yet (step 0 / + * tiny species), flat tail pass over [npart_partitioned, npart) for + * particles appended since the last sort, per-particle escape valve inside + * the harness for particles that drifted off their tile. + * + * @see kernels/hybrid/pusher.hpp, kernels/tiled_scatter.hpp and + * PIC/hybrid/pusher.md. + */ + +#ifndef ENGINES_HYBRID_PARTICLE_PUSHER_H +#define ENGINES_HYBRID_PARTICLE_PUSHER_H + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "traits/metric.h" +#include "utils/comparators.h" +#include "utils/error.h" +#include "utils/log.h" +#include "utils/param_container.h" + +#include "framework/domain/domain.h" +#include "framework/parameters/parameters.h" +#include "kernels/hybrid/pusher.hpp" +#include "kernels/pushers/context.h" // kernel::sr::PusherBoundaries +#include "kernels/tiled_scatter.hpp" + +#include +#include + +namespace ntt::hybrid { + + /** + * @brief Run the fused push+deposit kernel over all ion species into `aux`. + * Zeroes aux, deposits N -> aux::3 and V = Σ m v -> aux::0..2. + * Caller handles the subsequent ghost sync/comm. + * @tparam Mode MomentsOnly (no push), Predictor (no store), Corrector (store). + * @param dt time-step (unused for MomentsOnly). + * @param team_size_req explicit tiled team size (0 = Kokkos::AUTO); only + * read on the TEAM_POLICY path. + */ + template + void runPusher(Domain& domain, + const SimulationParams& params, + real_t dt, + int team_size_req = 0) { + const auto omegaB0 = params.template get("scales.omegaB0"); + const auto inv_n0 = ONE / params.template get("scales.n0"); + const auto use_weights = params.template get("particles.use_weights"); + + const auto pusher_boundaries = kernel::sr::PusherBoundaries { + domain.mesh.prtl_bc() + }; + + Kokkos::deep_copy(domain.fields.aux, ZERO); + +#if !defined(TEAM_POLICY) + (void)team_size_req; + auto scatter_aux = Kokkos::Experimental::create_scatter_view(domain.fields.aux); +#endif + + for (auto& species : domain.species) { + if ((species.npart() == 0) or cmp::AlmostZero_host(species.mass())) { + continue; + } + if constexpr (Mode != kernel::hybrid::PushMode::MomentsOnly) { + species.set_unsorted(); + } + + const kernel::hybrid::PusherContext ctx { + species.mass(), + species.charge(), + dt, + omegaB0, + inv_n0, + use_weights, + static_cast(domain.mesh.n_active(in::x1)), + static_cast(domain.mesh.n_active(in::x2)), + static_cast(domain.mesh.n_active(in::x3)) + }; + +#if defined(TEAM_POLICY) + // Tiled push+deposit. Coverage mirrors the tiled currents launcher + // (engines/srpic/currents.h): the harness handles a stale partition + // per-particle (escape valve for drifted particles, dead-tag skip in + // the kernel, slice clamp to the live npart), the flat tail pass + // below covers particles appended since the last sort, and species + // with no tile layout yet (first step, before any SortSpatially) + // take the flat scatter-view path for that call alone. + const auto& layout = species.tile_layout(); + if (layout.ntiles_total == 0u or layout.tile_offsets.extent(0) == 0u) { + auto scatter_aux = Kokkos::Experimental::create_scatter_view( + domain.fields.aux); + Kokkos::parallel_for( + "HybridPushDeposit", + species.rangeActiveParticles(), + kernel::hybrid::Pusher_kernel { ctx, + pusher_boundaries, + species, + domain.fields.bckp, + scatter_aux, + domain.mesh.metric }); + Kokkos::Experimental::contribute(domain.fields.aux, scatter_aux); + continue; + } + + // Sort-cadence sanity (see the plan's §"sort-cadence gotcha" and the + // halo derivation in kernels/tiled_scatter.hpp): with an interval + // K > DRIFT + 1, most particles drift past the scratch halo between + // sorts and take the global escape valve — correct, but the SLM + // scratch is silently bypassed and the tiled kernel performs like + // the flat one with extra overhead. Each push moves ions by <= 1 + // cell (CFL), and the pusher un-sorts them anyway: keep + // spatial_sorting_interval = 1 for ion species, or build with + // team_policy_drift = interval. + { +#if defined(TEAM_POLICY_DRIFT) + constexpr auto DRIFT = static_cast(TEAM_POLICY_DRIFT); +#else + constexpr auto DRIFT = static_cast(1); +#endif + static bool warned_cadence = false; + if ((not warned_cadence) and + (species.spatial_sorting_interval() > DRIFT + 1u)) { + warned_cadence = true; + raise::Warning( + fmt::format("hybrid tiled deposit: spatial_sorting_interval = %d " + "for species %d exceeds team_policy_drift + 1 = %d — " + "most particles will bypass the SLM scratch through " + "the escape valve; set the interval to 1 or rebuild " + "with a larger team_policy_drift", + static_cast(species.spatial_sorting_interval()), + species.index(), + static_cast(DRIFT + 1u)), + HERE); + } + } + + using body_t = kernel::hybrid::Pusher_kernel; + using tiled_t = kernel::TiledScatter_kernel( + TEAM_POLICY_TILE_SIZE), + body_t>; + const body_t body { ctx, + pusher_boundaries, + species, + domain.fields.bckp, + domain.mesh.metric }; + const tiled_t kern { domain.fields.aux, body, layout, species.npart() }; + const auto policy = kernel::MakeTiledPolicy(kern, + layout.ntiles_total, + team_size_req); + Kokkos::parallel_for("HybridPushDepositTiled", policy, kern); + + // Particles appended since the last sort (injection / MPI receive on + // a no-sort step) live past the partition and are not visited by any + // team above. Push+deposit that tail [npart_partitioned, npart) with + // the flat scatter-view kernel so every active particle is handled + // exactly once (the Corrector store-back included). + if (species.npart() > layout.npart_partitioned) { + auto scatter_aux = Kokkos::Experimental::create_scatter_view( + domain.fields.aux); + Kokkos::parallel_for( + "HybridPushDepositTail", + CreateParticleRangePolicy({ layout.npart_partitioned }, + { species.npart() }), + kernel::hybrid::Pusher_kernel { ctx, + pusher_boundaries, + species, + domain.fields.bckp, + scatter_aux, + domain.mesh.metric }); + Kokkos::Experimental::contribute(domain.fields.aux, scatter_aux); + } +#else + Kokkos::parallel_for( + "HybridPushDeposit", + species.rangeActiveParticles(), + kernel::hybrid::Pusher_kernel { ctx, + pusher_boundaries, + species, + domain.fields.bckp, + scatter_aux, + domain.mesh.metric }); +#endif + } +#if !defined(TEAM_POLICY) + Kokkos::Experimental::contribute(domain.fields.aux, scatter_aux); +#endif + } + + /** + * @brief Advance all ion species from x^(n),v^(n) to x^(n+1),v^(n+1) and deposit + * the corresponding ion moments into `aux` in the same pass. + * @param corrector false -> transient predictor push (no store, no particle BCs); + * true -> accepted corrector push (store-back + particle BCs). + */ + template + void ParticlePush(Domain& domain, + const prm::Parameters& engine_params, + const SimulationParams& params, + bool corrector) { + const auto dt = engine_params.get("dt"); + // Optional runtime override for the tiled team (work-group) size; + // 0 (default) keeps Kokkos::AUTO. Clamped to the backend max in + // kernel::MakeTiledPolicy. Ignored without TEAM_POLICY. + const auto team_size_req = static_cast( + engine_params.get("team_policy_team_size", + std::optional { 0u })); + if (corrector) { + runPusher(domain, + params, + dt, + team_size_req); + } else { + runPusher(domain, + params, + dt, + team_size_req); + } + } + + /** + * @brief Deposit ion moments N, V from the stored particles into `aux` without + * pushing. Used once at step 0 to seed aux with N^(0), V^(0). + */ + template + void DepositMoments(Domain& domain, + const SimulationParams& params) { + runPusher(domain, params, ZERO); + } + +} // namespace ntt::hybrid + +#endif // ENGINES_HYBRID_PARTICLE_PUSHER_H diff --git a/src/engines/srpic/currents.h b/src/engines/srpic/currents.h index 74946ed68..e2bacb135 100644 --- a/src/engines/srpic/currents.h +++ b/src/engines/srpic/currents.h @@ -87,41 +87,12 @@ namespace ntt { dt, layout, species.npart() }; - const auto scratch = Kokkos::PerTeam( - decltype(deposit_kernel)::scratch_bytes()); - - // Team (work-group) size. The default (team_size_req == 0) leaves - // Kokkos::AUTO, which sizes the team from the backend occupancy - // heuristic. A positive `algorithms.deposit.team_policy_team_size` - // overrides it, clamped to the scratch/backend-feasible maximum so an - // over-large request cannot abort the launch (Kokkos errors when - // team_size > team_size_max). No portable subgroup rounding is applied; - // pick a multiple of the device subgroup width (printed per arch by - // ideal_tile_size.py) for the best occupancy. - Kokkos::TeamPolicy<> policy(static_cast(layout.ntiles_total), - Kokkos::AUTO); - policy.set_scratch_size(0, scratch); - if (team_size_req > 0) { - const int ts_max = policy.team_size_max(deposit_kernel, - Kokkos::ParallelForTag {}); - int ts = team_size_req; - if (ts > ts_max) { - raise::Warning( - fmt::format("algorithms.deposit.team_policy_team_size = %d exceeds " - "the tiled-deposit maximum %d on this backend; clamping " - "to %d", - team_size_req, - ts_max, - ts_max), - HERE); - ts = ts_max; - } - policy = Kokkos::TeamPolicy<>(static_cast(layout.ntiles_total), ts); - policy.set_scratch_size(0, scratch); - logger::Checkpoint( - fmt::format("Tiled deposit: explicit team size %d", ts), - HERE); - } + // Policy boilerplate (scratch sizing, optional explicit team size with + // clamping) is shared by all tiled scatters — see MakeTiledPolicy in + // kernels/tiled_scatter.hpp. + const auto policy = kernel::MakeTiledPolicy(deposit_kernel, + layout.ntiles_total, + team_size_req); Kokkos::parallel_for("CurrentsDepositTiled", policy, deposit_kernel); // Particles appended since the last sort (injection / MPI receive on a @@ -364,7 +335,7 @@ namespace ntt { if (m < 0 or i == nfilter - 1u) { // refresh ghosts to distance G (and leave them valid for the // downstream field solver after the final pass) - metadomain.CommunicateFields(domain, Comm::J); + metadomain.CommunicateFields(domain, ::Comm::CUR); m = G - 1; } } diff --git a/src/engines/srpic/particles_bcs.h b/src/engines/srpic/particles_bcs.h index 5e79f3b78..eab19b63f 100644 --- a/src/engines/srpic/particles_bcs.h +++ b/src/engines/srpic/particles_bcs.h @@ -105,7 +105,7 @@ namespace ntt { prtl_spec.set_unsorted(); } Kokkos::Experimental::contribute(domain.fields.bckp, scatter_bckp); - metadomain.SynchronizeFields(domain, Comm::Bckp, { 0, 1 }); + metadomain.SynchronizeFields(domain, ::Comm::Bckp, { 0, 1 }); } const auto maxwellian = arch::energy_dist::Maxwellian { diff --git a/src/engines/srpic/srpic.hpp b/src/engines/srpic/srpic.hpp index d0352318e..4a48ff1c6 100644 --- a/src/engines/srpic/srpic.hpp +++ b/src/engines/srpic/srpic.hpp @@ -36,6 +36,12 @@ namespace ntt { + namespace srpic::Comm { + static constexpr auto E = ::Comm::EM_012; + static constexpr auto B = ::Comm::EM_345; + static constexpr auto J = ::Comm::CUR; + } // namespace srpic::Comm + template class SRPICEngine : public Engine { @@ -70,7 +76,7 @@ namespace ntt { if (step == 0) { // communicate fields and apply BCs on the first timestep - m_metadomain.CommunicateFields(dom, Comm::B | Comm::E); + m_metadomain.CommunicateFields(dom, srpic::Comm::B | srpic::Comm::E); srpic::FieldBoundaries(dom, m_metadomain.mesh().metric, m_metadomain.mesh(), @@ -87,7 +93,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); + m_metadomain.CommunicateFields(dom, srpic::Comm::B); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -118,8 +124,8 @@ namespace ntt { timers.stop("CurrentDeposit"); timers.start("Communications"); - m_metadomain.SynchronizeFields(dom, Comm::J); - m_metadomain.CommunicateFields(dom, Comm::J); + m_metadomain.SynchronizeFields(dom, srpic::Comm::J); + m_metadomain.CommunicateFields(dom, srpic::Comm::J); timers.stop("Communications"); timers.start("CurrentFiltering"); @@ -138,7 +144,7 @@ namespace ntt { timers.stop("FieldSolver"); timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::B); + m_metadomain.CommunicateFields(dom, srpic::Comm::B); timers.stop("Communications"); timers.start("FieldBoundaries"); @@ -162,7 +168,7 @@ namespace ntt { } timers.start("Communications"); - m_metadomain.CommunicateFields(dom, Comm::E | Comm::J); + m_metadomain.CommunicateFields(dom, srpic::Comm::E | srpic::Comm::J); timers.stop("Communications"); timers.start("FieldBoundaries"); diff --git a/src/entity.cpp b/src/entity.cpp index c05c6c18a..b7a32600e 100644 --- a/src/entity.cpp +++ b/src/entity.cpp @@ -10,6 +10,7 @@ #include "framework/specialization_registry.h" #include "engines/grpic/grpic.hpp" +#include "engines/hybrid/hybrid.hpp" #include "engines/srpic/srpic.hpp" #include "pgen.hpp" @@ -32,6 +33,12 @@ namespace ntt { template using type = GRPICEngine; }; + + template <> + struct EngineSelector { + template + using type = HYBRIDEngine; + }; } // namespace ntt template @@ -48,9 +55,7 @@ static constexpr bool should_compile { template class M, Dimension D> void dispatch_engine(ntt::Simulation& sim) { - if constexpr (S == SimEngine::SRPIC) { - sim.run::template type, M, D>(); - } else if constexpr (S == SimEngine::GRPIC) { + if constexpr (S != ntt::SimEngine::INVALID) { sim.run::template type, M, D>(); } else { static_assert(::traits::always_false>::value, diff --git a/src/framework/containers/fields.cpp b/src/framework/containers/fields.cpp index 6d379ee34..8207e90b7 100644 --- a/src/framework/containers/fields.cpp +++ b/src/framework/containers/fields.cpp @@ -3,6 +3,10 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" + +#include "framework/specialization_registry.h" + #include namespace ntt { @@ -23,14 +27,27 @@ namespace ntt { bckp = ndfield_t { "BCKP", nx1 }; cur = ndfield_t { "J", nx1 }; buff = ndfield_t { "BUFF", nx1 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { + cur0 = ndfield_t { "CUR0", nx1 }; + } } else if constexpr (D == Dim::_2D) { em = ndfield_t { "EM", nx1, nx2 }; bckp = ndfield_t { "BCKP", nx1, nx2 }; cur = ndfield_t { "J", nx1, nx2 }; buff = ndfield_t { "BUFF", nx1, nx2 }; - if constexpr (S == SimEngine::GRPIC) { - aux = ndfield_t { "AUX", nx1, nx2 }; - em0 = ndfield_t { "EM0", nx1, nx2 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1, nx2 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1, nx2 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { cur0 = ndfield_t { "CUR0", nx1, nx2 }; } } else if constexpr (D == Dim::_3D) { @@ -38,18 +55,23 @@ namespace ntt { bckp = ndfield_t { "BCKP", nx1, nx2, nx3 }; cur = ndfield_t { "J", nx1, nx2, nx3 }; buff = ndfield_t { "BUFF", nx1, nx2, nx3 }; - if constexpr (S == SimEngine::GRPIC) { - aux = ndfield_t { "AUX", nx1, nx2, nx3 }; - em0 = ndfield_t { "EM0", nx1, nx2, nx3 }; + if constexpr (::traits::engine::DefinesAuxFields) { + aux = ndfield_t { "AUX", nx1, nx2, nx3 }; + } + if constexpr (::traits::engine::DefinesEM0Fields) { + em0 = ndfield_t { "EM0", nx1, nx2, nx3 }; + } + if constexpr (::traits::engine::DefinesCur0Fields) { cur0 = ndfield_t { "CUR0", nx1, nx2, nx3 }; } } } - template struct Fields; - template struct Fields; - template struct Fields; - template struct Fields; - template struct Fields; + // NOLINTBEGIN(bugprone-macro-parentheses) +#define FIELDS_CONSTRUCTOR(D, S) template struct Fields; + + NTT_FOREACH_SPECIALIZATION_FIELDS(FIELDS_CONSTRUCTOR) +#undef FIELDS_CONSTRUCTOR + // NOLINTEND(bugprone-macro-parentheses) } // namespace ntt diff --git a/src/framework/containers/fields_io.cpp b/src/framework/containers/fields_io.cpp index a17ab1430..2025ee8de 100644 --- a/src/framework/containers/fields_io.cpp +++ b/src/framework/containers/fields_io.cpp @@ -1,9 +1,11 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/log.h" #include "framework/containers/fields.h" +#include "framework/specialization_registry.h" #include "output/utils/readers.h" #include "output/utils/writers.h" @@ -30,8 +32,10 @@ namespace ntt { ls6.push_back(6); io.DefineVariable("em", gs6, lo6, ls6); - if (S == ntt::SimEngine::GRPIC) { + if (::traits::engine::WriteEM0FieldToCheckpoint) { io.DefineVariable("em0", gs6, lo6, ls6); + } + if (::traits::engine::WriteCurFieldToCheckpoint) { auto gs3 = std::vector(global_shape.begin(), global_shape.end()); auto lo3 = std::vector(local_offset.begin(), local_offset.end()); auto ls3 = std::vector(local_shape.begin(), local_shape.end()); @@ -52,8 +56,10 @@ namespace ntt { range6.first.push_back(0); range6.second.push_back(6); out::ReadNDField(io, reader, "em", em, range6); - if (S == ntt::SimEngine::GRPIC) { + if (::traits::engine::WriteEM0FieldToCheckpoint) { out::ReadNDField(io, reader, "em0", em0, range6); + } + if (::traits::engine::WriteCurFieldToCheckpoint) { auto range3 = adios2::Box(range.first, range.second); range3.first.push_back(0); range3.second.push_back(3); @@ -86,6 +92,7 @@ namespace ntt { } } + // NOLINTBEGIN(bugprone-macro-parentheses) #define FIELDS_CHECKPOINTS(D, S) \ template void Fields::CheckpointDeclare(adios2::IO&, \ const std::vector&, \ @@ -100,11 +107,8 @@ namespace ntt { const std::vector&, \ const std::vector&) const; - FIELDS_CHECKPOINTS(Dim::_1D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_2D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_3D, SimEngine::SRPIC) - FIELDS_CHECKPOINTS(Dim::_2D, SimEngine::GRPIC) - FIELDS_CHECKPOINTS(Dim::_3D, SimEngine::GRPIC) + NTT_FOREACH_SPECIALIZATION_FIELDS(FIELDS_CHECKPOINTS) #undef FIELDS_CHECKPOINTS + // NOLINTEND(bugprone-macro-parentheses) } // namespace ntt diff --git a/src/framework/domain/metadomain_comm.cpp b/src/framework/domain/metadomain_comm.cpp index f8c0f60d7..2cd558a0c 100644 --- a/src/framework/domain/metadomain_comm.cpp +++ b/src/framework/domain/metadomain_comm.cpp @@ -3,6 +3,7 @@ #include "arch/directions.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/formatting.h" @@ -204,74 +205,66 @@ namespace ntt { template void Metadomain::CommunicateFields(Domain& domain, CommTags tags) const { - const auto comm_em = ((S == SimEngine::SRPIC) and - ((tags & Comm::E) or (tags & Comm::B))) or - ((S == SimEngine::GRPIC) and - ((tags & Comm::D) or (tags & Comm::B))); - const bool comm_em0 = (S == SimEngine::GRPIC) and - ((tags & Comm::B0) or (tags & Comm::D0)); - const bool comm_j = (tags & Comm::J); - const bool comm_aux = (S == SimEngine::GRPIC) and - ((tags & Comm::E) or (tags & Comm::H)); - raise::ErrorIf(not(comm_em or comm_em0 or comm_j or comm_aux), + const auto comm_em = (tags & Comm::EM_012) or (tags & Comm::EM_345); + const auto comm_em0 = (tags & Comm::EM0_012) or (tags & Comm::EM0_345); + const auto comm_cur = (tags & Comm::CUR); + const auto comm_cur0 = (tags & Comm::CUR0); + const auto comm_aux = (tags & Comm::AUX_012) or (tags & Comm::AUX_345); + const auto comm_bckp = (tags & Comm::Bckp); + + raise::ErrorIf(not(comm_em or comm_em0 or comm_cur or comm_aux or comm_cur0 or + comm_bckp), "CommunicateFields called with no task", HERE); + if constexpr (not ::traits::engine::DefinesEM0Fields) { + raise::ErrorIf(comm_em0, + "CommunicateFields called with EM0 communication " + "for an engine that does not define EM0 fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesAuxFields) { + raise::ErrorIf(comm_aux, + "CommunicateFields called with AUX communication " + "for an engine that does not define AUX fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesCur0Fields) { + raise::ErrorIf(comm_cur0, + "CommunicateFields called with CUR0 communication " + "for an engine that does not define CUR0 fields", + HERE); + } std::string comms; - if (tags & Comm::E) { - comms += "E "; + if (tags & Comm::EM_012) { + comms += "EM[0-2] "; } - if (tags & Comm::B) { - comms += "B "; + if (tags & Comm::EM_345) { + comms += "EM[3-5] "; } - if (tags & Comm::J) { - comms += "J "; + if (comm_cur) { + comms += "CUR "; } - if (tags & Comm::D) { - comms += "D "; + if (tags & Comm::AUX_012) { + comms += "AUX[0-2] "; } - if (tags & Comm::H) { - comms += "H "; + if (tags & Comm::AUX_345) { + comms += "AUX[3-5] "; } - if (tags & Comm::D0) { - comms += "D0 "; + if (tags & Comm::EM0_012) { + comms += "EM0[0-2] "; } - if (tags & Comm::B0) { - comms += "B0 "; + if (tags & Comm::EM0_345) { + comms += "EM0[3-5] "; } - logger::Checkpoint(fmt::format("Communicating %s\n", comms.c_str()), HERE); - - /** - * @note this block is designed to support in the future multiple domains - * on a single rank, however that is not yet implemented - */ - // establish the last index ranges for fields (i.e., components) - auto comp_range_fld = cell_range_t {}; - auto comp_range_cur = cell_range_t {}; - if constexpr (S == SimEngine::GRPIC) { - if (((tags & Comm::D) and (tags & Comm::B)) or - ((tags & Comm::D0) and (tags & Comm::B0)) or - ((tags & Comm::E) and (tags & Comm::H))) { - comp_range_fld = cell_range_t(em::dx1, em::bx3 + 1); - } else if ((tags & Comm::D) or (tags & Comm::D0) or (tags & Comm::E)) { - comp_range_fld = cell_range_t(em::dx1, em::dx3 + 1); - } else if ((tags & Comm::B) or (tags & Comm::B0) or (tags & Comm::H)) { - comp_range_fld = cell_range_t(em::bx1, em::bx3 + 1); - } - } else if constexpr (S == SimEngine::SRPIC) { - if ((tags & Comm::E) and (tags & Comm::B)) { - comp_range_fld = cell_range_t(em::ex1, em::bx3 + 1); - } else if (tags & Comm::E) { - comp_range_fld = cell_range_t(em::ex1, em::ex3 + 1); - } else if (tags & Comm::B) { - comp_range_fld = cell_range_t(em::bx1, em::bx3 + 1); - } - } else { - raise::Error("Unknown simulation engine", HERE); + if (comm_cur0) { + comms += "CUR0 "; } - if (comm_j) { - comp_range_cur = cell_range_t(cur::jx1, cur::jx3 + 1); + if (comm_bckp) { + comms += "Bckp "; } + logger::Checkpoint(fmt::format("Communicating %s\n", comms.c_str()), HERE); + // traverse in all directions and send/recv the fields for (auto& direction : dir::Directions::all) { const auto [send_params, @@ -284,6 +277,16 @@ namespace ntt { continue; } if (comm_em) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::EM_012) and (tags & Comm::EM_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::EM_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::EM_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); + } comm::CommunicateField(domain.index(), domain.fields.em, domain.fields.em, @@ -293,75 +296,95 @@ namespace ntt { recv_rank, send_slice, recv_slice, - comp_range_fld, + comp_range, false); } - if constexpr (S == SimEngine::GRPIC) { - if (comm_aux) { - comm::CommunicateField(domain.index(), - domain.fields.aux, - domain.fields.aux, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_fld, - false); - } - if (comm_em0) { - comm::CommunicateField(domain.index(), - domain.fields.em0, - domain.fields.em0, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_fld, - false); - // @HACK_GR_1.2.0 -- this has to be done carefully - // comm::CommunicateField(domain.index(), - // domain.fields.aux, - // domain.fields.aux, - // send_ind, - // recv_ind, - // send_rank, - // recv_rank, - // send_slice, - // recv_slice, - // comp_range_fld, - // false); - } - if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur0, - domain.fields.cur0, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - false); + if (comm_aux) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::AUX_012) and (tags & Comm::AUX_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::AUX_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::AUX_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); } - } else { - if (comm_j) { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.cur, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - false); + comm::CommunicateField(domain.index(), + domain.fields.aux, + domain.fields.aux, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range, + false); + } + if (comm_em0) { + auto comp_range = cell_range_t {}; + if ((tags & Comm::EM0_012) and (tags & Comm::EM0_345)) { + comp_range = cell_range_t(0, 6); + } else if (tags & Comm::EM0_012) { + comp_range = cell_range_t(0, 3); + } else if (tags & Comm::EM0_345) { + comp_range = cell_range_t(3, 6); + } else { + raise::Error("Incorrect logic", HERE); } + comm::CommunicateField(domain.index(), + domain.fields.em0, + domain.fields.em0, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + comp_range, + false); + } + if (comm_cur0) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.cur0, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + false); + } + if (comm_cur) { + auto comp_range = cell_range_t(0, 3); + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.cur, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + false); + } + if (comm_bckp) { + // copy active -> ghost of bckp (Ec/Bc); read by the hybrid pusher gather + comm::CommunicateField(domain.index(), + domain.fields.bckp, + domain.fields.bckp, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 6 }, + false); } } } @@ -409,20 +432,45 @@ namespace ntt { void Metadomain::SynchronizeFields(Domain& domain, CommTags tags, const cell_range_t& components) const { - const bool comm_j = (tags & Comm::J); + const bool comm_cur = (tags & Comm::CUR); + const bool comm_cur0 = (tags & Comm::CUR0); const bool comm_bckp = (tags & Comm::Bckp); const bool comm_buff = (tags & Comm::Buff); - raise::ErrorIf(not(comm_j || comm_bckp || comm_buff), - "SynchronizeFields called with no task or incorrect task", - HERE); - raise::ErrorIf(comm_j and comm_buff, - "SynchronizeFields cannot sync J and Buff at the same time", + const bool comm_aux = (tags & Comm::AUX_012) || (tags & Comm::AUX_345); + raise::ErrorIf( + not(comm_cur0 || comm_cur || comm_bckp || comm_buff || comm_aux), + "SynchronizeFields called with no task or incorrect task", + HERE); + raise::ErrorIf( + (comm_cur0 and comm_buff) or (comm_cur and comm_buff), + "SynchronizeFields cannot sync CUR/CUR0 and Buff at the same time", + HERE); + raise::ErrorIf((comm_cur0 and comm_cur), + "SynchronizeFields cannot sync CUR and CUR0 at the same " + "time (both use Buff as buffer)", HERE); - const auto synchronize = true; + + if constexpr (not ::traits::engine::DefinesCur0Fields) { + raise::ErrorIf(comm_cur0, + "CommunicateFields called with CUR0 communication " + "for an engine that does not define CUR0 fields", + HERE); + } + if constexpr (not ::traits::engine::DefinesAuxFields) { + raise::ErrorIf(comm_aux, + "SynchronizeFields called with AUX synchronization " + "for an engine that does not define AUX fields", + HERE); + } + + const auto SYNCHRONIZE = true; std::string comms; - if (comm_j) { - comms += "J "; + if (comm_cur) { + comms += "CUR "; + } + if (comm_cur0) { + comms += "CUR0 "; } if (comm_bckp) { comms += "Bckp "; @@ -430,14 +478,13 @@ namespace ntt { if (comm_buff) { comms += "Buff "; } + if (comm_aux) { + comms += "AUX "; + } logger::Checkpoint(fmt::format("Synchronizing %s\n", comms.c_str()), HERE); - auto comp_range_cur = cell_range_t {}; - if (comm_j) { - comp_range_cur = cell_range_t(cur::jx1, cur::jx3 + 1); - Kokkos::deep_copy(domain.fields.buff, ZERO); - } ndfield_t bckp_recv; + ndfield_t aux_recv; ndfield_t buff_recv; if (comm_bckp) { if constexpr (M::Dim == Dim::_1D) { @@ -454,6 +501,21 @@ namespace ntt { domain.fields.bckp.extent(2) }; } } + if (comm_aux) { + if constexpr (M::Dim == Dim::_1D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0) }; + } else if constexpr (M::Dim == Dim::_2D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0), + domain.fields.aux.extent(1) }; + } else if constexpr (M::Dim == Dim::_3D) { + aux_recv = ndfield_t { "aux_recv", + domain.fields.aux.extent(0), + domain.fields.aux.extent(1), + domain.fields.aux.extent(2) }; + } + } if (comm_buff) { if constexpr (M::Dim == Dim::_1D) { buff_recv = ndfield_t { "buff_recv", @@ -469,6 +531,13 @@ namespace ntt { domain.fields.buff.extent(2) }; } } + // buff accumulates the received deposit tails from every direction and is + // added into cur/cur0 once after the loop, so it must be zeroed exactly + // once, here, before the loop. Zeroing it per direction would keep only the + // last direction's contribution. + if (comm_cur or comm_cur0) { + Kokkos::deep_copy(domain.fields.buff, ZERO); + } // traverse in all directions and sync the fields for (auto& direction : dir::Directions::all) { const auto [send_params, @@ -480,32 +549,30 @@ namespace ntt { if (send_rank < 0 and recv_rank < 0) { continue; } - if (comm_j) { - if constexpr (S == SimEngine::GRPIC) { - comm::CommunicateField(domain.index(), - domain.fields.cur0, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); - } else { - comm::CommunicateField(domain.index(), - domain.fields.cur, - domain.fields.buff, - send_ind, - recv_ind, - send_rank, - recv_rank, - send_slice, - recv_slice, - comp_range_cur, - synchronize); - } + if (comm_cur) { + comm::CommunicateField(domain.index(), + domain.fields.cur, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + SYNCHRONIZE); + } else if (comm_cur0) { + comm::CommunicateField(domain.index(), + domain.fields.cur0, + domain.fields.buff, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 3 }, + SYNCHRONIZE); } if (comm_bckp) { comm::CommunicateField(domain.index(), @@ -518,7 +585,22 @@ namespace ntt { send_slice, recv_slice, components, - synchronize); + SYNCHRONIZE); + } + if (comm_aux) { + // additive remap of moment deposit tails (Pegasus §3.6): accumulate the + // ghost-cell contributions of aux (V in 0..2, N in 3) into aux_recv + comm::CommunicateField(domain.index(), + domain.fields.aux, + aux_recv, + send_ind, + recv_ind, + send_rank, + recv_rank, + send_slice, + recv_slice, + { 0, 6 }, + SYNCHRONIZE); } if (comm_buff) { comm::CommunicateField(domain.index(), @@ -531,21 +613,19 @@ namespace ntt { send_slice, recv_slice, components, - synchronize); + SYNCHRONIZE); } } - if (comm_j) { - if constexpr (S == SimEngine::GRPIC) { - AddBufferedFields(domain.fields.cur0, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); - } else { - AddBufferedFields(domain.fields.cur, - domain.fields.buff, - domain.mesh.rangeActiveCells(), - comp_range_cur); - } + if (comm_cur) { + AddBufferedFields(domain.fields.cur, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + { 0, 3 }); + } else if (comm_cur0) { + AddBufferedFields(domain.fields.cur0, + domain.fields.buff, + domain.mesh.rangeActiveCells(), + { 0, 3 }); } if (comm_bckp) { AddBufferedFields(domain.fields.bckp, @@ -553,6 +633,12 @@ namespace ntt { domain.mesh.rangeActiveCells(), components); } + if (comm_aux) { + AddBufferedFields(domain.fields.aux, + aux_recv, + domain.mesh.rangeActiveCells(), + { 0, 6 }); + } if (comm_buff) { AddBufferedFields(domain.fields.buff, buff_recv, diff --git a/src/framework/domain/metadomain_io.cpp b/src/framework/domain/metadomain_io.cpp index d53dc49a1..d242c57e1 100644 --- a/src/framework/domain/metadomain_io.cpp +++ b/src/framework/domain/metadomain_io.cpp @@ -548,7 +548,7 @@ namespace ntt { HERE); } } else if (fld.is_vpotential()) { - if constexpr (S == SimEngine::GRPIC && M::Dim == Dim::_2D) { + if constexpr (::traits::engine::IsGR && M::Dim == Dim::_2D) { const auto c = static_cast(addresses.back()); ComputeVectorPotential(local_domain->fields.bckp, local_domain->fields.em, @@ -611,7 +611,7 @@ namespace ntt { SynchronizeFields(*local_domain, Comm::Bckp, { addresses[0], addresses[2] + 1 }); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::IsSR) { if (fld.id() == FldsID::V) { // normalize 3vel * rho (combuted above) by rho ComputeMoments(params, @@ -646,7 +646,7 @@ namespace ntt { { cur::jx1, cur::jx3 + 1 }, copy_to); } else if (fld.is_field()) { - if (S == SimEngine::GRPIC && fld.is_gr_aux_field()) { + if (::traits::engine::IsGR && fld.is_gr_aux_field()) { if (fld.is_efield()) { // GR: E DeepCopyFields(local_domain->fields.aux, @@ -699,7 +699,7 @@ namespace ntt { } } } else if (fld.comp.size() == 4) { // 4-vector - if constexpr (S == SimEngine::GRPIC) { + if constexpr (::traits::engine::IsGR) { if (fld.is_moment() && fld.id() == FldsID::V) { // Compute 4-velocity: V^μ (u^0, u^1, u^2, u^3) for (auto i = 0; i < 4; ++i) { diff --git a/src/framework/domain/metadomain_loadbal.cpp b/src/framework/domain/metadomain_loadbal.cpp index a7bf97873..63954b4be 100644 --- a/src/framework/domain/metadomain_loadbal.cpp +++ b/src/framework/domain/metadomain_loadbal.cpp @@ -414,7 +414,7 @@ namespace ntt { } /* --- 8. Refill ghost zones from neighbors ---------------------------- */ - CommunicateFields(local_dom, Comm::E | Comm::B); + CommunicateFields(local_dom, Comm::EM); /* --- 9. Shift particle indices, retag, and migrate ------------------- */ // Particle indices i_d are in active-cell coordinates: i_d in [0, n_active) diff --git a/src/framework/domain/metadomain_stats.cpp b/src/framework/domain/metadomain_stats.cpp index 35ff26e75..76635fc61 100644 --- a/src/framework/domain/metadomain_stats.cpp +++ b/src/framework/domain/metadomain_stats.cpp @@ -1,6 +1,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/comparators.h" #include "utils/error.h" @@ -137,7 +138,7 @@ namespace ntt { raise::Error("Components not supported for JdotE", HERE); } } else if constexpr ( - (S == SimEngine::SRPIC) and + (::traits::engine::IsSR) and (F == StatsID::B2 or F == StatsID::E2 or F == StatsID::ExB)) { raise::ErrorIf(components.size() != 1, "Components must be of size 1 for B2, E2 or ExB stats", @@ -251,7 +252,7 @@ namespace ntt { } else if (stat.id() == StatsID::JdotE) { g_stats_writer.write( ReduceFields(local_domain, g_mesh.metric, {})); - } else if (S == SimEngine::SRPIC) { + } else if (::traits::engine::IsSR) { if (stat.id() == StatsID::E2) { for (const auto& comp : stat.comp) { g_stats_writer.write( diff --git a/src/framework/parameters/grid.cpp b/src/framework/parameters/grid.cpp index b701ab98e..84088f367 100644 --- a/src/framework/parameters/grid.cpp +++ b/src/framework/parameters/grid.cpp @@ -4,6 +4,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/error.h" #include "utils/formatting.h" #include "utils/numeric.h" @@ -122,10 +123,10 @@ namespace ntt { std::vector> flds_bc_enum; std::vector> prtl_bc_enum; if (coord_enum == Coord::Cartesian) { - raise::ErrorIf(flds_bc.size() != (std::size_t)dim, + raise::ErrorIf(flds_bc.size() != (size_t)dim, "invalid `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc.size() != (std::size_t)dim, + raise::ErrorIf(prtl_bc.size() != (size_t)dim, "invalid `grid.boundaries.particles`", HERE); for (auto d { 0u }; d < (dim_t)dim; ++d) { @@ -181,7 +182,7 @@ namespace ntt { raise::ErrorIf(prtl_bc.size() > 1, "invalid `grid.boundaries.particles`", HERE); - if (engine_enum == SimEngine::SRPIC) { + if (::traits::engine::isSR(engine_enum)) { raise::ErrorIf(flds_bc[0].size() != 2, "invalid `grid.boundaries.fields`", HERE); @@ -224,10 +225,10 @@ namespace ntt { } } - raise::ErrorIf(flds_bc_enum.size() != (std::size_t)dim, + raise::ErrorIf(flds_bc_enum.size() != (size_t)dim, "invalid inferred `grid.boundaries.fields`", HERE); - raise::ErrorIf(prtl_bc_enum.size() != (std::size_t)dim, + raise::ErrorIf(prtl_bc_enum.size() != (size_t)dim, "invalid inferred `grid.boundaries.particles`", HERE); boundaries_t flds_bc_pairwise; @@ -421,8 +422,8 @@ namespace ntt { metric_params_short_.emplace(); std::string coord; if (metric_enum == Metric::Minkowski) { - raise::ErrorIf(engine_enum != SimEngine::SRPIC, - "minkowski metric is only supported for SRPIC", + raise::ErrorIf(not ::traits::engine::isSR(engine_enum), + "minkowski metric is only supported for SR", HERE); coord = "cart"; } else if (metric_enum == Metric::QKerr_Schild or @@ -455,7 +456,7 @@ namespace ntt { HERE); coord = "sph"; } - if ((engine_enum == SimEngine::GRPIC) && + if (::traits::engine::isGR(engine_enum) && (metric_enum != Metric::Kerr_Schild_0)) { const auto ks_a = toml::find_or(toml_data, "grid", @@ -501,7 +502,7 @@ namespace ntt { (*metric_params_short_)["r0"] = (*metric_params)["qsph_r0"]; (*metric_params_short_)["h"] = (*metric_params)["qsph_h"]; } - if ((engine_enum == SimEngine::GRPIC) && + if (::traits::engine::isGR(engine_enum) && (metric_enum != Metric::Kerr_Schild_0)) { (*metric_params_short_)["a"] = (*metric_params)["ks_a"]; } diff --git a/src/framework/parameters/parameters.cpp b/src/framework/parameters/parameters.cpp index 8b525d49a..2f632c5a9 100644 --- a/src/framework/parameters/parameters.cpp +++ b/src/framework/parameters/parameters.cpp @@ -54,6 +54,99 @@ namespace ntt { set("scales.sigma0", SQR(skindepth0 / larmor0)); set("scales.B0", ONE / larmor0); set("scales.omegaB0", ONE / larmor0); + if (engine_enum == SimEngine::HYBRID) { + /** + * hybrid-engine parameters (read from the [hybrid] table; ignored by the + * SR/GR engines): + * gamma_ad - adiabatic index of the massless electron fluid (1 = isothermal) + * theta0 - electron temperature T_e (code units); 0 = cold electrons + * dens_min - vacuum threshold for the Ohm's law; below it E ramps to 0 (units of n0) + * hall_lim - cap on the local whistler Courant the Hall term may imply (<= 0 off) + * resist_vac - vacuum resistivity: E -> eta curl B below dens_min (0 = off) + * resist_hyper - hyper-resistivity eta_H grad^4 B, everywhere (0 = off) + * resist - uniform Ohmic resistivity: E += eta curl B everywhere (0 = off) + * subcycle - Pegasus-style sub-cycled field advance (default on) + * subcycle_courant - target whistler Courant per field sub-step + * subcycle_max - cap on the number of field sub-steps per advance + * v_max - characteristic flow speed for the hybrid CFL (code units) + */ + set("hybrid.gamma_ad", + toml::find_or(toml_data, + "hybrid", + "gamma_ad", + defaults::hybrid::gamma_ad)); + set("hybrid.theta0", + toml::find_or(toml_data, "hybrid", "theta0", ZERO)); + // vacuum threshold for the hybrid Ohm's law. Above it E follows the usual + // [...]/N; below it E ramps continuously to zero so that plasma-free cells + // keep B frozen instead of amplifying the right-hand side by 1/dens_min. + set("hybrid.dens_min", + toml::find_or(toml_data, + "hybrid", + "dens_min", + defaults::hybrid::dens_min)); + // per-cell limiter on the Ohm's-law Hall term: caps the implied local + // whistler Courant (v_w ~ |B|/N, which exceeds the ambient value at shock + // overshoots and magnetic cavities) at hall_lim * dx/dt. Identity in + // resolvable cells; <= 0 disables. + set("hybrid.hall_lim", + toml::find_or(toml_data, + "hybrid", + "hall_lim", + static_cast(0.5))); + // vacuum resistivity (code units). > 0 turns the E = 0 vacuum of dens_min + // into a resistive vacuum: below the threshold compute_Ee crossfades E to + // eta * curl B, so vacuum regions diffuse toward a curl-free (potential) + // field instead of freezing. Needed when fast plasma sweeps along a vacuum + // boundary, where a hard E = 0 mask makes an artificial tangential-E jump + // (a surface current) that pumps B at rate ~ u/dx. Internally clamped to + // the explicit-diffusion stability limit (0.4 dx^2/dt at the sub-step dt), + // so any value is safe; eta ~ u_sweep * dx suffices. + set("hybrid.resist_vac", + toml::find_or(toml_data, "hybrid", "resist_vac", ZERO)); + // hyper-resistivity (E -= eta_H grad^2 curl B => dB/dt -= eta_H grad^4 B), + // applied everywhere in the Faraday-side Ohm's law. The moments filter + // (algorithms.current_filters) attenuates the deposited N, V by + // cos^(2p)(k dx/2) -- near-zero for few-cell wavelengths -- so in that band + // the field evolves with no kinetic feedback and sustained compression or + // shear in the frozen moments pumps the band by induction. eta_H k^4 + // blankets the filtered band while leaving k dx < 0.3 untouched; size so + // eta_H (pi/dx)^4 exceeds the drive. Internally clamped to the explicit + // grad^4 stability limit at the actual sub-step dt. + set("hybrid.resist_hyper", + toml::find_or(toml_data, "hybrid", "resist_hyper", ZERO)); + // uniform Ohmic resistivity (code units). > 0 adds E += eta * curl B in the + // Faraday-side Ohm's law everywhere (=> dB/dt += eta grad^2 B), the k^2 + // companion to resist_hyper's k^4: it damps the intermediate few-cell band + // (k dx ~ 0.1-0.3) that hyper-resistivity barely reaches but a strong + // boundary/beam drive (e.g. a high-Mach reflecting-wall shock foot) pumps. + // Unlike resist_vac this fires in dense plasma too. Trades against physical + // magnetic diffusion, so keep it as small as stability allows. Internally + // clamped to the explicit grad^2 diffusion limit at the actual sub-step dt, + // so any value is safe. + set("hybrid.resist", + toml::find_or(toml_data, "hybrid", "resist", ZERO)); + // Pegasus-style sub-cycled magnetic-field advance (fieldsolvers.h + // ::SubcycledFaraday): the whistler-stiff Ohm/Faraday loop is integrated + // with adaptive SSP-RK3 sub-steps at its own CFL instead of full-dt Euler + // pushes. The sub-step count targets `subcycle_courant` whistler Courant + // per sub-step (from the global max |B|/N) and is capped at `subcycle_max`; + // beyond the cap the per-cell Hall limiter (which then sees the SUB-step + // dt) takes over. subcycle = false -> legacy 3-push scheme. + set("hybrid.subcycle", + toml::find_or(toml_data, "hybrid", "subcycle", true)); + set("hybrid.subcycle_courant", + toml::find_or(toml_data, + "hybrid", + "subcycle_courant", + static_cast(0.5))); + set("hybrid.subcycle_max", + toml::find_or(toml_data, "hybrid", "subcycle_max", 64)); + // optional user-set characteristic flow speed for the hybrid CFL (code + // units); 0 -> dt set purely by the Alfven + whistler signal speeds. + set("hybrid.v_max", + toml::find_or(toml_data, "hybrid", "v_max", ZERO)); + } /* [particles] ---------------------------------------------------------- */ const auto ppc0 = toml::find(toml_data, "particles", "ppc0"); @@ -239,6 +332,38 @@ namespace ntt { alg_params.read(get("scales.dx0"), alg_extra_flags, toml_data); alg_params.setParams(alg_extra_flags, this); + /* hybrid timestep — correct CFL --------------------------------------- */ + // The default light-crossing dt = CFL * dx0 assumes the fastest signal + // travels at ~1 code-velocity unit (as for SR/GR, where speeds are bounded + // by c = 1). A hybrid plasma has no displacement current; its fastest + // signals are the DISPERSIVE whistler/Hall mode and the bulk ion flow, both + // larger than 1 in code units. Using dt = CFL * dx0 then advects particles + // by > 1 cell/step (Courant > 1), which breaks the single-cell rebucket and + // the moment deposit. Replace it with the Pegasus CFL (Kunz, Stone & Bai + // 2014, §3.5): dt <= CFL * min( dx0 / v_max, 2*pi/Omega ), with the + // ground-frame max signal speed + // v_max = v_flow + v_A + v_whistler, + // v_A = d0 / rho0 (Alfven speed, code units), + // v_whistler = (d0^2 / rho0) * pi / dx0 (grid-scale Hall/whistler; + // d0^2/rho0 is the EMF Hall coefficient coeff_2, k_max~pi/dx0), + // and v_flow an optional user-set characteristic flow speed in code units + // (hybrid.v_max, default 0 — set it for super-whistler flows). + if (engine_enum == SimEngine::HYBRID) { + const auto cfl = get("algorithms.timestep.CFL"); + const auto dx0 = get("scales.dx0"); + const auto d0 = get("scales.skindepth0"); + const auto rho0 = get("scales.larmor0"); + const auto v_flow = get("hybrid.v_max"); + const auto v_alfven = d0 / rho0; + const auto v_whistler = (SQR(d0) / rho0) * + static_cast(constant::PI) / dx0; + const auto v_max = v_flow + v_alfven + v_whistler; + const auto dt_adv = dx0 / v_max; + // gyration limit 2*pi/Omega (Omega = 1/rho0); rarely binding + const auto dt_gyr = static_cast(constant::TWO_PI) * rho0; + const auto dt_hyb = cfl * ((dt_adv < dt_gyr) ? dt_adv : dt_gyr); + set("algorithms.timestep.dt", dt_hyb); + } /* [simulation.domain.load_balance] ------------------------------------- */ set("simulation.domain.load_balance.enable", toml::find_or(toml_data, "simulation", "domain", "load_balance", "enable", false)); diff --git a/src/framework/specialization_registry.h b/src/framework/specialization_registry.h index dea5772c6..d276d4a84 100644 --- a/src/framework/specialization_registry.h +++ b/src/framework/specialization_registry.h @@ -37,6 +37,16 @@ namespace ntt { static constexpr auto dimension = D; }; +#define NTT_FOREACH_SPECIALIZATION_FIELDS(MACRO) \ + MACRO(Dim::_1D, SimEngine::SRPIC) \ + MACRO(Dim::_2D, SimEngine::SRPIC) \ + MACRO(Dim::_3D, SimEngine::SRPIC) \ + MACRO(Dim::_2D, SimEngine::GRPIC) \ + MACRO(Dim::_3D, SimEngine::GRPIC) \ + MACRO(Dim::_1D, SimEngine::HYBRID) \ + MACRO(Dim::_2D, SimEngine::HYBRID) \ + MACRO(Dim::_3D, SimEngine::HYBRID) + #define NTT_FOREACH_SPECIALIZATION(MACRO) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_1D) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_2D) \ @@ -45,12 +55,18 @@ namespace ntt { MACRO(SimEngine::SRPIC, metric::QSpherical, Dim::_2D) \ MACRO(SimEngine::GRPIC, metric::KerrSchild, Dim::_2D) \ MACRO(SimEngine::GRPIC, metric::QKerrSchild, Dim::_2D) \ - MACRO(SimEngine::GRPIC, metric::KerrSchild0, Dim::_2D) + MACRO(SimEngine::GRPIC, metric::KerrSchild0, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_1D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_3D) #define NTT_FOREACH_CARTESIAN_SPECIALIZATION(MACRO) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_1D) \ MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_2D) \ - MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_3D) + MACRO(SimEngine::SRPIC, metric::Minkowski, Dim::_3D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_1D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_2D) \ + MACRO(SimEngine::HYBRID, metric::Minkowski, Dim::_3D) #define NTT_BUILD_SPECIALIZATION_ENTRY(S, M, D) SpecializationEntry {}, diff --git a/src/global/defaults.h b/src/global/defaults.h index c4f9b67f4..9b0074ab8 100644 --- a/src/global/defaults.h +++ b/src/global/defaults.h @@ -42,6 +42,11 @@ namespace ntt::defaults { const real_t beta_zy = 0.0; } // namespace fieldsolver + namespace hybrid { + const real_t gamma_ad = static_cast(5.0 / 3.0); + const real_t dens_min = static_cast(1e-3); + } // namespace hybrid + namespace qsph { const real_t r0 = 0.0; const real_t h = 0.0; diff --git a/src/global/enums.h b/src/global/enums.h index 5700a3a28..6136bd653 100644 --- a/src/global/enums.h +++ b/src/global/enums.h @@ -5,7 +5,7 @@ * - enum ntt::Coord // Cart, Sph, Qsph * - enum ntt::Metric // Minkowski, Spherical, QSpherical, * Kerr_Schild, QKerr_Schild, Kerr_Schild_0 - * - enum ntt::SimEngine // SRPIC, GRPIC + * - enum ntt::SimEngine // SRPIC, GRPIC, HYBRID * - enum ntt::PrtlBC // periodic, absorb, atmosphere, custom, * reflect, horizon, axis, sync * - enum ntt::FldsBC // periodic, match, fixed, atmosphere, @@ -59,8 +59,8 @@ namespace ntt { namespace enums_hidden { template class EnumBase { - constexpr auto idx() const -> std::size_t { - return static_cast(d().val) - 1; // assumes 1-indexed + constexpr auto idx() const -> size_t { + return static_cast(d().val) - 1; // assumes 1-indexed } constexpr auto d() const -> const Derived& { @@ -131,7 +131,7 @@ namespace ntt { static constexpr const char* label = "coord"; static constexpr type variants[] = { Cartesian, Spherical, Qspherical }; static constexpr const char* lookup[] = { "cart", "sph", "qsph" }; - static constexpr std::size_t total = std::size(variants); + static constexpr size_t total = std::size(variants); }; struct Metric : public enums_hidden::EnumBase { @@ -159,7 +159,7 @@ namespace ntt { static constexpr const char* lookup[] = { "minkowski", "spherical", "qspherical", "kerr_schild", "qkerr_schild", "kerr_schild_0" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct SimEngine : public enums_hidden::EnumBase { @@ -167,6 +167,7 @@ namespace ntt { INVALID = 0, SRPIC = 1, GRPIC = 2, + HYBRID = 3, }; using enum type; type val; @@ -177,9 +178,9 @@ namespace ntt { : val { v } {} static constexpr const char* label = "sim_engine"; - static constexpr type variants[] = { SRPIC, GRPIC }; - static constexpr const char* lookup[] = { "srpic", "grpic" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr type variants[] = { SRPIC, GRPIC, HYBRID }; + static constexpr const char* lookup[] = { "srpic", "grpic", "hybrid" }; + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct PrtlBC : public enums_hidden::EnumBase { @@ -209,7 +210,7 @@ namespace ntt { "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct FldsBC : public enums_hidden::EnumBase { @@ -242,7 +243,7 @@ namespace ntt { "periodic", "match", "fixed", "atmosphere", "custom", "horizon", "axis", "conductor", "sync" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct FldsID : public enums_hidden::EnumBase { @@ -280,7 +281,7 @@ namespace ntt { "b", "h", "j", "a", "t", "rho", "charge", "n", "nppc", "v", "custom" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; struct StatsID : public enums_hidden::EnumBase { @@ -311,7 +312,7 @@ namespace ntt { static constexpr const char* lookup[] = { "b^2", "e^2", "exb", "j.e", "t", "rho", "charge", "n", "npart", "custom" }; - static constexpr std::size_t total = sizeof(variants) / sizeof(variants[0]); + static constexpr size_t total = sizeof(variants) / sizeof(variants[0]); }; namespace ParticlePusher { diff --git a/src/global/global.h b/src/global/global.h index c3fbc5061..335be097f 100644 --- a/src/global/global.h +++ b/src/global/global.h @@ -154,6 +154,15 @@ namespace ntt { hx3 = 5 }; + enum comp : uint8_t { + c0 = 0, + c1 = 1, + c2 = 2, + c3 = 3, + c4 = 4, + c5 = 5, + }; + enum cur : uint8_t { jx1 = 0, jx2 = 1, @@ -276,16 +285,20 @@ using DiagFlags = uint8_t; namespace Comm { enum CommTags_ : uint16_t { - None = 0, - E = 1 << 0, - B = 1 << 1, - J = 1 << 2, - D = 1 << 3, - D0 = 1 << 4, - B0 = 1 << 5, - H = 1 << 6, - Bckp = 1 << 7, - Buff = 1 << 8, + None = 0, + EM_012 = 1 << 0, + EM_345 = 1 << 1, + CUR = 1 << 2, + AUX_012 = 1 << 3, + AUX_345 = 1 << 4, + EM0_012 = 1 << 5, + EM0_345 = 1 << 6, + CUR0 = 1 << 7, + Bckp = 1 << 8, + Buff = 1 << 9, + EM = EM_012 | EM_345, + AUX = AUX_012 | AUX_345, + EM0 = EM0_012 | EM0_345, }; } // namespace Comm diff --git a/src/global/traits/archetypes.h b/src/global/traits/archetypes.h index e170170fb..4c9c443c1 100644 --- a/src/global/traits/archetypes.h +++ b/src/global/traits/archetypes.h @@ -24,6 +24,8 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" + #include template @@ -202,11 +204,11 @@ namespace traits::fieldsetter { template concept FieldSetterClass = - ((S == ntt::SimEngine::SRPIC) and + ((::traits::engine::IsSR) and (::traits::fieldsetter::HasEx1 or ::traits::fieldsetter::HasEx2 or ::traits::fieldsetter::HasEx3 or ::traits::fieldsetter::HasBx1 or ::traits::fieldsetter::HasBx2 or ::traits::fieldsetter::HasBx3)) or - ((S == ntt::SimEngine::GRPIC) and + ((::traits::engine::IsGR) and ((::traits::fieldsetter::HasDx1 and ::traits::fieldsetter::HasDx2 and ::traits::fieldsetter::HasDx3) or (::traits::fieldsetter::HasBx1 and ::traits::fieldsetter::HasBx2 and diff --git a/src/global/traits/engine.h b/src/global/traits/engine.h new file mode 100644 index 000000000..1993c828d --- /dev/null +++ b/src/global/traits/engine.h @@ -0,0 +1,95 @@ +/** + * @file traits/engine.h + * @brief Defines a set of traits to check if an engine class satisfies certain conditions + * @implements + * - IsSR<> - checks if engine is a special relativistic engine + * - isSR() - checks if a SimEngine enum instance corresponds to a special relativistic engine + * - IsGR<> - checks if engine is a general relativistic engine + * - isGR() - checks if a SimEngine enum instance corresponds to a general relativistic engine + * - VelocitiesInCartesianBasis<> - checks if engine has velocities in Cartesian basis + * - VelocitiesInCovariantBasis<> - checks if engine has velocities in covariant basis + * - StressEnergyInTetradBasis<> - checks if engine needs stress-energy tensor in tetrad basis + * - StressEnergyInContravariantBasis<> - checks if engine needs stress-energy tensor in contravariant basis + * - UserFieldsInTetradBasis<> - checks if engine expects user-defined fields in tetrad basis + * - userFieldsInTetradBasis() - checks if a SimEngine enum instance corresponds + * to an engine that expects user-defined fields in tetrad basis + * - HasImplicitPhiCoordinate<> - checks if engine has particles with an implicit phi coordinate + * @namespaces: + * - ::traits::engine:: + */ +#ifndef TRAITS_ENGINE_H +#define TRAITS_ENGINE_H + +#include "enums.h" + +#include "traits/metric.h" + +namespace traits::engine { + + template + concept IsSR = (S == ntt::SimEngine::SRPIC) or (S == ntt::SimEngine::HYBRID); + + [[nodiscard]] + constexpr auto isSR(ntt::SimEngine s) noexcept -> bool { + return (s == ntt::SimEngine::SRPIC) or (s == ntt::SimEngine::HYBRID); + } + + template + concept IsGR = (S == ntt::SimEngine::GRPIC); + + [[nodiscard]] + constexpr auto isGR(ntt::SimEngine s) noexcept -> bool { + return s == ntt::SimEngine::GRPIC; + } + + template + concept VelocitiesInCartesianBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept VelocitiesInCovariantBasis = (S == ntt::SimEngine::GRPIC); + + template + concept StressEnergyInTetradBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept StressEnergyInContravariantBasis = (S == ntt::SimEngine::GRPIC); + + template + concept DefinesEM0Fields = (S == ntt::SimEngine::GRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept DefinesAuxFields = (S == ntt::SimEngine::GRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept DefinesCur0Fields = (S == ntt::SimEngine::GRPIC); + + template + concept WriteEM0FieldToCheckpoint = (S == ntt::SimEngine::GRPIC); + + template + concept WriteCurFieldToCheckpoint = (S == ntt::SimEngine::GRPIC); + + template + concept UserFieldsInTetradBasis = (S == ntt::SimEngine::SRPIC) or + (S == ntt::SimEngine::HYBRID); + + template + concept UserFieldsInContravariantBasis = (S == ntt::SimEngine::GRPIC); + + [[nodiscard]] + constexpr auto userFieldsInTetradBasis(ntt::SimEngine s) noexcept -> bool { + return (s == ntt::SimEngine::SRPIC) or (s == ntt::SimEngine::HYBRID); + } + + template + concept HasImplicitPhiCoordinate = IsSR and ::traits::metric::HasPrtlDim and + ::traits::metric::HasD and + M::Dim == Dim::_2D and M::PrtlDim == Dim::_3D; + +} // namespace traits::engine + +#endif // TRAITS_ENGINE_H diff --git a/src/global/utils/timer.h b/src/global/utils/timer.h index 1823005d3..95e28345a 100644 --- a/src/global/utils/timer.h +++ b/src/global/utils/timer.h @@ -58,6 +58,13 @@ namespace timer { Timers(std::initializer_list names, const std::function& synchronize = nullptr, const bool& blocking = false) + : Timers(std::vector { names }, synchronize, blocking) {} + + // vector overload: lets the caller pick the timer set at runtime + // (e.g. an engine-specific list built behind `if constexpr`) + Timers(const std::vector& names, + const std::function& synchronize = nullptr, + const bool& blocking = false) : m_blocking { blocking } , m_synchronize { synchronize } { raise::ErrorIf((synchronize == nullptr) && blocking, diff --git a/src/kernels/currents_deposit.hpp b/src/kernels/currents_deposit.hpp index 98fa0b661..6bce809eb 100644 --- a/src/kernels/currents_deposit.hpp +++ b/src/kernels/currents_deposit.hpp @@ -6,16 +6,21 @@ * (`kernel::DepositOneParticle`): * - `kernel::DepositCurrents_kernel` flat (RangePolicy over particles, * writes into a `Kokkos::Experimental::ScatterView`). Always available. - * - `kernel::DepositCurrents_kernel_tiled` team-policy + * - `kernel::DepositCurrentsTiled_kernel` team-policy * (one team per spatial tile, accumulates into team SLM scratch with * atomic adds, then flushes to global J). Available when `team_policy=ON` - * (`#if defined(TEAM_POLICY)`). Stream 2 of the Pattern A plan. + * (`#if defined(TEAM_POLICY)`). A thin wrapper over the generic + * `kernel::TiledScatter_kernel` harness (kernels/tiled_scatter.hpp): + * the currents-specific part is only `CurrentsDepositBody`, which + * computes the per-particle footprint from the stored i/i_prev and + * feeds `DepositOneParticle` through the harness sink. * * @implements * - kernel::deposit::PrtlPack<> * - kernel::DepositOneParticle<> * - kernel::DepositCurrents_kernel<> - * - kernel::DepositCurrents_kernel_tiled<> (TEAM_POLICY only) + * - kernel::CurrentsDepositBody<> (TEAM_POLICY only) + * - kernel::DepositCurrentsTiled_kernel<> (TEAM_POLICY only) * @namespaces: * - kernel:: * - kernel::deposit:: @@ -28,12 +33,14 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" #include "framework/containers/particles.h" #include "kernels/particle_shapes.hpp" +#include "kernels/tiled_scatter.hpp" #include #include @@ -752,21 +759,85 @@ namespace kernel { } }; +#if defined(TEAM_POLICY) + /** + * @brief Per-particle body of the tiled current deposit. + * + * Implements the `TiledScatter_kernel` body contract: computes the + * particle's conservative deposit footprint from the stored `i`/`i_prev` + * pair, `select()`s it on the sink (which decides SLM scratch vs the + * per-particle global escape valve), then runs the shared + * `DepositOneParticle` math with the sink as the `deposit_at` callback. + * + * One-sided footprint reach: the deposit writes at most FOOTPRINT_REACH + * cells above max(i,i_prev) (and fewer below min), so + * [min(i,i_prev) - FOOTPRINT_REACH, max(i,i_prev) + FOOTPRINT_REACH] in + * cell coords conservatively bounds every deposited cell for any order + * (Esirkepov reaches max+O; O=0 zigzag reaches max+1). When the whole + * footprint fits the tile scratch window, every deposited cell is + * provably inside it and the scratch writes need no per-cell bounds + * test; otherwise the WHOLE particle goes to the bounds-clipped global + * path (see tiled_scatter.hpp for why this is charge-conserving). + */ + template + struct CurrentsDepositBody { + static_assert(O <= 11u, "Shape order O must be <= 11"); + static constexpr int FOOTPRINT_REACH = (O == 0u) ? 1 : static_cast(O); + + ParticleArrays prtls; + const M metric; + const real_t charge, inv_dt; + + CurrentsDepositBody(const ParticleArrays& prtls, + const M& metric, + real_t charge, + real_t dt) + : prtls { prtls } + , metric { metric } + , charge { charge } + , inv_dt { ONE / dt } {} + + template + Inline void operator()(prtlidx_t p, Sink& sink) const { + constexpr auto D = M::Dim; + const int G = static_cast(N_GHOSTS); + const int i1c = prtls.i1(p), i1p = prtls.i1_prev(p); + if constexpr (D == Dim::_1D) { + sink.select((i1c < i1p ? i1c : i1p) + G - FOOTPRINT_REACH, + (i1c > i1p ? i1c : i1p) + G + FOOTPRINT_REACH); + } else if constexpr (D == Dim::_2D) { + const int i2c = prtls.i2(p), i2p = prtls.i2_prev(p); + sink.select((i1c < i1p ? i1c : i1p) + G - FOOTPRINT_REACH, + (i1c > i1p ? i1c : i1p) + G + FOOTPRINT_REACH, + (i2c < i2p ? i2c : i2p) + G - FOOTPRINT_REACH, + (i2c > i2p ? i2c : i2p) + G + FOOTPRINT_REACH); + } else { + const int i2c = prtls.i2(p), i2p = prtls.i2_prev(p); + const int i3c = prtls.i3(p), i3p = prtls.i3_prev(p); + sink.select((i1c < i1p ? i1c : i1p) + G - FOOTPRINT_REACH, + (i1c > i1p ? i1c : i1p) + G + FOOTPRINT_REACH, + (i2c < i2p ? i2c : i2p) + G - FOOTPRINT_REACH, + (i2c > i2p ? i2c : i2p) + G + FOOTPRINT_REACH, + (i3c < i3p ? i3c : i3p) + G - FOOTPRINT_REACH, + (i3c > i3p ? i3c : i3p) + G + FOOTPRINT_REACH); + } + DepositOneParticle(p, prtls, metric, charge, inv_dt, sink); + } + }; + /** * @brief Tiled current-deposition kernel. * - * One team per spatial tile (`league_size = ntiles_total`). Each team - * accumulates particle contributions into a per-team scratch buffer of - * shape `(T_TILE + 2*HALO)^D × 3` real_t, where `HALO = O + 1` cells per - * side. Scratch atomics live in SLM (PVC: ~5–10 cycles per - * `atomic_add`); the global J is touched only once per scratch cell at - * flush time. Compared with the flat scatter-view kernel: - * - global atomic pressure ~ (T_TILE + 2*HALO)^D × 3 per tile - * instead of (stencil writes per particle × particles) - * - per-particle stencil writes are tile-local (SLM) instead of - * scattering through global HBM + * A thin wrapper: `TiledScatter_kernel` (kernels/tiled_scatter.hpp) + * carries the whole team/scratch/flush harness — one team per spatial + * tile, per-team scratch of shape `(T_TILE + 2*HALO)^D x 3`, SLM atomics + * for in-tile particles, the per-particle global escape valve, the + * particle-slice clamp to the live `npart`, and the bounds-clipped + * cooperative flush to global J. This class only fixes the template + * arguments (NC = NG = 3, REACH = STENCIL_REACH(O)) and keeps the + * public name + constructor signature the engine launchers use. * - * Supports `O ∈ {0, ..., 11}`. `O == 0` (zigzag) is wired for + * Supports `O in {0, ..., 11}`. `O == 0` (zigzag) is wired for * A/B benchmarking against the flat scatter-view kernel — its narrow * stencil typically makes scratch alloc/zero/flush overhead a * regression there, but it's good to be able to measure the @@ -778,466 +849,58 @@ namespace kernel { * Particle iteration order is governed by `tile_offsets`: tile `t` * owns particles `[tile_offsets(t), tile_offsets(t+1))`, post-sort. * `SortSpatially` (`particles_sort.cpp`) is responsible for keeping - * the SoA arrays consistent with that. + * the SoA arrays consistent with that. Particles appended past the + * partition are deposited by the launcher's flat tail pass (see the + * partition-coverage note in tiled_scatter.hpp). * - * **Halo sizing and escape valve.** Sort runs at the end of a step - * (see `srpic.hpp`); a particle is pushed once per step thereafter, so - * its `min(i, i_prev)` may differ from the bin key by one cell of drift - * per step elapsed since the last sort. The scratch HALO is - * `STENCIL_REACH(O) + DRIFT`, where `STENCIL_REACH = 2` for zigzag - * (writes `{i_prev, i_prev+1, i, i+1}` ⇒ +2 above `min(i, i_prev)` with - * `|Δi|=1`) and `O` for Esirkepov. `DRIFT` is the `team_policy_drift` - * CMake knob (macro TEAM_POLICY_DRIFT) — the number of cells a particle - * may drift between two sorts that the halo is sized to absorb — and `1` - * by default (the every-step-sorted common case). It is independent of - * the sort cadence, which is set at runtime via `spatial_sorting_interval`; - * particles that drift past the halo take the escape valve below. + * **Halo sizing.** Sort runs at the end of a step (see `srpic.hpp`); a + * particle is pushed once per step thereafter, so its `min(i, i_prev)` + * may differ from the bin key by one cell of drift per step elapsed + * since the last sort. The scratch HALO is `STENCIL_REACH(O) + DRIFT`: * - * Correctness does **not** depend on the halo size. Any particle whose - * full stencil escapes the scratch tile — because it drifted further - * than `DRIFT`, was reordered far from its tile by a no-sort-step - * `CommunicateParticles`, or because the halo is otherwise undersized — - * is deposited *as a whole* via a direct, bounds-clipped - * `Kokkos::atomic_add` on the global J view (the per-particle escape - * valve). Each particle's stencil is therefore deposited exactly once - * (entirely to SLM scratch when it fits, entirely to global J when it - * does not), so the path is charge-conserving; it is merely slower per - * write. Sizing `DRIFT` to the typical between-sort drift keeps the - * common case in fast SLM; sorting less often (or drifting past the - * halo) only costs escape-valve traffic, never accuracy. + * stencil_reach(O) — maximum cells the deposit writes ABOVE + * min(i, i_prev) under CFL |v * dt/dx| <= 1/2: + * - O == 0 (zigzag): writes { i_prev, i_prev+1, i, i+1 } => +2 + * - O >= 1 Esirkepov: `for_deposit` returns an (O+2)-wide + * array but only O+1 entries are non-zero, and the union + * window satisfies `i_max - i_min <= O+1` (see + * particle_shapes.hpp::for_deposit). The genuine one-sided + * reach above min(i, i_prev) is therefore O, not O+1 — the + * old `O+1` carried one extra cell of conservative padding + * on top of the already-conservative drift term. * - * **Partition coverage.** The team iteration covers only the particles - * partitioned at the last sort, `[0, layout.npart_partitioned)`, clamped - * to the live `npart`. Particles appended past the partition since the - * sort are not seen here; the launcher (`engines/srpic/currents.h`) - * deposits that tail with the flat kernel so every active particle is - * covered exactly once regardless of sort cadence. + * `DRIFT` (the `team_policy_drift` CMake knob) and the escape-valve / + * charge-conservation argument are documented on the harness. */ template - class DepositCurrentsTiled_kernel { + class DepositCurrentsTiled_kernel + : public TiledScatter_kernel(O)), + T_TILE, + CurrentsDepositBody> { static_assert(O <= 11u, "Shape order O must be <= 11"); - static_assert(T_TILE > 0u, "T_TILE must be positive"); - static constexpr auto D = M::Dim; - /** - * Per-side scratch halo, derived from first principles. - * - * total halo = stencil_reach(O) + drift_between_sort_and_deposit - * - * stencil_reach(O) — maximum cells the deposit writes ABOVE - * min(i, i_prev) under CFL |v * dt/dx| <= 1/2: - * - O == 0 (zigzag): writes { i_prev, i_prev+1, i, i+1 } => +2 - * - O >= 1 Esirkepov: `for_deposit` returns an (O+2)-wide - * array but only O+1 entries are non-zero, and the union - * window satisfies `i_max - i_min <= O+1` (see - * particle_shapes.hpp::for_deposit). The genuine one-sided - * reach above min(i, i_prev) is therefore O, not O+1 — the - * old `O+1` carried one extra cell of conservative padding - * on top of the already-conservative drift term below. - * - * drift — sort runs at end-of-step (see srpic.hpp), so a particle is - * pushed once per step between its last sort and a given deposit. With - * a runtime sort interval of `K` (spatial_sorting_interval), a particle - * drifts at most `K` cells (CFL |v dt/dx| <= 1/2 ⇒ |Δi| <= 1 per step) - * before the next sort. The `team_policy_drift` CMake knob (macro - * TEAM_POLICY_DRIFT) sets DRIFT independently of `K`, sizing the halo so - * a particle that drifts up to DRIFT cells still deposits inside its - * tile scratch. DRIFT defaults to 1 (the sorted-every-step common case); - * any particle that drifts past the halo (e.g. a larger sort interval, - * or a CFL excursion) takes the per-particle global-J escape valve - * below — correct, only slower (see - * the class doc-comment for why this is charge-conserving). - */ - static constexpr int STENCIL_REACH = (O == 0u) ? 2 : static_cast(O); - // One-sided footprint reach for the per-particle escape valve: the - // deposit writes at most this many cells above max(i,i_prev) (and fewer - // below min), so [min - FOOTPRINT_REACH, max + FOOTPRINT_REACH] in cell - // coords conservatively bounds every deposited cell for any order - // (Esirkepov reaches max+O; O=0 zigzag reaches max+1). - static constexpr int FOOTPRINT_REACH = (O == 0u) ? 1 : static_cast(O); -#if defined(TEAM_POLICY_DRIFT) - static constexpr int DRIFT = static_cast(TEAM_POLICY_DRIFT); -#else - static constexpr int DRIFT = 1; -#endif - static constexpr int HALO = STENCIL_REACH + DRIFT; - static constexpr int TE = static_cast(T_TILE) + 2 * HALO; - - using exec_space = Kokkos::DefaultExecutionSpace; - using team_policy = Kokkos::TeamPolicy; - using member_t = typename team_policy::member_type; - - ndfield_t J; - ParticleArrays prtls; - const M metric; - const real_t charge, inv_dt; - - // Tile metadata produced by SortSpatially. - array_t tile_offsets; - ncells_t ntx1 { 1u }, ntx2 { 1u }, ntx3 { 1u }; - ncells_t total_tiles { 0u }; - - /** - * Current active-particle count. `tile_offsets` partitions only the - * particles that existed at the last sort ([0, layout.npart_partitioned)); - * `npart` may differ if the pusher dead-tagged particles in place since. - * Each team clamps its `[tile_offsets(t), tile_offsets(t+1))` slice to - * `npart` so stale slots past the live array are never read. Particles - * appended *beyond* the partition (npart > npart_partitioned) are not seen - * by any team here — the launcher deposits that tail separately. - */ - npart_t npart { 0u }; - - /** - * J's full storage extent including all ghost cells. Used to clip - * the cooperative flush so that a partial tile at the high end of - * the domain does not over-write past the J view. - */ - int j_ext1 { 0 }, j_ext2 { 0 }, j_ext3 { 0 }; + using body_t = CurrentsDepositBody; + using base_t = TiledScatter_kernel(O)), + T_TILE, + body_t>; public: - DepositCurrentsTiled_kernel(const ndfield_t& cur, - const ParticleArrays& prtls, - const M& metric, - real_t charge, - real_t dt, - const TileLayout& layout, - npart_t npart) - : J { cur } - , prtls { prtls } - , metric { metric } - , charge { charge } - , inv_dt { ONE / dt } - , tile_offsets { layout.tile_offsets } - , ntx1 { layout.ntiles_per_axis[0] } - , ntx2 { layout.ntiles_per_axis[1] } - , ntx3 { layout.ntiles_per_axis[2] } - , total_tiles { layout.ntiles_total } - , npart { npart } { - raise::ErrorIf( - layout.tile_size != T_TILE, - "Tiled deposit launched with mismatched T_TILE and runtime tile_size", - HERE); - /** - * @note: HALO is allowed to exceed N_GHOSTS. The cooperative - * scratch→J flush and the per-particle escape valve both bounds-clip - * their writes against `j_ext*` so writes that would land past J's - * ghost stripe are silently dropped (they only ever come from a - * particle whose stencil reaches into the domain ghost region, where - * CommunicateFields will re-supply the contribution). - */ - if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { - j_ext1 = static_cast(cur.extent(0)); - } - if constexpr (D == Dim::_2D or D == Dim::_3D) { - j_ext2 = static_cast(cur.extent(1)); - } - if constexpr (D == Dim::_3D) { - j_ext3 = static_cast(cur.extent(2)); - } - } - - /** - * @brief Per-team scratch size in bytes. Used by the launcher to set - * `team_policy.set_scratch_size(0, Kokkos::PerTeam(bytes))`. - */ - static constexpr size_t scratch_bytes() { - // The component count (3) is a *static* extent of scratch_ndfield_t - // (View / **[3] / ***[3]), so shmem_size() takes only the - // dynamic spatial extents — passing 3 as well trips Kokkos' - // `rank_dynamic != number of arguments` abort. This matches the - // scratch View construction below, which also omits the 3. - if constexpr (D == Dim::_1D) { - return scratch_ndfield_t::shmem_size(TE); - } else if constexpr (D == Dim::_2D) { - return scratch_ndfield_t::shmem_size(TE, TE); - } else { - return scratch_ndfield_t::shmem_size(TE, TE, TE); - } - } - - Inline void operator()(const member_t& team) const { - const auto tile_id = static_cast(team.league_rank()); - /** - * Tile coordinates (tile-grid indices) → tile origin in **active** - * cell coords (no ghost offset). Using ncells_t to match the linearised - * tile index produced by SortSpatially. - */ - ncells_t tx1 = 0, tx2 = 0, tx3 = 0; - if constexpr (D == Dim::_1D) { - tx1 = tile_id; - } else if constexpr (D == Dim::_2D) { - tx1 = tile_id / ntx2; - tx2 = tile_id - tx1 * ntx2; - } else { - const auto plane = ntx2 * ntx3; - tx1 = tile_id / plane; - const auto rem = tile_id - tx1 * plane; - tx2 = rem / ntx3; - tx3 = rem - tx2 * ntx3; - } - /** - * origin_active = lowest active-cell index in the tile (no ghost). - * origin_J = same value translated into J's storage coordinate - * (i.e. plus N_GHOSTS). - * origin_J_low = J coordinate of scratch index 0 (i.e. origin_J - HALO). - * local index `li` in scratch ↔ global J index `gi = li + origin_J_low`. - */ - const int origin_J1_low = static_cast(tx1 * T_TILE) + - static_cast(N_GHOSTS) - HALO; - const int origin_J2_low = static_cast(tx2 * T_TILE) + - static_cast(N_GHOSTS) - HALO; - const int origin_J3_low = static_cast(tx3 * T_TILE) + - static_cast(N_GHOSTS) - HALO; - - // Allocate scratch and cooperatively zero-fill it. - if constexpr (D == Dim::_1D) { - scratch_ndfield_t scr { team.team_scratch(0), TE }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, TE * 3), - [&](ncells_t idx) { - const auto li = idx / 3; - const auto c = idx - li * 3; - scr(li, c) = ZERO; - }); - team.team_barrier(); - - // Clamp the tile's particle slice to the live array: slots past - // `npart` may hold stale (possibly alive-tagged) data from a prior - // step's compaction and must not be re-deposited. - const auto t_lo = tile_offsets(tile_id); - const auto t_hi = tile_offsets(tile_id + 1u); - const auto p_begin = (t_lo < npart) ? t_lo : npart; - const auto p_end = (t_hi < npart) ? t_hi : npart; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, p_begin, p_end), - [&](prtlidx_t p) { - /** - * Per-particle escape valve: route the WHOLE particle to the - * global J view when its Esirkepov footprint does not fit - * inside this tile's scratch window [0,TE); only particles - * fully inside the tile touch SLM scratch. A particle drifts - * out of its tile when sorted less often than every step. - * - * The conservative footprint bound in cell coords, - * [min(i,i_prev) - O, max(i,i_prev) + O], covers - * prtl_shape::for_deposit for any order (i_min >= - * min-floor(O/2), i_max <= max+O), so when `to_scratch` is true - * every deposited cell is provably in [0,TE) and the scratch write - * needs no per-cell bounds test. The global path bounds-clips - * against J's storage extent (writes past the ghost stripe are - * re-supplied by SynchronizeFields(J)). - */ - const int i1c = prtls.i1(p), i1p = prtls.i1_prev(p); - const int lo1 = (i1c < i1p ? i1c : i1p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J1_low; - const int hi1 = (i1c > i1p ? i1c : i1p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J1_low; - const bool to_scratch = (lo1 >= 0 and hi1 < TE); - DepositOneParticle( - p, - prtls, - metric, - charge, - inv_dt, - [&](int g_i1, int comp, real_t v) { - if (to_scratch) { - Kokkos::atomic_add(&scr(g_i1 - origin_J1_low, comp), v); - } else if (g_i1 >= 0 and g_i1 < j_ext1) { - // Bounds-clip the escape-valve write against J's storage, - // exactly as the cooperative flush does. Cells past the - // ghost stripe are re-supplied by SynchronizeFields(J); an - // unclipped write here faults the GPU (an escaped boundary - // particle's stencil can reach past j_ext1). - Kokkos::atomic_add(&J(g_i1, comp), v); - } - }); - }); - team.team_barrier(); - - /** - * Cooperative flush of scratch to global J. Bounds-clip against - * the J view extent in case a partial high-end tile (or non-zero - * halo at domain edges) would otherwise write past J. - */ - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, TE * 3), - [&](const int idx) { - const auto li = idx / 3; - const auto c = idx - li * 3; - const auto gi = li + origin_J1_low; - if (gi < 0 or gi >= j_ext1) { - return; - } - const real_t v = scr(li, c); - if (v != ZERO) { - Kokkos::atomic_add(&J(gi, c), v); - } - }); - } else if constexpr (D == Dim::_2D) { - scratch_ndfield_t scr { team.team_scratch(0), TE, TE }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, SQR(TE) * 3), - [&](const int idx) { - const auto lij = idx / 3; - const auto c = idx - lij * 3; - const auto li = lij / TE; - const auto lj = lij - li * TE; - scr(li, lj, c) = ZERO; - }); - team.team_barrier(); - - // Clamp the tile's particle slice to the live array: slots past - // `npart` may hold stale (possibly alive-tagged) data from a prior - // step's compaction and must not be re-deposited. - const auto t_lo = tile_offsets(tile_id); - const auto t_hi = tile_offsets(tile_id + 1u); - const auto p_begin = (t_lo < npart) ? t_lo : npart; - const auto p_end = (t_hi < npart) ? t_hi : npart; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, p_begin, p_end), - [&](prtlidx_t p) { - // See 1D branch for rationale: route the whole particle to the - // global escape valve unless its full footprint fits in scratch. - const int i1c = prtls.i1(p), i1p = prtls.i1_prev(p); - const int i2c = prtls.i2(p), i2p = prtls.i2_prev(p); - const int lo1 = (i1c < i1p ? i1c : i1p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J1_low; - const int hi1 = (i1c > i1p ? i1c : i1p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J1_low; - const int lo2 = (i2c < i2p ? i2c : i2p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J2_low; - const int hi2 = (i2c > i2p ? i2c : i2p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J2_low; - const bool to_scratch = (lo1 >= 0 and hi1 < TE and lo2 >= 0 and - hi2 < TE); - DepositOneParticle( - p, - prtls, - metric, - charge, - inv_dt, - [&](const int g_i1, const int g_i2, int comp, real_t v) { - if (to_scratch) { - Kokkos::atomic_add( - &scr(g_i1 - origin_J1_low, g_i2 - origin_J2_low, comp), - v); - } else if (g_i1 >= 0 and g_i1 < j_ext1 and g_i2 >= 0 and - g_i2 < j_ext2) { - // Bounds-clip as the cooperative flush does; an unclipped - // escape-valve write faults the GPU when an escaped boundary - // particle's stencil reaches past j_ext. - Kokkos::atomic_add(&J(g_i1, g_i2, comp), v); - } - }); - }); - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, SQR(TE) * 3), - [&](const int idx) { - const auto lij = idx / 3; - const auto c = idx - lij * 3; - const auto li = lij / TE; - const auto lj = lij - li * TE; - const auto gi = li + origin_J1_low; - const auto gj = lj + origin_J2_low; - if ((gi < 0 or gi >= j_ext1) or - (gj < 0 or gj >= j_ext2)) { - return; - } - const real_t v = scr(li, lj, c); - if (v != ZERO) { - Kokkos::atomic_add(&J(gi, gj, c), v); - } - }); - } else if constexpr (D == Dim::_3D) { - scratch_ndfield_t scr { team.team_scratch(0), TE, TE, TE }; - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, CUBE(TE) * 3), - [&](const int idx) { - const auto lijk = idx / 3; - const auto c = idx - lijk * 3; - const auto li = lijk / (TE * TE); - const auto rem = lijk - li * TE * TE; - const auto lj = rem / TE; - const auto lk = rem - lj * TE; - scr(li, lj, lk, c) = ZERO; - }); - team.team_barrier(); - - // Clamp the tile's particle slice to the live array: slots past - // `npart` may hold stale (possibly alive-tagged) data from a prior - // step's compaction and must not be re-deposited. - const auto t_lo = tile_offsets(tile_id); - const auto t_hi = tile_offsets(tile_id + 1u); - const auto p_begin = (t_lo < npart) ? t_lo : npart; - const auto p_end = (t_hi < npart) ? t_hi : npart; - Kokkos::parallel_for( - Kokkos::TeamThreadRange(team, p_begin, p_end), - [&](prtlidx_t p) { - // See 1D branch for rationale: route the whole particle to the - // global escape valve unless its full footprint fits in scratch. - const int i1c = prtls.i1(p), i1p = prtls.i1_prev(p); - const int i2c = prtls.i2(p), i2p = prtls.i2_prev(p); - const int i3c = prtls.i3(p), i3p = prtls.i3_prev(p); - const int lo1 = (i1c < i1p ? i1c : i1p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J1_low; - const int hi1 = (i1c > i1p ? i1c : i1p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J1_low; - const int lo2 = (i2c < i2p ? i2c : i2p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J2_low; - const int hi2 = (i2c > i2p ? i2c : i2p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J2_low; - const int lo3 = (i3c < i3p ? i3c : i3p) + static_cast(N_GHOSTS) - - FOOTPRINT_REACH - origin_J3_low; - const int hi3 = (i3c > i3p ? i3c : i3p) + static_cast(N_GHOSTS) + - FOOTPRINT_REACH - origin_J3_low; - const bool to_scratch = (lo1 >= 0 and hi1 < TE and lo2 >= 0 and - hi2 < TE and lo3 >= 0 and hi3 < TE); - DepositOneParticle( - p, - prtls, - metric, - charge, - inv_dt, - [&](const int g_i1, const int g_i2, const int g_i3, int comp, real_t v) { - if (to_scratch) { - Kokkos::atomic_add(&scr(g_i1 - origin_J1_low, - g_i2 - origin_J2_low, - g_i3 - origin_J3_low, - comp), - v); - } else if (g_i1 >= 0 and g_i1 < j_ext1 and g_i2 >= 0 and - g_i2 < j_ext2 and g_i3 >= 0 and g_i3 < j_ext3) { - // Bounds-clip as the cooperative flush does; an unclipped - // escape-valve write faults the GPU when an escaped boundary - // particle's stencil reaches past j_ext. - Kokkos::atomic_add(&J(g_i1, g_i2, g_i3, comp), v); - } - }); - }); - team.team_barrier(); - - Kokkos::parallel_for(Kokkos::TeamThreadRange(team, CUBE(TE) * 3), - [&](const int idx) { - const int lijk = idx / 3; - const int c = idx - lijk * 3; - const int li = lijk / (TE * TE); - const int rem = lijk - li * TE * TE; - const int lj = rem / TE; - const int lk = rem - lj * TE; - const int gi = li + origin_J1_low; - const int gj = lj + origin_J2_low; - const int gk = lk + origin_J3_low; - if ((gi < 0 or gi >= j_ext1) or - (gj < 0 or gj >= j_ext2) or - (gk < 0 or gk >= j_ext3)) { - return; - } - const real_t v = scr(li, lj, lk, c); - if (v != ZERO) { - Kokkos::atomic_add(&J(gi, gj, gk, c), v); - } - }); - } - } + DepositCurrentsTiled_kernel(const ndfield_t& cur, + const ParticleArrays& prtls, + const M& metric, + real_t charge, + real_t dt, + const TileLayout& layout, + npart_t npart) + : base_t { cur, body_t { prtls, metric, charge, dt }, layout, npart } {} }; +#endif // TEAM_POLICY } // namespace kernel diff --git a/src/kernels/fields_bcs.hpp b/src/kernels/fields_bcs.hpp index 729a09d10..61e07f5c7 100644 --- a/src/kernels/fields_bcs.hpp +++ b/src/kernels/fields_bcs.hpp @@ -21,6 +21,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -94,7 +95,7 @@ namespace kernel::bc { if constexpr (M::Dim == Dim::_1D) { const auto i1_ = COORD(i1); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { coord_t x_Ph_0 { ZERO }; coord_t x_Ph_H { ZERO }; metric.template convert({ i1_ }, x_Ph_0); @@ -163,8 +164,7 @@ namespace kernel::bc { } } } else { - // GRPIC - raise::KernelError(HERE, "1D GRPIC not implemented"); + raise::KernelError(HERE, "not implemented"); } } else { raise::KernelError( @@ -196,13 +196,15 @@ namespace kernel::bc { if constexpr (HasEx1 or HasDx1) { if ((tags & BC::E) or (tags & BC::D)) { - if constexpr (HasEx1 and S == SimEngine::SRPIC) { + if constexpr (HasEx1 and + ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex1) = s * Fld(i1, i2, em::ex1) + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, fset.ex1(x_Ph_H0)); - } else if constexpr (HasDx1 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx1 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx1) = s * Fld(i1, i2, em::dx1) + (ONE - s) * fset.dx1(x_Ph_H0); } @@ -211,13 +213,13 @@ namespace kernel::bc { if constexpr (HasBx2) { if (tags & BC::B) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( { i1_ + HALF, i2_ }, fset.bx2(x_Ph_H0)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx2) = s * Fld(i1, i2, em::bx2) + (ONE - s) * fset.bx2(x_Ph_H0); } @@ -242,13 +244,15 @@ namespace kernel::bc { if constexpr (HasEx2 or HasDx2) { if ((tags & BC::E) or (tags & BC::D)) { - if constexpr (HasEx2 and S == SimEngine::SRPIC) { + if constexpr (HasEx2 and + ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex2) = s * Fld(i1, i2, em::ex2) + (ONE - s) * metric.template transform<2, Idx::T, Idx::U>( { i1_, i2_ + HALF }, fset.ex2(x_Ph_0H)); - } else if constexpr (HasDx2 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx2 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx2) = s * Fld(i1, i2, em::dx2) + (ONE - s) * fset.dx2(x_Ph_0H); } @@ -257,13 +261,13 @@ namespace kernel::bc { if constexpr (HasBx1) { if (tags & BC::B) { - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + (ONE - s) * metric.template transform<1, Idx::T, Idx::U>( { i1_, i2_ + HALF }, fset.bx1(x_Ph_0H)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx1) = s * Fld(i1, i2, em::bx1) + (ONE - s) * fset.bx1(x_Ph_0H); } @@ -288,7 +292,7 @@ namespace kernel::bc { coord_t x_Ph_00 { ZERO }; metric.template convert({ i1_, i2_ }, x_Ph_00); - if constexpr (HasEx3 and S == SimEngine::SRPIC) { + if constexpr (HasEx3 and ::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::ex3) = s * Fld(i1, i2, em::ex3); if ((!is_axis_i2min or (i2 > N_GHOSTS)) and (!is_axis_i2max or (i2 < extent_2 - N_GHOSTS))) { @@ -297,7 +301,8 @@ namespace kernel::bc { { i1_, i2_ }, fset.ex3(x_Ph_00)); } - } else if constexpr (HasDx3 and S == SimEngine::GRPIC) { + } else if constexpr ( + HasDx3 and (not ::traits::engine::UserFieldsInTetradBasis)) { Fld(i1, i2, em::dx3) = s * Fld(i1, i2, em::dx3); if ((!is_axis_i2min or (i2 > N_GHOSTS)) and (!is_axis_i2max or (i2 < extent_2 - N_GHOSTS))) { @@ -324,13 +329,13 @@ namespace kernel::bc { metric.template convert({ i1_ + HALF, i2_ + HALF }, x_Ph_HH); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + (ONE - s) * metric.template transform<3, Idx::T, Idx::U>( { i1_ + HALF, i2_ + HALF }, fset.bx3(x_Ph_HH)); - } else if constexpr (S == SimEngine::GRPIC) { + } else { Fld(i1, i2, em::bx3) = s * Fld(i1, i2, em::bx3) + (ONE - s) * fset.bx3(x_Ph_HH); } @@ -349,7 +354,7 @@ namespace kernel::bc { const auto i2_ = COORD(i2); const auto i3_ = COORD(i3); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::UserFieldsInTetradBasis) { // SRPIC if constexpr (HasEx1 or HasEx2 or HasEx3) { if (tags & BC::E) { @@ -509,8 +514,7 @@ namespace kernel::bc { } } } else { - // GRPIC - raise::KernelError(HERE, "GRPIC not implemented"); + raise::KernelError(HERE, "not implemented"); } } else { raise::KernelError( diff --git a/src/kernels/hybrid/EMF.hpp b/src/kernels/hybrid/EMF.hpp new file mode 100644 index 000000000..7810f7ca0 --- /dev/null +++ b/src/kernels/hybrid/EMF.hpp @@ -0,0 +1,1145 @@ +#ifndef KERNELS_HYBRID_EMF_HPP +#define KERNELS_HYBRID_EMF_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include + +namespace kernel::hybrid { + + /** + * Fields are stored in the contravariant (Idx::U) code basis of the Minkowski + * metric (cell size dx): components with index i <= D are physical/dx + * (h_ii = dx^2), out-of-plane components are physical (h_ii = 1). The + * deposited moments (PP = V, NN = N in `aux`) are physical (Cartesian XYZ + * velocity moments; N in units of n0). + * + * Inputs (Bf, Bfs, Ee_in) are U-basis. The outputs follow two conventions: + * - Ee_out (edge E, consumed by the Faraday curl and the trapezoidal + * average with Ee_in): U-basis, matching em/em0 storage. + * - Ec_out, Bc_out (cell-centered, read raw by the particle pusher and + * averaged with Ec = em0::012): physical (XYZ). + * + * Index-space differences of U-basis fields obey + * delta(b_inplane) = d(B_phys), delta(b_outofplane) = dx * d(B_phys), + * which is the origin of the per-term dx / dx_inv factors below. + */ + // EEONLY (requires INIT): compute and write ONLY the raw edge-E (Ee_out) -- + // used by the sub-cycled field advance, which refreshes Ee from frozen + // moments every sub-step and needs neither Ec nor Bc there. In this mode + // Bfs is the 3-component scratch (cur = the sub-stepped B) and Bf is unused + // (pass cur). + template + class EMF_kernel { + static_assert(INIT || not EEONLY, "EEONLY requires INIT (raw writes)"); + static constexpr uint8_t N1 = (INIT || EEONLY) ? 3 : 6; + static constexpr uint8_t N2 = (INIT && not EEONLY) ? 6 : 3; + + ndfield_t PP; + ndfield_t NN; + ndfield_t Ee_in; + ndfield_t Bf; + ndfield_t Ec; + ndfield_t Bfs; + + ndfield_t Ee_out; + ndfield_t Ec_out; + ndfield_t Bc_out; + + const uint8_t comp_PP; + const uint8_t comp_NN; + const uint8_t comp_Ee_in; + const uint8_t comp_Bf; + const uint8_t comp_Ec; + const uint8_t comp_Bfs; + + const uint8_t comp_Ee_out; + const uint8_t comp_Ec_out; + const uint8_t comp_Bc_out; + + const real_t dt; + const real_t gamma_ad; + const real_t theta; + const real_t d0; + const real_t rho0; + const real_t dens_min; + const real_t hall_lim; + const real_t resist_vac; + const real_t resist_hyper; + const real_t resist; + const real_t dx; + const real_t dx_inv; + + public: + EMF_kernel(const ndfield_t& PP, + const ndfield_t& NN, + const ndfield_t& Ee_in, + const ndfield_t& Bf, + const ndfield_t& Ec, + const ndfield_t& Bfs, + ndfield_t& Ee_out, + ndfield_t& Ec_out, + ndfield_t& Bc_out, + uint8_t comp_PP, + uint8_t comp_NN, + uint8_t comp_Ee_in, + uint8_t comp_Bf, + uint8_t comp_Ec, + uint8_t comp_Bfs, + uint8_t comp_Ee_out, + uint8_t comp_Ec_out, + uint8_t comp_Bc_out, + real_t dt, + real_t gamma_ad, + real_t theta, + real_t d0, + real_t rho0, + real_t dens_min, + real_t hall_lim, + real_t resist_vac, + real_t resist_hyper, + real_t resist, + real_t dx) + : PP { PP } + , NN { NN } + , Ee_in { Ee_in } + , Bf { Bf } + , Ec { Ec } + , Bfs { Bfs } + , Ee_out { Ee_out } + , Ec_out { Ec_out } + , Bc_out { Bc_out } + , comp_PP { comp_PP } + , comp_NN { comp_NN } + , comp_Ee_in { comp_Ee_in } + , comp_Bf { comp_Bf } + , comp_Ec { comp_Ec } + , comp_Bfs { comp_Bfs } + , comp_Ee_out { comp_Ee_out } + , comp_Ec_out { comp_Ec_out } + , comp_Bc_out { comp_Bc_out } + , dt { dt } + , gamma_ad { gamma_ad } + , theta { theta } + , d0 { d0 } + , rho0 { rho0 } + , dens_min { dens_min } + , hall_lim { hall_lim } + , resist_vac { resist_vac } + , resist_hyper { resist_hyper } + , resist { resist } + , dx { dx } + , dx_inv { ONE / dx } {} + + /** + * @brief Effective vacuum resistivity, clamped to the explicit-diffusion + * stability limit eta dt / dx^2 <= 0.4 for this kernel's dt (the + * sub-step dt inside the sub-cycled advance), so the user knob can + * never destabilize the update. + */ + Inline auto eta_vac() const -> real_t { + return math::min(resist_vac, static_cast(0.4) * SQR(dx) / dt); + } + + /** + * @brief Effective HYPER-resistivity (E -= eta_H grad^2 curl B, i.e. + * dB/dt -= eta_H grad^4 B), applied everywhere in compute_Ee. + * + * The moments filter creates a spectral gap: p binomial passes attenuate + * the deposited N, V by cos^(2p)(k dx / 2), which is ~zero at wavelengths + * of a few cells. In that band the field solver evolves B with no kinetic + * feedback, so the ion damping that would physically kill those modes never + * reaches Ohm's law, and sustained compression/shear in the frozen moments + * pumps the orphaned band by induction. Hyper-resistivity (damping ~ eta_H + * k^4) blankets the filtered band while leaving physical scales (k dx < 0.3) + * essentially untouched; size eta_H so eta_H (pi/dx)^4 exceeds the drive. + * + * Clamped to the explicit grad^4 stability limit for this kernel's dt (the + * sub-step dt inside the sub-cycled advance): the discrete grad^2 max + * eigenvalue is 4 D / dx^2, so eta_H (4 D / dx^2)^2 dt <= 1. + */ + Inline auto eta_hyper() const -> real_t { + constexpr real_t ndim = static_cast(static_cast(D)); + return math::min(resist_hyper, + SQR(SQR(dx)) / (SQR(static_cast(4) * ndim) * dt)); + } + + /** + * @brief Effective UNIFORM Ohmic resistivity, applied EVERYWHERE in + * compute_Ee: E += eta * curl B -> dB/dt += eta grad^2 B. + * + * The k^2 companion to the k^4 hyper-resistivity. Hyper-resistivity is + * concentrated at the grid cutoff (eta_H k^4) and barely touches the + * intermediate band (k dx ~ 0.1-0.3); a strong boundary/beam drive -- e.g. + * the counter-streaming foot of a high-Mach reflecting-wall shock -- pumps + * exactly that band, radiating a few-d_i wave train the filter+hyper stack + * leaves under-damped. A small uniform eta (damping ~ eta k^2) blankets it, + * as in dHybrid. Trades against physical magnetic diffusion, so keep it as + * small as stability allows (scan upward from ~0.05). + * + * Clamped to the explicit grad^2 diffusion stability limit for this kernel's + * dt (the sub-step dt inside the sub-cycled advance): the discrete grad^2 max + * eigenvalue is 4 D / dx^2, so eta (4 D / dx^2) dt <= 1 (a factor 2 below the + * hard <= 2 limit) makes any user knob safe. + */ + Inline auto eta_res() const -> real_t { + constexpr real_t ndim = static_cast(static_cast(D)); + return math::min(resist, + SQR(dx) / (static_cast(4) * ndim * dt)); + } + + /** + * @brief curl B at the Ee component locations, prefactored into the OUTPUT + * basis of compute_Ee (1D/2D: in-plane comps E/dx, out-of-plane + * physical; 3D: all E/dx). The resistive-vacuum E is +eta times + * these; the hyper-resistive E is -eta_H dx_inv^2 times their + * discrete Laplacian. + */ + Inline void res_curl(const tuple_t& i, + real_t& c0, + real_t& c1, + real_t& c2) const { + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + c0 = ZERO; // (curl B)_x = 0 in 1D + c1 = -dx_inv * (Bfs(i1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)); + c2 = dx_inv * (Bfs(i1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)); + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + c0 = SQR(dx_inv) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2)); + c1 = -SQR(dx_inv) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)); + c2 = (Bfs(i1, i2, comp_Bfs + 1) - Bfs(i1 - 1, i2, comp_Bfs + 1)) - + (Bfs(i1, i2, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0)); + } else { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + c0 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) - + (Bfs(i1, i2, i3, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1))); + c1 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) - + (Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 2))); + c2 = dx_inv * ((Bfs(i1, i2, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) - + (Bfs(i1, i2, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0))); + } + } + + /** + * @brief Per-cell limiter on the Hall term. The whistler speed implied at + * the grid cutoff, v_w = d0^2 pi |B| / (dx N), scales with the local + * |B|/N and can exceed the resolvable dx/dt at shock overshoots or + * magnetic cavities (large |B|, small N), where an explicit advance + * is unstable. The limiter scales the Hall term so the implied + * whistler Courant never exceeds `hall_lim`; it is the identity in + * resolvable cells. hall_lim <= 0 disables it. + */ + Inline auto hall_limiter(const tuple_t& i) const -> real_t { + if (hall_lim <= ZERO) { + return ONE; + } + // Worst case (max |B|, min N) over the +/-1 neighborhood the Hall stencil + // reads: a cell-local estimate misses a sharp front where the output + // cell's own B is small but the curl pulls in a much larger neighbor. + real_t bsq = ZERO; + real_t nn; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + nn = NN(i1, comp_NN); + for (int di = -1; di <= 1; ++di) { + const auto ii = i1 + static_cast(di); + // U-basis: b1 = Bx/dx, b2/b3 physical + const real_t b = SQR(dx * Bfs(ii, comp_Bfs + 0)) + + SQR(Bfs(ii, comp_Bfs + 1)) + + SQR(Bfs(ii, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, comp_NN)); + } + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + nn = NN(i1, i2, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + // U-basis: b1/b2 = B/dx, b3 physical + const real_t b = SQR(dx * Bfs(ii, jj, comp_Bfs + 0)) + + SQR(dx * Bfs(ii, jj, comp_Bfs + 1)) + + SQR(Bfs(ii, jj, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, comp_NN)); + } + } + } else { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + nn = NN(i1, i2, i3, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + for (int dk = -1; dk <= 1; ++dk) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + const auto kk = i3 + static_cast(dk); + // U-basis: all comps physical/dx + const real_t b = SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 0)) + + SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 1)) + + SQR(dx * Bfs(ii, jj, kk, comp_Bfs + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, kk, comp_NN)); + } + } + } + } + // v_w(B, N) = d0^2 pi |B| / (dx N); reduces to the hybrid-CFL formula + // (d0^2/rho0) pi / dx at B = B0 = 1/larmor0, N = 1 + const real_t v_w = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + // branch instead of min(1, x/y): avoids the 0/0 when B = 0 + return (v_w * dt > hall_lim * dx) ? (hall_lim * dx / (dt * v_w)) : ONE; + } + + /** + * @brief Vacuum cutoff: scales E by min(1, N_raw / dens_min). Below the + * density threshold a cell has no ions to define an E field, so E + * ramps to zero and B is left frozen there, instead of running the + * Ohm's law with a 1/dens_min-amplified right-hand side. The ramp is + * continuous, so cells do not flicker at the threshold and plasma-side + * interface cells keep evolving from their own E. + * + * A hard E = 0 vacuum is only safe when the interface is quiescent. + * When fast plasma sweeps ALONG a vacuum boundary, masking E to zero + * makes an artificial tangential-E discontinuity (a surface current) + * and Faraday pumps B at the interface at rate ~ u/dx. The optional + * resistive vacuum (resist_vac) crossfades E to eta * curl B below the + * threshold instead, so the vacuum diffuses toward a curl-free + * (potential) field and tangential E has a physical continuation. + */ + Inline auto vac_factor(real_t n_raw) const -> real_t { + // safe for dens_min <= 0: the branch is never taken + return (n_raw < dens_min) ? (n_raw / dens_min) : ONE; + } + + Inline void compute_Ee(const tuple_t& i, + real_t& E0, + real_t& E1, + real_t& E2) const { + // Hall/whistler coefficient d_i^2 * Omega_i = d0^2/rho0. Leading minus is + // INTENTIONAL: the (curl B) x B block below is summed into the E bracket, + // which is then multiplied by -1/N. A positive coeff would give + // E_Hall = -(d0^2/rho0)(curl B)xB/N -- the WRONG sign vs the generalized + // Ohm's law (Pegasus eq. 10 / standard Hall-MHD: +(d0^2/rho0)(curl B)xB/N). + // The minus restores the correct sign. (motional & e-pressure unaffected.) + // hall_limiter caps the implied local whistler Courant (see its doc). + const real_t coeff { -SQR(d0) / rho0 * hall_limiter(i) }; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + // Ee* = EMF(N^(n), P^(n), Bf*) + const real_t N0r { NN(i1, comp_NN) }; + const real_t N12r { INV_2 * (NN(i1, comp_NN) + NN(i1 - 1, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N12r, dens_min) }; + const real_t N2 { math::max(N12r, dens_min) }; + + // 1D basis: b1 = Bx/dx, b2/b3 physical; output e1 = Ex/dx, e2/e3 physical + E0 = dx_inv * (-Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 2) + + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 1)); + E1 = -INV_4 * (Bfs(i1, comp_Bfs + 2) + Bfs(i1 - 1, comp_Bfs + 2)) * + (PP(i1, comp_PP + 0) + PP(i1 - 1, comp_PP + 0)) + + dx * (INV_2) * (PP(i1, comp_PP + 2) + PP(i1 - 1, comp_PP + 2)) * + Bfs(i1, comp_Bfs + 0); + E2 = (INV_4) * (Bfs(i1, comp_Bfs + 1) + Bfs(i1 - 1, comp_Bfs + 1)) * + (PP(i1, comp_PP + 0) + PP(i1 - 1, comp_PP + 0)) - + dx * INV_2 * (PP(i1, comp_PP + 1) + PP(i1 - 1, comp_PP + 1)) * + Bfs(i1, comp_Bfs + 0); + + E0 += coeff * SQR(dx_inv) * + (-INV_2 * (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 1) - + INV_2 * (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 2)); + E1 += coeff * ((Bfs(i1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 0)); + E2 += coeff * ((Bfs(i1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 0)); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac12 { vac_factor(N12r) }; + E0 *= -vac0 / N0; + E1 *= -vac12 / N1; + E2 *= -vac12 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO or resist > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist > ZERO) { + // uniform Ohmic resistivity, everywhere: E += eta * curl B + // -> dB/dt += eta grad^2 B (the k^2 companion to the k^4 + // hyper-resistivity; see eta_res). c0 = 0 in 1D, so its E0 term + // is a harmless no-op there. + const real_t eta_u = eta_res(); + E0 += eta_u * c0; + E1 += eta_u * c1; + E2 += eta_u * c2; + } + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E1 += (ONE - vac12) * eta * c1; + E2 += (ONE - vac12) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t l0, l1, l2, r0, r1, r2; + res_curl({ i1 - 1 }, l0, l1, l2); + res_curl({ i1 + 1 }, r0, r1, r2); + const real_t etah = eta_hyper() * SQR(dx_inv); + E1 -= etah * (l1 + r1 - TWO * c1); + E2 -= etah * (l2 + r2 - TWO * c2); + } + } + } + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + + const real_t N0r { INV_2 * + (NN(i1, i2, comp_NN) + NN(i1, i2 - 1, comp_NN)) }; + const real_t N1r { INV_2 * + (NN(i1, i2, comp_NN) + NN(i1 - 1, i2, comp_NN)) }; + const real_t N2r { INV_4 * + (NN(i1, i2, comp_NN) + NN(i1, i2 - 1, comp_NN) + + NN(i1 - 1, i2, comp_NN) + + NN(i1 - 1, i2 - 1, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N1r, dens_min) }; + const real_t N2 { math::max(N2r, dens_min) }; + + // 2D basis: b1/b2 = B/dx, b3 physical; output e1/e2 = E/dx, e3 physical + E0 = dx_inv * INV_4 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2)) * + (PP(i1, i2, comp_PP + 1) + PP(i1, i2 - 1, comp_PP + 1)) - + INV_2 * (PP(i1, i2, comp_PP + 2) + PP(i1, i2 - 1, comp_PP + 2)) * + Bfs(i1, i2, comp_Bfs + 1); + E1 = -dx_inv * INV_4 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1 - 1, i2, comp_Bfs + 2)) * + (PP(i1, i2, comp_PP + 0) + PP(i1 - 1, i2, comp_PP + 0)) + + (INV_2) * (PP(i1, i2, comp_PP + 2) + PP(i1 - 1, i2, comp_PP + 2)) * + Bfs(i1, i2, comp_Bfs + 0); + E2 = dx * + (-INV_8 * + (Bfs(i1, i2, comp_Bfs + 0) + Bfs(i1, i2 - 1, comp_Bfs + 0)) * + (PP(i1, i2, comp_PP + 1) + PP(i1, i2 - 1, comp_PP + 1) + + PP(i1 - 1, i2, comp_PP + 1) + PP(i1 - 1, i2 - 1, comp_PP + 1)) + + (INV_8) * + (Bfs(i1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + (PP(i1, i2, comp_PP + 0) + PP(i1, i2 - 1, comp_PP + 0) + + PP(i1 - 1, i2, comp_PP + 0) + PP(i1 - 1, i2 - 1, comp_PP + 0))); + + E0 += + coeff * + (-SQR(dx_inv) * INV_8 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, comp_Bfs + 2) + Bfs(i1 + 1, i2 - 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) + + (INV_2) * + (Bfs(i1 + 1, i2, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + Bfs(i1, i2, comp_Bfs + 1)); + E1 += coeff * + (-SQR(dx_inv) * INV_8 * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1 - 1, i2, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) - + INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1, i2 + 1, comp_Bfs + 1) - Bfs(i1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + Bfs(i1, i2, comp_Bfs + 0)); + E2 += + coeff * + ((INV_4) * (Bfs(i1, i2, comp_Bfs + 0) + Bfs(i1, i2 - 1, comp_Bfs + 0)) * + (Bfs(i1, i2, comp_Bfs + 2) + Bfs(i1, i2 - 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2)) + + (INV_4) * (Bfs(i1, i2, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) * + (Bfs(i1, i2, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2 - 1, comp_Bfs + 2))); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac1 { vac_factor(N1r) }; + const real_t vac2 { vac_factor(N2r) }; + E0 *= -vac0 / N0; + E1 *= -vac1 / N1; + E2 *= -vac2 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO or resist > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist > ZERO) { + // uniform Ohmic resistivity, everywhere: E += eta * curl B + // -> dB/dt += eta grad^2 B (the k^2 companion to the k^4 + // hyper-resistivity; see eta_res). c0 = 0 in 1D, so its E0 term + // is a harmless no-op there. + const real_t eta_u = eta_res(); + E0 += eta_u * c0; + E1 += eta_u * c1; + E2 += eta_u * c2; + } + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E0 += (ONE - vac0) * eta * c0; + E1 += (ONE - vac1) * eta * c1; + E2 += (ONE - vac2) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t w0, w1, w2, e0, e1, e2, s0, s1, s2, n0, n1, n2; + res_curl({ i1 - 1, i2 }, w0, w1, w2); + res_curl({ i1 + 1, i2 }, e0, e1, e2); + res_curl({ i1, i2 - 1 }, s0, s1, s2); + res_curl({ i1, i2 + 1 }, n0, n1, n2); + const real_t etah = eta_hyper() * SQR(dx_inv); + E0 -= etah * (w0 + e0 + s0 + n0 - static_cast(4) * c0); + E1 -= etah * (w1 + e1 + s1 + n1 - static_cast(4) * c1); + E2 -= etah * (w2 + e2 + s2 + n2 - static_cast(4) * c2); + } + } + } + + } else if constexpr (D == Dim::_3D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + const real_t N0r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2, i3 - 1, comp_NN) + + NN(i1, i2 - 1, i3, comp_NN) + + NN(i1, i2 - 1, i3 - 1, comp_NN)) }; + const real_t N1r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2, i3 - 1, comp_NN) + + NN(i1 - 1, i2, i3, comp_NN) + + NN(i1 - 1, i2, i3 - 1, comp_NN)) }; + const real_t N2r { INV_4 * (NN(i1, i2, i3, comp_NN) + + NN(i1, i2 - 1, i3, comp_NN) + + NN(i1 - 1, i2, i3, comp_NN) + + NN(i1 - 1, i2 - 1, i3, comp_NN)) }; + const real_t N0 { math::max(N0r, dens_min) }; + const real_t N1 { math::max(N1r, dens_min) }; + const real_t N2 { math::max(N2r, dens_min) }; + E0 = -INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1, i2, i3 - 1, comp_Bfs + 1)) * + (PP(i1, i2, i3, comp_PP + 2) + PP(i1, i2, i3 - 1, comp_PP + 2) + + PP(i1, i2 - 1, i3, comp_PP + 2) + + PP(i1, i2 - 1, i3 - 1, comp_PP + 2)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) * + (PP(i1, i2, i3, comp_PP + 1) + PP(i1, i2, i3 - 1, comp_PP + 1) + + PP(i1, i2 - 1, i3, comp_PP + 1) + + PP(i1, i2 - 1, i3 - 1, comp_PP + 1)); + E1 = INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) * + (PP(i1, i2, i3, comp_PP + 2) + PP(i1, i2, i3 - 1, comp_PP + 2) + + PP(i1 - 1, i2, i3, comp_PP + 2) + + PP(i1 - 1, i2, i3 - 1, comp_PP + 2)) - + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) * + (PP(i1, i2, i3, comp_PP + 0) + PP(i1, i2, i3 - 1, comp_PP + 0) + + PP(i1 - 1, i2, i3, comp_PP + 0) + + PP(i1 - 1, i2, i3 - 1, comp_PP + 0)); + E2 = -INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2 - 1, i3, comp_Bfs + 0)) * + (PP(i1, i2, i3, comp_PP + 1) + PP(i1, i2 - 1, i3, comp_PP + 1) + + PP(i1 - 1, i2, i3, comp_PP + 1) + + PP(i1 - 1, i2 - 1, i3, comp_PP + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) * + (PP(i1, i2, i3, comp_PP + 0) + PP(i1, i2 - 1, i3, comp_PP + 0) + + PP(i1 - 1, i2, i3, comp_PP + 0) + + PP(i1 - 1, i2 - 1, i3, comp_PP + 0)); + + E0 += + coeff * + (INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1, i2, i3 - 1, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0) - Bfs(i1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + E1 += + coeff * + (-INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2, i3 - 1, comp_Bfs + 0)) * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2, i3, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 2) + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + E2 += + coeff * + (-INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 0) + Bfs(i1, i2 - 1, i3, comp_Bfs + 0)) * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - Bfs(i1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3 - 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) - Bfs(i1, i2, i3, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2)) - + INV_8 * + (Bfs(i1, i2, i3, comp_Bfs + 1) + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - Bfs(i1, i2, i3 - 1, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1 - 1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) - Bfs(i1, i2, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 - 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2 - 1, i3, comp_Bfs + 2))); + + { + const real_t vac0 { vac_factor(N0r) }; + const real_t vac1 { vac_factor(N1r) }; + const real_t vac2 { vac_factor(N2r) }; + E0 *= -vac0 / N0; + E1 *= -vac1 / N1; + E2 *= -vac2 / N2; + if (resist_vac > ZERO or resist_hyper > ZERO or resist > ZERO) { + real_t c0, c1, c2; + res_curl(i, c0, c1, c2); + if (resist > ZERO) { + // uniform Ohmic resistivity, everywhere: E += eta * curl B + // -> dB/dt += eta grad^2 B (the k^2 companion to the k^4 + // hyper-resistivity; see eta_res). c0 = 0 in 1D, so its E0 term + // is a harmless no-op there. + const real_t eta_u = eta_res(); + E0 += eta_u * c0; + E1 += eta_u * c1; + E2 += eta_u * c2; + } + if (resist_vac > ZERO) { + // resistive vacuum: crossfade to E = eta * curl B below the + // threshold (see the note at vac_factor) + const real_t eta = eta_vac(); + E0 += (ONE - vac0) * eta * c0; + E1 += (ONE - vac1) * eta * c1; + E2 += (ONE - vac2) * eta * c2; + } + if (resist_hyper > ZERO) { + // hyper-resistivity, everywhere: E -= eta_H grad^2(curl B) + // (see the note at eta_hyper) + real_t a0, a1, a2, lap0, lap1, lap2; + lap0 = -static_cast(6) * c0; + lap1 = -static_cast(6) * c1; + lap2 = -static_cast(6) * c2; + res_curl({ i1 - 1, i2, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1 + 1, i2, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2 - 1, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2 + 1, i3 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2, i3 - 1 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + res_curl({ i1, i2, i3 + 1 }, a0, a1, a2); + lap0 += a0; lap1 += a1; lap2 += a2; + const real_t etah = eta_hyper() * SQR(dx_inv); + E0 -= etah * lap0; + E1 -= etah * lap1; + E2 -= etah * lap2; + } + } + } + } + } + + Inline void compute_Ec(const tuple_t& i, + real_t& E0, + real_t& E1, + real_t& E2) const { + const real_t coeff_1 { rho0 * gamma_ad * theta }; + // Hall coefficient, same sign convention as `coeff` in compute_Ee above + // (leading minus corrects the (curl B) x B sign; see note there). + // hall_limiter caps the implied local whistler Courant (see its doc). + const real_t coeff_2 { -SQR(d0) / rho0 * hall_limiter(i) }; + if constexpr (D == Dim::_1D) { + const auto i1 = i[0]; + + // floored cell-centered density: the pusher reads Ec raw, so an + // unfloored empty cell would divide by zero into the particle push + const real_t Ncr { NN(i1, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); b1 = Bx/dx, b2/b3 physical + E0 = -Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 2) + + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 1); + E1 = dx * + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0)) * + PP(i1, comp_PP + 2) - + Bfs(i1, comp_Bfs + 2) * PP(i1, comp_PP + 0); + E2 = -dx * + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0)) * + PP(i1, comp_PP + 1) + + Bfs(i1, comp_Bfs + 1) * PP(i1, comp_PP + 0); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1 + 1, comp_NN) - NN(i1 - 1, comp_NN)); + + E0 += coeff_2 * dx_inv * + (-INV_2 * (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)) * + Bfs(i1, comp_Bfs + 1) - + INV_2 * (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)) * + Bfs(i1, comp_Bfs + 2)); + E1 += coeff_2 * INV_4 * + (Bfs(i1 + 1, comp_Bfs + 0) + Bfs(i1, comp_Bfs + 0)) * + (Bfs(i1 + 1, comp_Bfs + 1) - Bfs(i1 - 1, comp_Bfs + 1)); + E2 += coeff_2 * INV_4 * + (Bfs(i1 + 1, comp_Bfs + 0) + Bfs(i1, comp_Bfs + 0)) * + (Bfs(i1 + 1, comp_Bfs + 2) - Bfs(i1 - 1, comp_Bfs + 2)); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + + } else if constexpr (D == Dim::_2D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + + // floored cell-centered density (see 1D note above) + const real_t Ncr { NN(i1, i2, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); b1/b2 = B/dx, b3 physical + E0 = -dx * INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + PP(i1, i2, comp_PP + 2) + + Bfs(i1, i2, comp_Bfs + 2) * PP(i1, i2, comp_PP + 1); + E1 = dx * INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + PP(i1, i2, comp_PP + 2) - + Bfs(i1, i2, comp_Bfs + 2) * PP(i1, i2, comp_PP + 0); + E2 = dx * + (-INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + PP(i1, i2, comp_PP + 1) + + INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + PP(i1, i2, comp_PP + 0)); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1 + 1, i2, comp_NN) - NN(i1 - 1, i2, comp_NN)); + E1 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * INV_2 * + (NN(i1, i2 + 1, comp_NN) - NN(i1, i2 - 1, comp_NN)); + E0 += + coeff_2 * + (dx * (INV_8) * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2 + 1, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, comp_Bfs + 1) - Bfs(i1 + 1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) - + dx_inv * INV_2 * + (Bfs(i1 + 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)) * + Bfs(i1, i2, comp_Bfs + 2)); + E1 += + coeff_2 * + (-dx * INV_8 * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2 + 1, comp_Bfs + 0) - Bfs(i1 + 1, i2 - 1, comp_Bfs + 0) + + Bfs(i1, i2 + 1, comp_Bfs + 0) - Bfs(i1, i2 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, comp_Bfs + 1) - Bfs(i1 + 1, i2, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, comp_Bfs + 1) + Bfs(i1 - 1, i2, comp_Bfs + 1)) - + dx_inv * INV_2 * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2)) * + Bfs(i1, i2, comp_Bfs + 2)); + E2 += coeff_2 * + ((INV_4) * + (Bfs(i1 + 1, i2, comp_Bfs + 0) + Bfs(i1, i2, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2, comp_Bfs + 2) - Bfs(i1 - 1, i2, comp_Bfs + 2)) + + (INV_4) * + (Bfs(i1, i2 + 1, comp_Bfs + 1) + Bfs(i1, i2, comp_Bfs + 1)) * + (Bfs(i1, i2 + 1, comp_Bfs + 2) - Bfs(i1, i2 - 1, comp_Bfs + 2))); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + } else if constexpr (D == Dim::_3D) { + const auto i1 = i[0]; + const auto i2 = i[1]; + const auto i3 = i[2]; + + // floored cell-centered density (see 1D note above) + const real_t Ncr { NN(i1, i2, i3, comp_NN) }; + const real_t Nc { math::max(Ncr, dens_min) }; + // Ee* = EMF(N^(n), P^(n), Bf*) + + // Ec* = EMF(N^(n), P^(n), Bc*), where Bc* = interpolate Bf* + // output is PHYSICAL (pusher reads bckp raw); all b comps = B/dx in 3D + E0 = dx * (-INV_2 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + PP(i1, i2, i3, comp_PP + 2) + + INV_2 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + PP(i1, i2, i3, comp_PP + 1)); + E1 = dx * (INV_2 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + PP(i1, i2, i3, comp_PP + 2) - + INV_2 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + PP(i1, i2, i3, comp_PP + 0)); + E2 = dx * (-INV_2 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + PP(i1, i2, i3, comp_PP + 1) + + INV_2 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + PP(i1, i2, i3, comp_PP + 0)); + + E0 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1 + 1, i2, i3, comp_NN) - NN(i1 - 1, i2, i3, comp_NN)); + E1 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1, i2 + 1, i3, comp_NN) - NN(i1, i2 - 1, i3, comp_NN)); + E2 += coeff_1 * math::pow(Nc, gamma_ad - ONE) * dx_inv * + INV_2 * (NN(i1, i2, i3 + 1, comp_NN) - NN(i1, i2, i3 - 1, comp_NN)); + + E0 += coeff_2 * dx * (INV_8 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2))); + E1 += coeff_2 * dx * (-INV_8 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 - 1, i3, comp_Bfs + 0) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 0) - + Bfs(i1, i2 - 1, i3, comp_Bfs + 0) - + Bfs(i1 + 1, i2 + 1, i3, comp_Bfs + 1) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 1)) + + INV_8 * + (Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)) * + (Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2))); + E2 += coeff_2 * dx * (-INV_8 * + (Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)) * + (Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 - 1, comp_Bfs + 0) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 0) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 0) - + Bfs(i1 + 1, i2, i3 + 1, comp_Bfs + 2) - + Bfs(i1 + 1, i2, i3, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1 - 1, i2, i3, comp_Bfs + 2)) - + INV_8 * + (Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)) * + (Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 - 1, comp_Bfs + 1) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 1) - + Bfs(i1, i2, i3 - 1, comp_Bfs + 1) - + Bfs(i1, i2 + 1, i3 + 1, comp_Bfs + 2) - + Bfs(i1, i2 + 1, i3, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2 - 1, i3, comp_Bfs + 2))); + + E0 *= -vac_factor(Ncr) / Nc; + E1 *= -vac_factor(Ncr) / Nc; + E2 *= -vac_factor(Ncr) / Nc; + } + } + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, comp_Ec_out + 2) = Ecstar2; + } + } else { + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, comp_Ee_out + 0) = INV_2 * + (Eestar0 + Ee_in(i1, comp_Ee_in + 0)); + Ee_out(i1, comp_Ee_out + 1) = INV_2 * + (Eestar1 + Ee_in(i1, comp_Ee_in + 1)); + Ee_out(i1, comp_Ee_out + 2) = INV_2 * + (Eestar2 + Ee_in(i1, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, comp_Ec_out + 0) = INV_2 * (Ecstar0 + Ec(i1, comp_Ec + 0)); + Ec_out(i1, comp_Ec_out + 1) = INV_2 * (Ecstar1 + Ec(i1, comp_Ec + 1)); + Ec_out(i1, comp_Ec_out + 2) = INV_2 * (Ecstar2 + Ec(i1, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: b1 is stored as Bx/dx in 1D, b2/b3 physical) + Bc_out(i1, comp_Bc_out + 0) = dx * INV_2 * + ((INV_2 * Bf(i1 + 1, comp_Bf + 0) + + INV_2 * Bf(i1, comp_Bf + 0)) + + (INV_2 * Bfs(i1 + 1, comp_Bfs + 0) + + INV_2 * Bfs(i1, comp_Bfs + 0))); + + Bc_out(i1, comp_Bc_out + 1) = INV_2 * (Bf(i1, comp_Bf + 1) + + Bfs(i1, comp_Bfs + 1)); + Bc_out(i1, comp_Bc_out + 2) = INV_2 * (Bf(i1, comp_Bf + 2) + + Bfs(i1, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, i2, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, i2, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, i2, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, i2, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, i2, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, i2, comp_Ec_out + 2) = Ecstar2; + } + } else { + // Ee* = EMF(N^(n), P^(n), Bf*) + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, i2, comp_Ee_out + 0) = INV_2 * + (Eestar0 + + Ee_in(i1, i2, comp_Ee_in + 0)); + Ee_out(i1, i2, comp_Ee_out + 1) = INV_2 * + (Eestar1 + + Ee_in(i1, i2, comp_Ee_in + 1)); + Ee_out(i1, i2, comp_Ee_out + 2) = INV_2 * + (Eestar2 + + Ee_in(i1, i2, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, i2, comp_Ec_out + 0) = INV_2 * + (Ecstar0 + Ec(i1, i2, comp_Ec + 0)); + Ec_out(i1, i2, comp_Ec_out + 1) = INV_2 * + (Ecstar1 + Ec(i1, i2, comp_Ec + 1)); + Ec_out(i1, i2, comp_Ec_out + 2) = INV_2 * + (Ecstar2 + Ec(i1, i2, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: b1/b2 are stored as B/dx in 2D, b3 physical) + Bc_out(i1, i2, comp_Bc_out + 0) = dx * INV_4 * + (Bf(i1 + 1, i2, comp_Bf + 0) + + Bf(i1, i2, comp_Bf + 0) + + Bfs(i1 + 1, i2, comp_Bfs + 0) + + Bfs(i1, i2, comp_Bfs + 0)); + + Bc_out(i1, i2, comp_Bc_out + 1) = dx * INV_4 * + (Bf(i1, i2 + 1, comp_Bf + 1) + + Bf(i1, i2, comp_Bf + 1) + + Bfs(i1, i2 + 1, comp_Bfs + 1) + + Bfs(i1, i2, comp_Bfs + 1)); + Bc_out(i1, i2, comp_Bc_out + 2) = INV_2 * (Bf(i1, i2, comp_Bf + 2) + + Bfs(i1, i2, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if constexpr (INIT) { + /// terms of Ee and Ec + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2, i3 }, Eestar0, Eestar1, Eestar2); + + Ee_out(i1, i2, i3, comp_Ee_out + 0) = Eestar0; + Ee_out(i1, i2, i3, comp_Ee_out + 1) = Eestar1; + Ee_out(i1, i2, i3, comp_Ee_out + 2) = Eestar2; + + if constexpr (not EEONLY) { + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2, i3 }, Ecstar0, Ecstar1, Ecstar2); + + Ec_out(i1, i2, i3, comp_Ec_out + 0) = Ecstar0; + Ec_out(i1, i2, i3, comp_Ec_out + 1) = Ecstar1; + Ec_out(i1, i2, i3, comp_Ec_out + 2) = Ecstar2; + } + } else { + real_t Eestar0 { ZERO }, Eestar1 { ZERO }, Eestar2 { ZERO }; + compute_Ee({ i1, i2, i3 }, Eestar0, Eestar1, Eestar2); + real_t Ecstar0 { ZERO }, Ecstar1 { ZERO }, Ecstar2 { ZERO }; + compute_Ec({ i1, i2, i3 }, Ecstar0, Ecstar1, Ecstar2); + + // Ee' = 0.5 * (Ee* + Ee^(n)) + Ee_out(i1, i2, i3, comp_Ee_out + 0) = INV_2 * + (Eestar0 + + Ee_in(i1, i2, i3, comp_Ee_in + 0)); + Ee_out(i1, i2, i3, comp_Ee_out + 1) = INV_2 * + (Eestar1 + + Ee_in(i1, i2, i3, comp_Ee_in + 1)); + Ee_out(i1, i2, i3, comp_Ee_out + 2) = INV_2 * + (Eestar2 + + Ee_in(i1, i2, i3, comp_Ee_in + 2)); + + // Ec' = 0.5 * (Ec* + Ec^(n)) + Ec_out(i1, i2, i3, comp_Ec_out + 0) = INV_2 * + (Ecstar0 + + Ec(i1, i2, i3, comp_Ec + 0)); + Ec_out(i1, i2, i3, comp_Ec_out + 1) = INV_2 * + (Ecstar1 + + Ec(i1, i2, i3, comp_Ec + 1)); + Ec_out(i1, i2, i3, comp_Ec_out + 2) = INV_2 * + (Ecstar2 + + Ec(i1, i2, i3, comp_Ec + 2)); + + // Bc' = 0.5 * (Bc* + Bc^(n)), where Bc* = interpolate Bf* + // (PHYSICAL output: all b comps are stored as B/dx in 3D) + Bc_out(i1, i2, i3, comp_Bc_out + 0) = dx * INV_4 * + (Bf(i1 + 1, i2, i3, comp_Bf + 0) + + Bf(i1, i2, i3, comp_Bf + 0) + + Bfs(i1 + 1, i2, i3, comp_Bfs + 0) + + Bfs(i1, i2, i3, comp_Bfs + 0)); + + Bc_out(i1, i2, i3, comp_Bc_out + 1) = dx * INV_4 * + (Bf(i1, i2 + 1, i3, comp_Bf + 1) + + Bf(i1, i2, i3, comp_Bf + 1) + + Bfs(i1, i2 + 1, i3, comp_Bfs + 1) + + Bfs(i1, i2, i3, comp_Bfs + 1)); + Bc_out(i1, i2, i3, comp_Bc_out + 2) = dx * INV_4 * + (Bf(i1, i2, i3 + 1, comp_Bf + 2) + + Bf(i1, i2, i3, comp_Bf + 2) + + Bfs(i1, i2, i3 + 1, comp_Bfs + 2) + + Bfs(i1, i2, i3, comp_Bfs + 2)); + } + } else { + raise::KernelError(HERE, "EMF_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_EMF_HPP diff --git a/src/kernels/hybrid/faraday.hpp b/src/kernels/hybrid/faraday.hpp new file mode 100644 index 000000000..efd26023c --- /dev/null +++ b/src/kernels/hybrid/faraday.hpp @@ -0,0 +1,123 @@ +#ifndef KERNELS_HYBRID_FARADAY_HPP +#define KERNELS_HYBRID_FARADAY_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + + /** + * @note Fields are stored in the contravariant (Idx::U) code basis of the + * Minkowski metric: components with index i <= D carry a 1/dx + * (h_ii = dx^2), out-of-plane components are physical (h_ii = 1). The + * index-space curl therefore needs per-component coefficients: + * 1D: dt/dx for b2, b3 (from physical e2, e3); b1 untouched + * 2D: dt/dx^2 for b1, b2 (from physical e3); dt for b3 (from e1, e2) + * 3D: dt/dx for all components + */ + // NIN: number of components of Bin (6 = em, 3 = cur). Bin == Bout in-place is + // safe: the update reads only the cell's own B (the stencil is on Ein). + template + class Faraday_kernel { + ndfield_t Ein; + ndfield_t Bin; + ndfield_t Bout; + + const uint8_t comp_Ein; + const uint8_t comp_Bin; + const uint8_t comp_Bout; + + const real_t dt; + const real_t dx_inv; + + public: + Faraday_kernel(const ndfield_t& Ein, + const ndfield_t& Bin, + ndfield_t& Bout, + uint8_t comp_Ein, + uint8_t comp_Bin, + uint8_t comp_Bout, + real_t dt, + real_t dx) + : Ein { Ein } + , Bin { Bin } + , Bout { Bout } + , comp_Ein { comp_Ein } + , comp_Bin { comp_Bin } + , comp_Bout { comp_Bout } + , dt { dt } + , dx_inv { ONE / dx } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + // B_x is constant in 1D (dB_x/dt = -(curl E)_x = 0). Bout here is a + // SEPARATE scratch buffer (cur = Bf*/Bf** for the predictor pushes), + // NOT in-place on `em`, and is zero-initialized -- so B_x must be + // copied through, else the subsequent EMF (Ohm's-law Hall/motional + // terms) would read Bf*_x = 0 instead of B_x^n. + const real_t coeff { dt * dx_inv }; + Bout(i1, comp_Bout + 0) = Bin(i1, comp_Bin + 0); + Bout(i1, comp_Bout + 1) = Bin(i1, comp_Bin + 1) - + coeff * (-Ein(i1 + 1, comp_Ein + 2) + + Ein(i1, comp_Ein + 2)); + Bout(i1, comp_Bout + 2) = Bin(i1, comp_Bin + 2) - + coeff * (Ein(i1 + 1, comp_Ein + 1) - + Ein(i1, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const real_t coeff1 { dt * SQR(dx_inv) }; // in-plane b from physical e3 + const real_t coeff2 { dt }; // out-of-plane b3 from e1/dx, e2/dx + Bout(i1, i2, comp_Bout + 0) = Bin(i1, i2, comp_Bin + 0) - + coeff1 * (Ein(i1, i2 + 1, comp_Ein + 2) - + Ein(i1, i2, comp_Ein + 2)); + Bout(i1, i2, comp_Bout + 1) = Bin(i1, i2, comp_Bin + 1) - + coeff1 * (-Ein(i1 + 1, i2, comp_Ein + 2) + + Ein(i1, i2, comp_Ein + 2)); + Bout(i1, i2, comp_Bout + 2) = Bin(i1, i2, comp_Bin + 2) - + coeff2 * (-Ein(i1, i2 + 1, comp_Ein + 0) + + Ein(i1, i2, comp_Ein + 0) + + Ein(i1 + 1, i2, comp_Ein + 1) - + Ein(i1, i2, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const real_t coeff { dt * dx_inv }; // all comps stored as physical/dx + Bout(i1, i2, i3, comp_Bout + 0) = Bin(i1, i2, i3, comp_Bin + 0) - + coeff * + (-Ein(i1, i2, i3 + 1, comp_Ein + 1) + + Ein(i1, i2, i3, comp_Ein + 1) + + Ein(i1, i2 + 1, i3, comp_Ein + 2) - + Ein(i1, i2, i3, comp_Ein + 2)); + Bout(i1, i2, i3, comp_Bout + 1) = Bin(i1, i2, i3, comp_Bin + 1) - + coeff * + (Ein(i1, i2, i3 + 1, comp_Ein + 0) - + Ein(i1, i2, i3, comp_Ein + 0) - + Ein(i1 + 1, i2, i3, comp_Ein + 2) + + Ein(i1, i2, i3, comp_Ein + 2)); + Bout(i1, i2, i3, comp_Bout + 2) = Bin(i1, i2, i3, comp_Bin + 2) - + coeff * + (-Ein(i1, i2 + 1, i3, comp_Ein + 0) + + Ein(i1, i2, i3, comp_Ein + 0) + + Ein(i1 + 1, i2, i3, comp_Ein + 1) - + Ein(i1, i2, i3, comp_Ein + 1)); + } else { + raise::KernelError(HERE, "Faraday_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_FARADAY_HPP diff --git a/src/kernels/hybrid/moments_filter.hpp b/src/kernels/hybrid/moments_filter.hpp new file mode 100644 index 000000000..06e0c14e2 --- /dev/null +++ b/src/kernels/hybrid/moments_filter.hpp @@ -0,0 +1,118 @@ +#ifndef KERNELS_HYBRID_MOMENTS_FILTER_HPP +#define KERNELS_HYBRID_MOMENTS_FILTER_HPP + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + using namespace ntt; + + /** + * Binomial (1-2-1) smoothing of the deposited hybrid moments. + * + * The Ohm's-law E is built directly from the deposited number density N and + * momentum density V (aux::3 and aux::012). With cold, fast beams the per-cell + * shot noise in V (~ v_drift / sqrt(ppc)) is injected straight into E via the + * motional (-u x B) and Hall (J x B / N) terms and drives a grid-scale + * numerical instability. Smoothing the moments before the field solve removes + * the grid-scale (Nyquist) noise, exactly as current filtering does in the + * explicit PIC engines. + * + * Filters comps 0..3 (V0, V1, V2, N) of the 6-component `array` (aux), reading + * from `buffer` (a copy of `array`). Cartesian, interior stencil only: the + * i +/- 1 reads reach into the ghost layer, which the caller refills (periodic + * / MPI) between passes via CommunicateFields(AUX). N_GHOSTS >= 1 is required + * (always true for SHAPE_ORDER >= 0). + * + * @note Conductor/reflecting walls are NOT special-cased here (unlike + * DigitalFilter_kernel); the plain stencil smooths against whatever the + * wall ghost moments hold. Hybrid wall setups should revisit this if they + * enable filtering. + */ + template + class MomentsFilter_kernel { + ndfield_t array; + const ndfield_t buffer; + + static constexpr unsigned short NCOMP = 4; // V0, V1, V2, N + + public: + MomentsFilter_kernel(const ndfield_t& array, + const ndfield_t& buffer) + : array { array } + , buffer { buffer } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, c) = INV_2 * buffer(i1, c) + + INV_4 * (buffer(i1 - 1, c) + buffer(i1 + 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 1D called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, i2, c) = INV_4 * buffer(i1, i2, c) + + INV_8 * (buffer(i1 - 1, i2, c) + + buffer(i1 + 1, i2, c) + + buffer(i1, i2 - 1, c) + + buffer(i1, i2 + 1, c)) + + INV_16 * (buffer(i1 - 1, i2 - 1, c) + + buffer(i1 + 1, i2 + 1, c) + + buffer(i1 - 1, i2 + 1, c) + + buffer(i1 + 1, i2 - 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 2D called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { +#pragma unroll + for (auto c { 0u }; c < NCOMP; ++c) { + array(i1, i2, i3, c) = + INV_8 * buffer(i1, i2, i3, c) + + INV_16 * (buffer(i1 - 1, i2, i3, c) + buffer(i1 + 1, i2, i3, c) + + buffer(i1, i2 - 1, i3, c) + buffer(i1, i2 + 1, i3, c) + + buffer(i1, i2, i3 - 1, c) + buffer(i1, i2, i3 + 1, c)) + + INV_32 * (buffer(i1 - 1, i2 - 1, i3, c) + + buffer(i1 + 1, i2 + 1, i3, c) + + buffer(i1 - 1, i2 + 1, i3, c) + + buffer(i1 + 1, i2 - 1, i3, c) + + buffer(i1, i2 - 1, i3 - 1, c) + + buffer(i1, i2 + 1, i3 + 1, c) + + buffer(i1, i2 - 1, i3 + 1, c) + + buffer(i1, i2 + 1, i3 - 1, c) + + buffer(i1 - 1, i2, i3 - 1, c) + + buffer(i1 + 1, i2, i3 + 1, c) + + buffer(i1 - 1, i2, i3 + 1, c) + + buffer(i1 + 1, i2, i3 - 1, c)) + + INV_64 * (buffer(i1 - 1, i2 - 1, i3 - 1, c) + + buffer(i1 + 1, i2 + 1, i3 + 1, c) + + buffer(i1 - 1, i2 + 1, i3 + 1, c) + + buffer(i1 + 1, i2 - 1, i3 - 1, c) + + buffer(i1 - 1, i2 - 1, i3 + 1, c) + + buffer(i1 + 1, i2 + 1, i3 - 1, c) + + buffer(i1 - 1, i2 + 1, i3 - 1, c) + + buffer(i1 + 1, i2 - 1, i3 + 1, c)); + } + } else { + raise::KernelError(HERE, "MomentsFilter_kernel: 3D called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_MOMENTS_FILTER_HPP diff --git a/src/kernels/hybrid/pusher.hpp b/src/kernels/hybrid/pusher.hpp new file mode 100644 index 000000000..dcce5b0ac --- /dev/null +++ b/src/kernels/hybrid/pusher.hpp @@ -0,0 +1,720 @@ +/** + * @file kernels/hybrid/pusher.hpp + * @brief Non-relativistic modified-Boris ion pusher (+ fused moment deposit) + * for the HYBRID engine + * @implements + * - kernel::hybrid::PushMode + * - kernel::hybrid::PusherContext + * - kernel::hybrid::Pusher_kernel<> + * @namespaces: + * - kernel::hybrid:: + * + * Implements the Pegasus modified-Boris integrator (Kunz, Stone & Bai 2014, + * arXiv:1311.4865, eq. 12) for the base case sigma = varpi = 0. One push advances + * a particle from t^(n) to t^(n+1) using fields centered at t^(n+1/2): + * + * (12a) x* = x^(n) + (dt/2) v^(n) // first half drift + * (12b) v^- = v^(n) + (dt/2) c Ec // first electric half-kick + * (12c) v^+ = v^- + rotation(Bc) // Boris rotation (gamma = 1) + * (12d) v^(n+1) = v^+ + (dt/2) c Ec // second electric half-kick + * (12e) x^(n+1) = x* + (dt/2) v^(n+1) // second half drift + * + * with the coupling c = (q/m), folded with omegaB0*dt/2 into `normalized_dt_half`. + * Ec, Bc are *cell-centered*, *time-centered at n+1/2* fields (the `bckp` buffer: + * Ec in comps 0..2, Bc in comps 3..5), interpolated at the predicted position x* + * with the SAME shape function used by the deposit (paper §3.2). + * + * FUSED DEPOSIT. Immediately after advancing a particle, the kernel deposits the + * ion number density N -> aux comp 3 and momentum density V = sum(w*m*v) -> + * aux comps 0..2, with NO Lorentz factor (non-relativistic). This avoids a second + * pass through the particle table and lets the transient predictor produce its + * predicted moments WITHOUT writing the particle arrays (so no save/restore of + * x^(n), v^(n) is needed — both pushes start from the stored state). + * + * The template `PushMode Mode` selects the three uses (Pegasus Fig. 2): + * - MomentsOnly : deposit N,V from the stored x^(n), v^(n); no push, no store. + * Used once at step 0 to seed aux with N^(0), V^(0). + * - Predictor : push in registers (Fig. 2 steps 7+8), deposit predicted + * N', V'; do NOT write the particle arrays, no particle BCs. + * - Corrector : push (Fig. 2 step 12), deposit final N^(n+1), V^(n+1), AND + * write back x^(n+1), v^(n+1) (+ i/dx_prev) and apply particle BCs. + * + * SINK ROUTING. The per-particle flow is shared between the flat and the tiled + * (TEAM_POLICY) launches through a sink object (see the TiledScatter_kernel + * body contract in kernels/tiled_scatter.hpp): after the push, the kernel + * `select()`s the deposit-time footprint (computed from the post-push register + * position) on the sink and emits each stencil write through it. The flat + * launch uses the nested `FlatSink` (a bounds-clipping ScatterView accessor, + * `select()` a no-op); the tiled launch hands in the harness sink, which + * routes to per-tile SLM scratch when the footprint fits and to the + * bounds-clipped global escape valve when it does not. + * + * @note NON-RELATIVISTIC. ux1/ux2/ux3 store the 3-velocity v; gamma == 1 everywhere + * (no 1/sqrt(1+v^2) in the rotation, no dt/gamma in the drift). + * @note Cartesian Minkowski only. 1D gather/deposit are implemented; 2D/3D are + * stubs (the hybrid EMF solver is itself 1D-only for now). + * @see PIC/hybrid/pusher.md for the full plan and the correctness traps. + */ + +#ifndef KERNELS_HYBRID_PUSHER_HPP +#define KERNELS_HYBRID_PUSHER_HPP + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +#include "framework/containers/particles.h" +#include "kernels/particle_shapes.hpp" +#include "kernels/pushers/context.h" // kernel::sr::PusherBoundaries (BC flags) + +#include + +#if defined(MPI_ENABLED) + #include "arch/mpi_tags.h" +#endif + +namespace kernel::hybrid { + using namespace ntt; + + enum class PushMode { + MomentsOnly, // deposit only (step-0 seed); no push, no store + Predictor, // push in registers + deposit; no store, no BC + Corrector // push + deposit + store-back + BC + }; + + /** + * @brief Lean parameter bundle for the hybrid ion pusher / deposit. + * @note mass/charge live on the species, not on `ParticleArrays`, so they are + * passed in here (cf. kernel::sr::PusherContext). + */ + struct PusherContext { + const float mass, charge; + const real_t dt; + const real_t omegaB0; // = 1 / larmor0 + const real_t inv_n0; // = 1 / scales.n0 (number-density normalization) + const bool use_weights; + const int ni1, ni2, ni3; // # active cells per direction (periodic wrap) + + PusherContext(float mass, + float charge, + real_t dt, + real_t omegaB0, + real_t inv_n0, + bool use_weights, + int ni1, + int ni2, + int ni3) + : mass { mass } + , charge { charge } + , dt { dt } + , omegaB0 { omegaB0 } + , inv_n0 { inv_n0 } + , use_weights { use_weights } + , ni1 { ni1 } + , ni2 { ni2 } + , ni3 { ni3 } {} + }; + + /** + * @tparam M Metric (Cartesian Minkowski for the hybrid engine). + * @tparam Mode MomentsOnly / Predictor / Corrector — see file header. + */ + template + struct Pusher_kernel { + static constexpr auto D = M::Dim; + static_assert(M::CoordType == Coord::Cartesian, + "hybrid pusher: Cartesian (Minkowski) metric only"); + + // shape order and half-window; gather and deposit MUST share the cell-centering + static constexpr unsigned short O = SHAPE_ORDER; + static constexpr int window = (SHAPE_ORDER + 1) / 2; + + const PusherContext ctx; + const kernel::sr::PusherBoundaries bc; + ParticleArrays particles; + + // read-only, cell-centered, time-centered (n+1/2) fields: + // Ec -> comps 0..2, Bc -> comps 3..5 (the `bckp` buffer); unused in MomentsOnly + const randacc_ndfield_t EB; + // scatter view over `aux`: V -> comps 0..2, N -> comp 3 + scatter_ndfield_t moments; + + const M metric; + + // coefficients (precomputed once) + const real_t normalized_dt_half; // 1/2 * (q/m) * omegaB0 * dt (E-kick & rotation) + const real_t dt_half; // 1/2 * dt (each half drift) + + Pusher_kernel(const PusherContext& pusher_ctx, + const kernel::sr::PusherBoundaries& pusher_boundaries, + ParticleArrays& pusher_arrays, + const randacc_ndfield_t& EB, + const scatter_ndfield_t& moments, + const M& metric) + : ctx { pusher_ctx } + , bc { pusher_boundaries } + , particles { pusher_arrays } + , EB { EB } + , moments { moments } + , metric { metric } + , normalized_dt_half { HALF * (ctx.charge / ctx.mass) * ctx.omegaB0 * ctx.dt } + , dt_half { HALF * ctx.dt } {} + + /** + * Tiled-body constructor (TEAM_POLICY launch): the deposit goes through + * the TiledScatter_kernel sink instead of a ScatterView, so `moments` + * stays empty and is never accessed on this path (only the sink entry + * point below is called by the harness). + */ + Pusher_kernel(const PusherContext& pusher_ctx, + const kernel::sr::PusherBoundaries& pusher_boundaries, + ParticleArrays& pusher_arrays, + const randacc_ndfield_t& EB, + const M& metric) + : ctx { pusher_ctx } + , bc { pusher_boundaries } + , particles { pusher_arrays } + , EB { EB } + , moments {} + , metric { metric } + , normalized_dt_half { HALF * (ctx.charge / ctx.mass) * ctx.omegaB0 * ctx.dt } + , dt_half { HALF * ctx.dt } {} + + // ........................................................................ + // flat-path write sink: ScatterView accessor + per-write storage-extent + // clip — the same guards the flat deposit always had (see the comment + // above inX1; they are load-bearing here: an overshooting particle's + // stencil would otherwise scatter past the array end). select() is a + // no-op: the flat path has no scratch window to prove containment + // against. + // ........................................................................ + struct FlatSink { + using acc_t = decltype(std::declval&>().access()); + + acc_t buff; + int e1, e2, e3; // storage extents (= ni + 2*N_GHOSTS) + + Inline void select(int, int) {} + + Inline void select(int, int, int, int) {} + + Inline void select(int, int, int, int, int, int) {} + + Inline void operator()(int c1, int comp, real_t v) const { + if (c1 >= 0 and c1 < e1) { + buff(c1, comp) += v; + } + } + + Inline void operator()(int c1, int c2, int comp, real_t v) const { + if (c1 >= 0 and c1 < e1 and c2 >= 0 and c2 < e2) { + buff(c1, c2, comp) += v; + } + } + + Inline void operator()(int c1, int c2, int c3, int comp, real_t v) const { + if (c1 >= 0 and c1 < e1 and c2 >= 0 and c2 < e2 and c3 >= 0 and c3 < e3) { + buff(c1, c2, c3, comp) += v; + } + } + }; + + // ........................................................................ + // main per-particle update — flat entry (RangePolicy over particles) + // ........................................................................ + Inline void operator()(prtlidx_t p) const { + FlatSink sink { moments.access(), + ctx.ni1 + 2 * static_cast(N_GHOSTS), + ctx.ni2 + 2 * static_cast(N_GHOSTS), + ctx.ni3 + 2 * static_cast(N_GHOSTS) }; + (*this)(p, sink); + } + + // ........................................................................ + // main per-particle update — sink entry, shared by the flat launch + // (FlatSink above) and the tiled TEAM_POLICY launch (harness Sink; see + // the body contract in kernels/tiled_scatter.hpp) + // ........................................................................ + template + Inline void operator()(prtlidx_t p, SinkLike& sink) const { + if (particles.tag(p) != ParticleTag::alive) { + return; + } + + // load start-of-step state x^(n), v^(n); ux holds the 3-velocity v + const int i1n { particles.i1(p) }; + const prtldx_t dx1n { particles.dx1(p) }; + int i2n { 0 }, i3n { 0 }; + prtldx_t dx2n { ZERO }, dx3n { ZERO }; + if constexpr (D == Dim::_2D || D == Dim::_3D) { + i2n = particles.i2(p); + dx2n = particles.dx2(p); + } + if constexpr (D == Dim::_3D) { + i3n = particles.i3(p); + dx3n = particles.dx3(p); + } + + // working state in registers (real_t offset avoids float drift across substeps) + int i1 { i1n }, i2 { i2n }, i3 { i3n }; + real_t dx1 { static_cast(dx1n) }; + real_t dx2 { static_cast(dx2n) }; + real_t dx3 { static_cast(dx3n) }; + vec_t v { particles.ux1(p), particles.ux2(p), particles.ux3(p) }; + + if constexpr (Mode != PushMode::MomentsOnly) { + // (12a) first half drift to the predicted position x* + halfDrift(i1, dx1, i2, dx2, i3, dx3, v); + // interpolate cell-centered Ec, Bc at x* (Cartesian: already in XYZ basis) + vec_t e0 { ZERO }, b0 { ZERO }; + gather(i1, dx1, i2, dx2, i3, dx3, e0, b0); + // (12b-d) electric half-kick / Boris rotation / electric half-kick + velocityPush(e0, b0, v); + // (12e) second half drift to x^(n+1) + halfDrift(i1, dx1, i2, dx2, i3, dx3, v); + } + + // deposit-time footprint from the post-push register position (for + // MomentsOnly, the stored position), in global storage coordinates + // (incl. N_GHOSTS): the deposit below writes exactly + // [i - window, i + window] per axis. The tiled sink uses it to route + // the whole particle to SLM scratch (fits) or the global escape + // valve (drifted past the halo); FlatSink ignores it. + { + const int G = static_cast(N_GHOSTS); + if constexpr (D == Dim::_1D) { + sink.select(i1 + G - window, i1 + G + window); + } else if constexpr (D == Dim::_2D) { + sink.select(i1 + G - window, + i1 + G + window, + i2 + G - window, + i2 + G + window); + } else { + sink.select(i1 + G - window, + i1 + G + window, + i2 + G - window, + i2 + G + window, + i3 + G - window, + i3 + G + window); + } + } + + // fused moment deposit at the advanced (or, for MomentsOnly, the stored) state + deposit(p, i1, dx1, i2, dx2, i3, dx3, v, sink); + + // corrector commits the accepted move -> store back + particle boundaries + if constexpr (Mode == PushMode::Corrector) { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + particles.i1_prev(p) = i1n; + particles.dx1_prev(p) = dx1n; + particles.i1(p) = i1; + particles.dx1(p) = static_cast(dx1); + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + particles.i2_prev(p) = i2n; + particles.dx2_prev(p) = dx2n; + particles.i2(p) = i2; + particles.dx2(p) = static_cast(dx2); + } + if constexpr (D == Dim::_3D) { + particles.i3_prev(p) = i3n; + particles.dx3_prev(p) = dx3n; + particles.i3(p) = i3; + particles.dx3(p) = static_cast(dx3); + } + particles.ux1(p) = v[0]; + particles.ux2(p) = v[1]; + particles.ux3(p) = v[2]; + boundaryConditions(p); + } + } + + // ........................................................................ + // velocity update — eq. (12b)-(12d), gamma == 1 + // e0,b0 enter as the raw interpolated fields and are scaled here. + // ........................................................................ + Inline void velocityPush(vec_t& e0, + vec_t& b0, + vec_t& v) const { + e0[0] *= normalized_dt_half; + e0[1] *= normalized_dt_half; + e0[2] *= normalized_dt_half; + b0[0] *= normalized_dt_half; // t = (dt/2) c B (NO relativistic 1/gamma!) + b0[1] *= normalized_dt_half; + b0[2] *= normalized_dt_half; + + // (12b) v^- = v + (dt/2) c E + v[0] += e0[0]; + v[1] += e0[1]; + v[2] += e0[2]; + + // (12c) Boris rotation: v^+ = v^- + s (v^- + v^- x t) x t, s = 2/(1+|t|^2) + const real_t s { TWO / (ONE + NORM_SQR(b0[0], b0[1], b0[2])) }; + const vec_t vp { + (v[0] + CROSS_x1(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s, + (v[1] + CROSS_x2(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s, + (v[2] + CROSS_x3(v[0], v[1], v[2], b0[0], b0[1], b0[2])) * s + }; + v[0] += CROSS_x1(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + v[1] += CROSS_x2(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + v[2] += CROSS_x3(vp[0], vp[1], vp[2], b0[0], b0[1], b0[2]); + + // (12d) v^(n+1) = v^+ + (dt/2) c E + v[0] += e0[0]; + v[1] += e0[1]; + v[2] += e0[2]; + } + + // ........................................................................ + // half-step drift: x += (dt/2) v, expressed as i/dx increments. + // transform converts the Cartesian velocity to a coordinate + // (cell-fraction) velocity by dividing by the cell size dx. + // ........................................................................ + Inline void halfDrift(int& i1, + real_t& dx1, + int& i2, + real_t& dx2, + int& i3, + real_t& dx3, + const vec_t& v) const { + coord_t xp { ZERO }; + xCoord(i1, dx1, i2, dx2, i3, dx3, xp); + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + dx1 += metric.template transform<1, Idx::XYZ, Idx::U>(xp, v[0]) * dt_half; + rebucket(i1, dx1); + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + dx2 += metric.template transform<2, Idx::XYZ, Idx::U>(xp, v[1]) * dt_half; + rebucket(i2, dx2); + } + if constexpr (D == Dim::_3D) { + dx3 += metric.template transform<3, Idx::XYZ, Idx::U>(xp, v[2]) * dt_half; + rebucket(i3, dx3); + } + } + + // single-cell rebucketing of (i, dx); valid for |displacement| < 1 cell (CFL) + static Inline void rebucket(int& i, real_t& dx) { + const int shift { static_cast(dx >= ONE) - static_cast(dx < ZERO) }; + i += shift; + dx -= static_cast(shift); + } + + // fill the code-coordinate position xp from the working i/dx state + // (only consumed by metric.transform, position-independent for Minkowski) + Inline void xCoord(int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + coord_t& xp) const { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + xp[0] = static_cast(i1) + dx1; + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + xp[1] = static_cast(i2) + dx2; + } + if constexpr (D == Dim::_3D) { + xp[2] = static_cast(i3) + dx3; + } + } + + // valid cell-index guards. The field arrays span ni + 2*N_GHOSTS in each + // direction; a particle that overshoots a boundary by more than one ghost + // layer in a single step (e.g. bulk-flow Courant > 1, or near a reflecting + // wall before the BC is applied) would otherwise gather/scatter past the + // array end — an out-of-bounds atomic that page-faults on GPU. Skipping the + // out-of-range cell keeps the kernel safe; the lost edge contribution is + // negligible and the particle is wrapped/reflected by the boundary handler. + Inline auto inX1(int c) const -> bool { + return (c >= 0) && (c < ctx.ni1 + 2 * static_cast(N_GHOSTS)); + } + Inline auto inX2(int c) const -> bool { + return (c >= 0) && (c < ctx.ni2 + 2 * static_cast(N_GHOSTS)); + } + Inline auto inX3(int c) const -> bool { + return (c >= 0) && (c < ctx.ni3 + 2 * static_cast(N_GHOSTS)); + } + + // ........................................................................ + // cell-centered field gather at (i, dx) — transpose of the moment deposit. + // ........................................................................ + Inline void gather(int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + vec_t& e0, + vec_t& b0) const { + if constexpr (D == Dim::_1D) { + for (int di1 { -window }; di1 <= window; ++di1) { + const int c { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c)) { + continue; + } + const real_t S { prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c, 0); + e0[1] += S * EB(c, 1); + e0[2] += S * EB(c, 2); + b0[0] += S * EB(c, 3); + b0[1] += S * EB(c, 4); + b0[2] += S * EB(c, 5); + } + } else if constexpr (D == Dim::_2D) { + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx2 { prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx2 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c1, c2, 0); + e0[1] += S * EB(c1, c2, 1); + e0[2] += S * EB(c1, c2, 2); + b0[0] += S * EB(c1, c2, 3); + b0[1] += S * EB(c1, c2, 4); + b0[2] += S * EB(c1, c2, 5); + } + } + } else if constexpr (D == Dim::_3D) { + for (int di3 { -window }; di3 <= window; ++di3) { + const int c3 { i3 + di3 + static_cast(N_GHOSTS) }; + if (not inX3(c3)) { + continue; + } + const real_t sx3 { prtl_shape::particle_shape( + math::abs(dx3 - (static_cast(di3) + HALF))) }; + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + if (not inX2(c2)) { + continue; + } + const real_t sx23 { sx3 * prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + if (not inX1(c1)) { + continue; + } + const real_t S { sx23 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + e0[0] += S * EB(c1, c2, c3, 0); + e0[1] += S * EB(c1, c2, c3, 1); + e0[2] += S * EB(c1, c2, c3, 2); + b0[0] += S * EB(c1, c2, c3, 3); + b0[1] += S * EB(c1, c2, c3, 4); + b0[2] += S * EB(c1, c2, c3, 5); + } + } + } + } + } + + // ........................................................................ + // fused moment deposit: N -> comp 3, V = m*v -> comps 0..2 + // (non-relativistic). Cell-centered, shape-weighted, transpose of + // `gather`. Writes go through the sink's emit(c..., comp, val); bounds + // handling lives in the sink (FlatSink clips per write, the tiled sink + // routes per the select()ed footprint), so the loops here are + // unconditional. + // ........................................................................ + template + Inline void deposit(prtlidx_t p, + int i1, + real_t dx1, + int i2, + real_t dx2, + int i3, + real_t dx3, + const vec_t& v, + SinkLike& emit) const { + real_t w { ctx.inv_n0 }; + if constexpr (D == Dim::_1D) { + w /= metric.sqrt_det_h({ static_cast(i1) + HALF }); + } else if constexpr (D == Dim::_2D) { + w /= metric.sqrt_det_h( + { static_cast(i1) + HALF, static_cast(i2) + HALF }); + } else if constexpr (D == Dim::_3D) { + w /= metric.sqrt_det_h({ static_cast(i1) + HALF, + static_cast(i2) + HALF, + static_cast(i3) + HALF }); + } + if (ctx.use_weights) { + w *= particles.weight(p); + } + const real_t cN { w }; + const real_t cV0 { w * ctx.mass * v[0] }; + const real_t cV1 { w * ctx.mass * v[1] }; + const real_t cV2 { w * ctx.mass * v[2] }; + + if constexpr (D == Dim::_1D) { + for (int di1 { -window }; di1 <= window; ++di1) { + const int c { i1 + di1 + static_cast(N_GHOSTS) }; + const real_t S { prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + emit(c, 0, cV0 * S); + emit(c, 1, cV1 * S); + emit(c, 2, cV2 * S); + emit(c, 3, cN * S); + } + } else if constexpr (D == Dim::_2D) { + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + const real_t sx2 { prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + const real_t S { sx2 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + emit(c1, c2, 0, cV0 * S); + emit(c1, c2, 1, cV1 * S); + emit(c1, c2, 2, cV2 * S); + emit(c1, c2, 3, cN * S); + } + } + } else if constexpr (D == Dim::_3D) { + for (int di3 { -window }; di3 <= window; ++di3) { + const int c3 { i3 + di3 + static_cast(N_GHOSTS) }; + const real_t sx3 { prtl_shape::particle_shape( + math::abs(dx3 - (static_cast(di3) + HALF))) }; + for (int di2 { -window }; di2 <= window; ++di2) { + const int c2 { i2 + di2 + static_cast(N_GHOSTS) }; + const real_t sx23 { sx3 * prtl_shape::particle_shape( + math::abs(dx2 - (static_cast(di2) + HALF))) }; + for (int di1 { -window }; di1 <= window; ++di1) { + const int c1 { i1 + di1 + static_cast(N_GHOSTS) }; + const real_t S { sx23 * prtl_shape::particle_shape( + math::abs(dx1 - (static_cast(di1) + HALF))) }; + emit(c1, c2, c3, 0, cV0 * S); + emit(c1, c2, c3, 1, cV1 * S); + emit(c1, c2, c3, 2, cV2 * S); + emit(c1, c2, c3, 3, cN * S); + } + } + } + } + } + + // ........................................................................ + // particle boundaries — periodic / absorb / reflect (Cartesian Minkowski, + // so a reflection just negates the corresponding 3-velocity component), plus + // the MPI leaving-direction tagging. Modeled on kernels/pushers/sr.hpp:659-814. + // ........................................................................ + Inline void boundaryConditions(prtlidx_t p) const { + if constexpr (D == Dim::_1D || D == Dim::_2D || D == Dim::_3D) { + if (particles.i1(p) < 0) { + if (bc.is_periodic_i1min) { + particles.i1(p) += ctx.ni1; + particles.i1_prev(p) += ctx.ni1; + } else if (bc.is_absorb_i1min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i1min) { + particles.i1(p) = 0; + particles.dx1(p) = ONE - particles.dx1(p); + particles.ux1(p) = -particles.ux1(p); + } + } else if (particles.i1(p) >= ctx.ni1) { + if (bc.is_periodic_i1max) { + particles.i1(p) -= ctx.ni1; + particles.i1_prev(p) -= ctx.ni1; + } else if (bc.is_absorb_i1max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i1max) { + particles.i1(p) = ctx.ni1 - 1; + particles.dx1(p) = ONE - particles.dx1(p); + particles.ux1(p) = -particles.ux1(p); + } + } + } + if constexpr (D == Dim::_2D || D == Dim::_3D) { + if (particles.i2(p) < 0) { + if (bc.is_periodic_i2min) { + particles.i2(p) += ctx.ni2; + particles.i2_prev(p) += ctx.ni2; + } else if (bc.is_absorb_i2min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i2min) { + particles.i2(p) = 0; + particles.dx2(p) = ONE - particles.dx2(p); + particles.ux2(p) = -particles.ux2(p); + } + } else if (particles.i2(p) >= ctx.ni2) { + if (bc.is_periodic_i2max) { + particles.i2(p) -= ctx.ni2; + particles.i2_prev(p) -= ctx.ni2; + } else if (bc.is_absorb_i2max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i2max) { + particles.i2(p) = ctx.ni2 - 1; + particles.dx2(p) = ONE - particles.dx2(p); + particles.ux2(p) = -particles.ux2(p); + } + } + } + if constexpr (D == Dim::_3D) { + if (particles.i3(p) < 0) { + if (bc.is_periodic_i3min) { + particles.i3(p) += ctx.ni3; + particles.i3_prev(p) += ctx.ni3; + } else if (bc.is_absorb_i3min) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i3min) { + particles.i3(p) = 0; + particles.dx3(p) = ONE - particles.dx3(p); + particles.ux3(p) = -particles.ux3(p); + } + } else if (particles.i3(p) >= ctx.ni3) { + if (bc.is_periodic_i3max) { + particles.i3(p) -= ctx.ni3; + particles.i3_prev(p) -= ctx.ni3; + } else if (bc.is_absorb_i3max) { + particles.tag(p) = ParticleTag::dead; + } else if (bc.is_reflect_i3max) { + particles.i3(p) = ctx.ni3 - 1; + particles.dx3(p) = ONE - particles.dx3(p); + particles.ux3(p) = -particles.ux3(p); + } + } + } +#if defined(MPI_ENABLED) + // tag the particle with the direction it leaves the local subdomain so the + // metadomain particle exchange ships it to the right neighbor + if constexpr (D == Dim::_1D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1); + } else if constexpr (D == Dim::_2D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1, + particles.i2(p) < 0, + particles.i2(p) >= ctx.ni2); + } else if constexpr (D == Dim::_3D) { + particles.tag(p) = mpi::SendTag(particles.tag(p), + particles.i1(p) < 0, + particles.i1(p) >= ctx.ni1, + particles.i2(p) < 0, + particles.i2(p) >= ctx.ni2, + particles.i3(p) < 0, + particles.i3(p) >= ctx.ni3); + } +#endif + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_PUSHER_HPP diff --git a/src/kernels/hybrid/subcycle.hpp b/src/kernels/hybrid/subcycle.hpp new file mode 100644 index 000000000..97005c503 --- /dev/null +++ b/src/kernels/hybrid/subcycle.hpp @@ -0,0 +1,198 @@ +/** + * @file kernels/hybrid/subcycle.hpp + * @brief Support kernels for the sub-cycled (Pegasus-style) magnetic-field + * advance of the HYBRID engine. + * @implements + * - kernel::hybrid::FieldCombine_kernel<> -> dst = a*dst + b*src over a + * 3-component group (a = 0, b = 1 is a copy). Launched over the FULL + * extent so valid ghosts stay valid. + * - kernel::hybrid::VwMax_kernel<> -> parallel_reduce functor for + * max over active cells of the local whistler speed at the grid cutoff, + * v_w = d0^2 pi |B| / (dx max(N, dens_min)) -- used to pick the number of + * field sub-steps each advance needs. + * @namespaces: + * - kernel::hybrid:: + */ + +#ifndef KERNELS_HYBRID_SUBCYCLE_HPP +#define KERNELS_HYBRID_SUBCYCLE_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/numeric.h" + +namespace kernel::hybrid { + using namespace ntt; + + template + struct FieldCombine_kernel { + ndfield_t Dst; + ndfield_t Src; + const real_t a, b; + const uint8_t cd, cs; + + FieldCombine_kernel(ndfield_t& Dst, + const ndfield_t& Src, + real_t a, + real_t b, + uint8_t cd, + uint8_t cs) + : Dst { Dst } + , Src { Src } + , a { a } + , b { b } + , cd { cd } + , cs { cs } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, cd + c) = a * Dst(i1, cd + c) + b * Src(i1, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, i2, cd + c) = a * Dst(i1, i2, cd + c) + b * Src(i1, i2, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + for (uint8_t c = 0; c < 3; ++c) { + Dst(i1, i2, i3, cd + c) = a * Dst(i1, i2, i3, cd + c) + + b * Src(i1, i2, i3, cs + c); + } + } else { + raise::KernelError( + HERE, + "FieldCombine_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief max-reduce of v_w(B, N) = d0^2 pi |B|_phys / (dx max(N, dens_min)) + * over active cells. B is read from a 6-component field (em) at + * comp_B (U-basis: in-plane comps are physical/dx); N from aux comp 3. + * STENCIL-AWARE like EMF_kernel::hall_limiter: |B| and N are taken + * from the worst cells of the +/-1 neighborhood -- at a void / + * overshoot interface the large B and the small N sit in DIFFERENT + * cells, and a same-cell estimate misses exactly the edge whistler + * speed the discrete Hall operator feels. + */ + template + struct VwMax_kernel { + ndfield_t Bfld; + ndfield_t NN; + const uint8_t comp_B; + const uint8_t comp_NN; + const real_t d0, dens_min, dx; + + VwMax_kernel(const ndfield_t& Bfld, + const ndfield_t& NN, + uint8_t comp_B, + uint8_t comp_NN, + real_t d0, + real_t dens_min, + real_t dx) + : Bfld { Bfld } + , NN { NN } + , comp_B { comp_B } + , comp_NN { comp_NN } + , d0 { d0 } + , dens_min { dens_min } + , dx { dx } {} + + Inline void operator()(cellidx_t i1, real_t& lmax) const { + if constexpr (D == Dim::_1D) { + real_t bsq = ZERO; + real_t nn = NN(i1, comp_NN); + for (int di = -1; di <= 1; ++di) { + const auto ii = i1 + static_cast(di); + // U-basis: b1 = Bx/dx, b2/b3 physical + const real_t b = SQR(dx * Bfld(ii, comp_B + 0)) + + SQR(Bfld(ii, comp_B + 1)) + + SQR(Bfld(ii, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, comp_NN)); + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, real_t& lmax) const { + if constexpr (D == Dim::_2D) { + real_t bsq = ZERO; + real_t nn = NN(i1, i2, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + // U-basis: b1/b2 = B/dx, b3 physical + const real_t b = SQR(dx * Bfld(ii, jj, comp_B + 0)) + + SQR(dx * Bfld(ii, jj, comp_B + 1)) + + SQR(Bfld(ii, jj, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, comp_NN)); + } + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3, real_t& lmax) const { + if constexpr (D == Dim::_3D) { + real_t bsq = ZERO; + real_t nn = NN(i1, i2, i3, comp_NN); + for (int di = -1; di <= 1; ++di) { + for (int dj = -1; dj <= 1; ++dj) { + for (int dk = -1; dk <= 1; ++dk) { + const auto ii = i1 + static_cast(di); + const auto jj = i2 + static_cast(dj); + const auto kk = i3 + static_cast(dk); + // U-basis: all comps physical/dx + const real_t b = SQR(dx * Bfld(ii, jj, kk, comp_B + 0)) + + SQR(dx * Bfld(ii, jj, kk, comp_B + 1)) + + SQR(dx * Bfld(ii, jj, kk, comp_B + 2)); + bsq = math::max(bsq, b); + nn = math::min(nn, NN(ii, jj, kk, comp_NN)); + } + } + } + const real_t vw = SQR(d0) * static_cast(constant::PI) * + math::sqrt(bsq) / (dx * math::max(nn, dens_min)); + lmax = math::max(lmax, vw); + } else { + raise::KernelError(HERE, + "VwMax_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_SUBCYCLE_HPP diff --git a/src/kernels/hybrid/wall_bcs.hpp b/src/kernels/hybrid/wall_bcs.hpp new file mode 100644 index 000000000..000e7daa2 --- /dev/null +++ b/src/kernels/hybrid/wall_bcs.hpp @@ -0,0 +1,380 @@ +/** + * @file kernels/hybrid/wall_bcs.hpp + * @brief Reflecting-wall (perfect-conductor) boundary kernels for the HYBRID + * engine, x1 walls only. + * @implements + * - kernel::hybrid::WallEdgeE_kernel<> -> edge-E: E_tan = 0 on the wall + * plane + odd/even mirror into the ghosts + * - kernel::hybrid::WallFaceB_kernel<> -> face-B: even (zero-gradient) + * mirror into the ghosts; the wall-plane B_n is NOT touched + * - kernel::hybrid::WallMoments_kernel<> -> cell-centered moments (aux): + * additive fold of the ghost deposit tails (image plasma) + mirror fill + * - kernel::hybrid::WallBckp_kernel<> -> cell-centered Ec/Bc gather + * fields: conductor-sign mirror fill of the ghosts + * @namespaces: + * - kernel::hybrid:: + * + * Conventions (P = true is the +x1 wall): + * - i_edge is the wall-plane NODE index: i_min(x1) for -x1, i_max(x1) for + * +x1. Cell i is bounded by nodes i and i+1, so the ghost cells are + * i_edge-1-k (minus) / i_edge+k (plus), k = 0..N_GHOSTS-1. + * - Edge-E staggering in x1: comp +0 (E_x) is x1-cell-centered, comps +1/+2 + * (E_y/E_z) sit on x1 nodes -- at i1 == i_edge they lie ON the wall plane. + * - Face-B staggering in x1: comp +0 (B_x) sits on x1 nodes, comps +1/+2 are + * x1-cell-centered. + * + * Physics: + * - Perfect conductor: E_tan = 0 on the wall plane. Faraday then freezes the + * wall-plane B_n automatically (dB_n/dt = -(curl E)_n involves only + * in-plane E_tan derivatives), which is the correct condition for an + * OBLIQUE background field (B_n stays at its initial value). The wall-plane + * B_n must therefore never be overwritten -- in particular NOT zeroed, + * which is only valid when the initial B_n is zero (perpendicular shocks) + * and otherwise plants a div(B) monopole layer at the wall. + * - B ghosts are filled with an even (zero-gradient) mirror so the Hall term + * (curl B) x B sees no fake wall current sheet. Without this, scratch + * buffers whose wall ghosts stay zero (e.g. Bf*, Bf** in `cur`) produce a + * spurious J ~ B/dx at the wall every substage. + * - Moments: a reflecting wall is realized by image plasma (u_x -> -u_x). + * The shape-function tails a near-wall particle deposits into the ghost + * cells are exactly the image particle's contribution to the active cells: + * fold them back with V_x sign-flipped (V_y, V_z, N unchanged), then fill + * the ghosts with the same-sign mirror of the active cells for the EMF / + * filter stencils. Without the fold the wall-cell density is undercounted + * and 1/N in Ohm's law is overestimated. + * - bckp (cell-centered Ec/Bc the pusher gathers): conductor-sign mirror, + * E_x even, E_y/E_z odd (-> E_tan ~ 0 at the wall), B even. Without it the + * gather mixes in zero ghosts and near-wall particles feel ~50% fields. + */ + +#ifndef KERNELS_HYBRID_WALL_BCS_HPP +#define KERNELS_HYBRID_WALL_BCS_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" + +namespace kernel::hybrid { + using namespace ntt; + + /** + * @brief E_tan = 0 on the wall plane + conductor mirror into the ghosts. + * @tparam D dimension + * @tparam P true for the +x1 wall, false for the -x1 wall + * Launch range: i1 in [0, N_GHOSTS + 1) for -x1, [0, N_GHOSTS) for +x1 + * (i1 == 0 is the wall plane, i1 >= 1 the ghost layers); other indices span + * the full transverse extent. + */ + template + struct WallEdgeE_kernel { + ndfield_t Fld; + const ncells_t i_edge; + const uint8_t c0; + + WallEdgeE_kernel(ndfield_t& Fld, ncells_t i_edge, uint8_t c0) + : Fld { Fld } + , i_edge { i_edge } + , c0 { c0 } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if (i1 == 0) { + Fld(i_edge, c0 + 1) = ZERO; + Fld(i_edge, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, c0 + 0) = Fld(i_edge + i1 - 1, c0 + 0); + Fld(i_edge - i1, c0 + 1) = -Fld(i_edge + i1, c0 + 1); + Fld(i_edge - i1, c0 + 2) = -Fld(i_edge + i1, c0 + 2); + } else { + Fld(i_edge + i1 - 1, c0 + 0) = Fld(i_edge - i1, c0 + 0); + Fld(i_edge + i1, c0 + 1) = -Fld(i_edge - i1, c0 + 1); + Fld(i_edge + i1, c0 + 2) = -Fld(i_edge - i1, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if (i1 == 0) { + Fld(i_edge, i2, c0 + 1) = ZERO; + Fld(i_edge, i2, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, c0 + 0) = Fld(i_edge + i1 - 1, i2, c0 + 0); + Fld(i_edge - i1, i2, c0 + 1) = -Fld(i_edge + i1, i2, c0 + 1); + Fld(i_edge - i1, i2, c0 + 2) = -Fld(i_edge + i1, i2, c0 + 2); + } else { + Fld(i_edge + i1 - 1, i2, c0 + 0) = Fld(i_edge - i1, i2, c0 + 0); + Fld(i_edge + i1, i2, c0 + 1) = -Fld(i_edge - i1, i2, c0 + 1); + Fld(i_edge + i1, i2, c0 + 2) = -Fld(i_edge - i1, i2, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if (i1 == 0) { + Fld(i_edge, i2, i3, c0 + 1) = ZERO; + Fld(i_edge, i2, i3, c0 + 2) = ZERO; + } else { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, c0 + 0) = Fld(i_edge + i1 - 1, i2, i3, c0 + 0); + Fld(i_edge - i1, i2, i3, c0 + 1) = -Fld(i_edge + i1, i2, i3, c0 + 1); + Fld(i_edge - i1, i2, i3, c0 + 2) = -Fld(i_edge + i1, i2, i3, c0 + 2); + } else { + Fld(i_edge + i1 - 1, i2, i3, c0 + 0) = Fld(i_edge - i1, i2, i3, c0 + 0); + Fld(i_edge + i1, i2, i3, c0 + 1) = -Fld(i_edge - i1, i2, i3, c0 + 1); + Fld(i_edge + i1, i2, i3, c0 + 2) = -Fld(i_edge - i1, i2, i3, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallEdgeE_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Even (zero-gradient) mirror of face-B into the wall ghosts; the + * wall-plane B_n is left untouched (frozen by Faraday + E_tan = 0). + * @tparam D dimension + * @tparam P true for the +x1 wall + * @tparam N number of components of the field array (6 for em, 3 for cur) + * Launch range: as WallEdgeE_kernel (i1 == 0 is a no-op kept for range + * symmetry). + */ + template + struct WallFaceB_kernel { + ndfield_t Fld; + const ncells_t i_edge; + const uint8_t c0; + + WallFaceB_kernel(ndfield_t& Fld, ncells_t i_edge, uint8_t c0) + : Fld { Fld } + , i_edge { i_edge } + , c0 { c0 } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, c0 + 0) = Fld(i_edge + i1, c0 + 0); + Fld(i_edge - i1, c0 + 1) = Fld(i_edge + i1 - 1, c0 + 1); + Fld(i_edge - i1, c0 + 2) = Fld(i_edge + i1 - 1, c0 + 2); + } else { + Fld(i_edge + i1, c0 + 0) = Fld(i_edge - i1, c0 + 0); + Fld(i_edge + i1 - 1, c0 + 1) = Fld(i_edge - i1, c0 + 1); + Fld(i_edge + i1 - 1, c0 + 2) = Fld(i_edge - i1, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, i2, c0 + 0) = Fld(i_edge + i1, i2, c0 + 0); + Fld(i_edge - i1, i2, c0 + 1) = Fld(i_edge + i1 - 1, i2, c0 + 1); + Fld(i_edge - i1, i2, c0 + 2) = Fld(i_edge + i1 - 1, i2, c0 + 2); + } else { + Fld(i_edge + i1, i2, c0 + 0) = Fld(i_edge - i1, i2, c0 + 0); + Fld(i_edge + i1 - 1, i2, c0 + 1) = Fld(i_edge - i1, i2, c0 + 1); + Fld(i_edge + i1 - 1, i2, c0 + 2) = Fld(i_edge - i1, i2, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + if (i1 != 0) { + if constexpr (not P) { + Fld(i_edge - i1, i2, i3, c0 + 0) = Fld(i_edge + i1, i2, i3, c0 + 0); + Fld(i_edge - i1, i2, i3, c0 + 1) = Fld(i_edge + i1 - 1, i2, i3, c0 + 1); + Fld(i_edge - i1, i2, i3, c0 + 2) = Fld(i_edge + i1 - 1, i2, i3, c0 + 2); + } else { + Fld(i_edge + i1, i2, i3, c0 + 0) = Fld(i_edge - i1, i2, i3, c0 + 0); + Fld(i_edge + i1 - 1, i2, i3, c0 + 1) = Fld(i_edge - i1, i2, i3, c0 + 1); + Fld(i_edge + i1 - 1, i2, i3, c0 + 2) = Fld(i_edge - i1, i2, i3, c0 + 2); + } + } + } else { + raise::KernelError(HERE, + "WallFaceB_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Image-plasma treatment of the deposited moments at an x1 wall. + * FOLD = true: aux(active) += s_c * aux(ghost) -- folds the ghost + * deposit tails back as the image-particle contribution (must run + * exactly once per deposit, after SynchronizeFields(AUX)). + * FOLD = false: aux(ghost) = s_c * aux(active) -- idempotent mirror + * fill for stencils that read the wall ghosts. + * Signs: V_x odd (comp 0), V_y/V_z/N even (comps 1..3). + * @tparam D dimension + * @tparam P true for the +x1 wall + * Launch range: i1 in [0, N_GHOSTS) = ghost layer; other indices: active + * extent for FOLD (corner tails are already remapped by the transverse + * sync), full extent for fill. + */ + template + struct WallMoments_kernel { + ndfield_t Fld; + const ncells_t i_edge; + + WallMoments_kernel(ndfield_t& Fld, ncells_t i_edge) + : Fld { Fld } + , i_edge { i_edge } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, 0) += -Fld(ig, 0); + Fld(ia, 1) += Fld(ig, 1); + Fld(ia, 2) += Fld(ig, 2); + Fld(ia, 3) += Fld(ig, 3); + } else { + Fld(ig, 0) = -Fld(ia, 0); + Fld(ig, 1) = Fld(ia, 1); + Fld(ig, 2) = Fld(ia, 2); + Fld(ig, 3) = Fld(ia, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, i2, 0) += -Fld(ig, i2, 0); + Fld(ia, i2, 1) += Fld(ig, i2, 1); + Fld(ia, i2, 2) += Fld(ig, i2, 2); + Fld(ia, i2, 3) += Fld(ig, i2, 3); + } else { + Fld(ig, i2, 0) = -Fld(ia, i2, 0); + Fld(ig, i2, 1) = Fld(ia, i2, 1); + Fld(ig, i2, 2) = Fld(ia, i2, 2); + Fld(ig, i2, 3) = Fld(ia, i2, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + if constexpr (FOLD) { + Fld(ia, i2, i3, 0) += -Fld(ig, i2, i3, 0); + Fld(ia, i2, i3, 1) += Fld(ig, i2, i3, 1); + Fld(ia, i2, i3, 2) += Fld(ig, i2, i3, 2); + Fld(ia, i2, i3, 3) += Fld(ig, i2, i3, 3); + } else { + Fld(ig, i2, i3, 0) = -Fld(ia, i2, i3, 0); + Fld(ig, i2, i3, 1) = Fld(ia, i2, i3, 1); + Fld(ig, i2, i3, 2) = Fld(ia, i2, i3, 2); + Fld(ig, i2, i3, 3) = Fld(ia, i2, i3, 3); + } + } else { + raise::KernelError(HERE, + "WallMoments_kernel: 3D implementation called for D != 3"); + } + } + }; + + /** + * @brief Conductor-sign mirror fill of the cell-centered gather fields + * (bckp: Ec in comps 0..2, Bc in comps 3..5) into the x1-wall ghosts: + * E_x even, E_y/E_z odd, B even. + * @tparam D dimension + * @tparam P true for the +x1 wall + * Launch range: i1 in [0, N_GHOSTS); other indices: full extent. + */ + template + struct WallBckp_kernel { + ndfield_t Fld; + const ncells_t i_edge; + + WallBckp_kernel(ndfield_t& Fld, ncells_t i_edge) + : Fld { Fld } + , i_edge { i_edge } {} + + Inline void operator()(cellidx_t i1) const { + if constexpr (D == Dim::_1D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, 0) = Fld(ia, 0); + Fld(ig, 1) = -Fld(ia, 1); + Fld(ig, 2) = -Fld(ia, 2); + Fld(ig, 3) = Fld(ia, 3); + Fld(ig, 4) = Fld(ia, 4); + Fld(ig, 5) = Fld(ia, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 1D implementation called for D != 1"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2) const { + if constexpr (D == Dim::_2D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, i2, 0) = Fld(ia, i2, 0); + Fld(ig, i2, 1) = -Fld(ia, i2, 1); + Fld(ig, i2, 2) = -Fld(ia, i2, 2); + Fld(ig, i2, 3) = Fld(ia, i2, 3); + Fld(ig, i2, 4) = Fld(ia, i2, 4); + Fld(ig, i2, 5) = Fld(ia, i2, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 2D implementation called for D != 2"); + } + } + + Inline void operator()(cellidx_t i1, cellidx_t i2, cellidx_t i3) const { + if constexpr (D == Dim::_3D) { + const ncells_t ig = P ? (i_edge + i1) : (i_edge - 1 - i1); + const ncells_t ia = P ? (i_edge - 1 - i1) : (i_edge + i1); + Fld(ig, i2, i3, 0) = Fld(ia, i2, i3, 0); + Fld(ig, i2, i3, 1) = -Fld(ia, i2, i3, 1); + Fld(ig, i2, i3, 2) = -Fld(ia, i2, i3, 2); + Fld(ig, i2, i3, 3) = Fld(ia, i2, i3, 3); + Fld(ig, i2, i3, 4) = Fld(ia, i2, i3, 4); + Fld(ig, i2, i3, 5) = Fld(ia, i2, i3, 5); + } else { + raise::KernelError(HERE, + "WallBckp_kernel: 3D implementation called for D != 3"); + } + } + }; + +} // namespace kernel::hybrid + +#endif // KERNELS_HYBRID_WALL_BCS_HPP diff --git a/src/kernels/injectors.hpp b/src/kernels/injectors.hpp index 5d7904cd9..e623279fc 100644 --- a/src/kernels/injectors.hpp +++ b/src/kernels/injectors.hpp @@ -3,6 +3,8 @@ * @brief Kernels for injecting particles in different ways * @implements * - kernel::UniformInjector_kernel<> + * - kernel::UniformInjectorSingle_kernel<> + * - kernel::LatticeInjector_kernel<> * - kernel::GlobalInjector_kernel<> * - kernel::NonUniformInjector_kernel<> * @namespaces: @@ -17,6 +19,7 @@ #include "arch/kokkos_aliases.h" #include "traits/archetypes.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -222,17 +225,21 @@ namespace kernel { if constexpr (M::CoordType == Coord::Cartesian) { energy_dist_1(x_Ph, v1); energy_dist_2(x_Ph, v2); - } else if constexpr (S == SimEngine::SRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCartesianBasis) { coord_t x_Cd_ { ZERO }; x_Cd_[0] = x_Cd[0]; x_Cd_[1] = x_Cd[1]; - x_Cd_[2] = ZERO; // phi = 0 + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Cd_[2] = ZERO; // phi = 0 + } else { + x_Cd_[2] = x_Cd[2]; + } vec_t v_Ph { ZERO }; energy_dist_1(x_Ph, v_Ph); metric.template transform_xyz(x_Cd_, v_Ph, v1); energy_dist_2(x_Ph, v_Ph); metric.template transform_xyz(x_Cd_, v_Ph, v2); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { vec_t v_Ph { ZERO }; energy_dist_1(x_Ph, v_Ph); metric.template transform(x_Cd, v_Ph, v1); @@ -288,6 +295,351 @@ namespace kernel { } }; // struct UniformInjector_kernel + /** + * @brief Single-species variant of UniformInjector_kernel + * @note injects one species per cell (e.g. the lone ion species of the hybrid + * engine), where the charge-neutral pair injector does not apply + */ + template ED> + struct UniformInjectorSingle_kernel { + + array_t i1s, i2s, i3s; + array_t dx1s, dx2s, dx3s; + array_t ux1s, ux2s, ux3s; + array_t phis; + array_t weights; + array_t tags; + array_t pldis; + + const npart_t offset; + const npart_t domain_idx, cntr; + const bool use_tracking; + const M metric; + const array_t xi_min, xi_max; + const ED energy_dist; + const real_t inv_V0; + random_number_pool_t random_pool; + + UniformInjectorSingle_kernel(Particles& species, + npart_t domain_idx, + const M& metric, + const array_t& xi_min, + const array_t& xi_max, + const ED& energy_dist, + real_t inv_V0, + random_number_pool_t& random_pool) + : i1s { species.i1 } + , i2s { species.i2 } + , i3s { species.i3 } + , dx1s { species.dx1 } + , dx2s { species.dx2 } + , dx3s { species.dx3 } + , ux1s { species.ux1 } + , ux2s { species.ux2 } + , ux3s { species.ux3 } + , phis { species.phi } + , weights { species.weight } + , tags { species.tag } + , pldis { species.pld_i } + , offset { species.npart() } + , domain_idx { domain_idx } + , cntr { species.counter() } + , use_tracking { species.use_tracking() } + , metric { metric } + , xi_min { xi_min } + , xi_max { xi_max } + , energy_dist { energy_dist } + , inv_V0 { inv_V0 } + , random_pool { random_pool } { + if (use_tracking) { +#if !defined(MPI_ENABLED) + raise::ErrorIf(species.pld_i.extent(1) < 1, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 1", + HERE); +#else + raise::ErrorIf(species.pld_i.extent(1) < 2, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 2", + HERE); +#endif + } + } + + Inline void operator()(prtlidx_t p) const { + coord_t x_Cd { ZERO }; + tuple_t xi_Cd { 0 }; + tuple_t dxi_Cd { static_cast(0) }; + vec_t v { ZERO }; + { // generate a random coordinate + auto rand_gen = random_pool.get_state(); + if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or + M::Dim == Dim::_3D) { + x_Cd[0] = xi_min(0) + Random(rand_gen) * (xi_max(0) - xi_min(0)); + xi_Cd[0] = static_cast(x_Cd[0]); + dxi_Cd[0] = static_cast(x_Cd[0] - xi_Cd[0]); + } + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + x_Cd[1] = xi_min(1) + Random(rand_gen) * (xi_max(1) - xi_min(1)); + xi_Cd[1] = static_cast(x_Cd[1]); + dxi_Cd[1] = static_cast(x_Cd[1] - xi_Cd[1]); + } + if constexpr (M::Dim == Dim::_3D) { + x_Cd[2] = xi_min(2) + Random(rand_gen) * (xi_max(2) - xi_min(2)); + xi_Cd[2] = static_cast(x_Cd[2]); + dxi_Cd[2] = static_cast(x_Cd[2] - xi_Cd[2]); + } + random_pool.free_state(rand_gen); + } + { // generate the velocity + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + if constexpr (M::CoordType == Coord::Cartesian) { + energy_dist(x_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCartesianBasis) { + coord_t x_Cd_ { ZERO }; + x_Cd_[0] = x_Cd[0]; + x_Cd_[1] = x_Cd[1]; + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Cd_[2] = ZERO; // phi = 0 + } else { + x_Cd_[2] = x_Cd[2]; + } + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform_xyz(x_Cd_, v_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform(x_Cd, v_Ph, v); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); + } + } + real_t weight = ONE; + if constexpr (M::CoordType != Coord::Cartesian) { + const auto sqrt_det_h = metric.sqrt_det_h(x_Cd); + weight = sqrt_det_h * inv_V0; + } + // clang-format off + if (not use_tracking) { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO); + } else { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO, + domain_idx, cntr + p); + } + // clang-format on + } + }; // struct UniformInjectorSingle_kernel + + /** + * @brief Quiet-start injector: exactly `prod(nppd_i)` particles per cell placed + * on a deterministic sub-cell lattice (no random positions) + * @note Single species (as UniformInjectorSingle_kernel); call it once per + * species with the same lattice to inject an exactly neutral pair. + * @note The particle index decomposes as p = cell * ppc_cell + sub, with `cell` + * running over the cells of the (cell-aligned) injection region and `sub` + * over the nppd_1 x nppd_2 x nppd_3 sub-cell lattice sites of one cell, + * placed at dx_i = (s_i + 1/2) / nppd_i. A lattice commensurate with the + * grid deposits an EXACTLY uniform density for any B-spline shape (also + * for a non-cubic nppd_i), i.e. it + * carries no 1/sqrt(ppc) density shot noise -- unlike random placement, + * whose (multinomial) per-cell count noise is white down to the longest + * wavelengths and, e.g., Bragg-scatters a whistler into its + * counter-propagating partner. + * @note Velocities still come from `energy_dist`, so a finite-temperature draw + * remains noisy in velocity space; only the positional/density noise is + * removed. + */ + template ED> + struct LatticeInjector_kernel { + + array_t i1s, i2s, i3s; + array_t dx1s, dx2s, dx3s; + array_t ux1s, ux2s, ux3s; + array_t phis; + array_t weights; + array_t tags; + array_t pldis; + + const npart_t offset; + const npart_t domain_idx, cntr; + const bool use_tracking; + const M metric; + // cell-aligned injection region: first cell and # of cells per direction + const int i1_offset, i2_offset, i3_offset; + const npart_t ncells_1, ncells_2; + // sub-cell lattice: nppd_i sites in direction i, ppc_cell = prod(nppd_i) + const npart_t ppc_cell, nppd_1, nppd_2, nppd_3; + const real_t inv_nppd_1, inv_nppd_2, inv_nppd_3; + const ED energy_dist; + const real_t inv_V0; + + LatticeInjector_kernel(Particles& species, + npart_t domain_idx, + const M& metric, + int i1_offset, + int i2_offset, + int i3_offset, + npart_t ncells_1, + npart_t ncells_2, + npart_t nppd_1, + npart_t nppd_2, + npart_t nppd_3, + const ED& energy_dist, + real_t inv_V0) + : i1s { species.i1 } + , i2s { species.i2 } + , i3s { species.i3 } + , dx1s { species.dx1 } + , dx2s { species.dx2 } + , dx3s { species.dx3 } + , ux1s { species.ux1 } + , ux2s { species.ux2 } + , ux3s { species.ux3 } + , phis { species.phi } + , weights { species.weight } + , tags { species.tag } + , pldis { species.pld_i } + , offset { species.npart() } + , domain_idx { domain_idx } + , cntr { species.counter() } + , use_tracking { species.use_tracking() } + , metric { metric } + , i1_offset { i1_offset } + , i2_offset { i2_offset } + , i3_offset { i3_offset } + , ncells_1 { ncells_1 } + , ncells_2 { ncells_2 } + , ppc_cell { nppd_1 * nppd_2 * nppd_3 } + , nppd_1 { nppd_1 } + , nppd_2 { nppd_2 } + , nppd_3 { nppd_3 } + , inv_nppd_1 { ONE / static_cast(nppd_1) } + , inv_nppd_2 { ONE / static_cast(nppd_2) } + , inv_nppd_3 { ONE / static_cast(nppd_3) } + , energy_dist { energy_dist } + , inv_V0 { inv_V0 } { + raise::ErrorIf(nppd_1 == 0 or nppd_2 == 0 or nppd_3 == 0, + "LatticeInjector_kernel: empty lattice", + HERE); + if (use_tracking) { +#if !defined(MPI_ENABLED) + raise::ErrorIf(species.pld_i.extent(1) < 1, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 1", + HERE); +#else + raise::ErrorIf(species.pld_i.extent(1) < 2, + "Particle tracking is enabled but the " + "particle integer payload size is less " + "than 2", + HERE); +#endif + } + } + + Inline void operator()(prtlidx_t p) const { + coord_t x_Cd { ZERO }; + tuple_t xi_Cd { 0 }; + tuple_t dxi_Cd { static_cast(0) }; + vec_t v { ZERO }; + { // deterministic lattice coordinate (cell index & sub-cell site) + // the cell index and the sub-cell fraction are formed separately: no + // large-index cancellation, so dx is exact even for 10^5 cells in single + const npart_t cell { p / ppc_cell }; + const npart_t sub { p % ppc_cell }; + if constexpr (M::Dim == Dim::_1D or M::Dim == Dim::_2D or + M::Dim == Dim::_3D) { + xi_Cd[0] = i1_offset + static_cast(cell % ncells_1); + dxi_Cd[0] = static_cast( + (static_cast(sub % nppd_1) + HALF) * inv_nppd_1); + x_Cd[0] = static_cast(xi_Cd[0]) + + static_cast(dxi_Cd[0]); + } + if constexpr (M::Dim == Dim::_2D or M::Dim == Dim::_3D) { + xi_Cd[1] = i2_offset + static_cast((cell / ncells_1) % ncells_2); + dxi_Cd[1] = static_cast( + (static_cast((sub / nppd_1) % nppd_2) + HALF) * inv_nppd_2); + x_Cd[1] = static_cast(xi_Cd[1]) + + static_cast(dxi_Cd[1]); + } + if constexpr (M::Dim == Dim::_3D) { + xi_Cd[2] = i3_offset + static_cast(cell / (ncells_1 * ncells_2)); + dxi_Cd[2] = static_cast( + (static_cast(sub / (nppd_1 * nppd_2)) + HALF) * inv_nppd_3); + x_Cd[2] = static_cast(xi_Cd[2]) + + static_cast(dxi_Cd[2]); + } + } + { // generate the velocity (identical to UniformInjectorSingle_kernel) + coord_t x_Ph { ZERO }; + metric.template convert(x_Cd, x_Ph); + if constexpr (M::CoordType == Coord::Cartesian) { + energy_dist(x_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCartesianBasis) { + coord_t x_Cd_ { ZERO }; + x_Cd_[0] = x_Cd[0]; + x_Cd_[1] = x_Cd[1]; + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Cd_[2] = ZERO; // phi = 0 + } else { + x_Cd_[2] = x_Cd[2]; + } + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform_xyz(x_Cd_, v_Ph, v); + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { + vec_t v_Ph { ZERO }; + energy_dist(x_Ph, v_Ph); + metric.template transform(x_Cd, v_Ph, v); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); + } + } + real_t weight = ONE; + if constexpr (M::CoordType != Coord::Cartesian) { + const auto sqrt_det_h = metric.sqrt_det_h(x_Cd); + weight = sqrt_det_h * inv_V0; + } + // clang-format off + if (not use_tracking) { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO); + } else { + InjectParticle( + p + offset, + i1s, i2s, i3s, + dx1s, dx2s, dx3s, + ux1s, ux2s, ux3s, + phis, weights, tags, pldis, + xi_Cd, dxi_Cd, v, weight, ZERO, + domain_idx, cntr + p); + } + // clang-format on + } + }; // struct LatticeInjector_kernel + template struct GlobalInjector_kernel { static constexpr auto D = M::Dim; @@ -443,13 +795,13 @@ namespace kernel { if constexpr (M::CoordType != Coord::Cartesian) { phi = in_phi(p); } - if constexpr (S == SimEngine::SRPIC and M::CoordType != Coord::Cartesian) { + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Cd_[2] = phi; } - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { global_metric.template transform_xyz(x_Cd_, u_Ph, u_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); @@ -480,9 +832,9 @@ namespace kernel { vec_t u_Ph { in_ux1(p), in_ux2(p), in_ux3(p) }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { global_metric.template transform_xyz(x_Cd, u_Ph, u_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { global_metric.template transform(x_Cd, u_Ph, u_Cd); } else { raise::KernelError(HERE, "Unknown simulation engine"); @@ -733,7 +1085,7 @@ namespace kernel { coord_t x_Cd_ { ZERO }; x_Cd_[0] = x_Cd[0]; x_Cd_[1] = x_Cd[1]; - if constexpr (S == SimEngine::SRPIC and M::CoordType != Coord::Cartesian) { + if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Cd_[2] = ZERO; } metric.template convert(x_Cd, x_Ph); @@ -758,10 +1110,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_1(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd_, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd_, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject1(index, @@ -773,10 +1127,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_2(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd_, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd_, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject2(index, @@ -823,10 +1179,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_1(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject1( @@ -839,10 +1197,12 @@ namespace kernel { { vec_t v_T { ZERO }; energy_dist_2(x_Ph, v_T); - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { metric.template transform_xyz(x_Cd, v_T, v_Cd); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { metric.template transform(x_Cd, v_T, v_Cd); + } else { + raise::KernelError(HERE, "Unknown simulation engine"); } } inject2( diff --git a/src/kernels/particle_moments.hpp b/src/kernels/particle_moments.hpp index 1244ba21d..dea6bc213 100644 --- a/src/kernels/particle_moments.hpp +++ b/src/kernels/particle_moments.hpp @@ -17,6 +17,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/comparators.h" #include "utils/error.h" @@ -150,7 +151,7 @@ namespace kernel { Inline auto computeStressEnergyComponent(prtlidx_t p) const -> real_t { real_t u0 { ZERO }; vec_t u_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::StressEnergyInTetradBasis) { // stress-energy tensor for SR is computed in the tetrad (hatted) basis if constexpr (M::CoordType == Coord::Cartesian) { u_Phys[0] = particles.ux1(p); @@ -166,7 +167,7 @@ namespace kernel { if constexpr (D == Dim::_3D) { x_Code[2] = static_cast(particles.i3(p)) + static_cast(particles.dx3(p)); - } else { + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { x_Code[2] = particles.phi(p); } metric.template transform_xyz( @@ -177,7 +178,7 @@ namespace kernel { u0 = (mass == ZERO) ? (NORM(u_Phys[0], u_Phys[1], u_Phys[2])) : (math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2]))); - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::StressEnergyInContravariantBasis) { // stress-energy tensor for GR is computed in contravariant basis static_assert(D != Dim::_1D, "GRPIC 1D"); coord_t x_Code { ZERO }; @@ -247,12 +248,16 @@ namespace kernel { { particles.ux1(p), particles.ux2(p), particles.ux3(p) }, u_Phys); } - if (mass == ZERO) { - u0 = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + if constexpr (S == SimEngine::HYBRID) { + return mass * u_Phys[c1 - 1]; } else { - u0 = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + if (mass == ZERO) { + u0 = NORM(u_Phys[0], u_Phys[1], u_Phys[2]); + } else { + u0 = math::sqrt(ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); + } + return (mass == ZERO ? ONE : mass) * u_Phys[c1 - 1] / u0; } - return (mass == ZERO ? ONE : mass) * u_Phys[c1 - 1] / u0; } Inline auto computeEckartVelocityFluxComponent(prtlidx_t p) const -> real_t { @@ -295,7 +300,7 @@ namespace kernel { if constexpr (F == FldsID::T) { coeff = computeStressEnergyComponent(p); } else if constexpr (F == FldsID::V) { - if constexpr (S == SimEngine::GRPIC) { + if constexpr (::traits::engine::IsGR) { coeff = computeEckartVelocityFluxComponent(p); } else { coeff = computeBulk3VelocityTimesMass(p); @@ -744,13 +749,13 @@ namespace kernel { return; } real_t en; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { if (is_massive) { en = U2GAMMA(particles.ux1(p), particles.ux2(p), particles.ux3(p)) - ONE; } else { en = NORM(particles.ux1(p), particles.ux2(p), particles.ux3(p)); } - } else if constexpr (S == SimEngine::GRPIC) { + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { coord_t x_Code { ZERO }; x_Code[0] = static_cast(particles.i1(p)) + static_cast(particles.dx1(p)); diff --git a/src/kernels/particle_shapes.hpp b/src/kernels/particle_shapes.hpp index 8e155b795..507e3ded7 100644 --- a/src/kernels/particle_shapes.hpp +++ b/src/kernels/particle_shapes.hpp @@ -541,6 +541,43 @@ namespace prtl_shape { } } + /** + * @brief Spline shape S_O(x) with the order O selected at compile time. + * Compile-time analogue of ParticleMoments_kernel::shapeFunction. + * @tparam O shape order: 0 = NGP (constant), 1 = CIC, 2 = TSC, ... up to 11. + */ + template + Inline auto particle_shape(const real_t x) -> real_t { + if constexpr (O == 0u) { + return S1(x); + } else if constexpr (O == 1u) { + return S1(x); + } else if constexpr (O == 2u) { + return S2(x); + } else if constexpr (O == 3u) { + return S3(x); + } else if constexpr (O == 4u) { + return S4(x); + } else if constexpr (O == 5u) { + return S5(x); + } else if constexpr (O == 6u) { + return S6(x); + } else if constexpr (O == 7u) { + return S7(x); + } else if constexpr (O == 8u) { + return S8(x); + } else if constexpr (O == 9u) { + return S9(x); + } else if constexpr (O == 10u) { + return S10(x); + } else if constexpr (O == 11u) { + return S11(x); + } else { + static_assert(O <= 11u, "particle_shape: unsupported shape order (max 11)"); + return ZERO; + } + } + template Inline void order(const int& i, const real_t& di, int& i_min, real_t S[O + 1]) { if constexpr (O == 1u) { diff --git a/src/kernels/prtls_to_phys.hpp b/src/kernels/prtls_to_phys.hpp index b2f570a5d..eade079bb 100644 --- a/src/kernels/prtls_to_phys.hpp +++ b/src/kernels/prtls_to_phys.hpp @@ -19,6 +19,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/error.h" #include "utils/numeric.h" @@ -108,7 +109,7 @@ namespace kernel { if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { raise::ErrorIf(buff_x2.extent(0) == 0, "Invalid buffer size", HERE); } - if constexpr (((D == Dim::_2D) && (M::CoordType != Coord::Cartesian)) || + if constexpr (::traits::engine::HasImplicitPhiCoordinate or (D == Dim::_3D)) { raise::ErrorIf(buff_x3.extent(0) == 0, "Invalid buffer size", HERE); } @@ -132,75 +133,54 @@ namespace kernel { } Inline void bufferX(prtlidx_t p_from, prtlidx_t p_to) const { - if constexpr ((D == Dim::_1D) || (D == Dim::_2D) || (D == Dim::_3D)) { + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { buff_x1(p_to) = metric.template convert<1, Crd::Cd, Crd::Ph>( static_cast(i1(p_from)) + static_cast(dx1(p_from))); } - if constexpr ((D == Dim::_2D) || (D == Dim::_3D)) { + if constexpr (D == Dim::_2D or D == Dim::_3D) { buff_x2(p_to) = metric.template convert<2, Crd::Cd, Crd::Ph>( static_cast(i2(p_from)) + static_cast(dx2(p_from))); } - if constexpr ((D == Dim::_2D) && (M::CoordType != Coord::Cartesian)) { - buff_x3(p_to) = phi(p_from); - } else if constexpr (D == Dim::_3D) { + if constexpr (D == Dim::_3D) { buff_x3(p_to) = metric.template convert<3, Crd::Cd, Crd::Ph>( static_cast(i3(p_from)) + static_cast(dx3(p_from))); + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + buff_x3(p_to) = phi(p_from); } } Inline void bufferU(prtlidx_t p_from, prtlidx_t p_to) const { - vec_t u_Phys { ZERO }; - if constexpr (D == Dim::_1D) { - if constexpr (M::CoordType == Coord::Cartesian) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unsupported coordinate system in 1D"); - } - } else if constexpr (D == Dim::_2D) { - if constexpr (M::CoordType == Coord::Cartesian) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::SRPIC) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - phi(p_from) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::GRPIC) { - metric.template transform( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unrecognized simulation engine"); - } - } else if constexpr (D == Dim::_3D) { - if constexpr (S == SimEngine::SRPIC) { - metric.template transform_xyz( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - static_cast(i3(p_from)) + static_cast(dx3(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else if constexpr (S == SimEngine::GRPIC) { - metric.template transform( - { static_cast(i1(p_from)) + static_cast(dx1(p_from)), - static_cast(i2(p_from)) + static_cast(dx2(p_from)), - static_cast(i3(p_from)) + static_cast(dx3(p_from)) }, - { ux1(p_from), ux2(p_from), ux3(p_from) }, - u_Phys); - } else { - raise::KernelError(HERE, "Unrecognized simulation engine"); - } + vec_t u_Phys { ZERO }; + coord_t x_Code { ZERO }; + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { + x_Code[0] = static_cast(i1(p_from)) + + static_cast(dx1(p_from)); + } + if constexpr (D == Dim::_2D or D == Dim::_3D) { + x_Code[1] = static_cast(i2(p_from)) + + static_cast(dx2(p_from)); } + if constexpr (D == Dim::_3D) { + x_Code[2] = static_cast(i3(p_from)) + + static_cast(dx3(p_from)); + } else if constexpr (::traits::engine::HasImplicitPhiCoordinate) { + x_Code[2] = phi(p_from); + } + + if constexpr (::traits::engine::VelocitiesInCartesianBasis) { + metric.template transform_xyz( + x_Code, + { ux1(p_from), ux2(p_from), ux3(p_from) }, + u_Phys); + } else if constexpr (::traits::engine::VelocitiesInCovariantBasis) { + metric.template transform( + x_Code, + { ux1(p_from), ux2(p_from), ux3(p_from) }, + u_Phys); + } else { + raise::KernelError(HERE, "Unrecognized simulation engine"); + } + buff_ux1(p_to) = u_Phys[0]; buff_ux2(p_to) = u_Phys[1]; buff_ux3(p_to) = u_Phys[2]; diff --git a/src/kernels/reduced_stats.hpp b/src/kernels/reduced_stats.hpp index a6411f681..f80d7abcd 100644 --- a/src/kernels/reduced_stats.hpp +++ b/src/kernels/reduced_stats.hpp @@ -15,6 +15,7 @@ #include "global.h" #include "arch/kokkos_aliases.h" +#include "traits/engine.h" #include "traits/metric.h" #include "utils/numeric.h" @@ -461,7 +462,7 @@ namespace kernel { // for stress-energy tensor real_t energy { ZERO }; vec_t u_Phys { ZERO }; - if constexpr (S == SimEngine::SRPIC) { + if constexpr (::traits::engine::StressEnergyInTetradBasis) { // SR // stress-energy tensor for SR is computed in the tetrad (hatted) basis if constexpr (M::CoordType == Coord::Cartesian) { @@ -478,7 +479,8 @@ namespace kernel { if constexpr (D == Dim::_3D) { x_Code[2] = static_cast(particles.i3(p)) + static_cast(particles.dx3(p)); - } else { + } else if constexpr ( + ::traits::engine::HasImplicitPhiCoordinate) { x_Code[2] = particles.phi(p); } metric.template transform_xyz( @@ -492,7 +494,8 @@ namespace kernel { energy = mass * math::sqrt( ONE + NORM_SQR(u_Phys[0], u_Phys[1], u_Phys[2])); } - } else { + } else if constexpr ( + ::traits::engine::StressEnergyInContravariantBasis) { // GR // stress-energy tensor for GR is computed in contravariant basis static_assert(D != Dim::_1D, "GRPIC 1D"); @@ -519,6 +522,8 @@ namespace kernel { energy = mass * math::sqrt(ONE + energy); } metric.template transform(x_Code, u_Cntrv, u_Phys); + } else { + raise::KernelError(HERE, "Unsupported engine for stress-energy tensor"); } // compute the corresponding moment real_t coeff = ONE; diff --git a/src/kernels/tiled_scatter.hpp b/src/kernels/tiled_scatter.hpp new file mode 100644 index 000000000..bf39b8c77 --- /dev/null +++ b/src/kernels/tiled_scatter.hpp @@ -0,0 +1,517 @@ +/** + * @file kernels/tiled_scatter.hpp + * @brief Generic tiled (TeamPolicy + per-team SLM scratch) particle->grid + * scatter harness, shared by the current deposit and the hybrid + * moment deposit. + * + * The harness owns everything that is NOT per-particle math: tile-id + * decode, scratch allocation/zero, particle-slice clamping, the + * scratch-vs-global routing (per-particle escape valve), the team + * barrier, and the bounds-clipped cooperative flush to the global field. + * The per-particle physics lives in a `Body` functor supplied by the + * caller; flat and tiled kernels of one deposit differ only in the sink + * their body writes through. + * + * @implements + * - kernel::TiledScatter_kernel<> (TEAM_POLICY only) + * - kernel::MakeTiledPolicy<> (TEAM_POLICY only, host) + * @namespaces: + * - kernel:: + */ + +#ifndef KERNELS_TILED_SCATTER_HPP +#define KERNELS_TILED_SCATTER_HPP + +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" +#include "utils/log.h" +#include "utils/numeric.h" + +#include + +#if defined(TEAM_POLICY) + +namespace kernel { + using namespace ntt; + + /** + * @brief Generic tiled particle->grid scatter harness. + * + * One team per spatial tile (`league_size = ntiles_total`). Each team + * accumulates particle contributions into a per-team scratch buffer of + * shape `(T_TILE + 2*HALO)^D × NC` real_t, where `HALO = REACH + DRIFT` + * cells per side. Scratch atomics live in SLM (PVC: ~5–10 cycles per + * `atomic_add`); the global field is touched only once per scratch cell + * at flush time. Compared with a flat scatter-view kernel: + * - global atomic pressure ~ (T_TILE + 2*HALO)^D × NC per tile + * instead of (stencil writes per particle × particles) + * - per-particle stencil writes are tile-local (SLM) instead of + * scattering through global HBM + * + * D : dimension (must match the field view) + * NC : number of scratch components accumulated per cell + * NG : component count of the global field view (NC <= NG) + * REACH : one-sided stencil reach in cells, used ONLY for halo + * sizing: HALO = REACH + DRIFT (DRIFT = TEAM_POLICY_DRIFT, + * default 1) + * T_TILE : tile edge length (TEAM_POLICY_TILE_SIZE) + * Body : per-particle functor, see contract below + * + * **Body contract** (device-callable, captured by value): + * `template Inline void operator()(prtlidx_t p, Sink& sink) const;` + * The body must, for each particle: + * 1) decide its deposit-time footprint [lo, hi] per axis in GLOBAL + * field coordinates (including N_GHOSTS) and call + * `sink.select(lo1, hi1, ...)`; after select(), writes are routed + * to SLM scratch iff the whole footprint fits the tile window; + * 2) emit contributions via `sink(gi..., c, val)` with c in [0, NC) + * and gi... the global field storage index (incl. N_GHOSTS). + * The harness routes each write to tile scratch (no bounds test needed + * — select() proved containment) or, on escape, to the global view with + * a bounds clip against its storage extents. The global component + * written is `comp_offset + c` (comp_offset = 0 for currents / hybrid + * moments, buff_idx for single-component output moments). + * + * **Halo sizing and escape valve.** The scratch HALO is + * `REACH + DRIFT`: REACH is the one-sided stencil reach the caller + * instantiates with, DRIFT is the `team_policy_drift` CMake knob (macro + * TEAM_POLICY_DRIFT) — the number of cells a particle may drift between + * two sorts that the halo is sized to absorb — and `1` by default (the + * every-step-sorted common case). It is independent of the sort + * cadence, which is set at runtime via `spatial_sorting_interval`; + * particles that drift past the halo take the escape valve below. + * + * Correctness does **not** depend on the halo size. Any particle whose + * full footprint escapes the scratch tile — because it drifted further + * than `DRIFT`, was reordered far from its tile by a no-sort-step + * `CommunicateParticles`, or because the halo is otherwise undersized — + * is deposited *as a whole* via direct, bounds-clipped + * `Kokkos::atomic_add`s on the global field view (the per-particle + * escape valve, driven by the body's `select()`). Each particle's + * stencil is therefore deposited exactly once (entirely to SLM scratch + * when it fits, entirely to global memory when it does not), so the + * path is conservative; it is merely slower per write. Sizing `DRIFT` + * to the typical between-sort drift keeps the common case in fast SLM; + * sorting less often (or drifting past the halo) only costs + * escape-valve traffic, never accuracy. + * + * **Partition coverage.** The team iteration covers only the particles + * partitioned at the last sort, `[0, layout.npart_partitioned)`, clamped + * to the live `npart`. Particles appended past the partition since the + * sort are not seen here; the launcher deposits that tail with the + * corresponding flat kernel so every active particle is covered exactly + * once regardless of sort cadence. + */ + template + class TiledScatter_kernel { + static_assert(NC > 0u and NC <= NG, + "TiledScatter_kernel: need 0 < NC <= NG"); + static_assert(T_TILE > 0u, "T_TILE must be positive"); + static_assert(REACH >= 0, "REACH must be non-negative"); + + public: +#if defined(TEAM_POLICY_DRIFT) + static constexpr int DRIFT = static_cast(TEAM_POLICY_DRIFT); +#else + static constexpr int DRIFT = 1; +#endif + static constexpr int HALO = REACH + DRIFT; + static constexpr int TE = static_cast(T_TILE) + 2 * HALO; + + private: + static constexpr int NCi = static_cast(NC); + + using exec_space = Kokkos::DefaultExecutionSpace; + using team_policy = Kokkos::TeamPolicy; + using member_t = typename team_policy::member_type; + using scratch_t = scratch_ndfield_t; + + ndfield_t F; + Body body; + + // Tile metadata produced by SortSpatially. + array_t tile_offsets; + ncells_t ntx1 { 1u }, ntx2 { 1u }, ntx3 { 1u }; + ncells_t total_tiles { 0u }; + + /** + * Current active-particle count. `tile_offsets` partitions only the + * particles that existed at the last sort ([0, layout.npart_partitioned)); + * `npart` may differ if a pusher dead-tagged particles in place since. + * Each team clamps its `[tile_offsets(t), tile_offsets(t+1))` slice to + * `npart` so stale slots past the live array are never read. Particles + * appended *beyond* the partition (npart > npart_partitioned) are not seen + * by any team here — the launcher deposits that tail separately. + */ + npart_t npart { 0u }; + + /** + * F's full storage extent including all ghost cells. Used to clip the + * cooperative flush (and the escape-valve writes) so that a partial + * tile at the high end of the domain does not over-write past the view. + */ + int ext1 { 0 }, ext2 { 0 }, ext3 { 0 }; + + // Global component the scratch component 0 flushes into. + int comp_offset { 0 }; + + public: + /** + * @brief Per-particle write sink handed to the body. + * + * Carries the per-particle `to_scratch` routing flag, so it must be + * constructed per particle (inside the TeamThreadRange lambda, one + * thread per particle) and passed to the body by reference — never + * shared across particles/threads. + * + * `select(lo, hi, ...)` takes the footprint in GLOBAL field storage + * coordinates (incl. N_GHOSTS) and proves containment in the tile + * scratch window; `operator()(gi..., c, v)` then routes the write: + * scratch needs no per-cell bounds test (select() proved it), the + * global escape path bounds-clips against the storage extent (writes + * past the ghost stripe are re-supplied by SynchronizeFields; an + * unclipped write here faults the GPU when an escaped boundary + * particle's stencil reaches past the extent). + */ + struct Sink { + scratch_t scr; + ndfield_t F; + int o1, o2, o3; // global coord of scratch index 0 per axis + int e1, e2, e3; // global storage extents + int comp_offset; + bool to_scratch { false }; + + Inline void select(int lo1, int hi1) { + to_scratch = (lo1 - o1 >= 0) and (hi1 - o1 < TE); + } + + Inline void select(int lo1, int hi1, int lo2, int hi2) { + to_scratch = (lo1 - o1 >= 0) and (hi1 - o1 < TE) and + (lo2 - o2 >= 0) and (hi2 - o2 < TE); + } + + Inline void select(int lo1, int hi1, int lo2, int hi2, int lo3, int hi3) { + to_scratch = (lo1 - o1 >= 0) and (hi1 - o1 < TE) and + (lo2 - o2 >= 0) and (hi2 - o2 < TE) and + (lo3 - o3 >= 0) and (hi3 - o3 < TE); + } + + Inline void operator()(int g_i1, int c, real_t v) const { + if (to_scratch) { + Kokkos::atomic_add(&scr(g_i1 - o1, c), v); + } else if (g_i1 >= 0 and g_i1 < e1) { + Kokkos::atomic_add(&F(g_i1, comp_offset + c), v); + } + } + + Inline void operator()(int g_i1, int g_i2, int c, real_t v) const { + if (to_scratch) { + Kokkos::atomic_add(&scr(g_i1 - o1, g_i2 - o2, c), v); + } else if (g_i1 >= 0 and g_i1 < e1 and g_i2 >= 0 and g_i2 < e2) { + Kokkos::atomic_add(&F(g_i1, g_i2, comp_offset + c), v); + } + } + + Inline void operator()(int g_i1, int g_i2, int g_i3, int c, real_t v) const { + if (to_scratch) { + Kokkos::atomic_add(&scr(g_i1 - o1, g_i2 - o2, g_i3 - o3, c), v); + } else if (g_i1 >= 0 and g_i1 < e1 and g_i2 >= 0 and g_i2 < e2 and + g_i3 >= 0 and g_i3 < e3) { + Kokkos::atomic_add(&F(g_i1, g_i2, g_i3, comp_offset + c), v); + } + } + }; + + TiledScatter_kernel(const ndfield_t& F, + const Body& body, + const TileLayout& layout, + npart_t npart, + int comp_offset = 0) + : F { F } + , body { body } + , tile_offsets { layout.tile_offsets } + , ntx1 { layout.ntiles_per_axis[0] } + , ntx2 { layout.ntiles_per_axis[1] } + , ntx3 { layout.ntiles_per_axis[2] } + , total_tiles { layout.ntiles_total } + , npart { npart } + , comp_offset { comp_offset } { + raise::ErrorIf( + layout.tile_size != T_TILE, + "TiledScatter launched with mismatched T_TILE and runtime tile_size", + HERE); + raise::ErrorIf(comp_offset < 0 or + comp_offset + NCi > static_cast(NG), + "TiledScatter: comp_offset + NC exceeds the field view", + HERE); + /** + * @note: HALO is allowed to exceed N_GHOSTS. The cooperative + * scratch→F flush and the per-particle escape valve both bounds-clip + * their writes against `ext*` so writes that would land past F's + * ghost stripe are silently dropped (they only ever come from a + * particle whose stencil reaches into the domain ghost region, where + * the field synchronization will re-supply the contribution). + */ + if constexpr (D == Dim::_1D or D == Dim::_2D or D == Dim::_3D) { + ext1 = static_cast(F.extent(0)); + } + if constexpr (D == Dim::_2D or D == Dim::_3D) { + ext2 = static_cast(F.extent(1)); + } + if constexpr (D == Dim::_3D) { + ext3 = static_cast(F.extent(2)); + } + } + + /** + * @brief Per-team scratch size in bytes. Used by the launcher to set + * `team_policy.set_scratch_size(0, Kokkos::PerTeam(bytes))`. + */ + static constexpr size_t scratch_bytes() { + // The component count (NC) is a *static* extent of scratch_ndfield_t + // (View / **[NC] / ***[NC]), so shmem_size() takes only + // the dynamic spatial extents — passing NC as well trips Kokkos' + // `rank_dynamic != number of arguments` abort. This matches the + // scratch View construction below, which also omits the NC. + if constexpr (D == Dim::_1D) { + return scratch_t::shmem_size(TE); + } else if constexpr (D == Dim::_2D) { + return scratch_t::shmem_size(TE, TE); + } else { + return scratch_t::shmem_size(TE, TE, TE); + } + } + + Inline void operator()(const member_t& team) const { + const auto tile_id = static_cast(team.league_rank()); + /** + * Tile coordinates (tile-grid indices) → tile origin in **active** + * cell coords (no ghost offset). Using ncells_t to match the + * linearised tile index produced by SortSpatially. + */ + ncells_t tx1 = 0, tx2 = 0, tx3 = 0; + if constexpr (D == Dim::_1D) { + tx1 = tile_id; + } else if constexpr (D == Dim::_2D) { + tx1 = tile_id / ntx2; + tx2 = tile_id - tx1 * ntx2; + } else { + const auto plane = ntx2 * ntx3; + tx1 = tile_id / plane; + const auto rem = tile_id - tx1 * plane; + tx2 = rem / ntx3; + tx3 = rem - tx2 * ntx3; + } + /** + * origin_active = lowest active-cell index in the tile (no ghost). + * origin_F = same value translated into F's storage coordinate + * (i.e. plus N_GHOSTS). + * origin_F_low = F coordinate of scratch index 0 (i.e. origin_F - HALO). + * local index `li` in scratch ↔ global F index `gi = li + origin_F_low`. + */ + const int origin_F1_low = static_cast(tx1 * T_TILE) + + static_cast(N_GHOSTS) - HALO; + const int origin_F2_low = static_cast(tx2 * T_TILE) + + static_cast(N_GHOSTS) - HALO; + const int origin_F3_low = static_cast(tx3 * T_TILE) + + static_cast(N_GHOSTS) - HALO; + + // Clamp the tile's particle slice to the live array: slots past + // `npart` may hold stale (possibly alive-tagged) data from a prior + // step's compaction and must not be re-deposited. + const auto t_lo = tile_offsets(tile_id); + const auto t_hi = tile_offsets(tile_id + 1u); + const auto p_begin = (t_lo < npart) ? t_lo : npart; + const auto p_end = (t_hi < npart) ? t_hi : npart; + + // Allocate scratch, cooperatively zero-fill it, run the body over + // the tile's particles (one thread per particle, each through its + // own Sink), then cooperatively flush scratch to the global field + // with a bounds clip against the storage extent. + if constexpr (D == Dim::_1D) { + scratch_t scr { team.team_scratch(0), TE }; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, TE * NCi), + [&](const int idx) { + const int li = idx / NCi; + const int c = idx - li * NCi; + scr(li, c) = ZERO; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, p_begin, p_end), + [&](prtlidx_t p) { + Sink sink { scr, F, + origin_F1_low, 0, + 0, ext1, + 0, 0, + comp_offset }; + body(p, sink); + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, TE * NCi), + [&](const int idx) { + const int li = idx / NCi; + const int c = idx - li * NCi; + const int gi = li + origin_F1_low; + if (gi < 0 or gi >= ext1) { + return; + } + const real_t v = scr(li, c); + if (v != ZERO) { + Kokkos::atomic_add(&F(gi, comp_offset + c), v); + } + }); + } else if constexpr (D == Dim::_2D) { + scratch_t scr { team.team_scratch(0), TE, TE }; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, SQR(TE) * NCi), + [&](const int idx) { + const int lij = idx / NCi; + const int c = idx - lij * NCi; + const int li = lij / TE; + const int lj = lij - li * TE; + scr(li, lj, c) = ZERO; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, p_begin, p_end), + [&](prtlidx_t p) { + Sink sink { scr, F, + origin_F1_low, origin_F2_low, + 0, ext1, + ext2, 0, + comp_offset }; + body(p, sink); + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, SQR(TE) * NCi), + [&](const int idx) { + const int lij = idx / NCi; + const int c = idx - lij * NCi; + const int li = lij / TE; + const int lj = lij - li * TE; + const int gi = li + origin_F1_low; + const int gj = lj + origin_F2_low; + if ((gi < 0 or gi >= ext1) or + (gj < 0 or gj >= ext2)) { + return; + } + const real_t v = scr(li, lj, c); + if (v != ZERO) { + Kokkos::atomic_add(&F(gi, gj, comp_offset + c), + v); + } + }); + } else if constexpr (D == Dim::_3D) { + scratch_t scr { team.team_scratch(0), TE, TE, TE }; + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, CUBE(TE) * NCi), + [&](const int idx) { + const int lijk = idx / NCi; + const int c = idx - lijk * NCi; + const int li = lijk / (TE * TE); + const int rem = lijk - li * TE * TE; + const int lj = rem / TE; + const int lk = rem - lj * TE; + scr(li, lj, lk, c) = ZERO; + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, p_begin, p_end), + [&](prtlidx_t p) { + Sink sink { scr, F, + origin_F1_low, origin_F2_low, + origin_F3_low, ext1, + ext2, ext3, + comp_offset }; + body(p, sink); + }); + team.team_barrier(); + + Kokkos::parallel_for(Kokkos::TeamThreadRange(team, CUBE(TE) * NCi), + [&](const int idx) { + const int lijk = idx / NCi; + const int c = idx - lijk * NCi; + const int li = lijk / (TE * TE); + const int rem = lijk - li * TE * TE; + const int lj = rem / TE; + const int lk = rem - lj * TE; + const int gi = li + origin_F1_low; + const int gj = lj + origin_F2_low; + const int gk = lk + origin_F3_low; + if ((gi < 0 or gi >= ext1) or + (gj < 0 or gj >= ext2) or + (gk < 0 or gk >= ext3)) { + return; + } + const real_t v = scr(li, lj, lk, c); + if (v != ZERO) { + Kokkos::atomic_add( + &F(gi, gj, gk, comp_offset + c), + v); + } + }); + } + } + }; + + /** + * @brief Build the TeamPolicy for a tiled-scatter kernel (host). + * + * Factors the policy boilerplate shared by every tiled launcher: + * `TeamPolicy(ntiles, AUTO)` + per-team scratch from + * `Kern::scratch_bytes()`. The default (team_size_req == 0) leaves + * Kokkos::AUTO, which sizes the team from the backend occupancy + * heuristic. A positive request (runtime param + * `algorithms.deposit.team_policy_team_size`) overrides it, clamped to + * the scratch/backend-feasible maximum so an over-large request cannot + * abort the launch (Kokkos errors when team_size > team_size_max). No + * portable subgroup rounding is applied; pick a multiple of the device + * subgroup width (printed per arch by ideal_tile_size.py) for the best + * occupancy. + * + * The *tail pass* over `[npart_partitioned, npart)` and the *flat + * fallback* (empty tile layout) stay at each call site — they need the + * engine-specific flat kernel. + */ + template + inline auto MakeTiledPolicy(const Kern& kern, + ncells_t ntiles, + int team_size_req) -> Kokkos::TeamPolicy<> { + const auto scratch = Kokkos::PerTeam(Kern::scratch_bytes()); + Kokkos::TeamPolicy<> policy(static_cast(ntiles), Kokkos::AUTO); + policy.set_scratch_size(0, scratch); + if (team_size_req > 0) { + const int ts_max = policy.team_size_max(kern, Kokkos::ParallelForTag {}); + int ts = team_size_req; + if (ts > ts_max) { + raise::Warning( + fmt::format("algorithms.deposit.team_policy_team_size = %d exceeds " + "the tiled-scatter maximum %d on this backend; clamping " + "to %d", + team_size_req, + ts_max, + ts_max), + HERE); + ts = ts_max; + } + policy = Kokkos::TeamPolicy<>(static_cast(ntiles), ts); + policy.set_scratch_size(0, scratch); + logger::Checkpoint(fmt::format("Tiled scatter: explicit team size %d", ts), + HERE); + } + return policy; + } + +} // namespace kernel + +#endif // TEAM_POLICY + +#endif // KERNELS_TILED_SCATTER_HPP diff --git a/src/metrics/kerr_schild.h b/src/metrics/kerr_schild.h index 2f225a4f5..7338a0216 100644 --- a/src/metrics/kerr_schild.h +++ b/src/metrics/kerr_schild.h @@ -26,6 +26,7 @@ #include namespace metric { + using namespace ntt; template class KerrSchild : public MetricBase { @@ -59,10 +60,10 @@ namespace metric { } public: - static constexpr const char* Label { "kerr_schild" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; - static constexpr ntt::Metric MetricType { ntt::Metric::Kerr_Schild }; + static constexpr const char* Label { "kerr_schild" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Coord::type CoordType { Coord::Spherical }; + static constexpr Metric::type MetricType { Metric::Kerr_Schild }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/kerr_schild_0.h b/src/metrics/kerr_schild_0.h index fc4d04c9c..e6b3b9de2 100644 --- a/src/metrics/kerr_schild_0.h +++ b/src/metrics/kerr_schild_0.h @@ -27,6 +27,7 @@ #include namespace metric { + using namespace ntt; template class KerrSchild0 : public MetricBase { @@ -39,10 +40,10 @@ namespace metric { const real_t dr_inv, dtheta_inv, dphi_inv; public: - static constexpr const char* Label { "kerr_schild_0" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Kerr_Schild_0 }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; + static constexpr const char* Label { "kerr_schild_0" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::Kerr_Schild_0 }; + static constexpr Coord::type CoordType { Coord::Spherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/minkowski.h b/src/metrics/minkowski.h index 3a679b389..166c0624c 100644 --- a/src/metrics/minkowski.h +++ b/src/metrics/minkowski.h @@ -26,16 +26,17 @@ #include namespace metric { + using namespace ntt; template class Minkowski : public MetricBase { const real_t dx, dx_inv; public: - static constexpr const char* Label { "minkowski" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Minkowski }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Cartesian }; + static constexpr const char* Label { "minkowski" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::Minkowski }; + static constexpr Coord::type CoordType { Coord::Cartesian }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/qkerr_schild.h b/src/metrics/qkerr_schild.h index 7b7c3646e..87bea03a4 100644 --- a/src/metrics/qkerr_schild.h +++ b/src/metrics/qkerr_schild.h @@ -23,6 +23,7 @@ #include "metrics/metric_base.h" namespace metric { + using namespace ntt; template class QKerrSchild : public MetricBase { @@ -58,10 +59,10 @@ namespace metric { } public: - static constexpr const char* Label { "qkerr_schild" }; - static constexpr Dimension PrtlDim { D }; - static constexpr ntt::Metric MetricType { ntt::Metric::QKerr_Schild }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Qspherical }; + static constexpr const char* Label { "qkerr_schild" }; + static constexpr Dimension PrtlDim { D }; + static constexpr Metric::type MetricType { Metric::QKerr_Schild }; + static constexpr Coord::type CoordType { Coord::Qspherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/qspherical.h b/src/metrics/qspherical.h index 652991300..3f1500acf 100644 --- a/src/metrics/qspherical.h +++ b/src/metrics/qspherical.h @@ -28,6 +28,7 @@ #include namespace metric { + using namespace ntt; template class QSpherical : public MetricBase { @@ -41,10 +42,10 @@ namespace metric { const bool small_angle; public: - static constexpr const char* Label { "qspherical" }; - static constexpr Dimension PrtlDim = Dim::_3D; - static constexpr ntt::Metric MetricType { ntt::Metric::QSpherical }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Qspherical }; + static constexpr const char* Label { "qspherical" }; + static constexpr Dimension PrtlDim = Dim::_3D; + static constexpr Metric::type MetricType { Metric::QSpherical }; + static constexpr Coord::type CoordType { Coord::Qspherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/metrics/spherical.h b/src/metrics/spherical.h index 72ca69368..aef0df67c 100644 --- a/src/metrics/spherical.h +++ b/src/metrics/spherical.h @@ -25,6 +25,7 @@ #include namespace metric { + using namespace ntt; template class Spherical : public MetricBase { @@ -36,10 +37,10 @@ namespace metric { const bool small_angle { false }; public: - static constexpr const char* Label { "spherical" }; - static constexpr Dimension PrtlDim { Dim::_3D }; - static constexpr ntt::Metric MetricType { ntt::Metric::Spherical }; - static constexpr ntt::Coord::type CoordType { ntt::Coord::type::Spherical }; + static constexpr const char* Label { "spherical" }; + static constexpr Dimension PrtlDim { Dim::_3D }; + static constexpr Metric::type MetricType { Metric::Spherical }; + static constexpr Coord::type CoordType { Coord::Spherical }; using MetricBase::x1_min; using MetricBase::x1_max; using MetricBase::x2_min; diff --git a/src/output/fields.cpp b/src/output/fields.cpp index 2fa5b00ce..ffdf7e18d 100644 --- a/src/output/fields.cpp +++ b/src/output/fields.cpp @@ -3,6 +3,7 @@ #include "enums.h" #include "global.h" +#include "traits/engine.h" #include "utils/error.h" #include "utils/formatting.h" @@ -46,7 +47,7 @@ namespace out { comp = { { 1 }, { 2 }, { 3 } }; } else if (id() == FldsID::V) { // bulk velocity: 3D for SR, 4D for GR - if (S == SimEngine::SRPIC) { + if (::traits::engine::isSR(S)) { // SR: always 3-velocity components comp = { { 1 }, { 2 }, { 3 } }; } else { @@ -65,7 +66,7 @@ namespace out { } // data preparation flags if (not(is_moment() or is_custom() or is_divergence())) { - if (S == SimEngine::SRPIC) { + if (::traits::engine::userFieldsInTetradBasis(S)) { prepare_flag = PrepareOutput::ConvertToHat; } else { prepare_flag = is_gr_aux_field() ? PrepareOutput::ConvertToPhysCov diff --git a/tests/global/enums.cpp b/tests/global/enums.cpp index 66f1e2274..2a5772852 100644 --- a/tests/global/enums.cpp +++ b/tests/global/enums.cpp @@ -55,7 +55,7 @@ auto main() -> int { enum_str_t all_coords = { "cart", "sph", "qsph" }; enum_str_t all_metrics = { "minkowski", "spherical", "qspherical", "kerr_schild", "qkerr_schild", "kerr_schild_0" }; - enum_str_t all_simulation_engines = { "srpic", "grpic" }; + enum_str_t all_simulation_engines = { "srpic", "grpic", "hybrid" }; enum_str_t all_particle_bcs = { "periodic", "absorb", "atmosphere", "custom", "reflect", "horizon", "axis", "sync" }; enum_str_t all_fields_bcs = { "periodic", "match", "fixed", diff --git a/tests/kernels/CMakeLists.txt b/tests/kernels/CMakeLists.txt index 0fe23f578..6172e1d21 100644 --- a/tests/kernels/CMakeLists.txt +++ b/tests/kernels/CMakeLists.txt @@ -28,6 +28,7 @@ gen_test(ampere_mink) gen_test(deposit) if(${team_policy}) gen_test(deposit_tiled) + gen_test(moments_tiled) endif() gen_test(digital_filter) gen_test(particle_moments) @@ -41,3 +42,4 @@ gen_test(ext_force) gen_test(reduced_stats) gen_test(custom_emission) gen_test(custom_prtl_update) +gen_test(injector_lattice) diff --git a/tests/kernels/injector_lattice.cpp b/tests/kernels/injector_lattice.cpp new file mode 100644 index 000000000..c96b0f167 --- /dev/null +++ b/tests/kernels/injector_lattice.cpp @@ -0,0 +1,248 @@ +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" +#include "utils/error.h" + +#include "metrics/minkowski.h" + +#include "archetypes/particle_injector.h" +#include "kernels/injectors.hpp" + +#include + +#include +#include +#include +#include +#include +#include + +using namespace ntt; +using namespace metric; + +void errorIf(bool condition, const std::string& message) { + if (condition) { + throw std::runtime_error(message); + } +} + +/** + * @brief Constant-velocity energy distribution (satisfies EnrgDistClass) + */ +template +struct ConstVelocity { + ConstVelocity(real_t vx) : vx { vx } {} + + Inline void operator()(const coord_t&, vec_t& v) const { + v[0] = vx; + v[1] = ZERO; + v[2] = ZERO; + } + +private: + const real_t vx; +}; + +/** + * @brief Verifies kernel::LatticeInjector_kernel for a given dimension + * + * Checks that the injector is a proper quiet start: + * 1. every cell of the region receives EXACTLY prod(nppd) particles + * 2. within each cell the sub-cell positions are exactly the lattice sites + * dx_i = (s_i + 1/2) / nppd_i, each visited once -- with (1) this makes the + * deposited density exactly uniform for any shape function + * 3. particles land in the requested cells only (offsets are respected) + * 4. tags/velocities are set, and the placement is deterministic (bitwise + * reproducible between two identical launches) + */ +template +void testLatticeInjector(const std::vector& res, + const std::array& offsets, + const std::array& ncells, + const std::array& nppd) { + using M = Minkowski; + errorIf(res.size() != static_cast(D), "res.size() != D"); + + boundaries_t ext; + for (auto d { 0u }; d < static_cast(D); ++d) { + ext.push_back({ ZERO, static_cast(res[d]) }); // dx = 1 + } + M metric { res, ext }; + + const auto ppc_cell = nppd[0] * nppd[1] * nppd[2]; + npart_t ncells_tot { 1 }; + for (auto d { 0u }; d < static_cast(D); ++d) { + ncells_tot *= ncells[d]; + } + const auto nparticles = ppc_cell * ncells_tot; + + const auto make_species = [&](spidx_t idx) { + return Particles { idx, + "test", + 1.0f, + 1.0f, + nparticles, + 0, + 0, + ParticlePusher::NONE, + false, + RadiativeDrag::NONE, + EmissionType::NONE, + 0, + 0 }; + }; + + const auto inject = [&](Particles& sp1) { + Kokkos::parallel_for( + "InjectLattice", + nparticles, + kernel::LatticeInjector_kernel>( + sp1, + 0u, + metric, + offsets[0], + offsets[1], + offsets[2], + ncells[0], + ncells[1], + nppd[0], + nppd[1], + nppd[2], + ConstVelocity { ONE }, + ONE)); + Kokkos::fence(); + }; + + auto species1 = make_species(1u); + inject(species1); + + auto i1 = Kokkos::create_mirror_view(species1.i1); + auto i2 = Kokkos::create_mirror_view(species1.i2); + auto i3 = Kokkos::create_mirror_view(species1.i3); + auto d1 = Kokkos::create_mirror_view(species1.dx1); + auto d2 = Kokkos::create_mirror_view(species1.dx2); + auto d3 = Kokkos::create_mirror_view(species1.dx3); + auto tg = Kokkos::create_mirror_view(species1.tag); + auto u1 = Kokkos::create_mirror_view(species1.ux1); + Kokkos::deep_copy(i1, species1.i1); + Kokkos::deep_copy(i2, species1.i2); + Kokkos::deep_copy(i3, species1.i3); + Kokkos::deep_copy(d1, species1.dx1); + Kokkos::deep_copy(d2, species1.dx2); + Kokkos::deep_copy(d3, species1.dx3); + Kokkos::deep_copy(tg, species1.tag); + Kokkos::deep_copy(u1, species1.ux1); + // per-cell occupancy and per-cell lattice-site bookkeeping + std::vector counts(ncells_tot, 0); + std::vector sites(static_cast(ncells_tot) * ppc_cell, 0); + const auto eps = static_cast(1e-5); + + const auto site_of = [&](prtldx_t dx, npart_t n) -> npart_t { + const auto s = static_cast( + static_cast(dx) * static_cast(n)); + errorIf(s >= n, "sub-cell index out of range"); + const auto expected = (static_cast(s) + HALF) / + static_cast(n); + errorIf(math::abs(static_cast(dx) - expected) > eps, + "particle is not on a lattice site: dx = " + + std::to_string(static_cast(dx))); + return s; + }; + + for (npart_t p { 0 }; p < nparticles; ++p) { + errorIf(tg(p) != ParticleTag::alive, "particle is not alive"); + errorIf(math::abs(u1(p) - ONE) > eps, + "velocity was not set from the energy distribution"); + + npart_t cell { 0 }, site { 0 }; + { // x1 + const auto c = i1(p) - offsets[0]; + errorIf(c < 0 or static_cast(c) >= ncells[0], + "particle outside of the injection region in x1"); + cell = static_cast(c); + site = site_of(d1(p), nppd[0]); + } + if constexpr (D == Dim::_2D or D == Dim::_3D) { + const auto c = i2(p) - offsets[1]; + errorIf(c < 0 or static_cast(c) >= ncells[1], + "particle outside of the injection region in x2"); + cell += ncells[0] * static_cast(c); + site += nppd[0] * site_of(d2(p), nppd[1]); + } + if constexpr (D == Dim::_3D) { + const auto c = i3(p) - offsets[2]; + errorIf(c < 0 or static_cast(c) >= ncells[2], + "particle outside of the injection region in x3"); + cell += ncells[0] * ncells[1] * static_cast(c); + site += nppd[0] * nppd[1] * site_of(d3(p), nppd[2]); + } + counts[cell] += 1; + auto& visited = sites[static_cast(cell) * ppc_cell + site]; + errorIf(visited != 0, "two particles on the same lattice site"); + visited = 1; + } + for (npart_t c { 0 }; c < ncells_tot; ++c) { + errorIf(counts[c] != ppc_cell, + "cell " + std::to_string(c) + " holds " + + std::to_string(counts[c]) + " particles instead of " + + std::to_string(ppc_cell) + " (density shot noise!)"); + } + + // determinism: an identical launch must reproduce the placement bit for bit + auto species1b = make_species(1u); + inject(species1b); + auto i1b = Kokkos::create_mirror_view(species1b.i1); + auto d1b = Kokkos::create_mirror_view(species1b.dx1); + Kokkos::deep_copy(i1b, species1b.i1); + Kokkos::deep_copy(d1b, species1b.dx1); + for (npart_t p { 0 }; p < nparticles; ++p) { + errorIf(i1(p) != i1b(p) or d1(p) != d1b(p), + "the lattice injector is not deterministic"); + } +} + +/** + * @brief Verifies arch::BalancedLattice: the per-direction counts must multiply + * back to ppc exactly for every ppc, in every dimension + */ +void testBalancedLattice(npart_t ppc_max) { + for (dim_t dim { 1 }; dim <= 3; ++dim) { + for (npart_t ppc { 1 }; ppc <= ppc_max; ++ppc) { + const auto nppd = arch::BalancedLattice(ppc, dim); + errorIf(nppd[0] * nppd[1] * nppd[2] != ppc, + "BalancedLattice(" + std::to_string(ppc) + ", " + + std::to_string(dim) + ") does not multiply back to ppc"); + for (auto d { 0u }; d < 3u; ++d) { + errorIf(nppd[d] == 0, "BalancedLattice produced a zero count"); + errorIf(d >= static_cast(dim) and nppd[d] != 1, + "BalancedLattice populated an unused direction"); + } + } + } +} + +auto main(int argc, char* argv[]) -> int { + Kokkos::initialize(argc, argv); + try { + testBalancedLattice(1024); + // 1D: any ppc is a valid lattice + testLatticeInjector({ 16 }, { 0, 0, 0 }, { 16, 1, 1 }, { 8, 1, 1 }); + testLatticeInjector({ 16 }, { 0, 0, 0 }, { 16, 1, 1 }, { 1, 1, 1 }); + testLatticeInjector({ 16 }, { 0, 0, 0 }, { 16, 1, 1 }, { 256, 1, 1 }); + // sub-region (cell-aligned): offset respected, nothing spills over + testLatticeInjector({ 16 }, { 4, 0, 0 }, { 5, 1, 1 }, { 3, 1, 1 }); + // 2D/3D, including rectangular sub-cell lattices (ppc = 512 -> 16 x 32) + testLatticeInjector({ 8, 8 }, { 0, 0, 0 }, { 8, 8, 1 }, { 4, 4, 1 }); + testLatticeInjector({ 8, 8 }, { 2, 3, 0 }, { 3, 4, 1 }, { 2, 5, 1 }); + testLatticeInjector({ 4, 4 }, { 0, 0, 0 }, { 4, 4, 1 }, { 16, 32, 1 }); + testLatticeInjector({ 4, 4, 4 }, { 0, 0, 0 }, { 4, 4, 4 }, { 3, 3, 3 }); + testLatticeInjector({ 4, 4, 4 }, { 1, 0, 2 }, { 2, 4, 2 }, { 2, 3, 4 }); + } catch (std::exception& e) { + std::cerr << e.what() << std::endl; + Kokkos::finalize(); + return -1; + } + Kokkos::finalize(); + return 0; +} diff --git a/tests/kernels/moments_tiled.cpp b/tests/kernels/moments_tiled.cpp new file mode 100644 index 000000000..26d175223 --- /dev/null +++ b/tests/kernels/moments_tiled.cpp @@ -0,0 +1,448 @@ +/** + * @file tests/kernels/moments_tiled.cpp + * @brief Numerical-equivalence + conservation test for the tiled hybrid + * push + moment deposit. + * + * Runs the flat (`kernel::hybrid::Pusher_kernel` over a ScatterView) and + * tiled (`kernel::TiledScatter_kernel` harness with the same kernel as + * body) paths on identical particle SoA inputs and asserts: + * - the resulting aux moments (V -> comps 0..2, N -> comp 3) agree + * cell-by-cell within a small floating-point tolerance; + * - aux comps 4..5 are untouched by the tiled flush (NC = 4 < NG = 6); + * - exact conservation: sum_cells N and sum_cells V match the + * analytically summed particle weights (interior particles, so the + * shape function partitions unity); + * - the per-particle escape valve (maximally-stale tile layout) and the + * partitioned-prefix + flat-tail composition both reproduce the flat + * reference. + * + * Modes covered: MomentsOnly (stored positions, no push) and Predictor + * (in-register push with zero E/B — velocities unchanged, positions + * drift by dt*v — exercising the post-push footprint select without + * mutating the SoA, so flat and tiled share one particle set). + * + * Built only when `team_policy=ON` (`-D TEAM_POLICY` defined). + */ + +#include "enums.h" +#include "global.h" + +#include "arch/kokkos_aliases.h" + +#include "metrics/minkowski.h" + +#include "kernels/hybrid/pusher.hpp" +#include "kernels/tiled_scatter.hpp" + +#include +#include + +#include +#include +#include +#include +#include + +namespace { + + using namespace ntt; + + // Pack the per-test SoA arrays into a ParticleArrays — the struct the + // pusher kernel takes. Payload (pld_*) members stay default; these are + // 2D Cartesian Minkowski cases, so phi/i3/dx3 are present but unread. + ParticleArrays pack_arrays(const array_t& i1, + const array_t& i2, + const array_t& i3, + const array_t& i1_prev, + const array_t& i2_prev, + const array_t& i3_prev, + const array_t& dx1, + const array_t& dx2, + const array_t& dx3, + const array_t& dx1_prev, + const array_t& dx2_prev, + const array_t& dx3_prev, + const array_t& ux1, + const array_t& ux2, + const array_t& ux3, + const array_t& phi, + const array_t& weight, + const array_t& tag) { + ParticleArrays pa; + pa.i1 = i1, pa.i2 = i2, pa.i3 = i3; + pa.i1_prev = i1_prev, pa.i2_prev = i2_prev, pa.i3_prev = i3_prev; + pa.dx1 = dx1, pa.dx2 = dx2, pa.dx3 = dx3; + pa.dx1_prev = dx1_prev, pa.dx2_prev = dx2_prev, pa.dx3_prev = dx3_prev; + pa.ux1 = ux1, pa.ux2 = ux2, pa.ux3 = ux3; + pa.phi = phi, pa.weight = weight, pa.tag = tag; + return pa; + } + + // Host mirror of one particle's state, used to build the SoA and the + // analytic conservation sums. + struct TestPrtl { + int i1, i2; + real_t dx1, dx2; + real_t ux1, ux2, ux3; + real_t weight; + }; + + // Cell-by-cell comparison of the deposited moments (comps 0..3); throws + // on mismatch. Also asserts comps 4..5 of `aux_tiled` are exactly zero + // (the tiled flush must not touch components >= NC). + void compare_aux(const ndfield_t& aux_flat, + const ndfield_t& aux_tiled, + unsigned short T_TILE, + const char* label) { + auto h_flat = Kokkos::create_mirror_view(aux_flat); + auto h_tiled = Kokkos::create_mirror_view(aux_tiled); + Kokkos::deep_copy(h_flat, aux_flat); + Kokkos::deep_copy(h_tiled, aux_tiled); + + const real_t eps = static_cast(1.0e-5); + real_t max_diff = ZERO; + int fail_count = 0; + for (ncells_t i = 0; i < h_flat.extent(0); ++i) { + for (ncells_t j = 0; j < h_flat.extent(1); ++j) { + for (int c = 0; c < 4; ++c) { + const real_t a = h_flat(i, j, c); + const real_t b = h_tiled(i, j, c); + const real_t diff = math::fabs(a - b); + const real_t mag = math::max(math::fabs(a), math::fabs(b)); + if (diff > max_diff) { + max_diff = diff; + } + if (diff > eps * math::max(mag, static_cast(1.0))) { + if (fail_count < 5) { + std::cerr << " [" << label << "] aux(" << i << "," << j + << ",c=" << c << ") flat=" << a << " tiled=" << b + << " diff=" << diff << '\n'; + } + ++fail_count; + } + } + for (int c = 4; c < 6; ++c) { + if (h_tiled(i, j, c) != ZERO) { + if (fail_count < 5) { + std::cerr << " [" << label << "] aux(" << i << "," << j + << ",c=" << c << ") = " << h_tiled(i, j, c) + << " != 0 (comp >= NC must stay untouched)\n"; + } + ++fail_count; + } + } + } + } + if (fail_count > 0) { + std::cerr << "moments_tiled[" << label << "] FAILED for T_TILE=" << T_TILE + << " : " << fail_count << " mismatches; max_diff=" << max_diff + << '\n'; + throw std::logic_error("tiled hybrid moment deposit mismatch"); + } + std::cerr << "moments_tiled[" << label << "] OK T_TILE=" << T_TILE + << " max_diff=" << max_diff << '\n'; + } + + // Conservation: the shape function partitions unity for particles whose + // full stencil lies inside the storage array, so + // sum_cells N = sum_p inv_n0 / sqrt_det_h * w_p + // sum_cells V_c = sum_p inv_n0 / sqrt_det_h * w_p * mass * v_c(p) + // (sqrt_det_h is uniform for Minkowski). Accumulated in double. + void check_conservation(const ndfield_t& aux, + const std::vector& prtls, + double w_norm, // inv_n0/sqrt_det_h + double mass, + unsigned short T_TILE, + const char* label) { + auto h = Kokkos::create_mirror_view(aux); + Kokkos::deep_copy(h, aux); + + double got[4] = { 0.0, 0.0, 0.0, 0.0 }; + for (ncells_t i = 0; i < h.extent(0); ++i) { + for (ncells_t j = 0; j < h.extent(1); ++j) { + for (int c = 0; c < 4; ++c) { + got[c] += static_cast(h(i, j, c)); + } + } + } + double expect[4] = { 0.0, 0.0, 0.0, 0.0 }; + for (const auto& p : prtls) { + const double w = w_norm * static_cast(p.weight); + expect[0] += w * mass * static_cast(p.ux1); + expect[1] += w * mass * static_cast(p.ux2); + expect[2] += w * mass * static_cast(p.ux3); + expect[3] += w; + } + bool failed = false; + for (int c = 0; c < 4; ++c) { + const double scale = std::max({ std::fabs(expect[c]), std::fabs(got[c]), 1.0 }); + if (std::fabs(got[c] - expect[c]) > 1.0e-5 * scale) { + failed = true; + std::cerr << "moments_tiled[" << label << "] NON-CONSERVED comp " << c + << " for T_TILE=" << T_TILE << " : sum=" << got[c] + << " expected=" << expect[c] + << " ratio=" << (got[c] / expect[c]) << '\n'; + } + } + if (failed) { + throw std::logic_error("tiled hybrid moment deposit non-conservation"); + } + std::cerr << "moments_tiled[" << label << "] conserved T_TILE=" << T_TILE + << '\n'; + } + + /** + * One test case. + * + * @tparam T_TILE tile edge length + * @tparam Mode MomentsOnly or Predictor + * @param stale_layout bucket ALL partitioned particles into tile 0 (the + * maximally-stale layout): every particle off tile 0 + * must take the per-particle escape valve. + * @param n_partitioned how many of the alive particles the tile layout + * partitions; the remainder [n_partitioned, n_alive) + * is deposited by the flat tail pass, emulating the + * launcher's appended-particles handling. + */ + template + void run_case(bool stale_layout, int n_partitioned, const char* label) { + using metric_t = metric::Minkowski; + using kernel_t = kernel::hybrid::Pusher_kernel; + + constexpr ncells_t nx1 = 50u, nx2 = 50u; + metric_t metric { { nx1, nx2 }, { { 0.0, 55.0 }, { 0.0, 55.0 } }, {} }; + // Minkowski: sqrt_det_h = dx^2, dx = 55/50 + const double dxc = 55.0 / 50.0; + const double sqrt_det_h = dxc * dxc; + + // interior particle lattice (margins >= 4 cells so any build's + // window + the tiny Predictor drift stays inside the storage array — + // required for the exact conservation check) + constexpr int n_base = 5; + const int bases[n_base] = { 4, 13, 25, 37, 44 }; + constexpr int n_alive = n_base * n_base; // 25 + constexpr int n_slots = 32; // dead tail beyond n_alive + + std::vector prtls; + prtls.reserve(n_alive); + int p = 0; + for (int a = 0; a < n_base; ++a) { + for (int b = 0; b < n_base; ++b, ++p) { + TestPrtl tp; + tp.i1 = bases[a]; + tp.i2 = bases[b]; + tp.dx1 = static_cast(0.15 + 0.028 * p); + tp.dx2 = static_cast(0.85 - 0.026 * p); + tp.ux1 = static_cast(0.9 - 0.07 * p); + tp.ux2 = static_cast(-0.8 + 0.06 * p); + tp.ux3 = static_cast(2.5 - 0.1 * a); + tp.weight = static_cast(0.5 + 0.05 * p); + prtls.push_back(tp); + } + } + // NOTE: iterating a-major, b-minor makes (i1/T, i2/T) lexicographically + // nondecreasing, i.e. the natural order is already tile-sorted for the + // row-major tile index tx1 * ntx2 + tx2 — as SortSpatially guarantees. + + array_t i1 { "i1", n_slots }, i2 { "i2", n_slots }, + i3 { "i3", n_slots }; + array_t i1_prev { "i1_prev", n_slots }, + i2_prev { "i2_prev", n_slots }, i3_prev { "i3_prev", n_slots }; + array_t dx1 { "dx1", n_slots }, dx2 { "dx2", n_slots }, + dx3 { "dx3", n_slots }; + array_t dx1_prev { "dx1_prev", n_slots }, + dx2_prev { "dx2_prev", n_slots }, dx3_prev { "dx3_prev", n_slots }; + array_t ux1 { "ux1", n_slots }, ux2 { "ux2", n_slots }, + ux3 { "ux3", n_slots }; + array_t phi { "phi", n_slots }, weight { "weight", n_slots }; + array_t tag { "tag", n_slots }; + + { + auto h_i1 = Kokkos::create_mirror_view(i1); + auto h_i2 = Kokkos::create_mirror_view(i2); + auto h_dx1 = Kokkos::create_mirror_view(dx1); + auto h_dx2 = Kokkos::create_mirror_view(dx2); + auto h_ux1 = Kokkos::create_mirror_view(ux1); + auto h_ux2 = Kokkos::create_mirror_view(ux2); + auto h_ux3 = Kokkos::create_mirror_view(ux3); + auto h_w = Kokkos::create_mirror_view(weight); + auto h_tag = Kokkos::create_mirror_view(tag); + for (int q = 0; q < n_alive; ++q) { + h_i1(q) = prtls[q].i1; + h_i2(q) = prtls[q].i2; + h_dx1(q) = static_cast(prtls[q].dx1); + h_dx2(q) = static_cast(prtls[q].dx2); + h_ux1(q) = prtls[q].ux1; + h_ux2(q) = prtls[q].ux2; + h_ux3(q) = prtls[q].ux3; + h_w(q) = prtls[q].weight; + h_tag(q) = ParticleTag::alive; + } + // slots >= n_alive stay zero == dead + Kokkos::deep_copy(i1, h_i1); + Kokkos::deep_copy(i2, h_i2); + Kokkos::deep_copy(dx1, h_dx1); + Kokkos::deep_copy(dx2, h_dx2); + Kokkos::deep_copy(ux1, h_ux1); + Kokkos::deep_copy(ux2, h_ux2); + Kokkos::deep_copy(ux3, h_ux3); + Kokkos::deep_copy(weight, h_w); + Kokkos::deep_copy(tag, h_tag); + } + auto arrays = pack_arrays(i1, i2, i3, + i1_prev, i2_prev, i3_prev, + dx1, dx2, dx3, + dx1_prev, dx2_prev, dx3_prev, + ux1, ux2, ux3, + phi, weight, tag); + + // context: non-trivial mass / inv_n0 / weights; dt small enough that a + // Predictor push drifts every particle by << 1 cell (stays interior) + const float mass = 1.5f, charge = 1.0f; + const real_t dt = 0.01, omegaB0 = 0.8, inv_n0 = 0.7; + const kernel::hybrid::PusherContext ctx { mass, + charge, + dt, + omegaB0, + inv_n0, + /* use_weights */ true, + static_cast(nx1), + static_cast(nx2), + 1 }; + const boundaries_t prtl_bc { + { PrtlBC::PERIODIC, PrtlBC::PERIODIC }, + { PrtlBC::PERIODIC, PrtlBC::PERIODIC } + }; + const kernel::sr::PusherBoundaries bc { prtl_bc }; + + // zero E/B: the Predictor push leaves v unchanged and drifts x by dt*v + ndfield_t EB { "EB", nx1 + 2u * N_GHOSTS, nx2 + 2u * N_GHOSTS }; + + // ---------------- flat reference (all alive particles) ---------------- + ndfield_t aux_flat { "aux_flat", + nx1 + 2u * N_GHOSTS, + nx2 + 2u * N_GHOSTS }; + { + auto scatter_aux = Kokkos::Experimental::create_scatter_view(aux_flat); + Kokkos::parallel_for( + "FlatPushDeposit", + n_slots, + kernel_t { ctx, bc, arrays, EB, scatter_aux, metric }); + Kokkos::Experimental::contribute(aux_flat, scatter_aux); + Kokkos::fence("flat done"); + } + + // ---------------- tiled (+ flat tail over [n_partitioned, n_alive)) ---- + ndfield_t aux_tiled { "aux_tiled", + nx1 + 2u * N_GHOSTS, + nx2 + 2u * N_GHOSTS }; + { + const auto ntx1 = static_cast( + std::ceil(static_cast(nx1) / static_cast(T_TILE))); + const auto ntx2 = static_cast( + std::ceil(static_cast(nx2) / static_cast(T_TILE))); + const auto total_tiles = ntx1 * ntx2; + + TileLayout layout; + layout.ntiles_per_axis[0] = ntx1; + layout.ntiles_per_axis[1] = ntx2; + layout.ntiles_per_axis[2] = 1u; + layout.ntiles_total = total_tiles; + layout.tile_size = T_TILE; + layout.npart_partitioned = static_cast(n_partitioned); + layout.tile_offsets = array_t { "tile_offsets", + total_tiles + 1u }; + { + auto h = Kokkos::create_mirror_view(layout.tile_offsets); + if (stale_layout) { + // all partitioned particles in tile 0 — maximally stale + h(0) = 0u; + for (ncells_t t = 1; t <= total_tiles; ++t) { + h(t) = static_cast(n_partitioned); + } + } else { + // proper binning by (i1/T, i2/T) of the (tile-sorted) prefix + for (ncells_t t = 0; t <= total_tiles; ++t) { + h(t) = 0u; + } + for (int q = 0; q < n_partitioned; ++q) { + const auto tid = static_cast(prtls[q].i1 / T_TILE) * ntx2 + + static_cast(prtls[q].i2 / T_TILE); + h(tid + 1u) += 1u; + } + for (ncells_t t = 1; t <= total_tiles; ++t) { + h(t) += h(t - 1u); + } + } + Kokkos::deep_copy(layout.tile_offsets, h); + } + + using tiled_t = kernel::TiledScatter_kernel; + const kernel_t body { ctx, bc, arrays, EB, metric }; + const tiled_t kern { aux_tiled, + body, + layout, + static_cast(n_alive) }; + const auto policy = kernel::MakeTiledPolicy(kern, total_tiles, 0); + Kokkos::parallel_for("TiledPushDeposit", policy, kern); + Kokkos::fence("tiled done"); + + if (n_partitioned < n_alive) { + // flat tail pass, exactly as the launcher composes it + auto scatter_aux = Kokkos::Experimental::create_scatter_view(aux_tiled); + Kokkos::parallel_for( + "TiledPushDepositTail", + CreateParticleRangePolicy( + { static_cast(n_partitioned) }, + { static_cast(n_alive) }), + kernel_t { ctx, bc, arrays, EB, scatter_aux, metric }); + Kokkos::Experimental::contribute(aux_tiled, scatter_aux); + Kokkos::fence("tail done"); + } + } + + compare_aux(aux_flat, aux_tiled, T_TILE, label); + check_conservation(aux_tiled, + prtls, + static_cast(inv_n0) / sqrt_det_h, + static_cast(mass), + T_TILE, + label); + } + + template + void run_all() { + constexpr int n_alive = 25; + using kernel::hybrid::PushMode; + // in-tile fast path + run_case(false, n_alive, "moments"); + run_case(false, n_alive, "predictor"); + // per-particle escape valve (maximally-stale layout) + run_case(true, n_alive, "moments/stale"); + run_case(true, n_alive, "predictor/stale"); + // appended-particles tail: tiled prefix + flat tail == flat all + run_case(false, 15, "moments/tail"); + run_case(false, 15, "predictor/tail"); + } + +} // namespace + +auto main(int argc, char* argv[]) -> int { + Kokkos::initialize(argc, argv); + try { + run_all<4u>(); + run_all<8u>(); + run_all<16u>(); + } catch (std::exception& e) { + std::cerr << e.what() << '\n'; + Kokkos::finalize(); + return 1; + } + Kokkos::finalize(); + return 0; +} diff --git a/tests/kernels/particle_moments.cpp b/tests/kernels/particle_moments.cpp index cbda7546f..274280f74 100644 --- a/tests/kernels/particle_moments.cpp +++ b/tests/kernels/particle_moments.cpp @@ -79,9 +79,6 @@ void testParticleMoments(const std::vector& res, EmissionType::NONE, 0, 0 }; - - const float mass = 1.0; - const float charge = 1.0; const bool use_weights = false; const real_t inv_n0 = 1.0;