diff --git a/src/nfa/arm/truffle.hpp b/src/nfa/arm/truffle.hpp index 6f9a61fc9..7d2cfa6b9 100644 --- a/src/nfa/arm/truffle.hpp +++ b/src/nfa/arm/truffle.hpp @@ -230,6 +230,34 @@ svuint8_t blockSingleMaskWide(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_m #endif //HAVE_SVE2 #endif //HAVE_SVE +#if defined(CAN_USE_WIDE_TRUFFLE) && !defined(HAVE_SVE) +/* require wide truffle compilation. The 256b mask is split between the two parameters */ + +template +static really_inline +const SuperVector blockSingleMaskWide(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars) { + chars.print8("chars"); + shuf_mask_lo_highclear.print8("shuf_mask_lo_highclear"); + + uint8x16x2_t tbl = {{shuf_mask_lo_highclear.u.u8x16[0], shuf_mask_lo_highset.u.u8x16[0]}}; + + SuperVector idx = chars & SuperVector::dup_u8(31); + idx.print8("idx"); + SuperVector byte_select = SuperVector(vqtbl2q_u8(tbl, idx.u.u8x16[0])); + byte_select.print8("byte_select"); + + SuperVector bits = chars.template vshr_8_imm<5>(); + bits.print8("bits"); + SuperVector bit_select = SuperVector( + vshlq_u8(vdupq_n_u8(1), vreinterpretq_s8_u8(bits.u.u8x16[0]))); + bit_select.print8("bit_select"); + + SuperVector res = bit_select & byte_select; + res.print8("bit_select & byte_select"); + return !res.eq(SuperVector::Zeroes()); +} + +#endif /* require normal truffle compilation. The 256b mask is split between the two parameters */ template static really_inline diff --git a/src/nfa/truffle.cpp b/src/nfa/truffle.cpp index df3a4a3f5..054cb7f7e 100644 --- a/src/nfa/truffle.cpp +++ b/src/nfa/truffle.cpp @@ -58,7 +58,15 @@ const u8 *rtruffleExecWide(m256 mask, const u8 *buf, } } #else // HAVE_SVE -#error "Wide truffle enabled for the target architecture but no implementation found" +const u8 *truffleExecWide(m256 mask, const u8 *buf, + const u8 *buf_end) { + return truffleExecReal(mask.lo, mask.hi, buf, buf_end); +} + +const u8 *rtruffleExecWide(m256 mask, const u8 *buf, + const u8 *buf_end) { + return rtruffleExecReal(mask.lo, mask.hi, buf, buf_end); +} #endif // HAVE_SVE #endif // CAN_USE_WIDE_TRUFFLE @@ -84,11 +92,11 @@ const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf, #else const u8 *truffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *buf_end) { - return truffleExecReal(mask_lo, mask_hi, buf, buf_end); + return truffleExecReal(mask_lo, mask_hi, buf, buf_end); } const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf, const u8 *buf_end) { - return rtruffleExecReal(mask_lo, mask_hi, buf, buf_end); + return rtruffleExecReal(mask_lo, mask_hi, buf, buf_end); } #endif //HAVE_SVE diff --git a/src/nfa/truffle_simd.hpp b/src/nfa/truffle_simd.hpp index 8a2a06280..4b508ecf4 100644 --- a/src/nfa/truffle_simd.hpp +++ b/src/nfa/truffle_simd.hpp @@ -54,6 +54,11 @@ static really_inline svuint8_t blockSingleMaskWide(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars); #endif //HAVE_SVE2 #else +#ifdef CAN_USE_WIDE_TRUFFLE +template +static really_inline +const SuperVector blockSingleMaskWide(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars); +#endif template static really_inline const SuperVector blockSingleMask(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars); @@ -95,6 +100,7 @@ const u8 *scanBlock(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_hig #else DEBUG_PRINTF("Wide Truffle is not supported with 128b vectors unless SVE2 is enabled"); assert(false); + return nullptr; #endif } else { result_mask = blockSingleMaskWide32(shuf_mask_lo_highclear, chars); @@ -260,14 +266,24 @@ const u8 *rtruffleExecSVE(m256 shuf_mask_32, const u8 *buf, const u8 *buf_end){ return buf - 1; } #else -template +template static really_inline const u8 *fwdBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector chars, const u8 *buf) { - SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); - return first_zero_match_inverted(buf, res); + if constexpr (is_wide) { +#ifdef CAN_USE_WIDE_TRUFFLE + SuperVector res = blockSingleMaskWide(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); + return first_zero_match_inverted(buf, res); +#else + assert(false); + return nullptr; +#endif + } else { + SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars); + return first_zero_match_inverted(buf, res); + } } -template +template const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end) { assert(buf && buf_end); assert(buf < buf_end); @@ -289,7 +305,7 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_ if (!ISALIGNED_N(d, S)) { SuperVector chars = SuperVector::loadu(d); const u8 *dup = ROUNDUP_PTR(d, S); - rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); + rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); if (rv && rv < dup) return rv; d = dup; } @@ -298,7 +314,7 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_ __builtin_prefetch(d + 16*64); DEBUG_PRINTF("d %p \n", d); SuperVector chars = SuperVector::load(d); - rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); + rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); if (rv) return rv; d += S; } @@ -317,7 +333,7 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_ chars = SuperVector::loadu(buf_end - S); end_buf = buf_end - S; } - rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf); + rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } @@ -325,15 +341,25 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_ return buf_end; } -template +template static really_inline -const u8 *revBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, +const u8 *revBlock(SuperVector shuf_mask_lo_highclear, SuperVector shuf_mask_lo_highset, SuperVector v, const u8 *buf) { - SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); - return last_zero_match_inverted(buf, res); + if constexpr (is_wide) { +#ifdef CAN_USE_WIDE_TRUFFLE + SuperVector res = blockSingleMaskWide(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); + return last_zero_match_inverted(buf, res); +#else + assert(false); + return nullptr; +#endif + } else { + SuperVector res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v); + return last_zero_match_inverted(buf, res); + } } -template +template const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end){ assert(buf && buf_end); assert(buf < buf_end); @@ -355,7 +381,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse if (!ISALIGNED_N(d, S)) { SuperVector chars = SuperVector::loadu(d - S); const u8 *dbot = ROUNDDOWN_PTR(d, S); - rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d - S); + rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d - S); DEBUG_PRINTF("rv %p \n", rv); if (rv >= dbot) return rv; d = dbot; @@ -368,7 +394,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse d -= S; SuperVector chars = SuperVector::load(d); - rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); + rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d); if (rv) return rv; } } @@ -383,7 +409,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse } else { chars = SuperVector::loadu(buf); } - rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf); + rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf); DEBUG_PRINTF("rv %p \n", rv); if (rv && rv < buf_end) return rv; } diff --git a/src/util/arch/arm/arm.h b/src/util/arch/arm/arm.h index 3735d066e..9b743c1c0 100644 --- a/src/util/arch/arm/arm.h +++ b/src/util/arch/arm/arm.h @@ -57,6 +57,8 @@ #define CAN_USE_WIDE_TRUFFLE 1 #elif defined(HAVE_SVE) #define CAN_USE_WIDE_TRUFFLE (svcntb() >= 32) +#elif defined(HAVE_NEON) && defined(ARCH_AARCH64) && !defined(VS_SIMDE_BACKEND) +#define CAN_USE_WIDE_TRUFFLE 1 #endif #endif // UTIL_ARCH_ARM_H_ diff --git a/src/util/supervector/arch/arm/impl.cpp b/src/util/supervector/arch/arm/impl.cpp index b4cca4409..c19b281a4 100644 --- a/src/util/supervector/arch/arm/impl.cpp +++ b/src/util/supervector/arch/arm/impl.cpp @@ -363,6 +363,7 @@ template SuperVector<16> SuperVector<16>::vshl_128_imm<1>() const; template SuperVector<16> SuperVector<16>::vshl_128_imm<4>() const; template SuperVector<16> SuperVector<16>::vshr_8_imm<1>() const; template SuperVector<16> SuperVector<16>::vshr_8_imm<4>() const; +template SuperVector<16> SuperVector<16>::vshr_8_imm<5>() const; template SuperVector<16> SuperVector<16>::vshr_16_imm<1>() const; template SuperVector<16> SuperVector<16>::vshr_64_imm<1>() const; template SuperVector<16> SuperVector<16>::vshr_64_imm<4>() const; diff --git a/unit/internal/truffleWide.cpp b/unit/internal/truffleWide.cpp index 733e6ffb8..c5afb1726 100644 --- a/unit/internal/truffleWide.cpp +++ b/unit/internal/truffleWide.cpp @@ -36,7 +36,7 @@ #include "util/simd_utils.h" #include "util/arch.h" -#ifdef HAVE_SVE +#ifdef CAN_USE_WIDE_TRUFFLE using namespace ue2; #define SKIP_IF_NO_WIDE_AVAILABLE() \