Skip to content

truffle: add AArch64 wide path using vqtbl2q #406

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
a-eee wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

truffle: add AArch64 wide path using vqtbl2q #406

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions src/nfa/arm/truffle.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,6 +230,34 @@ svuint8_t blockSingleMaskWide(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_m
#endif //HAVE_SVE2
#endif //HAVE_SVE

#if defined(CAN_USE_WIDE_TRUFFLE) && !defined(HAVE_SVE)
/* require wide truffle compilation. The 256b mask is split between the two parameters */

template <uint16_t S>
static really_inline
const SuperVector<S> blockSingleMaskWide(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars) {
chars.print8("chars");
shuf_mask_lo_highclear.print8("shuf_mask_lo_highclear");

uint8x16x2_t tbl = {{shuf_mask_lo_highclear.u.u8x16[0], shuf_mask_lo_highset.u.u8x16[0]}};

SuperVector<S> idx = chars & SuperVector<S>::dup_u8(31);
idx.print8("idx");
SuperVector<S> byte_select = SuperVector<S>(vqtbl2q_u8(tbl, idx.u.u8x16[0]));
byte_select.print8("byte_select");

SuperVector<S> bits = chars.template vshr_8_imm<5>();
bits.print8("bits");
SuperVector<S> bit_select = SuperVector<S>(
vshlq_u8(vdupq_n_u8(1), vreinterpretq_s8_u8(bits.u.u8x16[0])));
bit_select.print8("bit_select");

SuperVector<S> res = bit_select & byte_select;
res.print8("bit_select & byte_select");
return !res.eq(SuperVector<S>::Zeroes());
}

#endif
/* require normal truffle compilation. The 256b mask is split between the two parameters */
template <uint16_t S>
static really_inline
Expand Down
14 changes: 11 additions & 3 deletions src/nfa/truffle.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,15 @@ const u8 *rtruffleExecWide(m256 mask, const u8 *buf,
}
}
#else // HAVE_SVE
#error "Wide truffle enabled for the target architecture but no implementation found"
const u8 *truffleExecWide(m256 mask, const u8 *buf,
const u8 *buf_end) {
return truffleExecReal<VECTORSIZE, true>(mask.lo, mask.hi, buf, buf_end);
}

const u8 *rtruffleExecWide(m256 mask, const u8 *buf,
const u8 *buf_end) {
return rtruffleExecReal<VECTORSIZE, true>(mask.lo, mask.hi, buf, buf_end);
}
#endif // HAVE_SVE
#endif // CAN_USE_WIDE_TRUFFLE

Expand All @@ -84,11 +92,11 @@ const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
#else
const u8 *truffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return truffleExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
return truffleExecReal<VECTORSIZE, false>(mask_lo, mask_hi, buf, buf_end);
}

const u8 *rtruffleExec(m128 mask_lo, m128 mask_hi, const u8 *buf,
const u8 *buf_end) {
return rtruffleExecReal<VECTORSIZE>(mask_lo, mask_hi, buf, buf_end);
return rtruffleExecReal<VECTORSIZE, false>(mask_lo, mask_hi, buf, buf_end);
}
#endif //HAVE_SVE
56 changes: 41 additions & 15 deletions src/nfa/truffle_simd.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ static really_inline
svuint8_t blockSingleMaskWide(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_highset, svuint8_t chars);
#endif //HAVE_SVE2
#else
#ifdef CAN_USE_WIDE_TRUFFLE
template <uint16_t S>
static really_inline
const SuperVector<S> blockSingleMaskWide(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars);
#endif
template <uint16_t S>
static really_inline
const SuperVector<S> blockSingleMask(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars);
Expand Down Expand Up @@ -95,6 +100,7 @@ const u8 *scanBlock(svuint8_t shuf_mask_lo_highclear, svuint8_t shuf_mask_lo_hig
#else
DEBUG_PRINTF("Wide Truffle is not supported with 128b vectors unless SVE2 is enabled");
assert(false);
return nullptr;
#endif
} else {
result_mask = blockSingleMaskWide32(shuf_mask_lo_highclear, chars);
Expand Down Expand Up @@ -260,14 +266,24 @@ const u8 *rtruffleExecSVE(m256 shuf_mask_32, const u8 *buf, const u8 *buf_end){
return buf - 1;
}
#else
template <uint16_t S>
template <uint16_t S, bool is_wide>
static really_inline
const u8 *fwdBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> chars, const u8 *buf) {
SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
return first_zero_match_inverted<S>(buf, res);
if constexpr (is_wide) {
#ifdef CAN_USE_WIDE_TRUFFLE
SuperVector<S> res = blockSingleMaskWide(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
return first_zero_match_inverted<S>(buf, res);
#else
assert(false);
return nullptr;
#endif
} else {
SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, chars);
return first_zero_match_inverted<S>(buf, res);
}
}

template <uint16_t S>
template <uint16_t S, bool is_wide>
const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end) {
assert(buf && buf_end);
assert(buf < buf_end);
Expand All @@ -289,7 +305,7 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_
if (!ISALIGNED_N(d, S)) {
SuperVector<S> chars = SuperVector<S>::loadu(d);
const u8 *dup = ROUNDUP_PTR(d, S);
rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
rv = fwdBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
if (rv && rv < dup) return rv;
d = dup;
}
Expand All @@ -298,7 +314,7 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_
__builtin_prefetch(d + 16*64);
DEBUG_PRINTF("d %p \n", d);
SuperVector<S> chars = SuperVector<S>::load(d);
rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
rv = fwdBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
if (rv) return rv;
d += S;
}
Expand All @@ -317,23 +333,33 @@ const u8 *truffleExecReal(const m128 &shuf_mask_lo_highclear, m128 shuf_mask_lo_
chars = SuperVector<S>::loadu(buf_end - S);
end_buf = buf_end - S;
}
rv = fwdBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf);
rv = fwdBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, end_buf);
DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv;
}

return buf_end;
}

template <uint16_t S>
template <uint16_t S, bool is_wide>
static really_inline
const u8 *revBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> v,
const u8 *revBlock(SuperVector<S> shuf_mask_lo_highclear, SuperVector<S> shuf_mask_lo_highset, SuperVector<S> v,
const u8 *buf) {
SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v);
return last_zero_match_inverted<S>(buf, res);
if constexpr (is_wide) {
#ifdef CAN_USE_WIDE_TRUFFLE
SuperVector<S> res = blockSingleMaskWide(shuf_mask_lo_highclear, shuf_mask_lo_highset, v);
return last_zero_match_inverted<S>(buf, res);
#else
assert(false);
return nullptr;
#endif
} else {
SuperVector<S> res = blockSingleMask(shuf_mask_lo_highclear, shuf_mask_lo_highset, v);
return last_zero_match_inverted<S>(buf, res);
}
}

template <uint16_t S>
template <uint16_t S, bool is_wide>
const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highset, const u8 *buf, const u8 *buf_end){
assert(buf && buf_end);
assert(buf < buf_end);
Expand All @@ -355,7 +381,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
if (!ISALIGNED_N(d, S)) {
SuperVector<S> chars = SuperVector<S>::loadu(d - S);
const u8 *dbot = ROUNDDOWN_PTR(d, S);
rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d - S);
rv = revBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d - S);
DEBUG_PRINTF("rv %p \n", rv);
if (rv >= dbot) return rv;
d = dbot;
Expand All @@ -368,7 +394,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse

d -= S;
SuperVector<S> chars = SuperVector<S>::load(d);
rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
rv = revBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, d);
if (rv) return rv;
}
}
Expand All @@ -383,7 +409,7 @@ const u8 *rtruffleExecReal(m128 shuf_mask_lo_highclear, m128 shuf_mask_lo_highse
} else {
chars = SuperVector<S>::loadu(buf);
}
rv = revBlock(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf);
rv = revBlock<S, is_wide>(wide_shuf_mask_lo_highclear, wide_shuf_mask_lo_highset, chars, buf);
DEBUG_PRINTF("rv %p \n", rv);
if (rv && rv < buf_end) return rv;
}
Expand Down
2 changes: 2 additions & 0 deletions src/util/arch/arm/arm.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@
#define CAN_USE_WIDE_TRUFFLE 1
#elif defined(HAVE_SVE)
#define CAN_USE_WIDE_TRUFFLE (svcntb() >= 32)
#elif defined(HAVE_NEON) && defined(ARCH_AARCH64) && !defined(VS_SIMDE_BACKEND)
#define CAN_USE_WIDE_TRUFFLE 1
#endif

#endif // UTIL_ARCH_ARM_H_
Expand Down
1 change: 1 addition & 0 deletions src/util/supervector/arch/arm/impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -363,6 +363,7 @@ template SuperVector<16> SuperVector<16>::vshl_128_imm<1>() const;
template SuperVector<16> SuperVector<16>::vshl_128_imm<4>() const;
template SuperVector<16> SuperVector<16>::vshr_8_imm<1>() const;
template SuperVector<16> SuperVector<16>::vshr_8_imm<4>() const;
template SuperVector<16> SuperVector<16>::vshr_8_imm<5>() const;
template SuperVector<16> SuperVector<16>::vshr_16_imm<1>() const;
template SuperVector<16> SuperVector<16>::vshr_64_imm<1>() const;
template SuperVector<16> SuperVector<16>::vshr_64_imm<4>() const;
Expand Down
2 changes: 1 addition & 1 deletion unit/internal/truffleWide.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
#include "util/simd_utils.h"

#include "util/arch.h"
#ifdef HAVE_SVE
#ifdef CAN_USE_WIDE_TRUFFLE
using namespace ue2;

#define SKIP_IF_NO_WIDE_AVAILABLE() \
Expand Down