18 #define HAVE_BITSCAN_FORWARD
19 #define HAVE_BITSCAN_REVERSE
22 #if defined(HAVE__BUILTIN_CTZ)
23 #define HAVE_BITSCAN_FORWARD
26 #if defined(HAVE__BUILTIN_CLZ)
27 #define HAVE_BITSCAN_REVERSE
43 #ifdef HAVE__BUILTIN_CLZ
46 return 31 - __builtin_clz(
word);
47 #elif defined(_MSC_VER)
53 non_zero = _BitScanReverse(&result,
word);
60 while ((
word >> shift) == 0)
74 #ifdef HAVE__BUILTIN_CLZ
77 #if defined(HAVE_LONG_INT_64)
78 return 63 - __builtin_clzl(
word);
79 #elif defined(HAVE_LONG_LONG_INT_64)
80 return 63 - __builtin_clzll(
word);
82 #error must have a working 64-bit integer datatype
85 #elif defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64))
91 non_zero = _BitScanReverse64(&result,
word);
98 while ((
word >> shift) == 0)
113 #ifdef HAVE__BUILTIN_CTZ
116 return __builtin_ctz(
word);
117 #elif defined(_MSC_VER)
118 unsigned long result;
123 non_zero = _BitScanForward(&result,
word);
130 while ((
word & 255) == 0)
147 #ifdef HAVE__BUILTIN_CTZ
150 #if defined(HAVE_LONG_INT_64)
151 return __builtin_ctzl(
word);
152 #elif defined(HAVE_LONG_LONG_INT_64)
153 return __builtin_ctzll(
word);
155 #error must have a working 64-bit integer datatype
158 #elif defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64))
159 unsigned long result;
164 non_zero = _BitScanForward64(&result,
word);
171 while ((
word & 255) == 0)
198 if ((num & (num - 1)) == 0)
221 if ((num & (num - 1)) == 0)
285 #if defined(_MSC_VER) && defined(_M_AMD64)
286 #define HAVE_X86_64_POPCNTQ
295 #ifdef HAVE_X86_64_POPCNTQ
296 #if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID)
297 #define TRY_POPCNT_FAST 1
301 #ifdef TRY_POPCNT_FAST
314 #ifdef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK
315 extern bool pg_popcount_avx512_available(
void);
316 extern uint64 pg_popcount_avx512(
const char *
buf,
int bytes);
317 extern uint64 pg_popcount_masked_avx512(
const char *
buf,
int bytes,
bits8 mask);
345 #if SIZEOF_VOID_P >= 8
351 if (bytes < threshold)
376 #if SIZEOF_VOID_P >= 8
382 if (bytes < threshold)
400 return (
word >> n) | (
word << (32 - n));
406 return (
word << n) | (
word >> (32 - n));
411 #if SIZEOF_SIZE_T == 4
412 #define pg_leftmost_one_pos_size_t pg_leftmost_one_pos32
413 #define pg_nextpower2_size_t pg_nextpower2_32
414 #define pg_prevpower2_size_t pg_prevpower2_32
416 #define pg_leftmost_one_pos_size_t pg_leftmost_one_pos64
417 #define pg_nextpower2_size_t pg_nextpower2_64
418 #define pg_prevpower2_size_t pg_prevpower2_64
#define Assert(condition)
uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
PGDLLIMPORT const uint8 pg_number_of_ones[256]
static uint32 pg_rotate_left32(uint32 word, int n)
static int pg_rightmost_one_pos64(uint64 word)
static uint32 pg_nextpower2_32(uint32 num)
static uint64 pg_ceil_log2_64(uint64 num)
static uint32 pg_rotate_right32(uint32 word, int n)
uint64 pg_popcount_optimized(const char *buf, int bytes)
int pg_popcount64(uint64 word)
int pg_popcount32(uint32 word)
static int pg_rightmost_one_pos32(uint32 word)
static uint64 pg_nextpower2_64(uint64 num)
static int pg_leftmost_one_pos32(uint32 word)
PGDLLIMPORT const uint8 pg_rightmost_one_pos[256]
static uint32 pg_ceil_log2_32(uint32 num)
static uint64 pg_popcount(const char *buf, int bytes)
static uint32 pg_prevpower2_32(uint32 num)
static uint64 pg_prevpower2_64(uint64 num)
static uint64 pg_popcount_masked(const char *buf, int bytes, bits8 mask)
PGDLLIMPORT const uint8 pg_leftmost_one_pos[256]
static int pg_leftmost_one_pos64(uint64 word)
static void word(struct vars *v, int dir, struct state *lp, struct state *rp)