18#define HAVE_BITSCAN_FORWARD
19#define HAVE_BITSCAN_REVERSE
22#if defined(HAVE__BUILTIN_CTZ)
23#define HAVE_BITSCAN_FORWARD
26#if defined(HAVE__BUILTIN_CLZ)
27#define HAVE_BITSCAN_REVERSE
43#ifdef HAVE__BUILTIN_CLZ
46 return 31 - __builtin_clz(
word);
47#elif defined(_MSC_VER)
53 non_zero = _BitScanReverse(&result,
word);
60 while ((
word >> shift) == 0)
74#ifdef HAVE__BUILTIN_CLZ
78 return 63 - __builtin_clzl(
word);
79#elif SIZEOF_LONG_LONG == 8
80 return 63 - __builtin_clzll(
word);
82#error "cannot find integer type of the same size as uint64_t"
85#elif defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64))
91 non_zero = _BitScanReverse64(&result,
word);
98 while ((
word >> shift) == 0)
113#ifdef HAVE__BUILTIN_CTZ
116 return __builtin_ctz(
word);
117#elif defined(_MSC_VER)
118 unsigned long result;
123 non_zero = _BitScanForward(&result,
word);
130 while ((
word & 255) == 0)
147#ifdef HAVE__BUILTIN_CTZ
151 return __builtin_ctzl(
word);
152#elif SIZEOF_LONG_LONG == 8
153 return __builtin_ctzll(
word);
155#error "cannot find integer type of the same size as uint64_t"
158#elif defined(_MSC_VER) && (defined(_M_AMD64) || defined(_M_ARM64))
159 unsigned long result;
164 non_zero = _BitScanForward64(&result,
word);
171 while ((
word & 255) == 0)
198 if ((num & (num - 1)) == 0)
221 if ((num & (num - 1)) == 0)
285#if defined(_MSC_VER) && defined(_M_AMD64)
286#define HAVE_X86_64_POPCNTQ
295#ifdef HAVE_X86_64_POPCNTQ
296#if defined(HAVE__GET_CPUID) || defined(HAVE__CPUID)
297#define TRY_POPCNT_FAST 1
301#ifdef TRY_POPCNT_FAST
314#ifdef USE_AVX512_POPCNT_WITH_RUNTIME_CHECK
315extern bool pg_popcount_avx512_available(
void);
316extern uint64 pg_popcount_avx512(
const char *
buf,
int bytes);
317extern uint64 pg_popcount_masked_avx512(
const char *
buf,
int bytes,
bits8 mask);
345#if SIZEOF_VOID_P >= 8
351 if (bytes < threshold)
376#if SIZEOF_VOID_P >= 8
382 if (bytes < threshold)
400 return (
word >> n) | (
word << (32 - n));
406 return (
word << n) | (
word >> (32 - n));
411#if SIZEOF_SIZE_T == 4
412#define pg_leftmost_one_pos_size_t pg_leftmost_one_pos32
413#define pg_nextpower2_size_t pg_nextpower2_32
414#define pg_prevpower2_size_t pg_prevpower2_32
416#define pg_leftmost_one_pos_size_t pg_leftmost_one_pos64
417#define pg_nextpower2_size_t pg_nextpower2_64
418#define pg_prevpower2_size_t pg_prevpower2_64
#define Assert(condition)
uint64 pg_popcount_masked_optimized(const char *buf, int bytes, bits8 mask)
PGDLLIMPORT const uint8 pg_number_of_ones[256]
static uint32 pg_rotate_left32(uint32 word, int n)
static int pg_rightmost_one_pos64(uint64 word)
static uint32 pg_nextpower2_32(uint32 num)
static uint64 pg_ceil_log2_64(uint64 num)
static uint32 pg_rotate_right32(uint32 word, int n)
uint64 pg_popcount_optimized(const char *buf, int bytes)
int pg_popcount64(uint64 word)
int pg_popcount32(uint32 word)
static int pg_rightmost_one_pos32(uint32 word)
static uint64 pg_nextpower2_64(uint64 num)
static int pg_leftmost_one_pos32(uint32 word)
PGDLLIMPORT const uint8 pg_rightmost_one_pos[256]
static uint32 pg_ceil_log2_32(uint32 num)
static uint64 pg_popcount(const char *buf, int bytes)
static uint32 pg_prevpower2_32(uint32 num)
static uint64 pg_prevpower2_64(uint64 num)
static uint64 pg_popcount_masked(const char *buf, int bytes, bits8 mask)
PGDLLIMPORT const uint8 pg_leftmost_one_pos[256]
static int pg_leftmost_one_pos64(uint64 word)
static void word(struct vars *v, int dir, struct state *lp, struct state *rp)