PostgreSQL Source Code  git master
simd.h File Reference
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Macros

#define USE_NO_SIMD
 

Typedefs

typedef uint64 Vector8
 

Functions

static void vector8_load (Vector8 *v, const uint8 *s)
 
static Vector8 vector8_broadcast (const uint8 c)
 
static bool vector8_has (const Vector8 v, const uint8 c)
 
static bool vector8_has_zero (const Vector8 v)
 
static bool vector8_has_le (const Vector8 v, const uint8 c)
 
static bool vector8_is_highbit_set (const Vector8 v)
 
static Vector8 vector8_or (const Vector8 v1, const Vector8 v2)
 

Macro Definition Documentation

◆ USE_NO_SIMD

#define USE_NO_SIMD

Definition at line 59 of file simd.h.

Typedef Documentation

◆ Vector8

typedef uint64 Vector8

Definition at line 60 of file simd.h.

Function Documentation

◆ vector8_broadcast()

static Vector8 vector8_broadcast ( const uint8  c)
inlinestatic

Definition at line 135 of file simd.h.

136 {
137 #if defined(USE_SSE2)
138  return _mm_set1_epi8(c);
139 #elif defined(USE_NEON)
140  return vdupq_n_u8(c);
141 #else
142  return ~UINT64CONST(0) / 0xFF * c;
143 #endif
144 }
#define UINT64CONST(x)
Definition: c.h:500
char * c

References UINT64CONST.

Referenced by is_valid_ascii(), RT_NODE_16_GET_INSERTPOS(), RT_NODE_16_SEARCH_EQ(), vector8_has(), vector8_has_le(), and vector8_is_highbit_set().

◆ vector8_has()

static bool vector8_has ( const Vector8  v,
const uint8  c 
)
inlinestatic

Definition at line 162 of file simd.h.

163 {
164  bool result;
165 
166  /* pre-compute the result for assert checking */
167 #ifdef USE_ASSERT_CHECKING
168  bool assert_result = false;
169 
170  for (Size i = 0; i < sizeof(Vector8); i++)
171  {
172  if (((const uint8 *) &v)[i] == c)
173  {
174  assert_result = true;
175  break;
176  }
177  }
178 #endif /* USE_ASSERT_CHECKING */
179 
180 #if defined(USE_NO_SIMD)
181  /* any bytes in v equal to c will evaluate to zero via XOR */
182  result = vector8_has_zero(v ^ vector8_broadcast(c));
183 #else
184  result = vector8_is_highbit_set(vector8_eq(v, vector8_broadcast(c)));
185 #endif
186 
187  Assert(assert_result == result);
188  return result;
189 }
uint8_t uint8
Definition: c.h:483
#define Assert(condition)
Definition: c.h:812
size_t Size
Definition: c.h:559
int i
Definition: isn.c:72
static Vector8 vector8_broadcast(const uint8 c)
Definition: simd.h:135
static bool vector8_has_zero(const Vector8 v)
Definition: simd.h:195
uint64 Vector8
Definition: simd.h:60
static bool vector8_is_highbit_set(const Vector8 v)
Definition: simd.h:271

References Assert, i, vector8_broadcast(), vector8_has_zero(), and vector8_is_highbit_set().

Referenced by escape_json_with_len(), pg_lfind8(), and vector8_has_zero().

◆ vector8_has_le()

static bool vector8_has_le ( const Vector8  v,
const uint8  c 
)
inlinestatic

Definition at line 213 of file simd.h.

214 {
215  bool result = false;
216 
217  /* pre-compute the result for assert checking */
218 #ifdef USE_ASSERT_CHECKING
219  bool assert_result = false;
220 
221  for (Size i = 0; i < sizeof(Vector8); i++)
222  {
223  if (((const uint8 *) &v)[i] <= c)
224  {
225  assert_result = true;
226  break;
227  }
228  }
229 #endif /* USE_ASSERT_CHECKING */
230 
231 #if defined(USE_NO_SIMD)
232 
233  /*
234  * To find bytes <= c, we can use bitwise operations to find bytes < c+1,
235  * but it only works if c+1 <= 128 and if the highest bit in v is not set.
236  * Adapted from
237  * https://graphics.stanford.edu/~seander/bithacks.html#HasLessInWord
238  */
239  if ((int64) v >= 0 && c < 0x80)
240  result = (v - vector8_broadcast(c + 1)) & ~v & vector8_broadcast(0x80);
241  else
242  {
243  /* one byte at a time */
244  for (Size i = 0; i < sizeof(Vector8); i++)
245  {
246  if (((const uint8 *) &v)[i] <= c)
247  {
248  result = true;
249  break;
250  }
251  }
252  }
253 #else
254 
255  /*
256  * Use saturating subtraction to find bytes <= c, which will present as
257  * NUL bytes. This approach is a workaround for the lack of unsigned
258  * comparison instructions on some architectures.
259  */
260  result = vector8_has_zero(vector8_ssub(v, vector8_broadcast(c)));
261 #endif
262 
263  Assert(assert_result == result);
264  return result;
265 }
int64_t int64
Definition: c.h:482

References Assert, i, vector8_broadcast(), and vector8_has_zero().

Referenced by escape_json_with_len(), pg_lfind8_le(), and vector8_has_zero().

◆ vector8_has_zero()

static bool vector8_has_zero ( const Vector8  v)
inlinestatic

Definition at line 195 of file simd.h.

196 {
197 #if defined(USE_NO_SIMD)
198  /*
199  * We cannot call vector8_has() here, because that would lead to a
200  * circular definition.
201  */
202  return vector8_has_le(v, 0);
203 #else
204  return vector8_has(v, 0);
205 #endif
206 }
static bool vector8_has_le(const Vector8 v, const uint8 c)
Definition: simd.h:213
static bool vector8_has(const Vector8 v, const uint8 c)
Definition: simd.h:162

References vector8_has(), and vector8_has_le().

Referenced by vector8_has(), and vector8_has_le().

◆ vector8_is_highbit_set()

static bool vector8_is_highbit_set ( const Vector8  v)
inlinestatic

Definition at line 271 of file simd.h.

272 {
273 #ifdef USE_SSE2
274  return _mm_movemask_epi8(v) != 0;
275 #elif defined(USE_NEON)
276  return vmaxvq_u8(v) > 0x7F;
277 #else
278  return v & vector8_broadcast(0x80);
279 #endif
280 }

References vector8_broadcast().

Referenced by is_valid_ascii(), and vector8_has().

◆ vector8_load()

static void vector8_load ( Vector8 v,
const uint8 s 
)
inlinestatic

Definition at line 108 of file simd.h.

109 {
110 #if defined(USE_SSE2)
111  *v = _mm_loadu_si128((const __m128i *) s);
112 #elif defined(USE_NEON)
113  *v = vld1q_u8(s);
114 #else
115  memcpy(v, s, sizeof(Vector8));
116 #endif
117 }

Referenced by escape_json_with_len(), is_valid_ascii(), pg_lfind8(), pg_lfind8_le(), RT_NODE_16_GET_INSERTPOS(), and RT_NODE_16_SEARCH_EQ().

◆ vector8_or()

static Vector8 vector8_or ( const Vector8  v1,
const Vector8  v2 
)
inlinestatic

Definition at line 338 of file simd.h.

339 {
340 #ifdef USE_SSE2
341  return _mm_or_si128(v1, v2);
342 #elif defined(USE_NEON)
343  return vorrq_u8(v1, v2);
344 #else
345  return v1 | v2;
346 #endif
347 }

Referenced by is_valid_ascii().