PostgreSQL Source Code  git master
ts_locale.h File Reference
#include <ctype.h>
#include <limits.h>
#include "utils/pg_locale.h"
#include "mb/pg_wchar.h"
Include dependency graph for ts_locale.h:
This graph shows which files directly or indirectly include this file:

Go to the source code of this file.

Data Structures

struct  tsearch_readline_state
 

Macros

#define TOUCHAR(x)   (*((const unsigned char *) (x)))
 
#define t_iseq(x, c)   (TOUCHAR(x) == (unsigned char) (c))
 
#define COPYCHAR(d, s)   memcpy(d, s, pg_mblen(s))
 

Functions

int t_isdigit (const char *ptr)
 
int t_isspace (const char *ptr)
 
int t_isalpha (const char *ptr)
 
int t_isprint (const char *ptr)
 
char * lowerstr (const char *str)
 
char * lowerstr_with_len (const char *str, int len)
 
bool tsearch_readline_begin (tsearch_readline_state *stp, const char *filename)
 
char * tsearch_readline (tsearch_readline_state *stp)
 
void tsearch_readline_end (tsearch_readline_state *stp)
 
char * t_readline (FILE *fp)
 

Macro Definition Documentation

◆ COPYCHAR

#define COPYCHAR (   d,
 
)    memcpy(d, s, pg_mblen(s))

◆ t_iseq

◆ TOUCHAR

#define TOUCHAR (   x)    (*((const unsigned char *) (x)))

Definition at line 42 of file ts_locale.h.

Referenced by lowerstr_with_len(), t_isalpha(), t_isdigit(), t_isprint(), and t_isspace().

Function Documentation

◆ lowerstr()

char* lowerstr ( const char *  str)

Definition at line 228 of file ts_locale.c.

References lowerstr_with_len().

Referenced by convertPgWchar(), dispell_init(), dsimple_init(), dsnowball_init(), dsynonym_init(), dxsyn_lexize(), lowerstr_ctx(), NIImportAffixes(), and read_dictionary().

229 {
230  return lowerstr_with_len(str, strlen(str));
231 }
char * lowerstr_with_len(const char *str, int len)
Definition: ts_locale.c:241

◆ lowerstr_with_len()

char* lowerstr_with_len ( const char *  str,
int  len 
)

Definition at line 241 of file ts_locale.c.

References Assert, char2wchar(), DEFAULT_COLLATION_OID, ereport, errcode(), errmsg(), ERROR, lc_ctype_is_c(), palloc(), pfree(), pg_database_encoding_max_length(), pstrdup(), generate_unaccent_rules::str, TOUCHAR, and wchar2char().

Referenced by dispell_lexize(), dsimple_lexize(), dsnowball_lexize(), dsynonym_lexize(), generate_trgm_only(), generate_wildcard_trgm(), and lowerstr().

242 {
243  char *out;
244  Oid collation = DEFAULT_COLLATION_OID; /* TODO */
245  pg_locale_t mylocale = 0; /* TODO */
246 
247  if (len == 0)
248  return pstrdup("");
249 
250  /*
251  * Use wide char code only when max encoding length > 1 and ctype != C.
252  * Some operating systems fail with multi-byte encodings and a C locale.
253  * Also, for a C locale there is no need to process as multibyte. From
254  * backend/utils/adt/oracle_compat.c Teodor
255  */
256  if (pg_database_encoding_max_length() > 1 && !lc_ctype_is_c(collation))
257  {
258  wchar_t *wstr,
259  *wptr;
260  int wlen;
261 
262  /*
263  * alloc number of wchar_t for worst case, len contains number of
264  * bytes >= number of characters and alloc 1 wchar_t for 0, because
265  * wchar2char wants zero-terminated string
266  */
267  wptr = wstr = (wchar_t *) palloc(sizeof(wchar_t) * (len + 1));
268 
269  wlen = char2wchar(wstr, len + 1, str, len, mylocale);
270  Assert(wlen <= len);
271 
272  while (*wptr)
273  {
274  *wptr = towlower((wint_t) *wptr);
275  wptr++;
276  }
277 
278  /*
279  * Alloc result string for worst case + '\0'
280  */
281  len = pg_database_encoding_max_length() * wlen + 1;
282  out = (char *) palloc(len);
283 
284  wlen = wchar2char(out, wstr, len, mylocale);
285 
286  pfree(wstr);
287 
288  if (wlen < 0)
289  ereport(ERROR,
290  (errcode(ERRCODE_CHARACTER_NOT_IN_REPERTOIRE),
291  errmsg("conversion from wchar_t to server encoding failed: %m")));
292  Assert(wlen < len);
293  }
294  else
295  {
296  const char *ptr = str;
297  char *outptr;
298 
299  outptr = out = (char *) palloc(sizeof(char) * (len + 1));
300  while ((ptr - str) < len && *ptr)
301  {
302  *outptr++ = tolower(TOUCHAR(ptr));
303  ptr++;
304  }
305  *outptr = '\0';
306  }
307 
308  return out;
309 }
char * pstrdup(const char *in)
Definition: mcxt.c:1076
int errcode(int sqlerrcode)
Definition: elog.c:575
unsigned int Oid
Definition: postgres_ext.h:31
void pfree(void *pointer)
Definition: mcxt.c:949
#define ERROR
Definition: elog.h:43
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:75
int pg_database_encoding_max_length(void)
Definition: wchar.c:1833
#define ereport(elevel, rest)
Definition: elog.h:122
size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, pg_locale_t locale)
Definition: pg_locale.c:1677
#define Assert(condition)
Definition: c.h:670
#define TOUCHAR(x)
Definition: ts_locale.h:42
void * palloc(Size size)
Definition: mcxt.c:848
int errmsg(const char *fmt,...)
Definition: elog.c:797
size_t wchar2char(char *to, const wchar_t *from, size_t tolen, pg_locale_t locale)
Definition: pg_locale.c:1606
bool lc_ctype_is_c(Oid collation)
Definition: pg_locale.c:1178

◆ t_isalpha()

int t_isalpha ( const char *  ptr)

Definition at line 57 of file ts_locale.c.

References char2wchar(), DEFAULT_COLLATION_OID, lc_ctype_is_c(), pg_mblen(), and TOUCHAR.

Referenced by parse_affentry(), RS_compile(), and RS_isRegis().

58 {
59  int clen = pg_mblen(ptr);
60  wchar_t character[2];
61  Oid collation = DEFAULT_COLLATION_OID; /* TODO */
62  pg_locale_t mylocale = 0; /* TODO */
63 
64  if (clen == 1 || lc_ctype_is_c(collation))
65  return isalpha(TOUCHAR(ptr));
66 
67  char2wchar(character, 2, ptr, clen, mylocale);
68 
69  return iswalpha((wint_t) character[0]);
70 }
unsigned int Oid
Definition: postgres_ext.h:31
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:75
size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, pg_locale_t locale)
Definition: pg_locale.c:1677
#define TOUCHAR(x)
Definition: ts_locale.h:42
int pg_mblen(const char *mbstr)
Definition: mbutils.c:760
bool lc_ctype_is_c(Oid collation)
Definition: pg_locale.c:1178

◆ t_isdigit()

int t_isdigit ( const char *  ptr)

Definition at line 25 of file ts_locale.c.

References char2wchar(), DEFAULT_COLLATION_OID, lc_ctype_is_c(), pg_mblen(), and TOUCHAR.

Referenced by getNextFlagFromString(), gettoken_tsvector(), lquery_in(), and parse_phrase_operator().

26 {
27  int clen = pg_mblen(ptr);
28  wchar_t character[2];
29  Oid collation = DEFAULT_COLLATION_OID; /* TODO */
30  pg_locale_t mylocale = 0; /* TODO */
31 
32  if (clen == 1 || lc_ctype_is_c(collation))
33  return isdigit(TOUCHAR(ptr));
34 
35  char2wchar(character, 2, ptr, clen, mylocale);
36 
37  return iswdigit((wint_t) character[0]);
38 }
unsigned int Oid
Definition: postgres_ext.h:31
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:75
size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, pg_locale_t locale)
Definition: pg_locale.c:1677
#define TOUCHAR(x)
Definition: ts_locale.h:42
int pg_mblen(const char *mbstr)
Definition: mbutils.c:760
bool lc_ctype_is_c(Oid collation)
Definition: pg_locale.c:1178

◆ t_isprint()

int t_isprint ( const char *  ptr)

Definition at line 73 of file ts_locale.c.

References char2wchar(), DEFAULT_COLLATION_OID, lc_ctype_is_c(), pg_mblen(), and TOUCHAR.

Referenced by NIImportDictionary().

74 {
75  int clen = pg_mblen(ptr);
76  wchar_t character[2];
77  Oid collation = DEFAULT_COLLATION_OID; /* TODO */
78  pg_locale_t mylocale = 0; /* TODO */
79 
80  if (clen == 1 || lc_ctype_is_c(collation))
81  return isprint(TOUCHAR(ptr));
82 
83  char2wchar(character, 2, ptr, clen, mylocale);
84 
85  return iswprint((wint_t) character[0]);
86 }
unsigned int Oid
Definition: postgres_ext.h:31
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:75
size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, pg_locale_t locale)
Definition: pg_locale.c:1677
#define TOUCHAR(x)
Definition: ts_locale.h:42
int pg_mblen(const char *mbstr)
Definition: mbutils.c:760
bool lc_ctype_is_c(Oid collation)
Definition: pg_locale.c:1178

◆ t_isspace()

int t_isspace ( const char *  ptr)

Definition at line 41 of file ts_locale.c.

References char2wchar(), DEFAULT_COLLATION_OID, lc_ctype_is_c(), pg_mblen(), and TOUCHAR.

Referenced by addCompoundAffixFlagValue(), find_word(), findwrd(), get_nextfield(), getNextFlagFromString(), gettoken_query(), gettoken_tsvector(), initTrie(), NIImportAffixes(), NIImportDictionary(), NIImportOOAffixes(), parse_affentry(), readstoplist(), and thesaurusRead().

42 {
43  int clen = pg_mblen(ptr);
44  wchar_t character[2];
45  Oid collation = DEFAULT_COLLATION_OID; /* TODO */
46  pg_locale_t mylocale = 0; /* TODO */
47 
48  if (clen == 1 || lc_ctype_is_c(collation))
49  return isspace(TOUCHAR(ptr));
50 
51  char2wchar(character, 2, ptr, clen, mylocale);
52 
53  return iswspace((wint_t) character[0]);
54 }
unsigned int Oid
Definition: postgres_ext.h:31
#define DEFAULT_COLLATION_OID
Definition: pg_collation.h:75
size_t char2wchar(wchar_t *to, size_t tolen, const char *from, size_t fromlen, pg_locale_t locale)
Definition: pg_locale.c:1677
#define TOUCHAR(x)
Definition: ts_locale.h:42
int pg_mblen(const char *mbstr)
Definition: mbutils.c:760
bool lc_ctype_is_c(Oid collation)
Definition: pg_locale.c:1178

◆ t_readline()

char* t_readline ( FILE *  fp)

Definition at line 194 of file ts_locale.c.

References buf, pg_any_to_server(), PG_UTF8, pg_verify_mbstr(), and pnstrdup().

Referenced by tsearch_readline().

195 {
196  int len;
197  char *recoded;
198  char buf[4096]; /* lines must not be longer than this */
199 
200  if (fgets(buf, sizeof(buf), fp) == NULL)
201  return NULL;
202 
203  len = strlen(buf);
204 
205  /* Make sure the input is valid UTF-8 */
206  (void) pg_verify_mbstr(PG_UTF8, buf, len, false);
207 
208  /* And convert */
209  recoded = pg_any_to_server(buf, len, PG_UTF8);
210  if (recoded == buf)
211  {
212  /*
213  * conversion didn't pstrdup, so we must. We can use the length of the
214  * original string, because no conversion was done.
215  */
216  recoded = pnstrdup(recoded, len);
217  }
218 
219  return recoded;
220 }
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1087
static char * buf
Definition: pg_test_fsync.c:67
bool pg_verify_mbstr(int encoding, const char *mbstr, int len, bool noError)
Definition: wchar.c:1877
char * pg_any_to_server(const char *s, int len, int encoding)
Definition: mbutils.c:561

◆ tsearch_readline()

char* tsearch_readline ( tsearch_readline_state stp)

Definition at line 135 of file ts_locale.c.

References tsearch_readline_state::curline, tsearch_readline_state::fp, tsearch_readline_state::lineno, and t_readline().

Referenced by dsynonym_init(), initTrie(), NIImportAffixes(), NIImportDictionary(), NIImportOOAffixes(), read_dictionary(), readstoplist(), and thesaurusRead().

136 {
137  char *result;
138 
139  stp->lineno++;
140  stp->curline = NULL;
141  result = t_readline(stp->fp);
142  stp->curline = result;
143  return result;
144 }
char * t_readline(FILE *fp)
Definition: ts_locale.c:194

◆ tsearch_readline_begin()

bool tsearch_readline_begin ( tsearch_readline_state stp,
const char *  filename 
)

Definition at line 113 of file ts_locale.c.

References AllocateFile(), ErrorContextCallback::arg, ErrorContextCallback::callback, tsearch_readline_state::cb, tsearch_readline_state::curline, error_context_stack, tsearch_readline_state::filename, filename, tsearch_readline_state::fp, tsearch_readline_state::lineno, ErrorContextCallback::previous, and tsearch_readline_callback().

Referenced by dsynonym_init(), initTrie(), NIImportAffixes(), NIImportDictionary(), NIImportOOAffixes(), read_dictionary(), readstoplist(), and thesaurusRead().

115 {
116  if ((stp->fp = AllocateFile(filename, "r")) == NULL)
117  return false;
118  stp->filename = filename;
119  stp->lineno = 0;
120  stp->curline = NULL;
121  /* Setup error traceback support for ereport() */
123  stp->cb.arg = (void *) stp;
125  error_context_stack = &stp->cb;
126  return true;
127 }
static void tsearch_readline_callback(void *arg)
Definition: ts_locale.c:162
void(* callback)(void *arg)
Definition: elog.h:239
struct ErrorContextCallback * previous
Definition: elog.h:238
ErrorContextCallback * error_context_stack
Definition: elog.c:88
FILE * AllocateFile(const char *name, const char *mode)
Definition: fd.c:2342
ErrorContextCallback cb
Definition: ts_locale.h:39
const char * filename
Definition: ts_locale.h:36
static char * filename
Definition: pg_dumpall.c:90

◆ tsearch_readline_end()

void tsearch_readline_end ( tsearch_readline_state stp)

Definition at line 150 of file ts_locale.c.

References tsearch_readline_state::cb, error_context_stack, tsearch_readline_state::fp, FreeFile(), and ErrorContextCallback::previous.

Referenced by dsynonym_init(), initTrie(), NIImportAffixes(), NIImportDictionary(), NIImportOOAffixes(), read_dictionary(), readstoplist(), and thesaurusRead().

151 {
152  FreeFile(stp->fp);
153  /* Pop the error context stack */
155 }
struct ErrorContextCallback * previous
Definition: elog.h:238
ErrorContextCallback * error_context_stack
Definition: elog.c:88
ErrorContextCallback cb
Definition: ts_locale.h:39
int FreeFile(FILE *file)
Definition: fd.c:2534