PostgreSQL Source Code  git master
spell.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * spell.c
4  * Normalizing word with ISpell
5  *
6  * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group
7  *
8  * Ispell dictionary
9  * -----------------
10  *
11  * Rules of dictionaries are defined in two files with .affix and .dict
12  * extensions. They are used by spell checker programs Ispell and Hunspell.
13  *
14  * An .affix file declares morphological rules to get a basic form of words.
15  * The format of an .affix file has different structure for Ispell and Hunspell
16  * dictionaries. The Hunspell format is more complicated. But when an .affix
17  * file is imported and compiled, it is stored in the same structure AffixNode.
18  *
19  * A .dict file stores a list of basic forms of words with references to
20  * affix rules. The format of a .dict file has the same structure for Ispell
21  * and Hunspell dictionaries.
22  *
23  * Compilation of a dictionary
24  * ---------------------------
25  *
26  * A compiled dictionary is stored in the IspellDict structure. Compilation of
27  * a dictionary is divided into the several steps:
28  * - NIImportDictionary() - stores each word of a .dict file in the
29  * temporary Spell field.
30  * - NIImportAffixes() - stores affix rules of an .affix file in the
31  * Affix field (not temporary) if an .affix file has the Ispell format.
32  * -> NIImportOOAffixes() - stores affix rules if an .affix file has the
33  * Hunspell format. The AffixData field is initialized if AF parameter
34  * is defined.
35  * - NISortDictionary() - builds a prefix tree (Trie) from the words list
36  * and stores it in the Dictionary field. The words list is got from the
37  * Spell field. The AffixData field is initialized if AF parameter is not
38  * defined.
39  * - NISortAffixes():
40  * - builds a list of compound affixes from the affix list and stores it
41  * in the CompoundAffix.
42  * - builds prefix trees (Trie) from the affix list for prefixes and suffixes
43  * and stores them in Suffix and Prefix fields.
44  * The affix list is got from the Affix field.
45  *
46  * Memory management
47  * -----------------
48  *
49  * The IspellDict structure has the Spell field which is used only in compile
50  * time. The Spell field stores a words list. It can take a lot of memory.
51  * Therefore when a dictionary is compiled this field is cleared by
52  * NIFinishBuild().
53  *
54  * All resources which should cleared by NIFinishBuild() is initialized using
55  * tmpalloc() and tmpalloc0().
56  *
57  * IDENTIFICATION
58  * src/backend/tsearch/spell.c
59  *
60  *-------------------------------------------------------------------------
61  */
62 
63 #include "postgres.h"
64 
65 #include "catalog/pg_collation.h"
66 #include "miscadmin.h"
67 #include "tsearch/dicts/spell.h"
68 #include "tsearch/ts_locale.h"
69 #include "utils/memutils.h"
70 
71 
72 /*
73  * Initialization requires a lot of memory that's not needed
74  * after the initialization is done. During initialization,
75  * CurrentMemoryContext is the long-lived memory context associated
76  * with the dictionary cache entry. We keep the short-lived stuff
77  * in the Conf->buildCxt context.
78  */
79 #define tmpalloc(sz) MemoryContextAlloc(Conf->buildCxt, (sz))
80 #define tmpalloc0(sz) MemoryContextAllocZero(Conf->buildCxt, (sz))
81 
82 /*
83  * Prepare for constructing an ISpell dictionary.
84  *
85  * The IspellDict struct is assumed to be zeroed when allocated.
86  */
87 void
89 {
90  /*
91  * The temp context is a child of CurTransactionContext, so that it will
92  * go away automatically on error.
93  */
95  "Ispell dictionary init context",
97 }
98 
99 /*
100  * Clean up when dictionary construction is complete.
101  */
102 void
104 {
105  /* Release no-longer-needed temp memory */
107  /* Just for cleanliness, zero the now-dangling pointers */
108  Conf->buildCxt = NULL;
109  Conf->Spell = NULL;
110  Conf->firstfree = NULL;
111  Conf->CompoundAffixFlags = NULL;
112 }
113 
114 
115 /*
116  * "Compact" palloc: allocate without extra palloc overhead.
117  *
118  * Since we have no need to free the ispell data items individually, there's
119  * not much value in the per-chunk overhead normally consumed by palloc.
120  * Getting rid of it is helpful since ispell can allocate a lot of small nodes.
121  *
122  * We currently pre-zero all data allocated this way, even though some of it
123  * doesn't need that. The cpalloc and cpalloc0 macros are just documentation
124  * to indicate which allocations actually require zeroing.
125  */
126 #define COMPACT_ALLOC_CHUNK 8192 /* amount to get from palloc at once */
127 #define COMPACT_MAX_REQ 1024 /* must be < COMPACT_ALLOC_CHUNK */
128 
129 static void *
131 {
132  void *result;
133 
134  /* Should only be called during init */
135  Assert(Conf->buildCxt != NULL);
136 
137  /* No point in this for large chunks */
138  if (size > COMPACT_MAX_REQ)
139  return palloc0(size);
140 
141  /* Keep everything maxaligned */
142  size = MAXALIGN(size);
143 
144  /* Need more space? */
145  if (size > Conf->avail)
146  {
148  Conf->avail = COMPACT_ALLOC_CHUNK;
149  }
150 
151  result = (void *) Conf->firstfree;
152  Conf->firstfree += size;
153  Conf->avail -= size;
154 
155  return result;
156 }
157 
158 #define cpalloc(size) compact_palloc0(Conf, size)
159 #define cpalloc0(size) compact_palloc0(Conf, size)
160 
161 static char *
162 cpstrdup(IspellDict *Conf, const char *str)
163 {
164  char *res = cpalloc(strlen(str) + 1);
165 
166  strcpy(res, str);
167  return res;
168 }
169 
170 
171 /*
172  * Apply lowerstr(), producing a temporary result (in the buildCxt).
173  */
174 static char *
175 lowerstr_ctx(IspellDict *Conf, const char *src)
176 {
177  MemoryContext saveCtx;
178  char *dst;
179 
180  saveCtx = MemoryContextSwitchTo(Conf->buildCxt);
181  dst = lowerstr(src);
182  MemoryContextSwitchTo(saveCtx);
183 
184  return dst;
185 }
186 
187 #define MAX_NORM 1024
188 #define MAXNORMLEN 256
189 
190 #define STRNCMP(s,p) strncmp( (s), (p), strlen(p) )
191 #define GETWCHAR(W,L,N,T) ( ((const uint8*)(W))[ ((T)==FF_PREFIX) ? (N) : ( (L) - 1 - (N) ) ] )
192 #define GETCHAR(A,N,T) GETWCHAR( (A)->repl, (A)->replen, N, T )
193 
194 static char *VoidString = "";
195 
196 static int
197 cmpspell(const void *s1, const void *s2)
198 {
199  return strcmp((*(SPELL *const *) s1)->word, (*(SPELL *const *) s2)->word);
200 }
201 
202 static int
203 cmpspellaffix(const void *s1, const void *s2)
204 {
205  return strcmp((*(SPELL *const *) s1)->p.flag,
206  (*(SPELL *const *) s2)->p.flag);
207 }
208 
209 static int
210 cmpcmdflag(const void *f1, const void *f2)
211 {
213  *fv2 = (CompoundAffixFlag *) f2;
214 
215  Assert(fv1->flagMode == fv2->flagMode);
216 
217  if (fv1->flagMode == FM_NUM)
218  {
219  if (fv1->flag.i == fv2->flag.i)
220  return 0;
221 
222  return (fv1->flag.i > fv2->flag.i) ? 1 : -1;
223  }
224 
225  return strcmp(fv1->flag.s, fv2->flag.s);
226 }
227 
228 static char *
229 findchar(char *str, int c)
230 {
231  while (*str)
232  {
233  if (t_iseq(str, c))
234  return str;
235  str += pg_mblen(str);
236  }
237 
238  return NULL;
239 }
240 
241 static char *
242 findchar2(char *str, int c1, int c2)
243 {
244  while (*str)
245  {
246  if (t_iseq(str, c1) || t_iseq(str, c2))
247  return str;
248  str += pg_mblen(str);
249  }
250 
251  return NULL;
252 }
253 
254 
255 /* backward string compare for suffix tree operations */
256 static int
257 strbcmp(const unsigned char *s1, const unsigned char *s2)
258 {
259  int l1 = strlen((const char *) s1) - 1,
260  l2 = strlen((const char *) s2) - 1;
261 
262  while (l1 >= 0 && l2 >= 0)
263  {
264  if (s1[l1] < s2[l2])
265  return -1;
266  if (s1[l1] > s2[l2])
267  return 1;
268  l1--;
269  l2--;
270  }
271  if (l1 < l2)
272  return -1;
273  if (l1 > l2)
274  return 1;
275 
276  return 0;
277 }
278 
279 static int
280 strbncmp(const unsigned char *s1, const unsigned char *s2, size_t count)
281 {
282  int l1 = strlen((const char *) s1) - 1,
283  l2 = strlen((const char *) s2) - 1,
284  l = count;
285 
286  while (l1 >= 0 && l2 >= 0 && l > 0)
287  {
288  if (s1[l1] < s2[l2])
289  return -1;
290  if (s1[l1] > s2[l2])
291  return 1;
292  l1--;
293  l2--;
294  l--;
295  }
296  if (l == 0)
297  return 0;
298  if (l1 < l2)
299  return -1;
300  if (l1 > l2)
301  return 1;
302  return 0;
303 }
304 
305 /*
306  * Compares affixes.
307  * First compares the type of an affix. Prefixes should go before affixes.
308  * If types are equal then compares replaceable string.
309  */
310 static int
311 cmpaffix(const void *s1, const void *s2)
312 {
313  const AFFIX *a1 = (const AFFIX *) s1;
314  const AFFIX *a2 = (const AFFIX *) s2;
315 
316  if (a1->type < a2->type)
317  return -1;
318  if (a1->type > a2->type)
319  return 1;
320  if (a1->type == FF_PREFIX)
321  return strcmp(a1->repl, a2->repl);
322  else
323  return strbcmp((const unsigned char *) a1->repl,
324  (const unsigned char *) a2->repl);
325 }
326 
327 /*
328  * Gets an affix flag from the set of affix flags (sflagset).
329  *
330  * Several flags can be stored in a single string. Flags can be represented by:
331  * - 1 character (FM_CHAR). A character may be Unicode.
332  * - 2 characters (FM_LONG). A character may be Unicode.
333  * - numbers from 1 to 65000 (FM_NUM).
334  *
335  * Depending on the flagMode an affix string can have the following format:
336  * - FM_CHAR: ABCD
337  * Here we have 4 flags: A, B, C and D
338  * - FM_LONG: ABCDE*
339  * Here we have 3 flags: AB, CD and E*
340  * - FM_NUM: 200,205,50
341  * Here we have 3 flags: 200, 205 and 50
342  *
343  * Conf: current dictionary.
344  * sflagset: the set of affix flags. Returns a reference to the start of a next
345  * affix flag.
346  * sflag: returns an affix flag from sflagset.
347  */
348 static void
349 getNextFlagFromString(IspellDict *Conf, char **sflagset, char *sflag)
350 {
351  int32 s;
352  char *next,
353  *sbuf = *sflagset;
354  int maxstep;
355  bool stop = false;
356  bool met_comma = false;
357 
358  maxstep = (Conf->flagMode == FM_LONG) ? 2 : 1;
359 
360  while (**sflagset)
361  {
362  switch (Conf->flagMode)
363  {
364  case FM_LONG:
365  case FM_CHAR:
366  COPYCHAR(sflag, *sflagset);
367  sflag += pg_mblen(*sflagset);
368 
369  /* Go to start of the next flag */
370  *sflagset += pg_mblen(*sflagset);
371 
372  /* Check if we get all characters of flag */
373  maxstep--;
374  stop = (maxstep == 0);
375  break;
376  case FM_NUM:
377  s = strtol(*sflagset, &next, 10);
378  if (*sflagset == next || errno == ERANGE)
379  ereport(ERROR,
380  (errcode(ERRCODE_CONFIG_FILE_ERROR),
381  errmsg("invalid affix flag \"%s\"", *sflagset)));
382  if (s < 0 || s > FLAGNUM_MAXSIZE)
383  ereport(ERROR,
384  (errcode(ERRCODE_CONFIG_FILE_ERROR),
385  errmsg("affix flag \"%s\" is out of range",
386  *sflagset)));
387  sflag += sprintf(sflag, "%0d", s);
388 
389  /* Go to start of the next flag */
390  *sflagset = next;
391  while (**sflagset)
392  {
393  if (t_isdigit(*sflagset))
394  {
395  if (!met_comma)
396  ereport(ERROR,
397  (errcode(ERRCODE_CONFIG_FILE_ERROR),
398  errmsg("invalid affix flag \"%s\"",
399  *sflagset)));
400  break;
401  }
402  else if (t_iseq(*sflagset, ','))
403  {
404  if (met_comma)
405  ereport(ERROR,
406  (errcode(ERRCODE_CONFIG_FILE_ERROR),
407  errmsg("invalid affix flag \"%s\"",
408  *sflagset)));
409  met_comma = true;
410  }
411  else if (!t_isspace(*sflagset))
412  {
413  ereport(ERROR,
414  (errcode(ERRCODE_CONFIG_FILE_ERROR),
415  errmsg("invalid character in affix flag \"%s\"",
416  *sflagset)));
417  }
418 
419  *sflagset += pg_mblen(*sflagset);
420  }
421  stop = true;
422  break;
423  default:
424  elog(ERROR, "unrecognized type of Conf->flagMode: %d",
425  Conf->flagMode);
426  }
427 
428  if (stop)
429  break;
430  }
431 
432  if (Conf->flagMode == FM_LONG && maxstep > 0)
433  ereport(ERROR,
434  (errcode(ERRCODE_CONFIG_FILE_ERROR),
435  errmsg("invalid affix flag \"%s\" with \"long\" flag value",
436  sbuf)));
437 
438  *sflag = '\0';
439 }
440 
441 /*
442  * Checks if the affix set Conf->AffixData[affix] contains affixflag.
443  * Conf->AffixData[affix] does not contain affixflag if this flag is not used
444  * actually by the .dict file.
445  *
446  * Conf: current dictionary.
447  * affix: index of the Conf->AffixData array.
448  * affixflag: the affix flag.
449  *
450  * Returns true if the string Conf->AffixData[affix] contains affixflag,
451  * otherwise returns false.
452  */
453 static bool
454 IsAffixFlagInUse(IspellDict *Conf, int affix, const char *affixflag)
455 {
456  char *flagcur;
457  char flag[BUFSIZ];
458 
459  if (*affixflag == 0)
460  return true;
461 
462  Assert(affix < Conf->nAffixData);
463 
464  flagcur = Conf->AffixData[affix];
465 
466  while (*flagcur)
467  {
468  getNextFlagFromString(Conf, &flagcur, flag);
469  /* Compare first affix flag in flagcur with affixflag */
470  if (strcmp(flag, affixflag) == 0)
471  return true;
472  }
473 
474  /* Could not find affixflag */
475  return false;
476 }
477 
478 /*
479  * Adds the new word into the temporary array Spell.
480  *
481  * Conf: current dictionary.
482  * word: new word.
483  * flag: set of affix flags. Single flag can be get by getNextFlagFromString().
484  */
485 static void
486 NIAddSpell(IspellDict *Conf, const char *word, const char *flag)
487 {
488  if (Conf->nspell >= Conf->mspell)
489  {
490  if (Conf->mspell)
491  {
492  Conf->mspell *= 2;
493  Conf->Spell = (SPELL **) repalloc(Conf->Spell, Conf->mspell * sizeof(SPELL *));
494  }
495  else
496  {
497  Conf->mspell = 1024 * 20;
498  Conf->Spell = (SPELL **) tmpalloc(Conf->mspell * sizeof(SPELL *));
499  }
500  }
501  Conf->Spell[Conf->nspell] = (SPELL *) tmpalloc(SPELLHDRSZ + strlen(word) + 1);
502  strcpy(Conf->Spell[Conf->nspell]->word, word);
503  Conf->Spell[Conf->nspell]->p.flag = (*flag != '\0')
504  ? cpstrdup(Conf, flag) : VoidString;
505  Conf->nspell++;
506 }
507 
508 /*
509  * Imports dictionary into the temporary array Spell.
510  *
511  * Note caller must already have applied get_tsearch_config_filename.
512  *
513  * Conf: current dictionary.
514  * filename: path to the .dict file.
515  */
516 void
518 {
520  char *line;
521 
522  if (!tsearch_readline_begin(&trst, filename))
523  ereport(ERROR,
524  (errcode(ERRCODE_CONFIG_FILE_ERROR),
525  errmsg("could not open dictionary file \"%s\": %m",
526  filename)));
527 
528  while ((line = tsearch_readline(&trst)) != NULL)
529  {
530  char *s,
531  *pstr;
532 
533  /* Set of affix flags */
534  const char *flag;
535 
536  /* Extract flag from the line */
537  flag = NULL;
538  if ((s = findchar(line, '/')))
539  {
540  *s++ = '\0';
541  flag = s;
542  while (*s)
543  {
544  /* we allow only single encoded flags for faster works */
545  if (pg_mblen(s) == 1 && t_isprint(s) && !t_isspace(s))
546  s++;
547  else
548  {
549  *s = '\0';
550  break;
551  }
552  }
553  }
554  else
555  flag = "";
556 
557  /* Remove trailing spaces */
558  s = line;
559  while (*s)
560  {
561  if (t_isspace(s))
562  {
563  *s = '\0';
564  break;
565  }
566  s += pg_mblen(s);
567  }
568  pstr = lowerstr_ctx(Conf, line);
569 
570  NIAddSpell(Conf, pstr, flag);
571  pfree(pstr);
572 
573  pfree(line);
574  }
575  tsearch_readline_end(&trst);
576 }
577 
578 /*
579  * Searches a basic form of word in the prefix tree. This word was generated
580  * using an affix rule. This rule may not be presented in an affix set of
581  * a basic form of word.
582  *
583  * For example, we have the entry in the .dict file:
584  * meter/GMD
585  *
586  * The affix rule with the flag S:
587  * SFX S y ies [^aeiou]y
588  * is not presented here.
589  *
590  * The affix rule with the flag M:
591  * SFX M 0 's .
592  * is presented here.
593  *
594  * Conf: current dictionary.
595  * word: basic form of word.
596  * affixflag: affix flag, by which a basic form of word was generated.
597  * flag: compound flag used to compare with StopMiddle->compoundflag.
598  *
599  * Returns 1 if the word was found in the prefix tree, else returns 0.
600  */
601 static int
602 FindWord(IspellDict *Conf, const char *word, const char *affixflag, int flag)
603 {
604  SPNode *node = Conf->Dictionary;
605  SPNodeData *StopLow,
606  *StopHigh,
607  *StopMiddle;
608  const uint8 *ptr = (const uint8 *) word;
609 
611 
612  while (node && *ptr)
613  {
614  StopLow = node->data;
615  StopHigh = node->data + node->length;
616  while (StopLow < StopHigh)
617  {
618  StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
619  if (StopMiddle->val == *ptr)
620  {
621  if (*(ptr + 1) == '\0' && StopMiddle->isword)
622  {
623  if (flag == 0)
624  {
625  /*
626  * The word can be formed only with another word. And
627  * in the flag parameter there is not a sign that we
628  * search compound words.
629  */
630  if (StopMiddle->compoundflag & FF_COMPOUNDONLY)
631  return 0;
632  }
633  else if ((flag & StopMiddle->compoundflag) == 0)
634  return 0;
635 
636  /*
637  * Check if this affix rule is presented in the affix set
638  * with index StopMiddle->affix.
639  */
640  if (IsAffixFlagInUse(Conf, StopMiddle->affix, affixflag))
641  return 1;
642  }
643  node = StopMiddle->node;
644  ptr++;
645  break;
646  }
647  else if (StopMiddle->val < *ptr)
648  StopLow = StopMiddle + 1;
649  else
650  StopHigh = StopMiddle;
651  }
652  if (StopLow >= StopHigh)
653  break;
654  }
655  return 0;
656 }
657 
658 /*
659  * Adds a new affix rule to the Affix field.
660  *
661  * Conf: current dictionary.
662  * flag: affix flag ('\' in the below example).
663  * flagflags: set of flags from the flagval field for this affix rule. This set
664  * is listed after '/' character in the added string (repl).
665  *
666  * For example L flag in the hunspell_sample.affix:
667  * SFX \ 0 Y/L [^Y]
668  *
669  * mask: condition for search ('[^Y]' in the above example).
670  * find: stripping characters from beginning (at prefix) or end (at suffix)
671  * of the word ('0' in the above example, 0 means that there is not
672  * stripping character).
673  * repl: adding string after stripping ('Y' in the above example).
674  * type: FF_SUFFIX or FF_PREFIX.
675  */
676 static void
677 NIAddAffix(IspellDict *Conf, const char *flag, char flagflags, const char *mask,
678  const char *find, const char *repl, int type)
679 {
680  AFFIX *Affix;
681 
682  if (Conf->naffixes >= Conf->maffixes)
683  {
684  if (Conf->maffixes)
685  {
686  Conf->maffixes *= 2;
687  Conf->Affix = (AFFIX *) repalloc(Conf->Affix, Conf->maffixes * sizeof(AFFIX));
688  }
689  else
690  {
691  Conf->maffixes = 16;
692  Conf->Affix = (AFFIX *) palloc(Conf->maffixes * sizeof(AFFIX));
693  }
694  }
695 
696  Affix = Conf->Affix + Conf->naffixes;
697 
698  /* This affix rule can be applied for words with any ending */
699  if (strcmp(mask, ".") == 0 || *mask == '\0')
700  {
701  Affix->issimple = 1;
702  Affix->isregis = 0;
703  }
704  /* This affix rule will use regis to search word ending */
705  else if (RS_isRegis(mask))
706  {
707  Affix->issimple = 0;
708  Affix->isregis = 1;
709  RS_compile(&(Affix->reg.regis), (type == FF_SUFFIX),
710  *mask ? mask : VoidString);
711  }
712  /* This affix rule will use regex_t to search word ending */
713  else
714  {
715  int masklen;
716  int wmasklen;
717  int err;
718  pg_wchar *wmask;
719  char *tmask;
720 
721  Affix->issimple = 0;
722  Affix->isregis = 0;
723  tmask = (char *) tmpalloc(strlen(mask) + 3);
724  if (type == FF_SUFFIX)
725  sprintf(tmask, "%s$", mask);
726  else
727  sprintf(tmask, "^%s", mask);
728 
729  masklen = strlen(tmask);
730  wmask = (pg_wchar *) tmpalloc((masklen + 1) * sizeof(pg_wchar));
731  wmasklen = pg_mb2wchar_with_len(tmask, wmask, masklen);
732 
733  /*
734  * The regex and all internal state created by pg_regcomp are
735  * allocated in the dictionary's memory context, and will be freed
736  * automatically when it is destroyed.
737  */
738  Affix->reg.pregex = palloc(sizeof(regex_t));
739  err = pg_regcomp(Affix->reg.pregex, wmask, wmasklen,
741  DEFAULT_COLLATION_OID);
742  if (err)
743  {
744  char errstr[100];
745 
746  pg_regerror(err, Affix->reg.pregex, errstr, sizeof(errstr));
747  ereport(ERROR,
748  (errcode(ERRCODE_INVALID_REGULAR_EXPRESSION),
749  errmsg("invalid regular expression: %s", errstr)));
750  }
751  }
752 
753  Affix->flagflags = flagflags;
754  if ((Affix->flagflags & FF_COMPOUNDONLY) || (Affix->flagflags & FF_COMPOUNDPERMITFLAG))
755  {
756  if ((Affix->flagflags & FF_COMPOUNDFLAG) == 0)
757  Affix->flagflags |= FF_COMPOUNDFLAG;
758  }
759  Affix->flag = cpstrdup(Conf, flag);
760  Affix->type = type;
761 
762  Affix->find = (find && *find) ? cpstrdup(Conf, find) : VoidString;
763  if ((Affix->replen = strlen(repl)) > 0)
764  Affix->repl = cpstrdup(Conf, repl);
765  else
766  Affix->repl = VoidString;
767  Conf->naffixes++;
768 }
769 
770 /* Parsing states for parse_affentry() and friends */
771 #define PAE_WAIT_MASK 0
772 #define PAE_INMASK 1
773 #define PAE_WAIT_FIND 2
774 #define PAE_INFIND 3
775 #define PAE_WAIT_REPL 4
776 #define PAE_INREPL 5
777 #define PAE_WAIT_TYPE 6
778 #define PAE_WAIT_FLAG 7
779 
780 /*
781  * Parse next space-separated field of an .affix file line.
782  *
783  * *str is the input pointer (will be advanced past field)
784  * next is where to copy the field value to, with null termination
785  *
786  * The buffer at "next" must be of size BUFSIZ; we truncate the input to fit.
787  *
788  * Returns true if we found a field, false if not.
789  */
790 static bool
791 get_nextfield(char **str, char *next)
792 {
793  int state = PAE_WAIT_MASK;
794  int avail = BUFSIZ;
795 
796  while (**str)
797  {
798  if (state == PAE_WAIT_MASK)
799  {
800  if (t_iseq(*str, '#'))
801  return false;
802  else if (!t_isspace(*str))
803  {
804  int clen = pg_mblen(*str);
805 
806  if (clen < avail)
807  {
808  COPYCHAR(next, *str);
809  next += clen;
810  avail -= clen;
811  }
812  state = PAE_INMASK;
813  }
814  }
815  else /* state == PAE_INMASK */
816  {
817  if (t_isspace(*str))
818  {
819  *next = '\0';
820  return true;
821  }
822  else
823  {
824  int clen = pg_mblen(*str);
825 
826  if (clen < avail)
827  {
828  COPYCHAR(next, *str);
829  next += clen;
830  avail -= clen;
831  }
832  }
833  }
834  *str += pg_mblen(*str);
835  }
836 
837  *next = '\0';
838 
839  return (state == PAE_INMASK); /* OK if we got a nonempty field */
840 }
841 
842 /*
843  * Parses entry of an .affix file of MySpell or Hunspell format.
844  *
845  * An .affix file entry has the following format:
846  * - header
847  * <type> <flag> <cross_flag> <flag_count>
848  * - fields after header:
849  * <type> <flag> <find> <replace> <mask>
850  *
851  * str is the input line
852  * field values are returned to type etc, which must be buffers of size BUFSIZ.
853  *
854  * Returns number of fields found; any omitted fields are set to empty strings.
855  */
856 static int
857 parse_ooaffentry(char *str, char *type, char *flag, char *find,
858  char *repl, char *mask)
859 {
860  int state = PAE_WAIT_TYPE;
861  int fields_read = 0;
862  bool valid = false;
863 
864  *type = *flag = *find = *repl = *mask = '\0';
865 
866  while (*str)
867  {
868  switch (state)
869  {
870  case PAE_WAIT_TYPE:
871  valid = get_nextfield(&str, type);
873  break;
874  case PAE_WAIT_FLAG:
875  valid = get_nextfield(&str, flag);
877  break;
878  case PAE_WAIT_FIND:
879  valid = get_nextfield(&str, find);
881  break;
882  case PAE_WAIT_REPL:
883  valid = get_nextfield(&str, repl);
885  break;
886  case PAE_WAIT_MASK:
887  valid = get_nextfield(&str, mask);
888  state = -1; /* force loop exit */
889  break;
890  default:
891  elog(ERROR, "unrecognized state in parse_ooaffentry: %d",
892  state);
893  break;
894  }
895  if (valid)
896  fields_read++;
897  else
898  break; /* early EOL */
899  if (state < 0)
900  break; /* got all fields */
901  }
902 
903  return fields_read;
904 }
905 
906 /*
907  * Parses entry of an .affix file of Ispell format
908  *
909  * An .affix file entry has the following format:
910  * <mask> > [-<find>,]<replace>
911  */
912 static bool
913 parse_affentry(char *str, char *mask, char *find, char *repl)
914 {
915  int state = PAE_WAIT_MASK;
916  char *pmask = mask,
917  *pfind = find,
918  *prepl = repl;
919 
920  *mask = *find = *repl = '\0';
921 
922  while (*str)
923  {
924  if (state == PAE_WAIT_MASK)
925  {
926  if (t_iseq(str, '#'))
927  return false;
928  else if (!t_isspace(str))
929  {
930  COPYCHAR(pmask, str);
931  pmask += pg_mblen(str);
932  state = PAE_INMASK;
933  }
934  }
935  else if (state == PAE_INMASK)
936  {
937  if (t_iseq(str, '>'))
938  {
939  *pmask = '\0';
941  }
942  else if (!t_isspace(str))
943  {
944  COPYCHAR(pmask, str);
945  pmask += pg_mblen(str);
946  }
947  }
948  else if (state == PAE_WAIT_FIND)
949  {
950  if (t_iseq(str, '-'))
951  {
952  state = PAE_INFIND;
953  }
954  else if (t_isalpha(str) || t_iseq(str, '\'') /* english 's */ )
955  {
956  COPYCHAR(prepl, str);
957  prepl += pg_mblen(str);
958  state = PAE_INREPL;
959  }
960  else if (!t_isspace(str))
961  ereport(ERROR,
962  (errcode(ERRCODE_CONFIG_FILE_ERROR),
963  errmsg("syntax error")));
964  }
965  else if (state == PAE_INFIND)
966  {
967  if (t_iseq(str, ','))
968  {
969  *pfind = '\0';
971  }
972  else if (t_isalpha(str))
973  {
974  COPYCHAR(pfind, str);
975  pfind += pg_mblen(str);
976  }
977  else if (!t_isspace(str))
978  ereport(ERROR,
979  (errcode(ERRCODE_CONFIG_FILE_ERROR),
980  errmsg("syntax error")));
981  }
982  else if (state == PAE_WAIT_REPL)
983  {
984  if (t_iseq(str, '-'))
985  {
986  break; /* void repl */
987  }
988  else if (t_isalpha(str))
989  {
990  COPYCHAR(prepl, str);
991  prepl += pg_mblen(str);
992  state = PAE_INREPL;
993  }
994  else if (!t_isspace(str))
995  ereport(ERROR,
996  (errcode(ERRCODE_CONFIG_FILE_ERROR),
997  errmsg("syntax error")));
998  }
999  else if (state == PAE_INREPL)
1000  {
1001  if (t_iseq(str, '#'))
1002  {
1003  *prepl = '\0';
1004  break;
1005  }
1006  else if (t_isalpha(str))
1007  {
1008  COPYCHAR(prepl, str);
1009  prepl += pg_mblen(str);
1010  }
1011  else if (!t_isspace(str))
1012  ereport(ERROR,
1013  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1014  errmsg("syntax error")));
1015  }
1016  else
1017  elog(ERROR, "unrecognized state in parse_affentry: %d", state);
1018 
1019  str += pg_mblen(str);
1020  }
1021 
1022  *pmask = *pfind = *prepl = '\0';
1023 
1024  return (*mask && (*find || *repl));
1025 }
1026 
1027 /*
1028  * Sets a Hunspell options depending on flag type.
1029  */
1030 static void
1032  char *s, uint32 val)
1033 {
1034  if (Conf->flagMode == FM_NUM)
1035  {
1036  char *next;
1037  int i;
1038 
1039  i = strtol(s, &next, 10);
1040  if (s == next || errno == ERANGE)
1041  ereport(ERROR,
1042  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1043  errmsg("invalid affix flag \"%s\"", s)));
1044  if (i < 0 || i > FLAGNUM_MAXSIZE)
1045  ereport(ERROR,
1046  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1047  errmsg("affix flag \"%s\" is out of range", s)));
1048 
1049  entry->flag.i = i;
1050  }
1051  else
1052  entry->flag.s = cpstrdup(Conf, s);
1053 
1054  entry->flagMode = Conf->flagMode;
1055  entry->value = val;
1056 }
1057 
1058 /*
1059  * Sets up a correspondence for the affix parameter with the affix flag.
1060  *
1061  * Conf: current dictionary.
1062  * s: affix flag in string.
1063  * val: affix parameter.
1064  */
1065 static void
1067 {
1068  CompoundAffixFlag *newValue;
1069  char sbuf[BUFSIZ];
1070  char *sflag;
1071  int clen;
1072 
1073  while (*s && t_isspace(s))
1074  s += pg_mblen(s);
1075 
1076  if (!*s)
1077  ereport(ERROR,
1078  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1079  errmsg("syntax error")));
1080 
1081  /* Get flag without \n */
1082  sflag = sbuf;
1083  while (*s && !t_isspace(s) && *s != '\n')
1084  {
1085  clen = pg_mblen(s);
1086  COPYCHAR(sflag, s);
1087  sflag += clen;
1088  s += clen;
1089  }
1090  *sflag = '\0';
1091 
1092  /* Resize array or allocate memory for array CompoundAffixFlag */
1093  if (Conf->nCompoundAffixFlag >= Conf->mCompoundAffixFlag)
1094  {
1095  if (Conf->mCompoundAffixFlag)
1096  {
1097  Conf->mCompoundAffixFlag *= 2;
1100  Conf->mCompoundAffixFlag * sizeof(CompoundAffixFlag));
1101  }
1102  else
1103  {
1104  Conf->mCompoundAffixFlag = 10;
1107  }
1108  }
1109 
1110  newValue = Conf->CompoundAffixFlags + Conf->nCompoundAffixFlag;
1111 
1112  setCompoundAffixFlagValue(Conf, newValue, sbuf, val);
1113 
1114  Conf->usecompound = true;
1115  Conf->nCompoundAffixFlag++;
1116 }
1117 
1118 /*
1119  * Returns a set of affix parameters which correspondence to the set of affix
1120  * flags s.
1121  */
1122 static int
1124 {
1125  uint32 flag = 0;
1126  CompoundAffixFlag *found,
1127  key;
1128  char sflag[BUFSIZ];
1129  char *flagcur;
1130 
1131  if (Conf->nCompoundAffixFlag == 0)
1132  return 0;
1133 
1134  flagcur = s;
1135  while (*flagcur)
1136  {
1137  getNextFlagFromString(Conf, &flagcur, sflag);
1138  setCompoundAffixFlagValue(Conf, &key, sflag, 0);
1139 
1140  found = (CompoundAffixFlag *)
1141  bsearch(&key, Conf->CompoundAffixFlags,
1142  Conf->nCompoundAffixFlag, sizeof(CompoundAffixFlag),
1143  cmpcmdflag);
1144  if (found != NULL)
1145  flag |= found->value;
1146  }
1147 
1148  return flag;
1149 }
1150 
1151 /*
1152  * Returns a flag set using the s parameter.
1153  *
1154  * If Conf->useFlagAliases is true then the s parameter is index of the
1155  * Conf->AffixData array and function returns its entry.
1156  * Else function returns the s parameter.
1157  */
1158 static char *
1160 {
1161  if (Conf->useFlagAliases && *s != '\0')
1162  {
1163  int curaffix;
1164  char *end;
1165 
1166  curaffix = strtol(s, &end, 10);
1167  if (s == end || errno == ERANGE)
1168  ereport(ERROR,
1169  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1170  errmsg("invalid affix alias \"%s\"", s)));
1171 
1172  if (curaffix > 0 && curaffix < Conf->nAffixData)
1173 
1174  /*
1175  * Do not subtract 1 from curaffix because empty string was added
1176  * in NIImportOOAffixes
1177  */
1178  return Conf->AffixData[curaffix];
1179  else if (curaffix > Conf->nAffixData)
1180  ereport(ERROR,
1181  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1182  errmsg("invalid affix alias \"%s\"", s)));
1183  return VoidString;
1184  }
1185  else
1186  return s;
1187 }
1188 
1189 /*
1190  * Import an affix file that follows MySpell or Hunspell format.
1191  *
1192  * Conf: current dictionary.
1193  * filename: path to the .affix file.
1194  */
1195 static void
1197 {
1198  char type[BUFSIZ],
1199  *ptype = NULL;
1200  char sflag[BUFSIZ];
1201  char mask[BUFSIZ],
1202  *pmask;
1203  char find[BUFSIZ],
1204  *pfind;
1205  char repl[BUFSIZ],
1206  *prepl;
1207  bool isSuffix = false;
1208  int naffix = 0,
1209  curaffix = 0;
1210  int sflaglen = 0;
1211  char flagflags = 0;
1213  char *recoded;
1214 
1215  /* read file to find any flag */
1216  Conf->usecompound = false;
1217  Conf->useFlagAliases = false;
1218  Conf->flagMode = FM_CHAR;
1219 
1220  if (!tsearch_readline_begin(&trst, filename))
1221  ereport(ERROR,
1222  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1223  errmsg("could not open affix file \"%s\": %m",
1224  filename)));
1225 
1226  while ((recoded = tsearch_readline(&trst)) != NULL)
1227  {
1228  if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
1229  {
1230  pfree(recoded);
1231  continue;
1232  }
1233 
1234  if (STRNCMP(recoded, "COMPOUNDFLAG") == 0)
1235  addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDFLAG"),
1236  FF_COMPOUNDFLAG);
1237  else if (STRNCMP(recoded, "COMPOUNDBEGIN") == 0)
1238  addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDBEGIN"),
1240  else if (STRNCMP(recoded, "COMPOUNDLAST") == 0)
1241  addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDLAST"),
1242  FF_COMPOUNDLAST);
1243  /* COMPOUNDLAST and COMPOUNDEND are synonyms */
1244  else if (STRNCMP(recoded, "COMPOUNDEND") == 0)
1245  addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDEND"),
1246  FF_COMPOUNDLAST);
1247  else if (STRNCMP(recoded, "COMPOUNDMIDDLE") == 0)
1248  addCompoundAffixFlagValue(Conf, recoded + strlen("COMPOUNDMIDDLE"),
1250  else if (STRNCMP(recoded, "ONLYINCOMPOUND") == 0)
1251  addCompoundAffixFlagValue(Conf, recoded + strlen("ONLYINCOMPOUND"),
1252  FF_COMPOUNDONLY);
1253  else if (STRNCMP(recoded, "COMPOUNDPERMITFLAG") == 0)
1255  recoded + strlen("COMPOUNDPERMITFLAG"),
1257  else if (STRNCMP(recoded, "COMPOUNDFORBIDFLAG") == 0)
1259  recoded + strlen("COMPOUNDFORBIDFLAG"),
1261  else if (STRNCMP(recoded, "FLAG") == 0)
1262  {
1263  char *s = recoded + strlen("FLAG");
1264 
1265  while (*s && t_isspace(s))
1266  s += pg_mblen(s);
1267 
1268  if (*s)
1269  {
1270  if (STRNCMP(s, "long") == 0)
1271  Conf->flagMode = FM_LONG;
1272  else if (STRNCMP(s, "num") == 0)
1273  Conf->flagMode = FM_NUM;
1274  else if (STRNCMP(s, "default") != 0)
1275  ereport(ERROR,
1276  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1277  errmsg("Ispell dictionary supports only "
1278  "\"default\", \"long\", "
1279  "and \"num\" flag values")));
1280  }
1281  }
1282 
1283  pfree(recoded);
1284  }
1285  tsearch_readline_end(&trst);
1286 
1287  if (Conf->nCompoundAffixFlag > 1)
1289  sizeof(CompoundAffixFlag), cmpcmdflag);
1290 
1291  if (!tsearch_readline_begin(&trst, filename))
1292  ereport(ERROR,
1293  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1294  errmsg("could not open affix file \"%s\": %m",
1295  filename)));
1296 
1297  while ((recoded = tsearch_readline(&trst)) != NULL)
1298  {
1299  int fields_read;
1300 
1301  if (*recoded == '\0' || t_isspace(recoded) || t_iseq(recoded, '#'))
1302  goto nextline;
1303 
1304  fields_read = parse_ooaffentry(recoded, type, sflag, find, repl, mask);
1305 
1306  if (ptype)
1307  pfree(ptype);
1308  ptype = lowerstr_ctx(Conf, type);
1309 
1310  /* First try to parse AF parameter (alias compression) */
1311  if (STRNCMP(ptype, "af") == 0)
1312  {
1313  /* First line is the number of aliases */
1314  if (!Conf->useFlagAliases)
1315  {
1316  Conf->useFlagAliases = true;
1317  naffix = atoi(sflag);
1318  if (naffix <= 0)
1319  ereport(ERROR,
1320  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1321  errmsg("invalid number of flag vector aliases")));
1322 
1323  /* Also reserve place for empty flag set */
1324  naffix++;
1325 
1326  Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
1327  Conf->lenAffixData = Conf->nAffixData = naffix;
1328 
1329  /* Add empty flag set into AffixData */
1330  Conf->AffixData[curaffix] = VoidString;
1331  curaffix++;
1332  }
1333  /* Other lines are aliases */
1334  else
1335  {
1336  if (curaffix < naffix)
1337  {
1338  Conf->AffixData[curaffix] = cpstrdup(Conf, sflag);
1339  curaffix++;
1340  }
1341  else
1342  ereport(ERROR,
1343  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1344  errmsg("number of aliases exceeds specified number %d",
1345  naffix - 1)));
1346  }
1347  goto nextline;
1348  }
1349  /* Else try to parse prefixes and suffixes */
1350  if (fields_read < 4 ||
1351  (STRNCMP(ptype, "sfx") != 0 && STRNCMP(ptype, "pfx") != 0))
1352  goto nextline;
1353 
1354  sflaglen = strlen(sflag);
1355  if (sflaglen == 0
1356  || (sflaglen > 1 && Conf->flagMode == FM_CHAR)
1357  || (sflaglen > 2 && Conf->flagMode == FM_LONG))
1358  goto nextline;
1359 
1360  /*--------
1361  * Affix header. For example:
1362  * SFX \ N 1
1363  *--------
1364  */
1365  if (fields_read == 4)
1366  {
1367  isSuffix = (STRNCMP(ptype, "sfx") == 0);
1368  if (t_iseq(find, 'y') || t_iseq(find, 'Y'))
1369  flagflags = FF_CROSSPRODUCT;
1370  else
1371  flagflags = 0;
1372  }
1373  /*--------
1374  * Affix fields. For example:
1375  * SFX \ 0 Y/L [^Y]
1376  *--------
1377  */
1378  else
1379  {
1380  char *ptr;
1381  int aflg = 0;
1382 
1383  /* Get flags after '/' (flags are case sensitive) */
1384  if ((ptr = strchr(repl, '/')) != NULL)
1385  aflg |= getCompoundAffixFlagValue(Conf,
1386  getAffixFlagSet(Conf,
1387  ptr + 1));
1388  /* Get lowercased version of string before '/' */
1389  prepl = lowerstr_ctx(Conf, repl);
1390  if ((ptr = strchr(prepl, '/')) != NULL)
1391  *ptr = '\0';
1392  pfind = lowerstr_ctx(Conf, find);
1393  pmask = lowerstr_ctx(Conf, mask);
1394  if (t_iseq(find, '0'))
1395  *pfind = '\0';
1396  if (t_iseq(repl, '0'))
1397  *prepl = '\0';
1398 
1399  NIAddAffix(Conf, sflag, flagflags | aflg, pmask, pfind, prepl,
1400  isSuffix ? FF_SUFFIX : FF_PREFIX);
1401  pfree(prepl);
1402  pfree(pfind);
1403  pfree(pmask);
1404  }
1405 
1406 nextline:
1407  pfree(recoded);
1408  }
1409 
1410  tsearch_readline_end(&trst);
1411  if (ptype)
1412  pfree(ptype);
1413 }
1414 
1415 /*
1416  * import affixes
1417  *
1418  * Note caller must already have applied get_tsearch_config_filename
1419  *
1420  * This function is responsible for parsing ispell ("old format") affix files.
1421  * If we realize that the file contains new-format commands, we pass off the
1422  * work to NIImportOOAffixes(), which will re-read the whole file.
1423  */
1424 void
1426 {
1427  char *pstr = NULL;
1428  char flag[BUFSIZ];
1429  char mask[BUFSIZ];
1430  char find[BUFSIZ];
1431  char repl[BUFSIZ];
1432  char *s;
1433  bool suffixes = false;
1434  bool prefixes = false;
1435  char flagflags = 0;
1437  bool oldformat = false;
1438  char *recoded = NULL;
1439 
1440  if (!tsearch_readline_begin(&trst, filename))
1441  ereport(ERROR,
1442  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1443  errmsg("could not open affix file \"%s\": %m",
1444  filename)));
1445 
1446  Conf->usecompound = false;
1447  Conf->useFlagAliases = false;
1448  Conf->flagMode = FM_CHAR;
1449 
1450  while ((recoded = tsearch_readline(&trst)) != NULL)
1451  {
1452  pstr = lowerstr(recoded);
1453 
1454  /* Skip comments and empty lines */
1455  if (*pstr == '#' || *pstr == '\n')
1456  goto nextline;
1457 
1458  if (STRNCMP(pstr, "compoundwords") == 0)
1459  {
1460  /* Find case-insensitive L flag in non-lowercased string */
1461  s = findchar2(recoded, 'l', 'L');
1462  if (s)
1463  {
1464  while (*s && !t_isspace(s))
1465  s += pg_mblen(s);
1466  while (*s && t_isspace(s))
1467  s += pg_mblen(s);
1468 
1469  if (*s && pg_mblen(s) == 1)
1470  {
1472  Conf->usecompound = true;
1473  }
1474  oldformat = true;
1475  goto nextline;
1476  }
1477  }
1478  if (STRNCMP(pstr, "suffixes") == 0)
1479  {
1480  suffixes = true;
1481  prefixes = false;
1482  oldformat = true;
1483  goto nextline;
1484  }
1485  if (STRNCMP(pstr, "prefixes") == 0)
1486  {
1487  suffixes = false;
1488  prefixes = true;
1489  oldformat = true;
1490  goto nextline;
1491  }
1492  if (STRNCMP(pstr, "flag") == 0)
1493  {
1494  s = recoded + 4; /* we need non-lowercased string */
1495  flagflags = 0;
1496 
1497  while (*s && t_isspace(s))
1498  s += pg_mblen(s);
1499 
1500  if (*s == '*')
1501  {
1502  flagflags |= FF_CROSSPRODUCT;
1503  s++;
1504  }
1505  else if (*s == '~')
1506  {
1507  flagflags |= FF_COMPOUNDONLY;
1508  s++;
1509  }
1510 
1511  if (*s == '\\')
1512  s++;
1513 
1514  /*
1515  * An old-format flag is a single ASCII character; we expect it to
1516  * be followed by EOL, whitespace, or ':'. Otherwise this is a
1517  * new-format flag command.
1518  */
1519  if (*s && pg_mblen(s) == 1)
1520  {
1521  COPYCHAR(flag, s);
1522  flag[1] = '\0';
1523 
1524  s++;
1525  if (*s == '\0' || *s == '#' || *s == '\n' || *s == ':' ||
1526  t_isspace(s))
1527  {
1528  oldformat = true;
1529  goto nextline;
1530  }
1531  }
1532  goto isnewformat;
1533  }
1534  if (STRNCMP(recoded, "COMPOUNDFLAG") == 0 ||
1535  STRNCMP(recoded, "COMPOUNDMIN") == 0 ||
1536  STRNCMP(recoded, "PFX") == 0 ||
1537  STRNCMP(recoded, "SFX") == 0)
1538  goto isnewformat;
1539 
1540  if ((!suffixes) && (!prefixes))
1541  goto nextline;
1542 
1543  if (!parse_affentry(pstr, mask, find, repl))
1544  goto nextline;
1545 
1546  NIAddAffix(Conf, flag, flagflags, mask, find, repl, suffixes ? FF_SUFFIX : FF_PREFIX);
1547 
1548 nextline:
1549  pfree(recoded);
1550  pfree(pstr);
1551  }
1552  tsearch_readline_end(&trst);
1553  return;
1554 
1555 isnewformat:
1556  if (oldformat)
1557  ereport(ERROR,
1558  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1559  errmsg("affix file contains both old-style and new-style commands")));
1560  tsearch_readline_end(&trst);
1561 
1562  NIImportOOAffixes(Conf, filename);
1563 }
1564 
1565 /*
1566  * Merges two affix flag sets and stores a new affix flag set into
1567  * Conf->AffixData.
1568  *
1569  * Returns index of a new affix flag set.
1570  */
1571 static int
1572 MergeAffix(IspellDict *Conf, int a1, int a2)
1573 {
1574  char **ptr;
1575 
1576  Assert(a1 < Conf->nAffixData && a2 < Conf->nAffixData);
1577 
1578  /* Do not merge affix flags if one of affix flags is empty */
1579  if (*Conf->AffixData[a1] == '\0')
1580  return a2;
1581  else if (*Conf->AffixData[a2] == '\0')
1582  return a1;
1583 
1584  /* Double the size of AffixData if there's not enough space */
1585  if (Conf->nAffixData + 1 >= Conf->lenAffixData)
1586  {
1587  Conf->lenAffixData *= 2;
1588  Conf->AffixData = (char **) repalloc(Conf->AffixData,
1589  sizeof(char *) * Conf->lenAffixData);
1590  }
1591 
1592  ptr = Conf->AffixData + Conf->nAffixData;
1593  if (Conf->flagMode == FM_NUM)
1594  {
1595  *ptr = cpalloc(strlen(Conf->AffixData[a1]) +
1596  strlen(Conf->AffixData[a2]) +
1597  1 /* comma */ + 1 /* \0 */ );
1598  sprintf(*ptr, "%s,%s", Conf->AffixData[a1], Conf->AffixData[a2]);
1599  }
1600  else
1601  {
1602  *ptr = cpalloc(strlen(Conf->AffixData[a1]) +
1603  strlen(Conf->AffixData[a2]) +
1604  1 /* \0 */ );
1605  sprintf(*ptr, "%s%s", Conf->AffixData[a1], Conf->AffixData[a2]);
1606  }
1607  ptr++;
1608  *ptr = NULL;
1609  Conf->nAffixData++;
1610 
1611  return Conf->nAffixData - 1;
1612 }
1613 
1614 /*
1615  * Returns a set of affix parameters which correspondence to the set of affix
1616  * flags with the given index.
1617  */
1618 static uint32
1620 {
1621  Assert(affix < Conf->nAffixData);
1622 
1623  return (getCompoundAffixFlagValue(Conf, Conf->AffixData[affix]) &
1625 }
1626 
1627 /*
1628  * Makes a prefix tree for the given level.
1629  *
1630  * Conf: current dictionary.
1631  * low: lower index of the Conf->Spell array.
1632  * high: upper index of the Conf->Spell array.
1633  * level: current prefix tree level.
1634  */
1635 static SPNode *
1636 mkSPNode(IspellDict *Conf, int low, int high, int level)
1637 {
1638  int i;
1639  int nchar = 0;
1640  char lastchar = '\0';
1641  SPNode *rs;
1642  SPNodeData *data;
1643  int lownew = low;
1644 
1645  for (i = low; i < high; i++)
1646  if (Conf->Spell[i]->p.d.len > level && lastchar != Conf->Spell[i]->word[level])
1647  {
1648  nchar++;
1649  lastchar = Conf->Spell[i]->word[level];
1650  }
1651 
1652  if (!nchar)
1653  return NULL;
1654 
1655  rs = (SPNode *) cpalloc0(SPNHDRSZ + nchar * sizeof(SPNodeData));
1656  rs->length = nchar;
1657  data = rs->data;
1658 
1659  lastchar = '\0';
1660  for (i = low; i < high; i++)
1661  if (Conf->Spell[i]->p.d.len > level)
1662  {
1663  if (lastchar != Conf->Spell[i]->word[level])
1664  {
1665  if (lastchar)
1666  {
1667  /* Next level of the prefix tree */
1668  data->node = mkSPNode(Conf, lownew, i, level + 1);
1669  lownew = i;
1670  data++;
1671  }
1672  lastchar = Conf->Spell[i]->word[level];
1673  }
1674  data->val = ((uint8 *) (Conf->Spell[i]->word))[level];
1675  if (Conf->Spell[i]->p.d.len == level + 1)
1676  {
1677  bool clearCompoundOnly = false;
1678 
1679  if (data->isword && data->affix != Conf->Spell[i]->p.d.affix)
1680  {
1681  /*
1682  * MergeAffix called a few times. If one of word is
1683  * allowed to be in compound word and another isn't, then
1684  * clear FF_COMPOUNDONLY flag.
1685  */
1686 
1687  clearCompoundOnly = (FF_COMPOUNDONLY & data->compoundflag
1688  & makeCompoundFlags(Conf, Conf->Spell[i]->p.d.affix))
1689  ? false : true;
1690  data->affix = MergeAffix(Conf, data->affix, Conf->Spell[i]->p.d.affix);
1691  }
1692  else
1693  data->affix = Conf->Spell[i]->p.d.affix;
1694  data->isword = 1;
1695 
1696  data->compoundflag = makeCompoundFlags(Conf, data->affix);
1697 
1698  if ((data->compoundflag & FF_COMPOUNDONLY) &&
1699  (data->compoundflag & FF_COMPOUNDFLAG) == 0)
1700  data->compoundflag |= FF_COMPOUNDFLAG;
1701 
1702  if (clearCompoundOnly)
1703  data->compoundflag &= ~FF_COMPOUNDONLY;
1704  }
1705  }
1706 
1707  /* Next level of the prefix tree */
1708  data->node = mkSPNode(Conf, lownew, high, level + 1);
1709 
1710  return rs;
1711 }
1712 
1713 /*
1714  * Builds the Conf->Dictionary tree and AffixData from the imported dictionary
1715  * and affixes.
1716  */
1717 void
1719 {
1720  int i;
1721  int naffix;
1722  int curaffix;
1723 
1724  /* compress affixes */
1725 
1726  /*
1727  * If we use flag aliases then we need to use Conf->AffixData filled in
1728  * the NIImportOOAffixes().
1729  */
1730  if (Conf->useFlagAliases)
1731  {
1732  for (i = 0; i < Conf->nspell; i++)
1733  {
1734  char *end;
1735 
1736  if (*Conf->Spell[i]->p.flag != '\0')
1737  {
1738  curaffix = strtol(Conf->Spell[i]->p.flag, &end, 10);
1739  if (Conf->Spell[i]->p.flag == end || errno == ERANGE)
1740  ereport(ERROR,
1741  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1742  errmsg("invalid affix alias \"%s\"",
1743  Conf->Spell[i]->p.flag)));
1744  if (curaffix < 0 || curaffix >= Conf->nAffixData)
1745  ereport(ERROR,
1746  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1747  errmsg("invalid affix alias \"%s\"",
1748  Conf->Spell[i]->p.flag)));
1749  if (*end != '\0' && !t_isdigit(end) && !t_isspace(end))
1750  ereport(ERROR,
1751  (errcode(ERRCODE_CONFIG_FILE_ERROR),
1752  errmsg("invalid affix alias \"%s\"",
1753  Conf->Spell[i]->p.flag)));
1754  }
1755  else
1756  {
1757  /*
1758  * If Conf->Spell[i]->p.flag is empty, then get empty value of
1759  * Conf->AffixData (0 index).
1760  */
1761  curaffix = 0;
1762  }
1763 
1764  Conf->Spell[i]->p.d.affix = curaffix;
1765  Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
1766  }
1767  }
1768  /* Otherwise fill Conf->AffixData here */
1769  else
1770  {
1771  /* Count the number of different flags used in the dictionary */
1772  qsort(Conf->Spell, Conf->nspell, sizeof(SPELL *),
1773  cmpspellaffix);
1774 
1775  naffix = 0;
1776  for (i = 0; i < Conf->nspell; i++)
1777  {
1778  if (i == 0 ||
1779  strcmp(Conf->Spell[i]->p.flag, Conf->Spell[i - 1]->p.flag) != 0)
1780  naffix++;
1781  }
1782 
1783  /*
1784  * Fill in Conf->AffixData with the affixes that were used in the
1785  * dictionary. Replace textual flag-field of Conf->Spell entries with
1786  * indexes into Conf->AffixData array.
1787  */
1788  Conf->AffixData = (char **) palloc0(naffix * sizeof(char *));
1789 
1790  curaffix = -1;
1791  for (i = 0; i < Conf->nspell; i++)
1792  {
1793  if (i == 0 ||
1794  strcmp(Conf->Spell[i]->p.flag, Conf->AffixData[curaffix]) != 0)
1795  {
1796  curaffix++;
1797  Assert(curaffix < naffix);
1798  Conf->AffixData[curaffix] = cpstrdup(Conf,
1799  Conf->Spell[i]->p.flag);
1800  }
1801 
1802  Conf->Spell[i]->p.d.affix = curaffix;
1803  Conf->Spell[i]->p.d.len = strlen(Conf->Spell[i]->word);
1804  }
1805 
1806  Conf->lenAffixData = Conf->nAffixData = naffix;
1807  }
1808 
1809  /* Start build a prefix tree */
1810  qsort(Conf->Spell, Conf->nspell, sizeof(SPELL *), cmpspell);
1811  Conf->Dictionary = mkSPNode(Conf, 0, Conf->nspell, 0);
1812 }
1813 
1814 /*
1815  * Makes a prefix tree for the given level using the repl string of an affix
1816  * rule. Affixes with empty replace string do not include in the prefix tree.
1817  * This affixes are included by mkVoidAffix().
1818  *
1819  * Conf: current dictionary.
1820  * low: lower index of the Conf->Affix array.
1821  * high: upper index of the Conf->Affix array.
1822  * level: current prefix tree level.
1823  * type: FF_SUFFIX or FF_PREFIX.
1824  */
1825 static AffixNode *
1826 mkANode(IspellDict *Conf, int low, int high, int level, int type)
1827 {
1828  int i;
1829  int nchar = 0;
1830  uint8 lastchar = '\0';
1831  AffixNode *rs;
1833  int lownew = low;
1834  int naff;
1835  AFFIX **aff;
1836 
1837  for (i = low; i < high; i++)
1838  if (Conf->Affix[i].replen > level && lastchar != GETCHAR(Conf->Affix + i, level, type))
1839  {
1840  nchar++;
1841  lastchar = GETCHAR(Conf->Affix + i, level, type);
1842  }
1843 
1844  if (!nchar)
1845  return NULL;
1846 
1847  aff = (AFFIX **) tmpalloc(sizeof(AFFIX *) * (high - low + 1));
1848  naff = 0;
1849 
1850  rs = (AffixNode *) cpalloc0(ANHRDSZ + nchar * sizeof(AffixNodeData));
1851  rs->length = nchar;
1852  data = rs->data;
1853 
1854  lastchar = '\0';
1855  for (i = low; i < high; i++)
1856  if (Conf->Affix[i].replen > level)
1857  {
1858  if (lastchar != GETCHAR(Conf->Affix + i, level, type))
1859  {
1860  if (lastchar)
1861  {
1862  /* Next level of the prefix tree */
1863  data->node = mkANode(Conf, lownew, i, level + 1, type);
1864  if (naff)
1865  {
1866  data->naff = naff;
1867  data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff);
1868  memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
1869  naff = 0;
1870  }
1871  data++;
1872  lownew = i;
1873  }
1874  lastchar = GETCHAR(Conf->Affix + i, level, type);
1875  }
1876  data->val = GETCHAR(Conf->Affix + i, level, type);
1877  if (Conf->Affix[i].replen == level + 1)
1878  { /* affix stopped */
1879  aff[naff++] = Conf->Affix + i;
1880  }
1881  }
1882 
1883  /* Next level of the prefix tree */
1884  data->node = mkANode(Conf, lownew, high, level + 1, type);
1885  if (naff)
1886  {
1887  data->naff = naff;
1888  data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * naff);
1889  memcpy(data->aff, aff, sizeof(AFFIX *) * naff);
1890  naff = 0;
1891  }
1892 
1893  pfree(aff);
1894 
1895  return rs;
1896 }
1897 
1898 /*
1899  * Makes the root void node in the prefix tree. The root void node is created
1900  * for affixes which have empty replace string ("repl" field).
1901  */
1902 static void
1903 mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
1904 {
1905  int i,
1906  cnt = 0;
1907  int start = (issuffix) ? startsuffix : 0;
1908  int end = (issuffix) ? Conf->naffixes : startsuffix;
1909  AffixNode *Affix = (AffixNode *) palloc0(ANHRDSZ + sizeof(AffixNodeData));
1910 
1911  Affix->length = 1;
1912  Affix->isvoid = 1;
1913 
1914  if (issuffix)
1915  {
1916  Affix->data->node = Conf->Suffix;
1917  Conf->Suffix = Affix;
1918  }
1919  else
1920  {
1921  Affix->data->node = Conf->Prefix;
1922  Conf->Prefix = Affix;
1923  }
1924 
1925  /* Count affixes with empty replace string */
1926  for (i = start; i < end; i++)
1927  if (Conf->Affix[i].replen == 0)
1928  cnt++;
1929 
1930  /* There is not affixes with empty replace string */
1931  if (cnt == 0)
1932  return;
1933 
1934  Affix->data->aff = (AFFIX **) cpalloc(sizeof(AFFIX *) * cnt);
1935  Affix->data->naff = (uint32) cnt;
1936 
1937  cnt = 0;
1938  for (i = start; i < end; i++)
1939  if (Conf->Affix[i].replen == 0)
1940  {
1941  Affix->data->aff[cnt] = Conf->Affix + i;
1942  cnt++;
1943  }
1944 }
1945 
1946 /*
1947  * Checks if the affixflag is used by dictionary. Conf->AffixData does not
1948  * contain affixflag if this flag is not used actually by the .dict file.
1949  *
1950  * Conf: current dictionary.
1951  * affixflag: affix flag.
1952  *
1953  * Returns true if the Conf->AffixData array contains affixflag, otherwise
1954  * returns false.
1955  */
1956 static bool
1957 isAffixInUse(IspellDict *Conf, char *affixflag)
1958 {
1959  int i;
1960 
1961  for (i = 0; i < Conf->nAffixData; i++)
1962  if (IsAffixFlagInUse(Conf, i, affixflag))
1963  return true;
1964 
1965  return false;
1966 }
1967 
1968 /*
1969  * Builds Conf->Prefix and Conf->Suffix trees from the imported affixes.
1970  */
1971 void
1973 {
1974  AFFIX *Affix;
1975  size_t i;
1976  CMPDAffix *ptr;
1977  int firstsuffix = Conf->naffixes;
1978 
1979  if (Conf->naffixes == 0)
1980  return;
1981 
1982  /* Store compound affixes in the Conf->CompoundAffix array */
1983  if (Conf->naffixes > 1)
1984  qsort(Conf->Affix, Conf->naffixes, sizeof(AFFIX), cmpaffix);
1985  Conf->CompoundAffix = ptr = (CMPDAffix *) palloc(sizeof(CMPDAffix) * Conf->naffixes);
1986  ptr->affix = NULL;
1987 
1988  for (i = 0; i < Conf->naffixes; i++)
1989  {
1990  Affix = &(((AFFIX *) Conf->Affix)[i]);
1991  if (Affix->type == FF_SUFFIX && i < firstsuffix)
1992  firstsuffix = i;
1993 
1994  if ((Affix->flagflags & FF_COMPOUNDFLAG) && Affix->replen > 0 &&
1995  isAffixInUse(Conf, Affix->flag))
1996  {
1997  bool issuffix = (Affix->type == FF_SUFFIX);
1998 
1999  if (ptr == Conf->CompoundAffix ||
2000  issuffix != (ptr - 1)->issuffix ||
2001  strbncmp((const unsigned char *) (ptr - 1)->affix,
2002  (const unsigned char *) Affix->repl,
2003  (ptr - 1)->len))
2004  {
2005  /* leave only unique and minimal suffixes */
2006  ptr->affix = Affix->repl;
2007  ptr->len = Affix->replen;
2008  ptr->issuffix = issuffix;
2009  ptr++;
2010  }
2011  }
2012  }
2013  ptr->affix = NULL;
2014  Conf->CompoundAffix = (CMPDAffix *) repalloc(Conf->CompoundAffix, sizeof(CMPDAffix) * (ptr - Conf->CompoundAffix + 1));
2015 
2016  /* Start build a prefix tree */
2017  Conf->Prefix = mkANode(Conf, 0, firstsuffix, 0, FF_PREFIX);
2018  Conf->Suffix = mkANode(Conf, firstsuffix, Conf->naffixes, 0, FF_SUFFIX);
2019  mkVoidAffix(Conf, true, firstsuffix);
2020  mkVoidAffix(Conf, false, firstsuffix);
2021 }
2022 
2023 static AffixNodeData *
2024 FindAffixes(AffixNode *node, const char *word, int wrdlen, int *level, int type)
2025 {
2026  AffixNodeData *StopLow,
2027  *StopHigh,
2028  *StopMiddle;
2029  uint8 symbol;
2030 
2031  if (node->isvoid)
2032  { /* search void affixes */
2033  if (node->data->naff)
2034  return node->data;
2035  node = node->data->node;
2036  }
2037 
2038  while (node && *level < wrdlen)
2039  {
2040  StopLow = node->data;
2041  StopHigh = node->data + node->length;
2042  while (StopLow < StopHigh)
2043  {
2044  StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
2045  symbol = GETWCHAR(word, wrdlen, *level, type);
2046 
2047  if (StopMiddle->val == symbol)
2048  {
2049  (*level)++;
2050  if (StopMiddle->naff)
2051  return StopMiddle;
2052  node = StopMiddle->node;
2053  break;
2054  }
2055  else if (StopMiddle->val < symbol)
2056  StopLow = StopMiddle + 1;
2057  else
2058  StopHigh = StopMiddle;
2059  }
2060  if (StopLow >= StopHigh)
2061  break;
2062  }
2063  return NULL;
2064 }
2065 
2066 static char *
2067 CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *newword, int *baselen)
2068 {
2069  /*
2070  * Check compound allow flags
2071  */
2072 
2073  if (flagflags == 0)
2074  {
2075  if (Affix->flagflags & FF_COMPOUNDONLY)
2076  return NULL;
2077  }
2078  else if (flagflags & FF_COMPOUNDBEGIN)
2079  {
2080  if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG)
2081  return NULL;
2082  if ((Affix->flagflags & FF_COMPOUNDBEGIN) == 0)
2083  if (Affix->type == FF_SUFFIX)
2084  return NULL;
2085  }
2086  else if (flagflags & FF_COMPOUNDMIDDLE)
2087  {
2088  if ((Affix->flagflags & FF_COMPOUNDMIDDLE) == 0 ||
2089  (Affix->flagflags & FF_COMPOUNDFORBIDFLAG))
2090  return NULL;
2091  }
2092  else if (flagflags & FF_COMPOUNDLAST)
2093  {
2094  if (Affix->flagflags & FF_COMPOUNDFORBIDFLAG)
2095  return NULL;
2096  if ((Affix->flagflags & FF_COMPOUNDLAST) == 0)
2097  if (Affix->type == FF_PREFIX)
2098  return NULL;
2099  }
2100 
2101  /*
2102  * make replace pattern of affix
2103  */
2104  if (Affix->type == FF_SUFFIX)
2105  {
2106  strcpy(newword, word);
2107  strcpy(newword + len - Affix->replen, Affix->find);
2108  if (baselen) /* store length of non-changed part of word */
2109  *baselen = len - Affix->replen;
2110  }
2111  else
2112  {
2113  /*
2114  * if prefix is an all non-changed part's length then all word
2115  * contains only prefix and suffix, so out
2116  */
2117  if (baselen && *baselen + strlen(Affix->find) <= Affix->replen)
2118  return NULL;
2119  strcpy(newword, Affix->find);
2120  strcat(newword, word + Affix->replen);
2121  }
2122 
2123  /*
2124  * check resulting word
2125  */
2126  if (Affix->issimple)
2127  return newword;
2128  else if (Affix->isregis)
2129  {
2130  if (RS_execute(&(Affix->reg.regis), newword))
2131  return newword;
2132  }
2133  else
2134  {
2135  pg_wchar *data;
2136  size_t data_len;
2137  int newword_len;
2138 
2139  /* Convert data string to wide characters */
2140  newword_len = strlen(newword);
2141  data = (pg_wchar *) palloc((newword_len + 1) * sizeof(pg_wchar));
2142  data_len = pg_mb2wchar_with_len(newword, data, newword_len);
2143 
2144  if (pg_regexec(Affix->reg.pregex, data, data_len,
2145  0, NULL, 0, NULL, 0) == REG_OKAY)
2146  {
2147  pfree(data);
2148  return newword;
2149  }
2150  pfree(data);
2151  }
2152 
2153  return NULL;
2154 }
2155 
2156 static int
2157 addToResult(char **forms, char **cur, char *word)
2158 {
2159  if (cur - forms >= MAX_NORM - 1)
2160  return 0;
2161  if (forms == cur || strcmp(word, *(cur - 1)) != 0)
2162  {
2163  *cur = pstrdup(word);
2164  *(cur + 1) = NULL;
2165  return 1;
2166  }
2167 
2168  return 0;
2169 }
2170 
2171 static char **
2173 {
2174  AffixNodeData *suffix = NULL,
2175  *prefix = NULL;
2176  int slevel = 0,
2177  plevel = 0;
2178  int wrdlen = strlen(word),
2179  swrdlen;
2180  char **forms;
2181  char **cur;
2182  char newword[2 * MAXNORMLEN] = "";
2183  char pnewword[2 * MAXNORMLEN] = "";
2184  AffixNode *snode = Conf->Suffix,
2185  *pnode;
2186  int i,
2187  j;
2188 
2189  if (wrdlen > MAXNORMLEN)
2190  return NULL;
2191  cur = forms = (char **) palloc(MAX_NORM * sizeof(char *));
2192  *cur = NULL;
2193 
2194 
2195  /* Check that the word itself is normal form */
2196  if (FindWord(Conf, word, VoidString, flag))
2197  {
2198  *cur = pstrdup(word);
2199  cur++;
2200  *cur = NULL;
2201  }
2202 
2203  /* Find all other NORMAL forms of the 'word' (check only prefix) */
2204  pnode = Conf->Prefix;
2205  plevel = 0;
2206  while (pnode)
2207  {
2208  prefix = FindAffixes(pnode, word, wrdlen, &plevel, FF_PREFIX);
2209  if (!prefix)
2210  break;
2211  for (j = 0; j < prefix->naff; j++)
2212  {
2213  if (CheckAffix(word, wrdlen, prefix->aff[j], flag, newword, NULL))
2214  {
2215  /* prefix success */
2216  if (FindWord(Conf, newword, prefix->aff[j]->flag, flag))
2217  cur += addToResult(forms, cur, newword);
2218  }
2219  }
2220  pnode = prefix->node;
2221  }
2222 
2223  /*
2224  * Find all other NORMAL forms of the 'word' (check suffix and then
2225  * prefix)
2226  */
2227  while (snode)
2228  {
2229  int baselen = 0;
2230 
2231  /* find possible suffix */
2232  suffix = FindAffixes(snode, word, wrdlen, &slevel, FF_SUFFIX);
2233  if (!suffix)
2234  break;
2235  /* foreach suffix check affix */
2236  for (i = 0; i < suffix->naff; i++)
2237  {
2238  if (CheckAffix(word, wrdlen, suffix->aff[i], flag, newword, &baselen))
2239  {
2240  /* suffix success */
2241  if (FindWord(Conf, newword, suffix->aff[i]->flag, flag))
2242  cur += addToResult(forms, cur, newword);
2243 
2244  /* now we will look changed word with prefixes */
2245  pnode = Conf->Prefix;
2246  plevel = 0;
2247  swrdlen = strlen(newword);
2248  while (pnode)
2249  {
2250  prefix = FindAffixes(pnode, newword, swrdlen, &plevel, FF_PREFIX);
2251  if (!prefix)
2252  break;
2253  for (j = 0; j < prefix->naff; j++)
2254  {
2255  if (CheckAffix(newword, swrdlen, prefix->aff[j], flag, pnewword, &baselen))
2256  {
2257  /* prefix success */
2258  char *ff = (prefix->aff[j]->flagflags & suffix->aff[i]->flagflags & FF_CROSSPRODUCT) ?
2259  VoidString : prefix->aff[j]->flag;
2260 
2261  if (FindWord(Conf, pnewword, ff, flag))
2262  cur += addToResult(forms, cur, pnewword);
2263  }
2264  }
2265  pnode = prefix->node;
2266  }
2267  }
2268  }
2269 
2270  snode = suffix->node;
2271  }
2272 
2273  if (cur == forms)
2274  {
2275  pfree(forms);
2276  return NULL;
2277  }
2278  return forms;
2279 }
2280 
2281 typedef struct SplitVar
2282 {
2283  int nstem;
2284  int lenstem;
2285  char **stem;
2286  struct SplitVar *next;
2288 
2289 static int
2290 CheckCompoundAffixes(CMPDAffix **ptr, char *word, int len, bool CheckInPlace)
2291 {
2292  bool issuffix;
2293 
2294  /* in case CompoundAffix is null: */
2295  if (*ptr == NULL)
2296  return -1;
2297 
2298  if (CheckInPlace)
2299  {
2300  while ((*ptr)->affix)
2301  {
2302  if (len > (*ptr)->len && strncmp((*ptr)->affix, word, (*ptr)->len) == 0)
2303  {
2304  len = (*ptr)->len;
2305  issuffix = (*ptr)->issuffix;
2306  (*ptr)++;
2307  return (issuffix) ? len : 0;
2308  }
2309  (*ptr)++;
2310  }
2311  }
2312  else
2313  {
2314  char *affbegin;
2315 
2316  while ((*ptr)->affix)
2317  {
2318  if (len > (*ptr)->len && (affbegin = strstr(word, (*ptr)->affix)) != NULL)
2319  {
2320  len = (*ptr)->len + (affbegin - word);
2321  issuffix = (*ptr)->issuffix;
2322  (*ptr)++;
2323  return (issuffix) ? len : 0;
2324  }
2325  (*ptr)++;
2326  }
2327  }
2328  return -1;
2329 }
2330 
2331 static SplitVar *
2332 CopyVar(SplitVar *s, int makedup)
2333 {
2334  SplitVar *v = (SplitVar *) palloc(sizeof(SplitVar));
2335 
2336  v->next = NULL;
2337  if (s)
2338  {
2339  int i;
2340 
2341  v->lenstem = s->lenstem;
2342  v->stem = (char **) palloc(sizeof(char *) * v->lenstem);
2343  v->nstem = s->nstem;
2344  for (i = 0; i < s->nstem; i++)
2345  v->stem[i] = (makedup) ? pstrdup(s->stem[i]) : s->stem[i];
2346  }
2347  else
2348  {
2349  v->lenstem = 16;
2350  v->stem = (char **) palloc(sizeof(char *) * v->lenstem);
2351  v->nstem = 0;
2352  }
2353  return v;
2354 }
2355 
2356 static void
2358 {
2359  if (v->nstem >= v->lenstem)
2360  {
2361  v->lenstem *= 2;
2362  v->stem = (char **) repalloc(v->stem, sizeof(char *) * v->lenstem);
2363  }
2364 
2365  v->stem[v->nstem] = word;
2366  v->nstem++;
2367 }
2368 
2369 static SplitVar *
2370 SplitToVariants(IspellDict *Conf, SPNode *snode, SplitVar *orig, char *word, int wordlen, int startpos, int minpos)
2371 {
2372  SplitVar *var = NULL;
2373  SPNodeData *StopLow,
2374  *StopHigh,
2375  *StopMiddle = NULL;
2376  SPNode *node = (snode) ? snode : Conf->Dictionary;
2377  int level = (snode) ? minpos : startpos; /* recursive
2378  * minpos==level */
2379  int lenaff;
2380  CMPDAffix *caff;
2381  char *notprobed;
2382  int compoundflag = 0;
2383 
2384  /* since this function recurses, it could be driven to stack overflow */
2386 
2387  notprobed = (char *) palloc(wordlen);
2388  memset(notprobed, 1, wordlen);
2389  var = CopyVar(orig, 1);
2390 
2391  while (level < wordlen)
2392  {
2393  /* find word with epenthetic or/and compound affix */
2394  caff = Conf->CompoundAffix;
2395  while (level > startpos && (lenaff = CheckCompoundAffixes(&caff, word + level, wordlen - level, (node) ? true : false)) >= 0)
2396  {
2397  /*
2398  * there is one of compound affixes, so check word for existings
2399  */
2400  char buf[MAXNORMLEN];
2401  char **subres;
2402 
2403  lenaff = level - startpos + lenaff;
2404 
2405  if (!notprobed[startpos + lenaff - 1])
2406  continue;
2407 
2408  if (level + lenaff - 1 <= minpos)
2409  continue;
2410 
2411  if (lenaff >= MAXNORMLEN)
2412  continue; /* skip too big value */
2413  if (lenaff > 0)
2414  memcpy(buf, word + startpos, lenaff);
2415  buf[lenaff] = '\0';
2416 
2417  if (level == 0)
2418  compoundflag = FF_COMPOUNDBEGIN;
2419  else if (level == wordlen - 1)
2420  compoundflag = FF_COMPOUNDLAST;
2421  else
2422  compoundflag = FF_COMPOUNDMIDDLE;
2423  subres = NormalizeSubWord(Conf, buf, compoundflag);
2424  if (subres)
2425  {
2426  /* Yes, it was a word from dictionary */
2427  SplitVar *new = CopyVar(var, 0);
2428  SplitVar *ptr = var;
2429  char **sptr = subres;
2430 
2431  notprobed[startpos + lenaff - 1] = 0;
2432 
2433  while (*sptr)
2434  {
2435  AddStem(new, *sptr);
2436  sptr++;
2437  }
2438  pfree(subres);
2439 
2440  while (ptr->next)
2441  ptr = ptr->next;
2442  ptr->next = SplitToVariants(Conf, NULL, new, word, wordlen, startpos + lenaff, startpos + lenaff);
2443 
2444  pfree(new->stem);
2445  pfree(new);
2446  }
2447  }
2448 
2449  if (!node)
2450  break;
2451 
2452  StopLow = node->data;
2453  StopHigh = node->data + node->length;
2454  while (StopLow < StopHigh)
2455  {
2456  StopMiddle = StopLow + ((StopHigh - StopLow) >> 1);
2457  if (StopMiddle->val == ((uint8 *) (word))[level])
2458  break;
2459  else if (StopMiddle->val < ((uint8 *) (word))[level])
2460  StopLow = StopMiddle + 1;
2461  else
2462  StopHigh = StopMiddle;
2463  }
2464 
2465  if (StopLow < StopHigh)
2466  {
2467  if (startpos == 0)
2468  compoundflag = FF_COMPOUNDBEGIN;
2469  else if (level == wordlen - 1)
2470  compoundflag = FF_COMPOUNDLAST;
2471  else
2472  compoundflag = FF_COMPOUNDMIDDLE;
2473 
2474  /* find infinitive */
2475  if (StopMiddle->isword &&
2476  (StopMiddle->compoundflag & compoundflag) &&
2477  notprobed[level])
2478  {
2479  /* ok, we found full compoundallowed word */
2480  if (level > minpos)
2481  {
2482  /* and its length more than minimal */
2483  if (wordlen == level + 1)
2484  {
2485  /* well, it was last word */
2486  AddStem(var, pnstrdup(word + startpos, wordlen - startpos));
2487  pfree(notprobed);
2488  return var;
2489  }
2490  else
2491  {
2492  /* then we will search more big word at the same point */
2493  SplitVar *ptr = var;
2494 
2495  while (ptr->next)
2496  ptr = ptr->next;
2497  ptr->next = SplitToVariants(Conf, node, var, word, wordlen, startpos, level);
2498  /* we can find next word */
2499  level++;
2500  AddStem(var, pnstrdup(word + startpos, level - startpos));
2501  node = Conf->Dictionary;
2502  startpos = level;
2503  continue;
2504  }
2505  }
2506  }
2507  node = StopMiddle->node;
2508  }
2509  else
2510  node = NULL;
2511  level++;
2512  }
2513 
2514  AddStem(var, pnstrdup(word + startpos, wordlen - startpos));
2515  pfree(notprobed);
2516  return var;
2517 }
2518 
2519 static void
2520 addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant)
2521 {
2522  if (*lres == NULL)
2523  *lcur = *lres = (TSLexeme *) palloc(MAX_NORM * sizeof(TSLexeme));
2524 
2525  if (*lcur - *lres < MAX_NORM - 1)
2526  {
2527  (*lcur)->lexeme = word;
2528  (*lcur)->flags = flags;
2529  (*lcur)->nvariant = NVariant;
2530  (*lcur)++;
2531  (*lcur)->lexeme = NULL;
2532  }
2533 }
2534 
2535 TSLexeme *
2537 {
2538  char **res;
2539  TSLexeme *lcur = NULL,
2540  *lres = NULL;
2541  uint16 NVariant = 1;
2542 
2543  res = NormalizeSubWord(Conf, word, 0);
2544 
2545  if (res)
2546  {
2547  char **ptr = res;
2548 
2549  while (*ptr && (lcur - lres) < MAX_NORM)
2550  {
2551  addNorm(&lres, &lcur, *ptr, 0, NVariant++);
2552  ptr++;
2553  }
2554  pfree(res);
2555  }
2556 
2557  if (Conf->usecompound)
2558  {
2559  int wordlen = strlen(word);
2560  SplitVar *ptr,
2561  *var = SplitToVariants(Conf, NULL, NULL, word, wordlen, 0, -1);
2562  int i;
2563 
2564  while (var)
2565  {
2566  if (var->nstem > 1)
2567  {
2568  char **subres = NormalizeSubWord(Conf, var->stem[var->nstem - 1], FF_COMPOUNDLAST);
2569 
2570  if (subres)
2571  {
2572  char **subptr = subres;
2573 
2574  while (*subptr)
2575  {
2576  for (i = 0; i < var->nstem - 1; i++)
2577  {
2578  addNorm(&lres, &lcur, (subptr == subres) ? var->stem[i] : pstrdup(var->stem[i]), 0, NVariant);
2579  }
2580 
2581  addNorm(&lres, &lcur, *subptr, 0, NVariant);
2582  subptr++;
2583  NVariant++;
2584  }
2585 
2586  pfree(subres);
2587  var->stem[0] = NULL;
2588  pfree(var->stem[var->nstem - 1]);
2589  }
2590  }
2591 
2592  for (i = 0; i < var->nstem && var->stem[i]; i++)
2593  pfree(var->stem[i]);
2594  ptr = var->next;
2595  pfree(var->stem);
2596  pfree(var);
2597  var = ptr;
2598  }
2599  }
2600 
2601  return lres;
2602 }
unsigned char symbol
Definition: api.h:2
static int32 next
Definition: blutils.c:221
unsigned short uint16
Definition: c.h:505
unsigned int uint32
Definition: c.h:506
#define MAXALIGN(LEN)
Definition: c.h:811
signed int int32
Definition: c.h:494
#define Assert(condition)
Definition: c.h:858
unsigned char uint8
Definition: c.h:504
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:855
int errmsg(const char *fmt,...)
Definition: elog.c:1068
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
#define ereport(elevel,...)
Definition: elog.h:149
void err(int eval, const char *fmt,...)
Definition: err.c:43
return str start
const char * str
static const FormData_pg_attribute a1
Definition: heap.c:142
static const FormData_pg_attribute a2
Definition: heap.c:156
long val
Definition: informix.c:670
int j
Definition: isn.c:74
int i
Definition: isn.c:73
if(TABLE==NULL||TABLE_index==NULL)
Definition: isn.c:77
unsigned int pg_wchar
Definition: mbprint.c:31
int pg_mb2wchar_with_len(const char *from, pg_wchar *to, int len)
Definition: mbutils.c:986
int pg_mblen(const char *mbstr)
Definition: mbutils.c:1023
char * pnstrdup(const char *in, Size len)
Definition: mcxt.c:1707
char * pstrdup(const char *in)
Definition: mcxt.c:1696
void pfree(void *pointer)
Definition: mcxt.c:1521
void * palloc0(Size size)
Definition: mcxt.c:1347
MemoryContext CurTransactionContext
Definition: mcxt.c:155
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1541
void MemoryContextDelete(MemoryContext context)
Definition: mcxt.c:454
void * palloc(Size size)
Definition: mcxt.c:1317
#define AllocSetContextCreate
Definition: memutils.h:129
#define ALLOCSET_DEFAULT_SIZES
Definition: memutils.h:160
const void size_t len
const void * data
static char * filename
Definition: pg_dumpall.c:119
static XLogRecPtr startpos
static char * buf
Definition: pg_test_fsync.c:73
#define sprintf
Definition: port.h:240
#define qsort(a, b, c, d)
Definition: port.h:449
void check_stack_depth(void)
Definition: postgres.c:3531
char * c
char * s1
char * s2
MemoryContextSwitchTo(old_ctx)
static void prefixes(struct vars *v)
Definition: regc_lex.c:99
int pg_regcomp(regex_t *re, const chr *string, size_t len, int flags, Oid collation)
Definition: regcomp.c:370
static void word(struct vars *v, int dir, struct state *lp, struct state *rp)
Definition: regcomp.c:1474
size_t pg_regerror(int errcode, const regex_t *preg, char *errbuf, size_t errbuf_size)
Definition: regerror.c:60
#define REG_ADVANCED
Definition: regex.h:181
#define REG_OKAY
Definition: regex.h:215
#define REG_NOSUB
Definition: regex.h:185
#define regex_t
Definition: regex.h:245
static int find(struct vars *v, struct cnfa *cnfa, struct colormap *cm)
Definition: regexec.c:419
int pg_regexec(regex_t *re, const chr *string, size_t len, size_t search_start, rm_detail_t *details, size_t nmatch, regmatch_t pmatch[], int flags)
Definition: regexec.c:185
void RS_compile(Regis *r, bool issuffix, const char *str)
Definition: regis.c:85
bool RS_execute(Regis *r, char *str)
Definition: regis.c:213
bool RS_isRegis(const char *str)
Definition: regis.c:31
static pg_noinline void Size size
Definition: slab.c:607
void NIStartBuild(IspellDict *Conf)
Definition: spell.c:88
#define GETWCHAR(W, L, N, T)
Definition: spell.c:191
static int strbcmp(const unsigned char *s1, const unsigned char *s2)
Definition: spell.c:257
void NIFinishBuild(IspellDict *Conf)
Definition: spell.c:103
void NIImportAffixes(IspellDict *Conf, const char *filename)
Definition: spell.c:1425
static char * findchar2(char *str, int c1, int c2)
Definition: spell.c:242
#define GETCHAR(A, N, T)
Definition: spell.c:192
static SplitVar * SplitToVariants(IspellDict *Conf, SPNode *snode, SplitVar *orig, char *word, int wordlen, int startpos, int minpos)
Definition: spell.c:2370
static int parse_ooaffentry(char *str, char *type, char *flag, char *find, char *repl, char *mask)
Definition: spell.c:857
static int cmpspell(const void *s1, const void *s2)
Definition: spell.c:197
#define MAX_NORM
Definition: spell.c:187
#define PAE_WAIT_REPL
Definition: spell.c:775
static bool get_nextfield(char **str, char *next)
Definition: spell.c:791
void NISortDictionary(IspellDict *Conf)
Definition: spell.c:1718
static int FindWord(IspellDict *Conf, const char *word, const char *affixflag, int flag)
Definition: spell.c:602
static char * cpstrdup(IspellDict *Conf, const char *str)
Definition: spell.c:162
static char * VoidString
Definition: spell.c:194
static void NIAddSpell(IspellDict *Conf, const char *word, const char *flag)
Definition: spell.c:486
static SPNode * mkSPNode(IspellDict *Conf, int low, int high, int level)
Definition: spell.c:1636
static char * lowerstr_ctx(IspellDict *Conf, const char *src)
Definition: spell.c:175
static void NIImportOOAffixes(IspellDict *Conf, const char *filename)
Definition: spell.c:1196
#define COMPACT_ALLOC_CHUNK
Definition: spell.c:126
static void addNorm(TSLexeme **lres, TSLexeme **lcur, char *word, int flags, uint16 NVariant)
Definition: spell.c:2520
static AffixNodeData * FindAffixes(AffixNode *node, const char *word, int wrdlen, int *level, int type)
Definition: spell.c:2024
static void NIAddAffix(IspellDict *Conf, const char *flag, char flagflags, const char *mask, const char *find, const char *repl, int type)
Definition: spell.c:677
static AffixNode * mkANode(IspellDict *Conf, int low, int high, int level, int type)
Definition: spell.c:1826
static void getNextFlagFromString(IspellDict *Conf, char **sflagset, char *sflag)
Definition: spell.c:349
#define MAXNORMLEN
Definition: spell.c:188
#define STRNCMP(s, p)
Definition: spell.c:190
void NISortAffixes(IspellDict *Conf)
Definition: spell.c:1972
static int cmpcmdflag(const void *f1, const void *f2)
Definition: spell.c:210
#define PAE_INREPL
Definition: spell.c:776
struct SplitVar SplitVar
static SplitVar * CopyVar(SplitVar *s, int makedup)
Definition: spell.c:2332
static void setCompoundAffixFlagValue(IspellDict *Conf, CompoundAffixFlag *entry, char *s, uint32 val)
Definition: spell.c:1031
static int CheckCompoundAffixes(CMPDAffix **ptr, char *word, int len, bool CheckInPlace)
Definition: spell.c:2290
void NIImportDictionary(IspellDict *Conf, const char *filename)
Definition: spell.c:517
static bool IsAffixFlagInUse(IspellDict *Conf, int affix, const char *affixflag)
Definition: spell.c:454
static bool parse_affentry(char *str, char *mask, char *find, char *repl)
Definition: spell.c:913
static char * findchar(char *str, int c)
Definition: spell.c:229
static bool isAffixInUse(IspellDict *Conf, char *affixflag)
Definition: spell.c:1957
static uint32 makeCompoundFlags(IspellDict *Conf, int affix)
Definition: spell.c:1619
static int cmpspellaffix(const void *s1, const void *s2)
Definition: spell.c:203
#define cpalloc(size)
Definition: spell.c:158
TSLexeme * NINormalizeWord(IspellDict *Conf, char *word)
Definition: spell.c:2536
static int strbncmp(const unsigned char *s1, const unsigned char *s2, size_t count)
Definition: spell.c:280
static char ** NormalizeSubWord(IspellDict *Conf, char *word, int flag)
Definition: spell.c:2172
static int getCompoundAffixFlagValue(IspellDict *Conf, char *s)
Definition: spell.c:1123
#define tmpalloc(sz)
Definition: spell.c:79
#define cpalloc0(size)
Definition: spell.c:159
static int MergeAffix(IspellDict *Conf, int a1, int a2)
Definition: spell.c:1572
#define PAE_WAIT_FIND
Definition: spell.c:773
static char * getAffixFlagSet(IspellDict *Conf, char *s)
Definition: spell.c:1159
static void AddStem(SplitVar *v, char *word)
Definition: spell.c:2357
#define PAE_INMASK
Definition: spell.c:772
static void * compact_palloc0(IspellDict *Conf, size_t size)
Definition: spell.c:130
#define PAE_INFIND
Definition: spell.c:774
static int addToResult(char **forms, char **cur, char *word)
Definition: spell.c:2157
static char * CheckAffix(const char *word, size_t len, AFFIX *Affix, int flagflags, char *newword, int *baselen)
Definition: spell.c:2067
#define COMPACT_MAX_REQ
Definition: spell.c:127
#define PAE_WAIT_FLAG
Definition: spell.c:778
#define PAE_WAIT_MASK
Definition: spell.c:771
static void mkVoidAffix(IspellDict *Conf, bool issuffix, int startsuffix)
Definition: spell.c:1903
static void addCompoundAffixFlagValue(IspellDict *Conf, char *s, uint32 val)
Definition: spell.c:1066
#define PAE_WAIT_TYPE
Definition: spell.c:777
static int cmpaffix(const void *s1, const void *s2)
Definition: spell.c:311
#define FLAGNUM_MAXSIZE
Definition: spell.h:182
#define FF_SUFFIX
Definition: spell.h:121
#define FF_COMPOUNDFLAG
Definition: spell.h:46
#define FF_PREFIX
Definition: spell.h:122
#define ANHRDSZ
Definition: spell.h:145
#define FF_COMPOUNDFLAGMASK
Definition: spell.h:48
#define SPELLHDRSZ
Definition: spell.h:82
#define FF_COMPOUNDFORBIDFLAG
Definition: spell.h:114
#define FF_COMPOUNDBEGIN
Definition: spell.h:43
#define FF_COMPOUNDPERMITFLAG
Definition: spell.h:113
#define FF_CROSSPRODUCT
Definition: spell.h:115
#define FF_COMPOUNDMIDDLE
Definition: spell.h:44
@ FM_LONG
Definition: spell.h:160
@ FM_CHAR
Definition: spell.h:159
@ FM_NUM
Definition: spell.h:161
#define SPNHDRSZ
Definition: spell.h:56
#define FF_COMPOUNDONLY
Definition: spell.h:42
#define FF_COMPOUNDLAST
Definition: spell.h:45
int f1[ARRAY_SIZE]
Definition: sql-declare.c:113
int f2[ARRAY_SIZE]
Definition: sql-declare.c:116
uint32 naff
Definition: spell.h:133
AFFIX ** aff
Definition: spell.h:134
uint32 val
Definition: spell.h:132
struct AffixNode * node
Definition: spell.h:135
uint32 isvoid
Definition: spell.h:140
AffixNodeData data[FLEXIBLE_ARRAY_MEMBER]
Definition: spell.h:142
uint32 length
Definition: spell.h:141
int len
Definition: spell.h:150
bool issuffix
Definition: spell.h:151
char * affix
Definition: spell.h:149
uint32 value
Definition: spell.h:179
FlagMode flagMode
Definition: spell.h:178
union CompoundAffixFlag::@129 flag
int maffixes
Definition: spell.h:186
int lenAffixData
Definition: spell.h:196
MemoryContext buildCxt
Definition: spell.h:220
int mspell
Definition: spell.h:225
AffixNode * Suffix
Definition: spell.h:190
char ** AffixData
Definition: spell.h:195
int naffixes
Definition: spell.h:187
bool usecompound
Definition: spell.h:202
CompoundAffixFlag * CompoundAffixFlags
Definition: spell.h:210
AFFIX * Affix
Definition: spell.h:188
int nAffixData
Definition: spell.h:197
int nCompoundAffixFlag
Definition: spell.h:212
CMPDAffix * CompoundAffix
Definition: spell.h:200
bool useFlagAliases
Definition: spell.h:198
SPNode * Dictionary
Definition: spell.h:193
int mCompoundAffixFlag
Definition: spell.h:214
int nspell
Definition: spell.h:224
char * firstfree
Definition: spell.h:228
FlagMode flagMode
Definition: spell.h:203
size_t avail
Definition: spell.h:229
AffixNode * Prefix
Definition: spell.h:191
SPELL ** Spell
Definition: spell.h:223
struct SPNode * node
Definition: spell.h:35
uint32 val
Definition: spell.h:29
uint32 compoundflag
Definition: spell.h:32
uint32 isword
Definition: spell.h:30
uint32 affix
Definition: spell.h:34
Definition: spell.h:51
SPNodeData data[FLEXIBLE_ARRAY_MEMBER]
Definition: spell.h:53
uint32 length
Definition: spell.h:52
int nstem
Definition: spell.c:2283
struct SplitVar * next
Definition: spell.c:2286
int lenstem
Definition: spell.c:2284
char ** stem
Definition: spell.c:2285
uint32 isregis
Definition: spell.h:94
char * flag
Definition: spell.h:89
uint32 type
Definition: spell.h:91
Regis regis
Definition: spell.h:106
uint32 replen
Definition: spell.h:95
regex_t * pregex
Definition: spell.h:105
char * repl
Definition: spell.h:97
uint32 flagflags
Definition: spell.h:92
union aff_struct::@128 reg
uint32 issimple
Definition: spell.h:93
char * find
Definition: spell.h:96
char * flag
Definition: spell.h:69
union spell_struct::@126 p
char word[FLEXIBLE_ARRAY_MEMBER]
Definition: spell.h:79
struct spell_struct::@126::@127 d
Definition: regguts.h:323
char * flag(int b)
Definition: test-ctype.c:33
bool tsearch_readline_begin(tsearch_readline_state *stp, const char *filename)
Definition: ts_locale.c:134
char * tsearch_readline(tsearch_readline_state *stp)
Definition: ts_locale.c:157
int t_isspace(const char *ptr)
Definition: ts_locale.c:50
int t_isdigit(const char *ptr)
Definition: ts_locale.c:35
int t_isalpha(const char *ptr)
Definition: ts_locale.c:65
int t_isprint(const char *ptr)
Definition: ts_locale.c:95
void tsearch_readline_end(tsearch_readline_state *stp)
Definition: ts_locale.c:202
char * lowerstr(const char *str)
Definition: ts_locale.c:253
#define t_iseq(x, c)
Definition: ts_locale.h:38
#define COPYCHAR(d, s)
Definition: ts_locale.h:40
const char * type