PostgreSQL Source Code  git master
ts_parse.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * ts_parse.c
4  * main parse functions for tsearch
5  *
6  * Portions Copyright (c) 1996-2022, PostgreSQL Global Development Group
7  *
8  *
9  * IDENTIFICATION
10  * src/backend/tsearch/ts_parse.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 
15 #include "postgres.h"
16 
17 #include "tsearch/ts_cache.h"
18 #include "tsearch/ts_utils.h"
19 
20 #define IGNORE_LONGLEXEME 1
21 
22 /*
23  * Lexize subsystem
24  */
25 
26 typedef struct ParsedLex
27 {
28  int type;
29  char *lemm;
30  int lenlemm;
31  struct ParsedLex *next;
33 
34 typedef struct ListParsedLex
35 {
39 
40 typedef struct
41 {
44  int posDict;
47  ListParsedLex towork; /* current list to work */
48  ListParsedLex waste; /* list of lexemes that already lexized */
49 
50  /*
51  * fields to store last variant to lexize (basically, thesaurus or similar
52  * to, which wants several lexemes
53  */
54 
57 } LexizeData;
58 
59 static void
61 {
62  ld->cfg = cfg;
63  ld->curDictId = InvalidOid;
64  ld->posDict = 0;
65  ld->towork.head = ld->towork.tail = ld->curSub = NULL;
66  ld->waste.head = ld->waste.tail = NULL;
67  ld->lastRes = NULL;
68  ld->tmpRes = NULL;
69 }
70 
71 static void
73 {
74  if (list->tail)
75  {
76  list->tail->next = newpl;
77  list->tail = newpl;
78  }
79  else
80  list->head = list->tail = newpl;
81  newpl->next = NULL;
82 }
83 
84 static ParsedLex *
86 {
87  ParsedLex *res = list->head;
88 
89  if (list->head)
90  list->head = list->head->next;
91 
92  if (list->head == NULL)
93  list->tail = NULL;
94 
95  return res;
96 }
97 
98 static void
99 LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm)
100 {
101  ParsedLex *newpl = (ParsedLex *) palloc(sizeof(ParsedLex));
102 
103  newpl->type = type;
104  newpl->lemm = lemm;
105  newpl->lenlemm = lenlemm;
106  LPLAddTail(&ld->towork, newpl);
107  ld->curSub = ld->towork.tail;
108 }
109 
110 static void
112 {
113  LPLAddTail(&ld->waste, LPLRemoveHead(&ld->towork));
114 
115  ld->posDict = 0;
116 }
117 
118 static void
119 setCorrLex(LexizeData *ld, ParsedLex **correspondLexem)
120 {
121  if (correspondLexem)
122  {
123  *correspondLexem = ld->waste.head;
124  }
125  else
126  {
127  ParsedLex *tmp,
128  *ptr = ld->waste.head;
129 
130  while (ptr)
131  {
132  tmp = ptr->next;
133  pfree(ptr);
134  ptr = tmp;
135  }
136  }
137  ld->waste.head = ld->waste.tail = NULL;
138 }
139 
140 static void
142 {
143  bool go = true;
144 
145  while (ld->towork.head && go)
146  {
147  if (ld->towork.head == stop)
148  {
149  ld->curSub = stop->next;
150  go = false;
151  }
152  RemoveHead(ld);
153  }
154 }
155 
156 static void
158 {
159  if (ld->tmpRes)
160  {
161  TSLexeme *ptr;
162 
163  for (ptr = ld->tmpRes; ptr->lexeme; ptr++)
164  pfree(ptr->lexeme);
165  pfree(ld->tmpRes);
166  }
167  ld->tmpRes = res;
168  ld->lastRes = lex;
169 }
170 
171 static TSLexeme *
172 LexizeExec(LexizeData *ld, ParsedLex **correspondLexem)
173 {
174  int i;
175  ListDictionary *map;
177  TSLexeme *res;
178 
179  if (ld->curDictId == InvalidOid)
180  {
181  /*
182  * usual mode: dictionary wants only one word, but we should keep in
183  * mind that we should go through all stack
184  */
185 
186  while (ld->towork.head)
187  {
188  ParsedLex *curVal = ld->towork.head;
189  char *curValLemm = curVal->lemm;
190  int curValLenLemm = curVal->lenlemm;
191 
192  map = ld->cfg->map + curVal->type;
193 
194  if (curVal->type == 0 || curVal->type >= ld->cfg->lenmap || map->len == 0)
195  {
196  /* skip this type of lexeme */
197  RemoveHead(ld);
198  continue;
199  }
200 
201  for (i = ld->posDict; i < map->len; i++)
202  {
203  dict = lookup_ts_dictionary_cache(map->dictIds[i]);
204 
205  ld->dictState.isend = ld->dictState.getnext = false;
206  ld->dictState.private_state = NULL;
208  PointerGetDatum(dict->dictData),
209  PointerGetDatum(curValLemm),
210  Int32GetDatum(curValLenLemm),
211  PointerGetDatum(&ld->dictState)));
212 
213  if (ld->dictState.getnext)
214  {
215  /*
216  * dictionary wants next word, so setup and store current
217  * position and go to multiword mode
218  */
219 
220  ld->curDictId = DatumGetObjectId(map->dictIds[i]);
221  ld->posDict = i + 1;
222  ld->curSub = curVal->next;
223  if (res)
224  setNewTmpRes(ld, curVal, res);
225  return LexizeExec(ld, correspondLexem);
226  }
227 
228  if (!res) /* dictionary doesn't know this lexeme */
229  continue;
230 
231  if (res->flags & TSL_FILTER)
232  {
233  curValLemm = res->lexeme;
234  curValLenLemm = strlen(res->lexeme);
235  continue;
236  }
237 
238  RemoveHead(ld);
239  setCorrLex(ld, correspondLexem);
240  return res;
241  }
242 
243  RemoveHead(ld);
244  }
245  }
246  else
247  { /* curDictId is valid */
249 
250  /*
251  * Dictionary ld->curDictId asks us about following words
252  */
253 
254  while (ld->curSub)
255  {
256  ParsedLex *curVal = ld->curSub;
257 
258  map = ld->cfg->map + curVal->type;
259 
260  if (curVal->type != 0)
261  {
262  bool dictExists = false;
263 
264  if (curVal->type >= ld->cfg->lenmap || map->len == 0)
265  {
266  /* skip this type of lexeme */
267  ld->curSub = curVal->next;
268  continue;
269  }
270 
271  /*
272  * We should be sure that current type of lexeme is recognized
273  * by our dictionary: we just check is it exist in list of
274  * dictionaries ?
275  */
276  for (i = 0; i < map->len && !dictExists; i++)
277  if (ld->curDictId == DatumGetObjectId(map->dictIds[i]))
278  dictExists = true;
279 
280  if (!dictExists)
281  {
282  /*
283  * Dictionary can't work with current type of lexeme,
284  * return to basic mode and redo all stored lexemes
285  */
286  ld->curDictId = InvalidOid;
287  return LexizeExec(ld, correspondLexem);
288  }
289  }
290 
291  ld->dictState.isend = (curVal->type == 0);
292  ld->dictState.getnext = false;
293 
295  PointerGetDatum(dict->dictData),
296  PointerGetDatum(curVal->lemm),
297  Int32GetDatum(curVal->lenlemm),
298  PointerGetDatum(&ld->dictState)));
299 
300  if (ld->dictState.getnext)
301  {
302  /* Dictionary wants one more */
303  ld->curSub = curVal->next;
304  if (res)
305  setNewTmpRes(ld, curVal, res);
306  continue;
307  }
308 
309  if (res || ld->tmpRes)
310  {
311  /*
312  * Dictionary normalizes lexemes, so we remove from stack all
313  * used lexemes, return to basic mode and redo end of stack
314  * (if it exists)
315  */
316  if (res)
317  {
318  moveToWaste(ld, ld->curSub);
319  }
320  else
321  {
322  res = ld->tmpRes;
323  moveToWaste(ld, ld->lastRes);
324  }
325 
326  /* reset to initial state */
327  ld->curDictId = InvalidOid;
328  ld->posDict = 0;
329  ld->lastRes = NULL;
330  ld->tmpRes = NULL;
331  setCorrLex(ld, correspondLexem);
332  return res;
333  }
334 
335  /*
336  * Dict don't want next lexem and didn't recognize anything, redo
337  * from ld->towork.head
338  */
339  ld->curDictId = InvalidOid;
340  return LexizeExec(ld, correspondLexem);
341  }
342  }
343 
344  setCorrLex(ld, correspondLexem);
345  return NULL;
346 }
347 
348 /*
349  * Parse string and lexize words.
350  *
351  * prs will be filled in.
352  */
353 void
354 parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
355 {
356  int type,
357  lenlemm = 0; /* silence compiler warning */
358  char *lemm = NULL;
359  LexizeData ldata;
360  TSLexeme *norms;
361  TSConfigCacheEntry *cfg;
362  TSParserCacheEntry *prsobj;
363  void *prsdata;
364 
365  cfg = lookup_ts_config_cache(cfgId);
366  prsobj = lookup_ts_parser_cache(cfg->prsId);
367 
368  prsdata = (void *) DatumGetPointer(FunctionCall2(&prsobj->prsstart,
370  Int32GetDatum(buflen)));
371 
372  LexizeInit(&ldata, cfg);
373 
374  do
375  {
377  PointerGetDatum(prsdata),
378  PointerGetDatum(&lemm),
379  PointerGetDatum(&lenlemm)));
380 
381  if (type > 0 && lenlemm >= MAXSTRLEN)
382  {
383 #ifdef IGNORE_LONGLEXEME
384  ereport(NOTICE,
385  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
386  errmsg("word is too long to be indexed"),
387  errdetail("Words longer than %d characters are ignored.",
388  MAXSTRLEN)));
389  continue;
390 #else
391  ereport(ERROR,
392  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
393  errmsg("word is too long to be indexed"),
394  errdetail("Words longer than %d characters are ignored.",
395  MAXSTRLEN)));
396 #endif
397  }
398 
399  LexizeAddLemm(&ldata, type, lemm, lenlemm);
400 
401  while ((norms = LexizeExec(&ldata, NULL)) != NULL)
402  {
403  TSLexeme *ptr = norms;
404 
405  prs->pos++; /* set pos */
406 
407  while (ptr->lexeme)
408  {
409  if (prs->curwords == prs->lenwords)
410  {
411  prs->lenwords *= 2;
412  prs->words = (ParsedWord *) repalloc((void *) prs->words, prs->lenwords * sizeof(ParsedWord));
413  }
414 
415  if (ptr->flags & TSL_ADDPOS)
416  prs->pos++;
417  prs->words[prs->curwords].len = strlen(ptr->lexeme);
418  prs->words[prs->curwords].word = ptr->lexeme;
419  prs->words[prs->curwords].nvariant = ptr->nvariant;
420  prs->words[prs->curwords].flags = ptr->flags & TSL_PREFIX;
421  prs->words[prs->curwords].alen = 0;
422  prs->words[prs->curwords].pos.pos = LIMITPOS(prs->pos);
423  ptr++;
424  prs->curwords++;
425  }
426  pfree(norms);
427  }
428  } while (type > 0);
429 
430  FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
431 }
432 
433 /*
434  * Headline framework
435  */
436 
437 /* Add a word to prs->words[] */
438 static void
439 hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type)
440 {
441  if (prs->curwords >= prs->lenwords)
442  {
443  prs->lenwords *= 2;
444  prs->words = (HeadlineWordEntry *) repalloc((void *) prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
445  }
446  memset(&(prs->words[prs->curwords]), 0, sizeof(HeadlineWordEntry));
447  prs->words[prs->curwords].type = (uint8) type;
448  prs->words[prs->curwords].len = buflen;
449  prs->words[prs->curwords].word = palloc(buflen);
450  memcpy(prs->words[prs->curwords].word, buf, buflen);
451  prs->curwords++;
452 }
453 
454 /*
455  * Add pos and matching-query-item data to the just-added word.
456  * Here, buf/buflen represent a processed lexeme, not raw token text.
457  *
458  * If the query contains more than one matching item, we replicate
459  * the last-added word so that each item can be pointed to. The
460  * duplicate entries are marked with repeated = 1.
461  */
462 static void
463 hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen)
464 {
465  int i;
466  QueryItem *item = GETQUERY(query);
468 
469  while (prs->curwords + query->size >= prs->lenwords)
470  {
471  prs->lenwords *= 2;
472  prs->words = (HeadlineWordEntry *) repalloc((void *) prs->words, prs->lenwords * sizeof(HeadlineWordEntry));
473  }
474 
475  word = &(prs->words[prs->curwords - 1]);
476  word->pos = LIMITPOS(pos);
477  for (i = 0; i < query->size; i++)
478  {
479  if (item->type == QI_VAL &&
481  buf, buflen, item->qoperand.prefix) == 0)
482  {
483  if (word->item)
484  {
485  memcpy(&(prs->words[prs->curwords]), word, sizeof(HeadlineWordEntry));
486  prs->words[prs->curwords].item = &item->qoperand;
487  prs->words[prs->curwords].repeated = 1;
488  prs->curwords++;
489  }
490  else
491  word->item = &item->qoperand;
492  }
493  item++;
494  }
495 }
496 
497 static void
499 {
500  ParsedLex *tmplexs;
501  TSLexeme *ptr;
502  int32 savedpos;
503 
504  while (lexs)
505  {
506  if (lexs->type > 0)
507  hladdword(prs, lexs->lemm, lexs->lenlemm, lexs->type);
508 
509  ptr = norms;
510  savedpos = prs->vectorpos;
511  while (ptr && ptr->lexeme)
512  {
513  if (ptr->flags & TSL_ADDPOS)
514  savedpos++;
515  hlfinditem(prs, query, savedpos, ptr->lexeme, strlen(ptr->lexeme));
516  ptr++;
517  }
518 
519  tmplexs = lexs->next;
520  pfree(lexs);
521  lexs = tmplexs;
522  }
523 
524  if (norms)
525  {
526  ptr = norms;
527  while (ptr->lexeme)
528  {
529  if (ptr->flags & TSL_ADDPOS)
530  prs->vectorpos++;
531  pfree(ptr->lexeme);
532  ptr++;
533  }
534  pfree(norms);
535  }
536 }
537 
538 void
539 hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen)
540 {
541  int type,
542  lenlemm = 0; /* silence compiler warning */
543  char *lemm = NULL;
544  LexizeData ldata;
545  TSLexeme *norms;
546  ParsedLex *lexs;
547  TSConfigCacheEntry *cfg;
548  TSParserCacheEntry *prsobj;
549  void *prsdata;
550 
551  cfg = lookup_ts_config_cache(cfgId);
552  prsobj = lookup_ts_parser_cache(cfg->prsId);
553 
554  prsdata = (void *) DatumGetPointer(FunctionCall2(&(prsobj->prsstart),
556  Int32GetDatum(buflen)));
557 
558  LexizeInit(&ldata, cfg);
559 
560  do
561  {
563  PointerGetDatum(prsdata),
564  PointerGetDatum(&lemm),
565  PointerGetDatum(&lenlemm)));
566 
567  if (type > 0 && lenlemm >= MAXSTRLEN)
568  {
569 #ifdef IGNORE_LONGLEXEME
570  ereport(NOTICE,
571  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
572  errmsg("word is too long to be indexed"),
573  errdetail("Words longer than %d characters are ignored.",
574  MAXSTRLEN)));
575  continue;
576 #else
577  ereport(ERROR,
578  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
579  errmsg("word is too long to be indexed"),
580  errdetail("Words longer than %d characters are ignored.",
581  MAXSTRLEN)));
582 #endif
583  }
584 
585  LexizeAddLemm(&ldata, type, lemm, lenlemm);
586 
587  do
588  {
589  if ((norms = LexizeExec(&ldata, &lexs)) != NULL)
590  {
591  prs->vectorpos++;
592  addHLParsedLex(prs, query, lexs, norms);
593  }
594  else
595  addHLParsedLex(prs, query, lexs, NULL);
596  } while (norms);
597  } while (type > 0);
598 
599  FunctionCall1(&(prsobj->prsend), PointerGetDatum(prsdata));
600 }
601 
602 /*
603  * Generate the headline, as a text object, from HeadlineParsedText.
604  */
605 text *
607 {
608  text *out;
609  char *ptr;
610  int len = 128;
611  int numfragments = 0;
612  int16 infrag = 0;
613 
614  HeadlineWordEntry *wrd = prs->words;
615 
616  out = (text *) palloc(len);
617  ptr = ((char *) out) + VARHDRSZ;
618 
619  while (wrd - prs->words < prs->curwords)
620  {
621  while (wrd->len + prs->stopsellen + prs->startsellen + prs->fragdelimlen + (ptr - ((char *) out)) >= len)
622  {
623  int dist = ptr - ((char *) out);
624 
625  len *= 2;
626  out = (text *) repalloc(out, len);
627  ptr = ((char *) out) + dist;
628  }
629 
630  if (wrd->in && !wrd->repeated)
631  {
632  if (!infrag)
633  {
634 
635  /* start of a new fragment */
636  infrag = 1;
637  numfragments++;
638  /* add a fragment delimiter if this is after the first one */
639  if (numfragments > 1)
640  {
641  memcpy(ptr, prs->fragdelim, prs->fragdelimlen);
642  ptr += prs->fragdelimlen;
643  }
644  }
645  if (wrd->replace)
646  {
647  *ptr = ' ';
648  ptr++;
649  }
650  else if (!wrd->skip)
651  {
652  if (wrd->selected)
653  {
654  memcpy(ptr, prs->startsel, prs->startsellen);
655  ptr += prs->startsellen;
656  }
657  memcpy(ptr, wrd->word, wrd->len);
658  ptr += wrd->len;
659  if (wrd->selected)
660  {
661  memcpy(ptr, prs->stopsel, prs->stopsellen);
662  ptr += prs->stopsellen;
663  }
664  }
665  }
666  else if (!wrd->repeated)
667  {
668  if (infrag)
669  infrag = 0;
670  pfree(wrd->word);
671  }
672 
673  wrd++;
674  }
675 
676  SET_VARSIZE(out, ptr - ((char *) out));
677  return out;
678 }
#define GETQUERY(x)
Definition: _int.h:157
signed short int16
Definition: c.h:429
signed int int32
Definition: c.h:430
#define VARHDRSZ
Definition: c.h:628
unsigned char uint8
Definition: c.h:440
int errdetail(const char *fmt,...)
Definition: elog.c:1039
int errcode(int sqlerrcode)
Definition: elog.c:695
int errmsg(const char *fmt,...)
Definition: elog.c:906
#define ERROR
Definition: elog.h:35
#define NOTICE
Definition: elog.h:31
#define ereport(elevel,...)
Definition: elog.h:145
#define FunctionCall4(flinfo, arg1, arg2, arg3, arg4)
Definition: fmgr.h:666
#define FunctionCall1(flinfo, arg1)
Definition: fmgr.h:660
#define FunctionCall2(flinfo, arg1, arg2)
Definition: fmgr.h:662
#define FunctionCall3(flinfo, arg1, arg2, arg3)
Definition: fmgr.h:664
int i
Definition: isn.c:73
#define GETOPERAND(x)
Definition: ltree.h:164
void pfree(void *pointer)
Definition: mcxt.c:1306
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1321
void * palloc(Size size)
Definition: mcxt.c:1199
const void size_t len
static char * buf
Definition: pg_test_fsync.c:67
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:670
static Oid DatumGetObjectId(Datum X)
Definition: postgres.h:590
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:660
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:343
static Datum Int32GetDatum(int32 X)
Definition: postgres.h:560
static int32 DatumGetInt32(Datum X)
Definition: postgres.h:550
#define InvalidOid
Definition: postgres_ext.h:36
unsigned int Oid
Definition: postgres_ext.h:31
static void word(struct vars *v, int dir, struct state *lp, struct state *rp)
Definition: regcomp.c:1476
void * private_state
Definition: ts_public.h:155
bool getnext
Definition: ts_public.h:154
HeadlineWordEntry * words
Definition: ts_public.h:76
QueryOperand * item
Definition: ts_public.h:70
TSLexeme * tmpRes
Definition: ts_parse.c:56
Oid curDictId
Definition: ts_parse.c:43
ListParsedLex towork
Definition: ts_parse.c:47
ParsedLex * curSub
Definition: ts_parse.c:46
ParsedLex * lastRes
Definition: ts_parse.c:55
int posDict
Definition: ts_parse.c:44
ListParsedLex waste
Definition: ts_parse.c:48
DictSubState dictState
Definition: ts_parse.c:45
TSConfigCacheEntry * cfg
Definition: ts_parse.c:42
Oid * dictIds
Definition: ts_cache.h:68
ParsedLex * tail
Definition: ts_parse.c:37
ParsedLex * head
Definition: ts_parse.c:36
char * lemm
Definition: ts_parse.c:29
int lenlemm
Definition: ts_parse.c:30
struct ParsedLex * next
Definition: ts_parse.c:31
int type
Definition: ts_parse.c:28
int32 pos
Definition: ts_utils.h:103
int32 lenwords
Definition: ts_utils.h:101
int32 curwords
Definition: ts_utils.h:102
ParsedWord * words
Definition: ts_utils.h:100
uint16 flags
Definition: ts_utils.h:93
uint16 nvariant
Definition: ts_utils.h:81
uint16 len
Definition: ts_utils.h:80
uint16 pos
Definition: ts_utils.h:84
uint32 alen
Definition: ts_utils.h:95
char * word
Definition: ts_utils.h:94
bool prefix
Definition: ts_type.h:163
uint32 distance
Definition: ts_type.h:172
uint32 length
Definition: ts_type.h:171
ListDictionary * map
Definition: ts_cache.h:80
uint16 nvariant
Definition: ts_public.h:134
char * lexeme
Definition: ts_public.h:138
uint16 flags
Definition: ts_public.h:136
FmgrInfo prstoken
Definition: ts_cache.h:46
FmgrInfo prsstart
Definition: ts_cache.h:45
FmgrInfo prsend
Definition: ts_cache.h:47
int32 size
Definition: ts_type.h:221
Definition: c.h:623
TSDictionaryCacheEntry * lookup_ts_dictionary_cache(Oid dictId)
Definition: ts_cache.c:209
TSConfigCacheEntry * lookup_ts_config_cache(Oid cfgId)
Definition: ts_cache.c:386
TSParserCacheEntry * lookup_ts_parser_cache(Oid prsId)
Definition: ts_cache.c:112
static void hlfinditem(HeadlineParsedText *prs, TSQuery query, int32 pos, char *buf, int buflen)
Definition: ts_parse.c:463
static ParsedLex * LPLRemoveHead(ListParsedLex *list)
Definition: ts_parse.c:85
static void setNewTmpRes(LexizeData *ld, ParsedLex *lex, TSLexeme *res)
Definition: ts_parse.c:157
struct ListParsedLex ListParsedLex
static void RemoveHead(LexizeData *ld)
Definition: ts_parse.c:111
static void setCorrLex(LexizeData *ld, ParsedLex **correspondLexem)
Definition: ts_parse.c:119
void parsetext(Oid cfgId, ParsedText *prs, char *buf, int buflen)
Definition: ts_parse.c:354
void hlparsetext(Oid cfgId, HeadlineParsedText *prs, TSQuery query, char *buf, int buflen)
Definition: ts_parse.c:539
static TSLexeme * LexizeExec(LexizeData *ld, ParsedLex **correspondLexem)
Definition: ts_parse.c:172
static void hladdword(HeadlineParsedText *prs, char *buf, int buflen, int type)
Definition: ts_parse.c:439
static void LPLAddTail(ListParsedLex *list, ParsedLex *newpl)
Definition: ts_parse.c:72
text * generateHeadline(HeadlineParsedText *prs)
Definition: ts_parse.c:606
static void moveToWaste(LexizeData *ld, ParsedLex *stop)
Definition: ts_parse.c:141
struct ParsedLex ParsedLex
static void addHLParsedLex(HeadlineParsedText *prs, TSQuery query, ParsedLex *lexs, TSLexeme *norms)
Definition: ts_parse.c:498
static void LexizeAddLemm(LexizeData *ld, int type, char *lemm, int lenlemm)
Definition: ts_parse.c:99
static void LexizeInit(LexizeData *ld, TSConfigCacheEntry *cfg)
Definition: ts_parse.c:60
#define TSL_PREFIX
Definition: ts_public.h:143
#define TSL_FILTER
Definition: ts_public.h:144
#define TSL_ADDPOS
Definition: ts_public.h:142
#define QI_VAL
Definition: ts_type.h:149
#define MAXSTRLEN
Definition: ts_type.h:49
#define LIMITPOS(x)
Definition: ts_type.h:87
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1148
QueryOperand qoperand
Definition: ts_type.h:210
QueryItemType type
Definition: ts_type.h:208