PostgreSQL Source Code  git master
tsrank.c
Go to the documentation of this file.
1 /*-------------------------------------------------------------------------
2  *
3  * tsrank.c
4  * rank tsvector by tsquery
5  *
6  * Portions Copyright (c) 1996-2020, PostgreSQL Global Development Group
7  *
8  *
9  * IDENTIFICATION
10  * src/backend/utils/adt/tsrank.c
11  *
12  *-------------------------------------------------------------------------
13  */
14 #include "postgres.h"
15 
16 #include <limits.h>
17 #include <math.h>
18 
19 #include "miscadmin.h"
20 #include "tsearch/ts_utils.h"
21 #include "utils/array.h"
22 #include "utils/builtins.h"
23 
24 static const float weights[] = {0.1f, 0.2f, 0.4f, 1.0f};
25 
26 #define wpos(wep) ( w[ WEP_GETWEIGHT(wep) ] )
27 
28 #define RANK_NO_NORM 0x00
29 #define RANK_NORM_LOGLENGTH 0x01
30 #define RANK_NORM_LENGTH 0x02
31 #define RANK_NORM_EXTDIST 0x04
32 #define RANK_NORM_UNIQ 0x08
33 #define RANK_NORM_LOGUNIQ 0x10
34 #define RANK_NORM_RDIVRPLUS1 0x20
35 #define DEF_NORM_METHOD RANK_NO_NORM
36 
37 static float calc_rank_or(const float *w, TSVector t, TSQuery q);
38 static float calc_rank_and(const float *w, TSVector t, TSQuery q);
39 
40 /*
41  * Returns a weight of a word collocation
42  */
43 static float4
45 {
46  if (w > 100)
47  return 1e-30f;
48 
49  return 1.0 / (1.005 + 0.05 * exp(((float4) w) / 1.5 - 2));
50 }
51 
52 static int
54 {
55  WordEntry *ptr = ARRPTR(t),
56  *end = (WordEntry *) STRPTR(t);
57  int len = 0;
58 
59  while (ptr < end)
60  {
61  int clen = POSDATALEN(t, ptr);
62 
63  if (clen == 0)
64  len += 1;
65  else
66  len += clen;
67 
68  ptr++;
69  }
70 
71  return len;
72 }
73 
74 
75 #define WordECompareQueryItem(e,q,p,i,m) \
76  tsCompareString((q) + (i)->distance, (i)->length, \
77  (e) + (p)->pos, (p)->len, (m))
78 
79 
80 /*
81  * Returns a pointer to a WordEntry's array corresponding to 'item' from
82  * tsvector 't'. 'q' is the TSQuery containing 'item'.
83  * Returns NULL if not found.
84  */
85 static WordEntry *
87 {
88  WordEntry *StopLow = ARRPTR(t);
89  WordEntry *StopHigh = (WordEntry *) STRPTR(t);
90  WordEntry *StopMiddle = StopHigh;
91  int difference;
92 
93  *nitem = 0;
94 
95  /* Loop invariant: StopLow <= item < StopHigh */
96  while (StopLow < StopHigh)
97  {
98  StopMiddle = StopLow + (StopHigh - StopLow) / 2;
99  difference = WordECompareQueryItem(STRPTR(t), GETOPERAND(q), StopMiddle, item, false);
100  if (difference == 0)
101  {
102  StopHigh = StopMiddle;
103  *nitem = 1;
104  break;
105  }
106  else if (difference > 0)
107  StopLow = StopMiddle + 1;
108  else
109  StopHigh = StopMiddle;
110  }
111 
112  if (item->prefix)
113  {
114  if (StopLow >= StopHigh)
115  StopMiddle = StopHigh;
116 
117  *nitem = 0;
118 
119  while (StopMiddle < (WordEntry *) STRPTR(t) &&
120  WordECompareQueryItem(STRPTR(t), GETOPERAND(q), StopMiddle, item, true) == 0)
121  {
122  (*nitem)++;
123  StopMiddle++;
124  }
125  }
126 
127  return (*nitem > 0) ? StopHigh : NULL;
128 }
129 
130 
131 /*
132  * sort QueryOperands by (length, word)
133  */
134 static int
135 compareQueryOperand(const void *a, const void *b, void *arg)
136 {
137  char *operand = (char *) arg;
138  QueryOperand *qa = (*(QueryOperand *const *) a);
139  QueryOperand *qb = (*(QueryOperand *const *) b);
140 
141  return tsCompareString(operand + qa->distance, qa->length,
142  operand + qb->distance, qb->length,
143  false);
144 }
145 
146 /*
147  * Returns a sorted, de-duplicated array of QueryOperands in a query.
148  * The returned QueryOperands are pointers to the original QueryOperands
149  * in the query.
150  *
151  * Length of the returned array is stored in *size
152  */
153 static QueryOperand **
155 {
156  char *operand = GETOPERAND(q);
157  QueryItem *item = GETQUERY(q);
158  QueryOperand **res,
159  **ptr,
160  **prevptr;
161 
162  ptr = res = (QueryOperand **) palloc(sizeof(QueryOperand *) * *size);
163 
164  /* Collect all operands from the tree to res */
165  while ((*size)--)
166  {
167  if (item->type == QI_VAL)
168  {
169  *ptr = (QueryOperand *) item;
170  ptr++;
171  }
172  item++;
173  }
174 
175  *size = ptr - res;
176  if (*size < 2)
177  return res;
178 
179  qsort_arg(res, *size, sizeof(QueryOperand *), compareQueryOperand, (void *) operand);
180 
181  ptr = res + 1;
182  prevptr = res;
183 
184  /* remove duplicates */
185  while (ptr - res < *size)
186  {
187  if (compareQueryOperand((void *) ptr, (void *) prevptr, (void *) operand) != 0)
188  {
189  prevptr++;
190  *prevptr = *ptr;
191  }
192  ptr++;
193  }
194 
195  *size = prevptr + 1 - res;
196  return res;
197 }
198 
199 static float
200 calc_rank_and(const float *w, TSVector t, TSQuery q)
201 {
202  WordEntryPosVector **pos;
203  WordEntryPosVector1 posnull;
204  WordEntryPosVector *POSNULL;
205  int i,
206  k,
207  l,
208  p;
209  WordEntry *entry,
210  *firstentry;
211  WordEntryPos *post,
212  *ct;
213  int32 dimt,
214  lenct,
215  dist,
216  nitem;
217  float res = -1.0;
218  QueryOperand **item;
219  int size = q->size;
220 
221  item = SortAndUniqItems(q, &size);
222  if (size < 2)
223  {
224  pfree(item);
225  return calc_rank_or(w, t, q);
226  }
227  pos = (WordEntryPosVector **) palloc0(sizeof(WordEntryPosVector *) * q->size);
228 
229  /* A dummy WordEntryPos array to use when haspos is false */
230  posnull.npos = 1;
231  posnull.pos[0] = 0;
232  WEP_SETPOS(posnull.pos[0], MAXENTRYPOS - 1);
233  POSNULL = (WordEntryPosVector *) &posnull;
234 
235  for (i = 0; i < size; i++)
236  {
237  firstentry = entry = find_wordentry(t, q, item[i], &nitem);
238  if (!entry)
239  continue;
240 
241  while (entry - firstentry < nitem)
242  {
243  if (entry->haspos)
244  pos[i] = _POSVECPTR(t, entry);
245  else
246  pos[i] = POSNULL;
247 
248  dimt = pos[i]->npos;
249  post = pos[i]->pos;
250  for (k = 0; k < i; k++)
251  {
252  if (!pos[k])
253  continue;
254  lenct = pos[k]->npos;
255  ct = pos[k]->pos;
256  for (l = 0; l < dimt; l++)
257  {
258  for (p = 0; p < lenct; p++)
259  {
260  dist = Abs((int) WEP_GETPOS(post[l]) - (int) WEP_GETPOS(ct[p]));
261  if (dist || (dist == 0 && (pos[i] == POSNULL || pos[k] == POSNULL)))
262  {
263  float curw;
264 
265  if (!dist)
266  dist = MAXENTRYPOS;
267  curw = sqrt(wpos(post[l]) * wpos(ct[p]) * word_distance(dist));
268  res = (res < 0) ? curw : 1.0 - (1.0 - res) * (1.0 - curw);
269  }
270  }
271  }
272  }
273 
274  entry++;
275  }
276  }
277  pfree(pos);
278  pfree(item);
279  return res;
280 }
281 
282 static float
283 calc_rank_or(const float *w, TSVector t, TSQuery q)
284 {
285  WordEntry *entry,
286  *firstentry;
287  WordEntryPosVector1 posnull;
288  WordEntryPos *post;
289  int32 dimt,
290  j,
291  i,
292  nitem;
293  float res = 0.0;
294  QueryOperand **item;
295  int size = q->size;
296 
297  /* A dummy WordEntryPos array to use when haspos is false */
298  posnull.npos = 1;
299  posnull.pos[0] = 0;
300 
301  item = SortAndUniqItems(q, &size);
302 
303  for (i = 0; i < size; i++)
304  {
305  float resj,
306  wjm;
307  int32 jm;
308 
309  firstentry = entry = find_wordentry(t, q, item[i], &nitem);
310  if (!entry)
311  continue;
312 
313  while (entry - firstentry < nitem)
314  {
315  if (entry->haspos)
316  {
317  dimt = POSDATALEN(t, entry);
318  post = POSDATAPTR(t, entry);
319  }
320  else
321  {
322  dimt = posnull.npos;
323  post = posnull.pos;
324  }
325 
326  resj = 0.0;
327  wjm = -1.0;
328  jm = 0;
329  for (j = 0; j < dimt; j++)
330  {
331  resj = resj + wpos(post[j]) / ((j + 1) * (j + 1));
332  if (wpos(post[j]) > wjm)
333  {
334  wjm = wpos(post[j]);
335  jm = j;
336  }
337  }
338 /*
339  limit (sum(1/i^2),i=1,inf) = pi^2/6
340  resj = sum(wi/i^2),i=1,noccurence,
341  wi - should be sorted desc,
342  don't sort for now, just choose maximum weight. This should be corrected
343  Oleg Bartunov
344 */
345  res = res + (wjm + resj - wjm / ((jm + 1) * (jm + 1))) / 1.64493406685;
346 
347  entry++;
348  }
349  }
350  if (size > 0)
351  res = res / size;
352  pfree(item);
353  return res;
354 }
355 
356 static float
357 calc_rank(const float *w, TSVector t, TSQuery q, int32 method)
358 {
359  QueryItem *item = GETQUERY(q);
360  float res = 0.0;
361  int len;
362 
363  if (!t->size || !q->size)
364  return 0.0;
365 
366  /* XXX: What about NOT? */
367  res = (item->type == QI_OPR && (item->qoperator.oper == OP_AND ||
368  item->qoperator.oper == OP_PHRASE)) ?
369  calc_rank_and(w, t, q) :
370  calc_rank_or(w, t, q);
371 
372  if (res < 0)
373  res = 1e-20f;
374 
375  if ((method & RANK_NORM_LOGLENGTH) && t->size > 0)
376  res /= log((double) (cnt_length(t) + 1)) / log(2.0);
377 
378  if (method & RANK_NORM_LENGTH)
379  {
380  len = cnt_length(t);
381  if (len > 0)
382  res /= (float) len;
383  }
384 
385  /* RANK_NORM_EXTDIST not applicable */
386 
387  if ((method & RANK_NORM_UNIQ) && t->size > 0)
388  res /= (float) (t->size);
389 
390  if ((method & RANK_NORM_LOGUNIQ) && t->size > 0)
391  res /= log((double) (t->size + 1)) / log(2.0);
392 
393  if (method & RANK_NORM_RDIVRPLUS1)
394  res /= (res + 1);
395 
396  return res;
397 }
398 
399 static const float *
401 {
402  static float ws[lengthof(weights)];
403  int i;
404  float4 *arrdata;
405 
406  if (win == NULL)
407  return weights;
408 
409  if (ARR_NDIM(win) != 1)
410  ereport(ERROR,
411  (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR),
412  errmsg("array of weight must be one-dimensional")));
413 
414  if (ArrayGetNItems(ARR_NDIM(win), ARR_DIMS(win)) < lengthof(weights))
415  ereport(ERROR,
416  (errcode(ERRCODE_ARRAY_SUBSCRIPT_ERROR),
417  errmsg("array of weight is too short")));
418 
419  if (array_contains_nulls(win))
420  ereport(ERROR,
421  (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED),
422  errmsg("array of weight must not contain nulls")));
423 
424  arrdata = (float4 *) ARR_DATA_PTR(win);
425  for (i = 0; i < lengthof(weights); i++)
426  {
427  ws[i] = (arrdata[i] >= 0) ? arrdata[i] : weights[i];
428  if (ws[i] > 1.0)
429  ereport(ERROR,
430  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
431  errmsg("weight out of range")));
432  }
433 
434  return ws;
435 }
436 
437 Datum
439 {
441  TSVector txt = PG_GETARG_TSVECTOR(1);
442  TSQuery query = PG_GETARG_TSQUERY(2);
443  int method = PG_GETARG_INT32(3);
444  float res;
445 
446  res = calc_rank(getWeights(win), txt, query, method);
447 
448  PG_FREE_IF_COPY(win, 0);
449  PG_FREE_IF_COPY(txt, 1);
450  PG_FREE_IF_COPY(query, 2);
451  PG_RETURN_FLOAT4(res);
452 }
453 
454 Datum
456 {
458  TSVector txt = PG_GETARG_TSVECTOR(1);
459  TSQuery query = PG_GETARG_TSQUERY(2);
460  float res;
461 
462  res = calc_rank(getWeights(win), txt, query, DEF_NORM_METHOD);
463 
464  PG_FREE_IF_COPY(win, 0);
465  PG_FREE_IF_COPY(txt, 1);
466  PG_FREE_IF_COPY(query, 2);
467  PG_RETURN_FLOAT4(res);
468 }
469 
470 Datum
472 {
473  TSVector txt = PG_GETARG_TSVECTOR(0);
474  TSQuery query = PG_GETARG_TSQUERY(1);
475  int method = PG_GETARG_INT32(2);
476  float res;
477 
478  res = calc_rank(getWeights(NULL), txt, query, method);
479 
480  PG_FREE_IF_COPY(txt, 0);
481  PG_FREE_IF_COPY(query, 1);
482  PG_RETURN_FLOAT4(res);
483 }
484 
485 Datum
487 {
488  TSVector txt = PG_GETARG_TSVECTOR(0);
489  TSQuery query = PG_GETARG_TSQUERY(1);
490  float res;
491 
492  res = calc_rank(getWeights(NULL), txt, query, DEF_NORM_METHOD);
493 
494  PG_FREE_IF_COPY(txt, 0);
495  PG_FREE_IF_COPY(query, 1);
496  PG_RETURN_FLOAT4(res);
497 }
498 
499 typedef struct
500 {
501  union
502  {
503  struct
504  { /* compiled doc representation */
507  } query;
508  struct
509  { /* struct is used for preparing doc
510  * representation */
513  } map;
514  } data;
517 
518 static int
519 compareDocR(const void *va, const void *vb)
520 {
521  const DocRepresentation *a = (const DocRepresentation *) va;
522  const DocRepresentation *b = (const DocRepresentation *) vb;
523 
524  if (WEP_GETPOS(a->pos) == WEP_GETPOS(b->pos))
525  {
526  if (WEP_GETWEIGHT(a->pos) == WEP_GETWEIGHT(b->pos))
527  {
528  if (a->data.map.entry == b->data.map.entry)
529  return 0;
530 
531  return (a->data.map.entry > b->data.map.entry) ? 1 : -1;
532  }
533 
534  return (WEP_GETWEIGHT(a->pos) > WEP_GETWEIGHT(b->pos)) ? 1 : -1;
535  }
536 
537  return (WEP_GETPOS(a->pos) > WEP_GETPOS(b->pos)) ? 1 : -1;
538 }
539 
540 #define MAXQROPOS MAXENTRYPOS
541 typedef struct
542 {
544  bool reverseinsert; /* indicates insert order, true means
545  * descending order */
549 
550 typedef struct
551 {
555 
556 #define QR_GET_OPERAND_DATA(q, v) \
557  ( (q)->operandData + (((QueryItem*)(v)) - GETQUERY((q)->query)) )
558 
559 /*
560  * TS_execute callback for matching a tsquery operand to QueryRepresentation
561  */
562 static TSTernaryValue
564  ExecPhraseData *data)
565 {
566  QueryRepresentation *qr = (QueryRepresentation *) checkval;
568 
569  if (!opData->operandexists)
570  return TS_NO;
571 
572  if (data)
573  {
574  data->npos = opData->npos;
575  data->pos = opData->pos;
576  if (opData->reverseinsert)
577  data->pos += MAXQROPOS - opData->npos;
578  }
579 
580  return TS_YES;
581 }
582 
583 typedef struct
584 {
585  int pos;
586  int p;
587  int q;
590 } CoverExt;
591 
592 static void
594 {
595  int i;
596 
597  for (i = 0; i < qr->query->size; i++)
598  {
599  qr->operandData[i].operandexists = false;
600  qr->operandData[i].reverseinsert = reverseinsert;
601  qr->operandData[i].npos = 0;
602  }
603 }
604 
605 static void
607 {
608  int i;
609  int lastPos;
611 
612  for (i = 0; i < entry->data.query.nitem; i++)
613  {
614  if (entry->data.query.items[i]->type != QI_VAL)
615  continue;
616 
617  opData = QR_GET_OPERAND_DATA(qr, entry->data.query.items[i]);
618 
619  opData->operandexists = true;
620 
621  if (opData->npos == 0)
622  {
623  lastPos = (opData->reverseinsert) ? (MAXQROPOS - 1) : 0;
624  opData->pos[lastPos] = entry->pos;
625  opData->npos++;
626  continue;
627  }
628 
629  lastPos = opData->reverseinsert ?
630  (MAXQROPOS - opData->npos) :
631  (opData->npos - 1);
632 
633  if (WEP_GETPOS(opData->pos[lastPos]) != WEP_GETPOS(entry->pos))
634  {
635  lastPos = opData->reverseinsert ?
636  (MAXQROPOS - 1 - opData->npos) :
637  (opData->npos);
638 
639  opData->pos[lastPos] = entry->pos;
640  opData->npos++;
641  }
642  }
643 }
644 
645 static bool
647 {
648  DocRepresentation *ptr;
649  int lastpos = ext->pos;
650  bool found = false;
651 
652  /*
653  * since this function recurses, it could be driven to stack overflow.
654  * (though any decent compiler will optimize away the tail-recursion.
655  */
657 
658  resetQueryRepresentation(qr, false);
659 
660  ext->p = INT_MAX;
661  ext->q = 0;
662  ptr = doc + ext->pos;
663 
664  /* find upper bound of cover from current position, move up */
665  while (ptr - doc < len)
666  {
668 
669  if (TS_execute(GETQUERY(qr->query), (void *) qr,
671  {
672  if (WEP_GETPOS(ptr->pos) > ext->q)
673  {
674  ext->q = WEP_GETPOS(ptr->pos);
675  ext->end = ptr;
676  lastpos = ptr - doc;
677  found = true;
678  }
679  break;
680  }
681  ptr++;
682  }
683 
684  if (!found)
685  return false;
686 
687  resetQueryRepresentation(qr, true);
688 
689  ptr = doc + lastpos;
690 
691  /* find lower bound of cover from found upper bound, move down */
692  while (ptr >= doc + ext->pos)
693  {
694  /*
695  * we scan doc from right to left, so pos info in reverse order!
696  */
698 
699  if (TS_execute(GETQUERY(qr->query), (void *) qr,
701  {
702  if (WEP_GETPOS(ptr->pos) < ext->p)
703  {
704  ext->begin = ptr;
705  ext->p = WEP_GETPOS(ptr->pos);
706  }
707  break;
708  }
709  ptr--;
710  }
711 
712  if (ext->p <= ext->q)
713  {
714  /*
715  * set position for next try to next lexeme after beginning of found
716  * cover
717  */
718  ext->pos = (ptr - doc) + 1;
719  return true;
720  }
721 
722  ext->pos++;
723  return Cover(doc, len, qr, ext);
724 }
725 
726 static DocRepresentation *
728 {
729  QueryItem *item = GETQUERY(qr->query);
730  WordEntry *entry,
731  *firstentry;
732  WordEntryPos *post;
733  int32 dimt, /* number of 'post' items */
734  j,
735  i,
736  nitem;
737  int len = qr->query->size * 4,
738  cur = 0;
739  DocRepresentation *doc;
740 
741  doc = (DocRepresentation *) palloc(sizeof(DocRepresentation) * len);
742 
743  /*
744  * Iterate through query to make DocRepresentation for words and it's
745  * entries satisfied by query
746  */
747  for (i = 0; i < qr->query->size; i++)
748  {
749  QueryOperand *curoperand;
750 
751  if (item[i].type != QI_VAL)
752  continue;
753 
754  curoperand = &item[i].qoperand;
755 
756  firstentry = entry = find_wordentry(txt, qr->query, curoperand, &nitem);
757  if (!entry)
758  continue;
759 
760  /* iterations over entries in tsvector */
761  while (entry - firstentry < nitem)
762  {
763  if (entry->haspos)
764  {
765  dimt = POSDATALEN(txt, entry);
766  post = POSDATAPTR(txt, entry);
767  }
768  else
769  {
770  /* ignore words without positions */
771  entry++;
772  continue;
773  }
774 
775  while (cur + dimt >= len)
776  {
777  len *= 2;
778  doc = (DocRepresentation *) repalloc(doc, sizeof(DocRepresentation) * len);
779  }
780 
781  /* iterations over entry's positions */
782  for (j = 0; j < dimt; j++)
783  {
784  if (curoperand->weight == 0 ||
785  curoperand->weight & (1 << WEP_GETWEIGHT(post[j])))
786  {
787  doc[cur].pos = post[j];
788  doc[cur].data.map.entry = entry;
789  doc[cur].data.map.item = (QueryItem *) curoperand;
790  cur++;
791  }
792  }
793 
794  entry++;
795  }
796  }
797 
798  if (cur > 0)
799  {
800  DocRepresentation *rptr = doc + 1,
801  *wptr = doc,
802  storage;
803 
804  /*
805  * Sort representation in ascending order by pos and entry
806  */
807  qsort((void *) doc, cur, sizeof(DocRepresentation), compareDocR);
808 
809  /*
810  * Join QueryItem per WordEntry and it's position
811  */
812  storage.pos = doc->pos;
813  storage.data.query.items = palloc(sizeof(QueryItem *) * qr->query->size);
814  storage.data.query.items[0] = doc->data.map.item;
815  storage.data.query.nitem = 1;
816 
817  while (rptr - doc < cur)
818  {
819  if (rptr->pos == (rptr - 1)->pos &&
820  rptr->data.map.entry == (rptr - 1)->data.map.entry)
821  {
822  storage.data.query.items[storage.data.query.nitem] = rptr->data.map.item;
823  storage.data.query.nitem++;
824  }
825  else
826  {
827  *wptr = storage;
828  wptr++;
829  storage.pos = rptr->pos;
830  storage.data.query.items = palloc(sizeof(QueryItem *) * qr->query->size);
831  storage.data.query.items[0] = rptr->data.map.item;
832  storage.data.query.nitem = 1;
833  }
834 
835  rptr++;
836  }
837 
838  *wptr = storage;
839  wptr++;
840 
841  *doclen = wptr - doc;
842  return doc;
843  }
844 
845  pfree(doc);
846  return NULL;
847 }
848 
849 static float4
850 calc_rank_cd(const float4 *arrdata, TSVector txt, TSQuery query, int method)
851 {
852  DocRepresentation *doc;
853  int len,
854  i,
855  doclen = 0;
856  CoverExt ext;
857  double Wdoc = 0.0;
858  double invws[lengthof(weights)];
859  double SumDist = 0.0,
860  PrevExtPos = 0.0;
861  int NExtent = 0;
863 
864 
865  for (i = 0; i < lengthof(weights); i++)
866  {
867  invws[i] = ((double) ((arrdata[i] >= 0) ? arrdata[i] : weights[i]));
868  if (invws[i] > 1.0)
869  ereport(ERROR,
870  (errcode(ERRCODE_INVALID_PARAMETER_VALUE),
871  errmsg("weight out of range")));
872  invws[i] = 1.0 / invws[i];
873  }
874 
875  qr.query = query;
877  palloc0(sizeof(QueryRepresentationOperand) * query->size);
878 
879  doc = get_docrep(txt, &qr, &doclen);
880  if (!doc)
881  {
882  pfree(qr.operandData);
883  return 0.0;
884  }
885 
886  MemSet(&ext, 0, sizeof(CoverExt));
887  while (Cover(doc, doclen, &qr, &ext))
888  {
889  double Cpos = 0.0;
890  double InvSum = 0.0;
891  double CurExtPos;
892  int nNoise;
893  DocRepresentation *ptr = ext.begin;
894 
895  while (ptr <= ext.end)
896  {
897  InvSum += invws[WEP_GETWEIGHT(ptr->pos)];
898  ptr++;
899  }
900 
901  Cpos = ((double) (ext.end - ext.begin + 1)) / InvSum;
902 
903  /*
904  * if doc are big enough then ext.q may be equal to ext.p due to limit
905  * of positional information. In this case we approximate number of
906  * noise word as half cover's length
907  */
908  nNoise = (ext.q - ext.p) - (ext.end - ext.begin);
909  if (nNoise < 0)
910  nNoise = (ext.end - ext.begin) / 2;
911  Wdoc += Cpos / ((double) (1 + nNoise));
912 
913  CurExtPos = ((double) (ext.q + ext.p)) / 2.0;
914  if (NExtent > 0 && CurExtPos > PrevExtPos /* prevent division by
915  * zero in a case of
916  * multiple lexize */ )
917  SumDist += 1.0 / (CurExtPos - PrevExtPos);
918 
919  PrevExtPos = CurExtPos;
920  NExtent++;
921  }
922 
923  if ((method & RANK_NORM_LOGLENGTH) && txt->size > 0)
924  Wdoc /= log((double) (cnt_length(txt) + 1));
925 
926  if (method & RANK_NORM_LENGTH)
927  {
928  len = cnt_length(txt);
929  if (len > 0)
930  Wdoc /= (double) len;
931  }
932 
933  if ((method & RANK_NORM_EXTDIST) && NExtent > 0 && SumDist > 0)
934  Wdoc /= ((double) NExtent) / SumDist;
935 
936  if ((method & RANK_NORM_UNIQ) && txt->size > 0)
937  Wdoc /= (double) (txt->size);
938 
939  if ((method & RANK_NORM_LOGUNIQ) && txt->size > 0)
940  Wdoc /= log((double) (txt->size + 1)) / log(2.0);
941 
942  if (method & RANK_NORM_RDIVRPLUS1)
943  Wdoc /= (Wdoc + 1);
944 
945  pfree(doc);
946 
947  pfree(qr.operandData);
948 
949  return (float4) Wdoc;
950 }
951 
952 Datum
954 {
956  TSVector txt = PG_GETARG_TSVECTOR(1);
957  TSQuery query = PG_GETARG_TSQUERY(2);
958  int method = PG_GETARG_INT32(3);
959  float res;
960 
961  res = calc_rank_cd(getWeights(win), txt, query, method);
962 
963  PG_FREE_IF_COPY(win, 0);
964  PG_FREE_IF_COPY(txt, 1);
965  PG_FREE_IF_COPY(query, 2);
966  PG_RETURN_FLOAT4(res);
967 }
968 
969 Datum
971 {
973  TSVector txt = PG_GETARG_TSVECTOR(1);
974  TSQuery query = PG_GETARG_TSQUERY(2);
975  float res;
976 
977  res = calc_rank_cd(getWeights(win), txt, query, DEF_NORM_METHOD);
978 
979  PG_FREE_IF_COPY(win, 0);
980  PG_FREE_IF_COPY(txt, 1);
981  PG_FREE_IF_COPY(query, 2);
982  PG_RETURN_FLOAT4(res);
983 }
984 
985 Datum
987 {
988  TSVector txt = PG_GETARG_TSVECTOR(0);
989  TSQuery query = PG_GETARG_TSQUERY(1);
990  int method = PG_GETARG_INT32(2);
991  float res;
992 
993  res = calc_rank_cd(getWeights(NULL), txt, query, method);
994 
995  PG_FREE_IF_COPY(txt, 0);
996  PG_FREE_IF_COPY(query, 1);
997  PG_RETURN_FLOAT4(res);
998 }
999 
1000 Datum
1002 {
1003  TSVector txt = PG_GETARG_TSVECTOR(0);
1004  TSQuery query = PG_GETARG_TSQUERY(1);
1005  float res;
1006 
1007  res = calc_rank_cd(getWeights(NULL), txt, query, DEF_NORM_METHOD);
1008 
1009  PG_FREE_IF_COPY(txt, 0);
1010  PG_FREE_IF_COPY(query, 1);
1011  PG_RETURN_FLOAT4(res);
1012 }
uint16 WordEntryPos
Definition: ts_type.h:63
signed short int16
Definition: c.h:361
#define RANK_NORM_RDIVRPLUS1
Definition: tsrank.c:34
bool TS_execute(QueryItem *curitem, void *arg, uint32 flags, TSExecuteCallback chkcond)
Definition: tsvector_op.c:1846
#define DEF_NORM_METHOD
Definition: tsrank.c:35
QueryOperator qoperator
Definition: ts_type.h:196
static float calc_rank_or(const float *w, TSVector t, TSQuery q)
Definition: tsrank.c:283
DocRepresentation * end
Definition: tsrank.c:589
#define PG_GETARG_INT32(n)
Definition: fmgr.h:269
#define RANK_NORM_EXTDIST
Definition: tsrank.c:31
#define PG_RETURN_FLOAT4(x)
Definition: fmgr.h:364
static TSTernaryValue checkcondition_QueryOperand(void *checkval, QueryOperand *val, ExecPhraseData *data)
Definition: tsrank.c:563
Datum ts_rankcd_tt(PG_FUNCTION_ARGS)
Definition: tsrank.c:1001
Datum ts_rank_wttf(PG_FUNCTION_ARGS)
Definition: tsrank.c:438
static bool Cover(DocRepresentation *doc, int len, QueryRepresentation *qr, CoverExt *ext)
Definition: tsrank.c:646
struct DocRepresentation::@32::@33 query
#define POSDATALEN(x, e)
Definition: ts_type.h:110
#define PG_GETARG_DATUM(n)
Definition: fmgr.h:268
#define RANK_NORM_LOGLENGTH
Definition: tsrank.c:29
#define PG_GETARG_TSQUERY(n)
Definition: ts_type.h:238
#define QR_GET_OPERAND_DATA(q, v)
Definition: tsrank.c:556
int ArrayGetNItems(int ndim, const int *dims)
Definition: arrayutils.c:75
#define _POSVECPTR(x, e)
Definition: ts_type.h:109
Datum ts_rankcd_wttf(PG_FUNCTION_ARGS)
Definition: tsrank.c:953
QueryItem ** items
Definition: tsrank.c:505
struct cursor * cur
Definition: ecpg.c:28
int errcode(int sqlerrcode)
Definition: elog.c:610
#define MemSet(start, val, len)
Definition: c.h:949
#define QI_VAL
Definition: ts_type.h:134
#define lengthof(array)
Definition: c.h:675
static DocRepresentation * get_docrep(TSVector txt, QueryRepresentation *qr, int *doclen)
Definition: tsrank.c:727
uint32 distance
Definition: ts_type.h:158
Datum ts_rankcd_ttf(PG_FUNCTION_ARGS)
Definition: tsrank.c:986
static int compareDocR(const void *va, const void *vb)
Definition: tsrank.c:519
#define WEP_SETPOS(x, v)
Definition: ts_type.h:83
static void fillQueryRepresentationData(QueryRepresentation *qr, DocRepresentation *entry)
Definition: tsrank.c:606
#define GETQUERY(x)
Definition: _int.h:157
#define RANK_NORM_UNIQ
Definition: tsrank.c:32
signed int int32
Definition: c.h:362
#define GETOPERAND(x)
Definition: ltree.h:151
WordEntryPos pos[1]
Definition: ts_type.h:75
#define POSDATAPTR(x, e)
Definition: ts_type.h:111
#define Abs(x)
Definition: c.h:933
struct DocRepresentation::@32::@34 map
Datum ts_rank_wtt(PG_FUNCTION_ARGS)
Definition: tsrank.c:455
#define OP_AND
Definition: ts_type.h:167
static int cnt_length(TSVector t)
Definition: tsrank.c:53
void pfree(void *pointer)
Definition: mcxt.c:1057
#define ERROR
Definition: elog.h:43
int p
Definition: tsrank.c:586
#define ARR_DIMS(a)
Definition: array.h:282
uint8 weight
Definition: ts_type.h:146
#define WEP_GETPOS(x)
Definition: ts_type.h:80
int pos
Definition: tsrank.c:585
WordEntryPos pos[FLEXIBLE_ARRAY_MEMBER]
Definition: ts_type.h:68
#define ARR_DATA_PTR(a)
Definition: array.h:310
static int compareQueryOperand(const void *a, const void *b, void *arg)
Definition: tsrank.c:135
void check_stack_depth(void)
Definition: postgres.c:3312
int q
Definition: tsrank.c:587
int32 size
Definition: ts_type.h:93
static float4 word_distance(int32 w)
Definition: tsrank.c:44
WordEntryPos * pos
Definition: ts_utils.h:162
unsigned int uint32
Definition: c.h:374
#define WordECompareQueryItem(e, q, p, i, m)
Definition: tsrank.c:75
static float calc_rank(const float *w, TSVector t, TSQuery q, int32 method)
Definition: tsrank.c:357
#define RANK_NORM_LOGUNIQ
Definition: tsrank.c:33
uint32 haspos
Definition: ts_type.h:44
#define wpos(wep)
Definition: tsrank.c:26
void qsort_arg(void *base, size_t nel, size_t elsize, qsort_arg_comparator cmp, void *arg)
Definition: qsort_arg.c:113
static QueryOperand ** SortAndUniqItems(TSQuery q, int *size)
Definition: tsrank.c:154
float float4
Definition: c.h:497
#define QI_OPR
Definition: ts_type.h:135
QueryItemType type
Definition: ts_type.h:195
void * palloc0(Size size)
Definition: mcxt.c:981
uintptr_t Datum
Definition: postgres.h:367
Datum difference(PG_FUNCTION_ARGS)
static void resetQueryRepresentation(QueryRepresentation *qr, bool reverseinsert)
Definition: tsrank.c:593
QueryRepresentationOperand * operandData
Definition: tsrank.c:553
union DocRepresentation::@32 data
Datum ts_rank_ttf(PG_FUNCTION_ARGS)
Definition: tsrank.c:471
#define ereport(elevel,...)
Definition: elog.h:144
Datum ts_rank_tt(PG_FUNCTION_ARGS)
Definition: tsrank.c:486
static float calc_rank_and(const float *w, TSVector t, TSQuery q)
Definition: tsrank.c:200
#define OP_PHRASE
Definition: ts_type.h:169
#define MAXQROPOS
Definition: tsrank.c:540
#define RANK_NORM_LENGTH
Definition: tsrank.c:30
#define PG_FREE_IF_COPY(ptr, n)
Definition: fmgr.h:260
int32 tsCompareString(char *a, int lena, char *b, int lenb, bool prefix)
Definition: tsvector_op.c:1147
WordEntry * entry
Definition: tsrank.c:512
static float4 calc_rank_cd(const float4 *arrdata, TSVector txt, TSQuery query, int method)
Definition: tsrank.c:850
QueryItem * item
Definition: tsrank.c:511
#define ARR_NDIM(a)
Definition: array.h:278
void * repalloc(void *pointer, Size size)
Definition: mcxt.c:1070
uint32 length
Definition: ts_type.h:158
static WordEntry * find_wordentry(TSVector t, TSQuery q, QueryOperand *item, int32 *nitem)
Definition: tsrank.c:86
e
Definition: preproc-init.c:82
void * palloc(Size size)
Definition: mcxt.c:950
int errmsg(const char *fmt,...)
Definition: elog.c:824
#define STRPTR(x)
Definition: hstore.h:76
int32 size
Definition: ts_type.h:208
int i
void * arg
TSTernaryValue
Definition: ts_utils.h:128
#define PG_DETOAST_DATUM(datum)
Definition: fmgr.h:240
#define PG_GETARG_TSVECTOR(n)
Definition: ts_type.h:120
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
bool prefix
Definition: ts_type.h:150
#define ARRPTR(x)
Definition: cube.c:24
#define WEP_GETWEIGHT(x)
Definition: ts_type.h:79
QueryOperand qoperand
Definition: ts_type.h:197
#define qsort(a, b, c, d)
Definition: port.h:475
bool array_contains_nulls(ArrayType *array)
Definition: arrayfuncs.c:3550
#define MAXENTRYPOS
Definition: ts_type.h:85
Datum ts_rankcd_wtt(PG_FUNCTION_ARGS)
Definition: tsrank.c:970
#define TS_EXEC_EMPTY
Definition: ts_utils.h:184
static const float * getWeights(ArrayType *win)
Definition: tsrank.c:400
long val
Definition: informix.c:664
WordEntryPos pos[MAXQROPOS]
Definition: tsrank.c:547
DocRepresentation * begin
Definition: tsrank.c:588
WordEntryPos pos
Definition: tsrank.c:515
static const float weights[]
Definition: tsrank.c:24