PostgreSQL Source Code  git master
trgm_op.c
Go to the documentation of this file.
1 /*
2  * contrib/pg_trgm/trgm_op.c
3  */
4 #include "postgres.h"
5 
6 #include <ctype.h>
7 
8 #include "catalog/pg_type.h"
9 #include "lib/qunique.h"
10 #include "trgm.h"
11 #include "tsearch/ts_locale.h"
12 #include "utils/lsyscache.h"
13 #include "utils/memutils.h"
14 #include "utils/pg_crc.h"
15 
17 
18 /* GUC variables */
19 double similarity_threshold = 0.3f;
22 
23 void _PG_init(void);
24 
41 
42 /* Trigram with position */
43 typedef struct
44 {
46  int index;
47 } pos_trgm;
48 
49 /* Trigram bound type */
50 typedef uint8 TrgmBound;
51 #define TRGM_BOUND_LEFT 0x01 /* trigram is left bound of word */
52 #define TRGM_BOUND_RIGHT 0x02 /* trigram is right bound of word */
53 
54 /* Word similarity flags */
55 #define WORD_SIMILARITY_CHECK_ONLY 0x01 /* only check existence of similar
56  * search pattern in text */
57 #define WORD_SIMILARITY_STRICT 0x02 /* force bounds of extent to match
58  * word bounds */
59 
60 /*
61  * Module load callback
62  */
63 void
64 _PG_init(void)
65 {
66  /* Define custom GUC variables. */
67  DefineCustomRealVariable("pg_trgm.similarity_threshold",
68  "Sets the threshold used by the % operator.",
69  "Valid range is 0.0 .. 1.0.",
71  0.3,
72  0.0,
73  1.0,
75  0,
76  NULL,
77  NULL,
78  NULL);
79  DefineCustomRealVariable("pg_trgm.word_similarity_threshold",
80  "Sets the threshold used by the <% operator.",
81  "Valid range is 0.0 .. 1.0.",
83  0.6,
84  0.0,
85  1.0,
87  0,
88  NULL,
89  NULL,
90  NULL);
91  DefineCustomRealVariable("pg_trgm.strict_word_similarity_threshold",
92  "Sets the threshold used by the <<% operator.",
93  "Valid range is 0.0 .. 1.0.",
95  0.5,
96  0.0,
97  1.0,
99  0,
100  NULL,
101  NULL,
102  NULL);
103 
104  MarkGUCPrefixReserved("pg_trgm");
105 }
106 
107 /*
108  * Deprecated function.
109  * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
110  */
111 Datum
113 {
114  float4 nlimit = PG_GETARG_FLOAT4(0);
115  char *nlimit_str;
116  Oid func_out_oid;
117  bool is_varlena;
118 
119  getTypeOutputInfo(FLOAT4OID, &func_out_oid, &is_varlena);
120 
121  nlimit_str = OidOutputFunctionCall(func_out_oid, Float4GetDatum(nlimit));
122 
123  SetConfigOption("pg_trgm.similarity_threshold", nlimit_str,
125 
127 }
128 
129 
130 /*
131  * Get similarity threshold for given index scan strategy number.
132  */
133 double
135 {
136  switch (strategy)
137  {
139  return similarity_threshold;
144  default:
145  elog(ERROR, "unrecognized strategy number: %d", strategy);
146  break;
147  }
148 
149  return 0.0; /* keep compiler quiet */
150 }
151 
152 /*
153  * Deprecated function.
154  * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
155  */
156 Datum
158 {
160 }
161 
162 static int
163 comp_trgm(const void *a, const void *b)
164 {
165  return CMPTRGM(a, b);
166 }
167 
168 /*
169  * Finds first word in string, returns pointer to the word,
170  * endword points to the character after word
171  */
172 static char *
173 find_word(char *str, int lenstr, char **endword, int *charlen)
174 {
175  char *beginword = str;
176 
177  while (beginword - str < lenstr && !ISWORDCHR(beginword))
178  beginword += pg_mblen(beginword);
179 
180  if (beginword - str >= lenstr)
181  return NULL;
182 
183  *endword = beginword;
184  *charlen = 0;
185  while (*endword - str < lenstr && ISWORDCHR(*endword))
186  {
187  *endword += pg_mblen(*endword);
188  (*charlen)++;
189  }
190 
191  return beginword;
192 }
193 
194 /*
195  * Reduce a trigram (three possibly multi-byte characters) to a trgm,
196  * which is always exactly three bytes. If we have three single-byte
197  * characters, we just use them as-is; otherwise we form a hash value.
198  */
199 void
200 compact_trigram(trgm *tptr, char *str, int bytelen)
201 {
202  if (bytelen == 3)
203  {
204  CPTRGM(tptr, str);
205  }
206  else
207  {
208  pg_crc32 crc;
209 
211  COMP_LEGACY_CRC32(crc, str, bytelen);
213 
214  /*
215  * use only 3 upper bytes from crc, hope, it's good enough hashing
216  */
217  CPTRGM(tptr, &crc);
218  }
219 }
220 
221 /*
222  * Adds trigrams from words (already padded).
223  */
224 static trgm *
225 make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
226 {
227  char *ptr = str;
228 
229  if (charlen < 3)
230  return tptr;
231 
232  if (bytelen > charlen)
233  {
234  /* Find multibyte character boundaries and apply compact_trigram */
235  int lenfirst = pg_mblen(str),
236  lenmiddle = pg_mblen(str + lenfirst),
237  lenlast = pg_mblen(str + lenfirst + lenmiddle);
238 
239  while ((ptr - str) + lenfirst + lenmiddle + lenlast <= bytelen)
240  {
241  compact_trigram(tptr, ptr, lenfirst + lenmiddle + lenlast);
242 
243  ptr += lenfirst;
244  tptr++;
245 
246  lenfirst = lenmiddle;
247  lenmiddle = lenlast;
248  lenlast = pg_mblen(ptr + lenfirst + lenmiddle);
249  }
250  }
251  else
252  {
253  /* Fast path when there are no multibyte characters */
254  Assert(bytelen == charlen);
255 
256  while (ptr - str < bytelen - 2 /* number of trigrams = strlen - 2 */ )
257  {
258  CPTRGM(tptr, ptr);
259  ptr++;
260  tptr++;
261  }
262  }
263 
264  return tptr;
265 }
266 
267 /*
268  * Make array of trigrams without sorting and removing duplicate items.
269  *
270  * trg: where to return the array of trigrams.
271  * str: source string, of length slen bytes.
272  * bounds: where to return bounds of trigrams (if needed).
273  *
274  * Returns length of the generated array.
275  */
276 static int
277 generate_trgm_only(trgm *trg, char *str, int slen, TrgmBound *bounds)
278 {
279  trgm *tptr;
280  char *buf;
281  int charlen,
282  bytelen;
283  char *bword,
284  *eword;
285 
286  if (slen + LPADDING + RPADDING < 3 || slen == 0)
287  return 0;
288 
289  tptr = trg;
290 
291  /* Allocate a buffer for case-folded, blank-padded words */
292  buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
293 
294  if (LPADDING > 0)
295  {
296  *buf = ' ';
297  if (LPADDING > 1)
298  *(buf + 1) = ' ';
299  }
300 
301  eword = str;
302  while ((bword = find_word(eword, slen - (eword - str), &eword, &charlen)) != NULL)
303  {
304 #ifdef IGNORECASE
305  bword = lowerstr_with_len(bword, eword - bword);
306  bytelen = strlen(bword);
307 #else
308  bytelen = eword - bword;
309 #endif
310 
311  memcpy(buf + LPADDING, bword, bytelen);
312 
313 #ifdef IGNORECASE
314  pfree(bword);
315 #endif
316 
317  buf[LPADDING + bytelen] = ' ';
318  buf[LPADDING + bytelen + 1] = ' ';
319 
320  /* Calculate trigrams marking their bounds if needed */
321  if (bounds)
322  bounds[tptr - trg] |= TRGM_BOUND_LEFT;
323  tptr = make_trigrams(tptr, buf, bytelen + LPADDING + RPADDING,
324  charlen + LPADDING + RPADDING);
325  if (bounds)
326  bounds[tptr - trg - 1] |= TRGM_BOUND_RIGHT;
327  }
328 
329  pfree(buf);
330 
331  return tptr - trg;
332 }
333 
334 /*
335  * Guard against possible overflow in the palloc requests below. (We
336  * don't worry about the additive constants, since palloc can detect
337  * requests that are a little above MaxAllocSize --- we just need to
338  * prevent integer overflow in the multiplications.)
339  */
340 static void
341 protect_out_of_mem(int slen)
342 {
343  if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
345  ereport(ERROR,
346  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
347  errmsg("out of memory")));
348 }
349 
350 /*
351  * Make array of trigrams with sorting and removing duplicate items.
352  *
353  * str: source string, of length slen bytes.
354  *
355  * Returns the sorted array of unique trigrams.
356  */
357 TRGM *
358 generate_trgm(char *str, int slen)
359 {
360  TRGM *trg;
361  int len;
362 
363  protect_out_of_mem(slen);
364 
365  trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
366  trg->flag = ARRKEY;
367 
368  len = generate_trgm_only(GETARR(trg), str, slen, NULL);
370 
371  if (len == 0)
372  return trg;
373 
374  /*
375  * Make trigrams unique.
376  */
377  if (len > 1)
378  {
379  qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
380  len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
381  }
382 
384 
385  return trg;
386 }
387 
388 /*
389  * Make array of positional trigrams from two trigram arrays trg1 and trg2.
390  *
391  * trg1: trigram array of search pattern, of length len1. trg1 is required
392  * word which positions don't matter and replaced with -1.
393  * trg2: trigram array of text, of length len2. trg2 is haystack where we
394  * search and have to store its positions.
395  *
396  * Returns concatenated trigram array.
397  */
398 static pos_trgm *
399 make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
400 {
401  pos_trgm *result;
402  int i,
403  len = len1 + len2;
404 
405  result = (pos_trgm *) palloc(sizeof(pos_trgm) * len);
406 
407  for (i = 0; i < len1; i++)
408  {
409  memcpy(&result[i].trg, &trg1[i], sizeof(trgm));
410  result[i].index = -1;
411  }
412 
413  for (i = 0; i < len2; i++)
414  {
415  memcpy(&result[i + len1].trg, &trg2[i], sizeof(trgm));
416  result[i + len1].index = i;
417  }
418 
419  return result;
420 }
421 
422 /*
423  * Compare position trigrams: compare trigrams first and position second.
424  */
425 static int
426 comp_ptrgm(const void *v1, const void *v2)
427 {
428  const pos_trgm *p1 = (const pos_trgm *) v1;
429  const pos_trgm *p2 = (const pos_trgm *) v2;
430  int cmp;
431 
432  cmp = CMPTRGM(p1->trg, p2->trg);
433  if (cmp != 0)
434  return cmp;
435 
436  if (p1->index < p2->index)
437  return -1;
438  else if (p1->index == p2->index)
439  return 0;
440  else
441  return 1;
442 }
443 
444 /*
445  * Iterative search function which calculates maximum similarity with word in
446  * the string. But maximum similarity is calculated only if check_only == false.
447  *
448  * trg2indexes: array which stores indexes of the array "found".
449  * found: array which stores true of false values.
450  * ulen1: count of unique trigrams of array "trg1".
451  * len2: length of array "trg2" and array "trg2indexes".
452  * len: length of the array "found".
453  * lags: set of boolean flags parameterizing similarity calculation.
454  * bounds: whether each trigram is left/right bound of word.
455  *
456  * Returns word similarity.
457  */
458 static float4
459 iterate_word_similarity(int *trg2indexes,
460  bool *found,
461  int ulen1,
462  int len2,
463  int len,
464  uint8 flags,
465  TrgmBound *bounds)
466 {
467  int *lastpos,
468  i,
469  ulen2 = 0,
470  count = 0,
471  upper = -1,
472  lower;
473  float4 smlr_cur,
474  smlr_max = 0.0f;
475  double threshold;
476 
477  Assert(bounds || !(flags & WORD_SIMILARITY_STRICT));
478 
479  /* Select appropriate threshold */
480  threshold = (flags & WORD_SIMILARITY_STRICT) ?
483 
484  /*
485  * Consider first trigram as initial lower bound for strict word
486  * similarity, or initialize it later with first trigram present for plain
487  * word similarity.
488  */
489  lower = (flags & WORD_SIMILARITY_STRICT) ? 0 : -1;
490 
491  /* Memorise last position of each trigram */
492  lastpos = (int *) palloc(sizeof(int) * len);
493  memset(lastpos, -1, sizeof(int) * len);
494 
495  for (i = 0; i < len2; i++)
496  {
497  /* Get index of next trigram */
498  int trgindex = trg2indexes[i];
499 
500  /* Update last position of this trigram */
501  if (lower >= 0 || found[trgindex])
502  {
503  if (lastpos[trgindex] < 0)
504  {
505  ulen2++;
506  if (found[trgindex])
507  count++;
508  }
509  lastpos[trgindex] = i;
510  }
511 
512  /*
513  * Adjust upper bound if trigram is upper bound of word for strict
514  * word similarity, or if trigram is present in required substring for
515  * plain word similarity
516  */
517  if ((flags & WORD_SIMILARITY_STRICT) ? (bounds[i] & TRGM_BOUND_RIGHT)
518  : found[trgindex])
519  {
520  int prev_lower,
521  tmp_ulen2,
522  tmp_lower,
523  tmp_count;
524 
525  upper = i;
526  if (lower == -1)
527  {
528  lower = i;
529  ulen2 = 1;
530  }
531 
532  smlr_cur = CALCSML(count, ulen1, ulen2);
533 
534  /* Also try to adjust lower bound for greater similarity */
535  tmp_count = count;
536  tmp_ulen2 = ulen2;
537  prev_lower = lower;
538  for (tmp_lower = lower; tmp_lower <= upper; tmp_lower++)
539  {
540  float smlr_tmp;
541  int tmp_trgindex;
542 
543  /*
544  * Adjust lower bound only if trigram is lower bound of word
545  * for strict word similarity, or consider every trigram as
546  * lower bound for plain word similarity.
547  */
548  if (!(flags & WORD_SIMILARITY_STRICT)
549  || (bounds[tmp_lower] & TRGM_BOUND_LEFT))
550  {
551  smlr_tmp = CALCSML(tmp_count, ulen1, tmp_ulen2);
552  if (smlr_tmp > smlr_cur)
553  {
554  smlr_cur = smlr_tmp;
555  ulen2 = tmp_ulen2;
556  lower = tmp_lower;
557  count = tmp_count;
558  }
559 
560  /*
561  * If we only check that word similarity is greater than
562  * threshold we do not need to calculate a maximum
563  * similarity.
564  */
565  if ((flags & WORD_SIMILARITY_CHECK_ONLY)
566  && smlr_cur >= threshold)
567  break;
568  }
569 
570  tmp_trgindex = trg2indexes[tmp_lower];
571  if (lastpos[tmp_trgindex] == tmp_lower)
572  {
573  tmp_ulen2--;
574  if (found[tmp_trgindex])
575  tmp_count--;
576  }
577  }
578 
579  smlr_max = Max(smlr_max, smlr_cur);
580 
581  /*
582  * if we only check that word similarity is greater than threshold
583  * we do not need to calculate a maximum similarity.
584  */
585  if ((flags & WORD_SIMILARITY_CHECK_ONLY) && smlr_max >= threshold)
586  break;
587 
588  for (tmp_lower = prev_lower; tmp_lower < lower; tmp_lower++)
589  {
590  int tmp_trgindex;
591 
592  tmp_trgindex = trg2indexes[tmp_lower];
593  if (lastpos[tmp_trgindex] == tmp_lower)
594  lastpos[tmp_trgindex] = -1;
595  }
596  }
597  }
598 
599  pfree(lastpos);
600 
601  return smlr_max;
602 }
603 
604 /*
605  * Calculate word similarity.
606  * This function prepare two arrays: "trg2indexes" and "found". Then this arrays
607  * are used to calculate word similarity using iterate_word_similarity().
608  *
609  * "trg2indexes" is array which stores indexes of the array "found".
610  * In other words:
611  * trg2indexes[j] = i;
612  * found[i] = true (or false);
613  * If found[i] == true then there is trigram trg2[j] in array "trg1".
614  * If found[i] == false then there is not trigram trg2[j] in array "trg1".
615  *
616  * str1: search pattern string, of length slen1 bytes.
617  * str2: text in which we are looking for a word, of length slen2 bytes.
618  * flags: set of boolean flags parameterizing similarity calculation.
619  *
620  * Returns word similarity.
621  */
622 static float4
623 calc_word_similarity(char *str1, int slen1, char *str2, int slen2,
624  uint8 flags)
625 {
626  bool *found;
627  pos_trgm *ptrg;
628  trgm *trg1;
629  trgm *trg2;
630  int len1,
631  len2,
632  len,
633  i,
634  j,
635  ulen1;
636  int *trg2indexes;
637  float4 result;
638  TrgmBound *bounds;
639 
640  protect_out_of_mem(slen1 + slen2);
641 
642  /* Make positional trigrams */
643  trg1 = (trgm *) palloc(sizeof(trgm) * (slen1 / 2 + 1) * 3);
644  trg2 = (trgm *) palloc(sizeof(trgm) * (slen2 / 2 + 1) * 3);
645  if (flags & WORD_SIMILARITY_STRICT)
646  bounds = (TrgmBound *) palloc0(sizeof(TrgmBound) * (slen2 / 2 + 1) * 3);
647  else
648  bounds = NULL;
649 
650  len1 = generate_trgm_only(trg1, str1, slen1, NULL);
651  len2 = generate_trgm_only(trg2, str2, slen2, bounds);
652 
653  ptrg = make_positional_trgm(trg1, len1, trg2, len2);
654  len = len1 + len2;
655  qsort(ptrg, len, sizeof(pos_trgm), comp_ptrgm);
656 
657  pfree(trg1);
658  pfree(trg2);
659 
660  /*
661  * Merge positional trigrams array: enumerate each trigram and find its
662  * presence in required word.
663  */
664  trg2indexes = (int *) palloc(sizeof(int) * len2);
665  found = (bool *) palloc0(sizeof(bool) * len);
666 
667  ulen1 = 0;
668  j = 0;
669  for (i = 0; i < len; i++)
670  {
671  if (i > 0)
672  {
673  int cmp = CMPTRGM(ptrg[i - 1].trg, ptrg[i].trg);
674 
675  if (cmp != 0)
676  {
677  if (found[j])
678  ulen1++;
679  j++;
680  }
681  }
682 
683  if (ptrg[i].index >= 0)
684  {
685  trg2indexes[ptrg[i].index] = j;
686  }
687  else
688  {
689  found[j] = true;
690  }
691  }
692  if (found[j])
693  ulen1++;
694 
695  /* Run iterative procedure to find maximum similarity with word */
696  result = iterate_word_similarity(trg2indexes, found, ulen1, len2, len,
697  flags, bounds);
698 
699  pfree(trg2indexes);
700  pfree(found);
701  pfree(ptrg);
702 
703  return result;
704 }
705 
706 
707 /*
708  * Extract the next non-wildcard part of a search string, i.e. a word bounded
709  * by '_' or '%' meta-characters, non-word characters or string end.
710  *
711  * str: source string, of length lenstr bytes (need not be null-terminated)
712  * buf: where to return the substring (must be long enough)
713  * *bytelen: receives byte length of the found substring
714  * *charlen: receives character length of the found substring
715  *
716  * Returns pointer to end+1 of the found substring in the source string.
717  * Returns NULL if no word found (in which case buf, bytelen, charlen not set)
718  *
719  * If the found word is bounded by non-word characters or string boundaries
720  * then this function will include corresponding padding spaces into buf.
721  */
722 static const char *
723 get_wildcard_part(const char *str, int lenstr,
724  char *buf, int *bytelen, int *charlen)
725 {
726  const char *beginword = str;
727  const char *endword;
728  char *s = buf;
729  bool in_leading_wildcard_meta = false;
730  bool in_trailing_wildcard_meta = false;
731  bool in_escape = false;
732  int clen;
733 
734  /*
735  * Find the first word character, remembering whether preceding character
736  * was wildcard meta-character. Note that the in_escape state persists
737  * from this loop to the next one, since we may exit at a word character
738  * that is in_escape.
739  */
740  while (beginword - str < lenstr)
741  {
742  if (in_escape)
743  {
744  if (ISWORDCHR(beginword))
745  break;
746  in_escape = false;
747  in_leading_wildcard_meta = false;
748  }
749  else
750  {
751  if (ISESCAPECHAR(beginword))
752  in_escape = true;
753  else if (ISWILDCARDCHAR(beginword))
754  in_leading_wildcard_meta = true;
755  else if (ISWORDCHR(beginword))
756  break;
757  else
758  in_leading_wildcard_meta = false;
759  }
760  beginword += pg_mblen(beginword);
761  }
762 
763  /*
764  * Handle string end.
765  */
766  if (beginword - str >= lenstr)
767  return NULL;
768 
769  /*
770  * Add left padding spaces if preceding character wasn't wildcard
771  * meta-character.
772  */
773  *charlen = 0;
774  if (!in_leading_wildcard_meta)
775  {
776  if (LPADDING > 0)
777  {
778  *s++ = ' ';
779  (*charlen)++;
780  if (LPADDING > 1)
781  {
782  *s++ = ' ';
783  (*charlen)++;
784  }
785  }
786  }
787 
788  /*
789  * Copy data into buf until wildcard meta-character, non-word character or
790  * string boundary. Strip escapes during copy.
791  */
792  endword = beginword;
793  while (endword - str < lenstr)
794  {
795  clen = pg_mblen(endword);
796  if (in_escape)
797  {
798  if (ISWORDCHR(endword))
799  {
800  memcpy(s, endword, clen);
801  (*charlen)++;
802  s += clen;
803  }
804  else
805  {
806  /*
807  * Back up endword to the escape character when stopping at an
808  * escaped char, so that subsequent get_wildcard_part will
809  * restart from the escape character. We assume here that
810  * escape chars are single-byte.
811  */
812  endword--;
813  break;
814  }
815  in_escape = false;
816  }
817  else
818  {
819  if (ISESCAPECHAR(endword))
820  in_escape = true;
821  else if (ISWILDCARDCHAR(endword))
822  {
823  in_trailing_wildcard_meta = true;
824  break;
825  }
826  else if (ISWORDCHR(endword))
827  {
828  memcpy(s, endword, clen);
829  (*charlen)++;
830  s += clen;
831  }
832  else
833  break;
834  }
835  endword += clen;
836  }
837 
838  /*
839  * Add right padding spaces if next character isn't wildcard
840  * meta-character.
841  */
842  if (!in_trailing_wildcard_meta)
843  {
844  if (RPADDING > 0)
845  {
846  *s++ = ' ';
847  (*charlen)++;
848  if (RPADDING > 1)
849  {
850  *s++ = ' ';
851  (*charlen)++;
852  }
853  }
854  }
855 
856  *bytelen = s - buf;
857  return endword;
858 }
859 
860 /*
861  * Generates trigrams for wildcard search string.
862  *
863  * Returns array of trigrams that must occur in any string that matches the
864  * wildcard string. For example, given pattern "a%bcd%" the trigrams
865  * " a", "bcd" would be extracted.
866  */
867 TRGM *
868 generate_wildcard_trgm(const char *str, int slen)
869 {
870  TRGM *trg;
871  char *buf,
872  *buf2;
873  trgm *tptr;
874  int len,
875  charlen,
876  bytelen;
877  const char *eword;
878 
879  protect_out_of_mem(slen);
880 
881  trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
882  trg->flag = ARRKEY;
883  SET_VARSIZE(trg, TRGMHDRSIZE);
884 
885  if (slen + LPADDING + RPADDING < 3 || slen == 0)
886  return trg;
887 
888  tptr = GETARR(trg);
889 
890  /* Allocate a buffer for blank-padded, but not yet case-folded, words */
891  buf = palloc(sizeof(char) * (slen + 4));
892 
893  /*
894  * Extract trigrams from each substring extracted by get_wildcard_part.
895  */
896  eword = str;
897  while ((eword = get_wildcard_part(eword, slen - (eword - str),
898  buf, &bytelen, &charlen)) != NULL)
899  {
900 #ifdef IGNORECASE
901  buf2 = lowerstr_with_len(buf, bytelen);
902  bytelen = strlen(buf2);
903 #else
904  buf2 = buf;
905 #endif
906 
907  /*
908  * count trigrams
909  */
910  tptr = make_trigrams(tptr, buf2, bytelen, charlen);
911 
912 #ifdef IGNORECASE
913  pfree(buf2);
914 #endif
915  }
916 
917  pfree(buf);
918 
919  if ((len = tptr - GETARR(trg)) == 0)
920  return trg;
921 
922  /*
923  * Make trigrams unique.
924  */
925  if (len > 1)
926  {
927  qsort((void *) GETARR(trg), len, sizeof(trgm), comp_trgm);
928  len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
929  }
930 
932 
933  return trg;
934 }
935 
936 uint32
937 trgm2int(trgm *ptr)
938 {
939  uint32 val = 0;
940 
941  val |= *(((unsigned char *) ptr));
942  val <<= 8;
943  val |= *(((unsigned char *) ptr) + 1);
944  val <<= 8;
945  val |= *(((unsigned char *) ptr) + 2);
946 
947  return val;
948 }
949 
950 Datum
952 {
953  text *in = PG_GETARG_TEXT_PP(0);
954  TRGM *trg;
955  Datum *d;
956  ArrayType *a;
957  trgm *ptr;
958  int i;
959 
961  d = (Datum *) palloc(sizeof(Datum) * (1 + ARRNELEM(trg)));
962 
963  for (i = 0, ptr = GETARR(trg); i < ARRNELEM(trg); i++, ptr++)
964  {
965  text *item = (text *) palloc(VARHDRSZ + Max(12, pg_database_encoding_max_length() * 3));
966 
968  {
969  snprintf(VARDATA(item), 12, "0x%06x", trgm2int(ptr));
970  SET_VARSIZE(item, VARHDRSZ + strlen(VARDATA(item)));
971  }
972  else
973  {
974  SET_VARSIZE(item, VARHDRSZ + 3);
975  CPTRGM(VARDATA(item), ptr);
976  }
977  d[i] = PointerGetDatum(item);
978  }
979 
980  a = construct_array_builtin(d, ARRNELEM(trg), TEXTOID);
981 
982  for (i = 0; i < ARRNELEM(trg); i++)
983  pfree(DatumGetPointer(d[i]));
984 
985  pfree(d);
986  pfree(trg);
987  PG_FREE_IF_COPY(in, 0);
988 
990 }
991 
992 float4
993 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
994 {
995  trgm *ptr1,
996  *ptr2;
997  int count = 0;
998  int len1,
999  len2;
1000 
1001  ptr1 = GETARR(trg1);
1002  ptr2 = GETARR(trg2);
1003 
1004  len1 = ARRNELEM(trg1);
1005  len2 = ARRNELEM(trg2);
1006 
1007  /* explicit test is needed to avoid 0/0 division when both lengths are 0 */
1008  if (len1 <= 0 || len2 <= 0)
1009  return (float4) 0.0;
1010 
1011  while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
1012  {
1013  int res = CMPTRGM(ptr1, ptr2);
1014 
1015  if (res < 0)
1016  ptr1++;
1017  else if (res > 0)
1018  ptr2++;
1019  else
1020  {
1021  ptr1++;
1022  ptr2++;
1023  count++;
1024  }
1025  }
1026 
1027  /*
1028  * If inexact then len2 is equal to count, because we don't know actual
1029  * length of second string in inexact search and we can assume that count
1030  * is a lower bound of len2.
1031  */
1032  return CALCSML(count, len1, inexact ? count : len2);
1033 }
1034 
1035 
1036 /*
1037  * Returns whether trg2 contains all trigrams in trg1.
1038  * This relies on the trigram arrays being sorted.
1039  */
1040 bool
1041 trgm_contained_by(TRGM *trg1, TRGM *trg2)
1042 {
1043  trgm *ptr1,
1044  *ptr2;
1045  int len1,
1046  len2;
1047 
1048  ptr1 = GETARR(trg1);
1049  ptr2 = GETARR(trg2);
1050 
1051  len1 = ARRNELEM(trg1);
1052  len2 = ARRNELEM(trg2);
1053 
1054  while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
1055  {
1056  int res = CMPTRGM(ptr1, ptr2);
1057 
1058  if (res < 0)
1059  return false;
1060  else if (res > 0)
1061  ptr2++;
1062  else
1063  {
1064  ptr1++;
1065  ptr2++;
1066  }
1067  }
1068  if (ptr1 - GETARR(trg1) < len1)
1069  return false;
1070  else
1071  return true;
1072 }
1073 
1074 /*
1075  * Return a palloc'd boolean array showing, for each trigram in "query",
1076  * whether it is present in the trigram array "key".
1077  * This relies on the "key" array being sorted, but "query" need not be.
1078  */
1079 bool *
1080 trgm_presence_map(TRGM *query, TRGM *key)
1081 {
1082  bool *result;
1083  trgm *ptrq = GETARR(query),
1084  *ptrk = GETARR(key);
1085  int lenq = ARRNELEM(query),
1086  lenk = ARRNELEM(key),
1087  i;
1088 
1089  result = (bool *) palloc0(lenq * sizeof(bool));
1090 
1091  /* for each query trigram, do a binary search in the key array */
1092  for (i = 0; i < lenq; i++)
1093  {
1094  int lo = 0;
1095  int hi = lenk;
1096 
1097  while (lo < hi)
1098  {
1099  int mid = (lo + hi) / 2;
1100  int res = CMPTRGM(ptrq, ptrk + mid);
1101 
1102  if (res < 0)
1103  hi = mid;
1104  else if (res > 0)
1105  lo = mid + 1;
1106  else
1107  {
1108  result[i] = true;
1109  break;
1110  }
1111  }
1112  ptrq++;
1113  }
1114 
1115  return result;
1116 }
1118 Datum
1120 {
1121  text *in1 = PG_GETARG_TEXT_PP(0);
1122  text *in2 = PG_GETARG_TEXT_PP(1);
1123  TRGM *trg1,
1124  *trg2;
1125  float4 res;
1126 
1127  trg1 = generate_trgm(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1));
1128  trg2 = generate_trgm(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2));
1129 
1130  res = cnt_sml(trg1, trg2, false);
1131 
1132  pfree(trg1);
1133  pfree(trg2);
1134  PG_FREE_IF_COPY(in1, 0);
1135  PG_FREE_IF_COPY(in2, 1);
1136 
1138 }
1140 Datum
1142 {
1143  text *in1 = PG_GETARG_TEXT_PP(0);
1144  text *in2 = PG_GETARG_TEXT_PP(1);
1145  float4 res;
1146 
1148  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1149  0);
1150 
1151  PG_FREE_IF_COPY(in1, 0);
1152  PG_FREE_IF_COPY(in2, 1);
1154 }
1156 Datum
1158 {
1159  text *in1 = PG_GETARG_TEXT_PP(0);
1160  text *in2 = PG_GETARG_TEXT_PP(1);
1161  float4 res;
1162 
1164  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1166 
1167  PG_FREE_IF_COPY(in1, 0);
1168  PG_FREE_IF_COPY(in2, 1);
1170 }
1172 Datum
1174 {
1176  PG_GETARG_DATUM(0),
1177  PG_GETARG_DATUM(1)));
1178 
1179  PG_RETURN_FLOAT4(1.0 - res);
1180 }
1182 Datum
1184 {
1186  PG_GETARG_DATUM(0),
1187  PG_GETARG_DATUM(1)));
1188 
1190 }
1192 Datum
1194 {
1195  text *in1 = PG_GETARG_TEXT_PP(0);
1196  text *in2 = PG_GETARG_TEXT_PP(1);
1197  float4 res;
1198 
1200  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1202 
1203  PG_FREE_IF_COPY(in1, 0);
1204  PG_FREE_IF_COPY(in2, 1);
1206 }
1208 Datum
1210 {
1211  text *in1 = PG_GETARG_TEXT_PP(0);
1212  text *in2 = PG_GETARG_TEXT_PP(1);
1213  float4 res;
1214 
1216  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1218 
1219  PG_FREE_IF_COPY(in1, 0);
1220  PG_FREE_IF_COPY(in2, 1);
1222 }
1224 Datum
1226 {
1227  text *in1 = PG_GETARG_TEXT_PP(0);
1228  text *in2 = PG_GETARG_TEXT_PP(1);
1229  float4 res;
1230 
1232  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1233  0);
1234 
1235  PG_FREE_IF_COPY(in1, 0);
1236  PG_FREE_IF_COPY(in2, 1);
1237  PG_RETURN_FLOAT4(1.0 - res);
1238 }
1240 Datum
1242 {
1243  text *in1 = PG_GETARG_TEXT_PP(0);
1244  text *in2 = PG_GETARG_TEXT_PP(1);
1245  float4 res;
1246 
1248  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1249  0);
1250 
1251  PG_FREE_IF_COPY(in1, 0);
1252  PG_FREE_IF_COPY(in2, 1);
1253  PG_RETURN_FLOAT4(1.0 - res);
1254 }
1256 Datum
1258 {
1259  text *in1 = PG_GETARG_TEXT_PP(0);
1260  text *in2 = PG_GETARG_TEXT_PP(1);
1261  float4 res;
1262 
1264  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1266 
1267  PG_FREE_IF_COPY(in1, 0);
1268  PG_FREE_IF_COPY(in2, 1);
1270 }
1272 Datum
1274 {
1275  text *in1 = PG_GETARG_TEXT_PP(0);
1276  text *in2 = PG_GETARG_TEXT_PP(1);
1277  float4 res;
1278 
1280  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1282 
1283  PG_FREE_IF_COPY(in1, 0);
1284  PG_FREE_IF_COPY(in2, 1);
1286 }
1288 Datum
1290 {
1291  text *in1 = PG_GETARG_TEXT_PP(0);
1292  text *in2 = PG_GETARG_TEXT_PP(1);
1293  float4 res;
1294 
1296  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1298 
1299  PG_FREE_IF_COPY(in1, 0);
1300  PG_FREE_IF_COPY(in2, 1);
1301  PG_RETURN_FLOAT4(1.0 - res);
1302 }
1304 Datum
1306 {
1307  text *in1 = PG_GETARG_TEXT_PP(0);
1308  text *in2 = PG_GETARG_TEXT_PP(1);
1309  float4 res;
1310 
1312  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1314 
1315  PG_FREE_IF_COPY(in1, 0);
1316  PG_FREE_IF_COPY(in2, 1);
1317  PG_RETURN_FLOAT4(1.0 - res);
1318 }
ArrayType * construct_array_builtin(Datum *elems, int nelems, Oid elmtype)
Definition: arrayfuncs.c:3339
unsigned int uint32
Definition: c.h:441
#define Max(x, y)
Definition: c.h:980
#define VARHDRSZ
Definition: c.h:627
float float4
Definition: c.h:564
unsigned char uint8
Definition: c.h:439
size_t Size
Definition: c.h:540
int errcode(int sqlerrcode)
Definition: elog.c:693
int errmsg(const char *fmt,...)
Definition: elog.c:904
#define ERROR
Definition: elog.h:33
#define elog(elevel,...)
Definition: elog.h:218
#define ereport(elevel,...)
Definition: elog.h:143
char * OidOutputFunctionCall(Oid functionId, Datum val)
Definition: fmgr.c:1639
#define PG_FREE_IF_COPY(ptr, n)
Definition: fmgr.h:260
#define PG_GETARG_TEXT_PP(n)
Definition: fmgr.h:309
#define DirectFunctionCall2(func, arg1, arg2)
Definition: fmgr.h:633
#define PG_GETARG_DATUM(n)
Definition: fmgr.h:268
#define PG_GETARG_FLOAT4(n)
Definition: fmgr.h:281
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:361
#define PG_RETURN_FLOAT4(x)
Definition: fmgr.h:366
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
#define PG_RETURN_BOOL(x)
Definition: fmgr.h:359
void DefineCustomRealVariable(const char *name, const char *short_desc, const char *long_desc, double *valueAddr, double bootValue, double minValue, double maxValue, GucContext context, int flags, GucRealCheckHook check_hook, GucRealAssignHook assign_hook, GucShowHook show_hook)
Definition: guc.c:9538
void SetConfigOption(const char *name, const char *value, GucContext context, GucSource source)
Definition: guc.c:8376
void MarkGUCPrefixReserved(const char *className)
Definition: guc.c:9629
@ PGC_S_SESSION
Definition: guc.h:123
@ PGC_USERSET
Definition: guc.h:76
#define CALCGTSIZE(flag, siglen)
Definition: hstore_gist.c:59
long val
Definition: informix.c:664
int b
Definition: isn.c:70
int a
Definition: isn.c:69
int j
Definition: isn.c:74
int i
Definition: isn.c:73
Assert(fmt[strlen(fmt) - 1] !='\n')
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2864
int pg_database_encoding_max_length(void)
Definition: mbutils.c:1495
int pg_mblen(const char *mbstr)
Definition: mbutils.c:966
void pfree(void *pointer)
Definition: mcxt.c:1175
void * palloc0(Size size)
Definition: mcxt.c:1099
void * palloc(Size size)
Definition: mcxt.c:1068
#define MaxAllocSize
Definition: memutils.h:40
Datum lower(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:48
Datum upper(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:79
const void size_t len
return crc
uint32 pg_crc32
Definition: pg_crc.h:37
#define INIT_LEGACY_CRC32(crc)
Definition: pg_crc.h:79
#define COMP_LEGACY_CRC32(crc, data, len)
Definition: pg_crc.h:81
#define FIN_LEGACY_CRC32(crc)
Definition: pg_crc.h:80
static char * buf
Definition: pg_test_fsync.c:67
#define snprintf
Definition: port.h:225
#define qsort(a, b, c, d)
Definition: port.h:495
static Datum Float4GetDatum(float4 X)
Definition: postgres.h:725
uintptr_t Datum
Definition: postgres.h:411
static float4 DatumGetFloat4(Datum X)
Definition: postgres.h:708
#define DatumGetPointer(X)
Definition: postgres.h:593
#define VARDATA(PTR)
Definition: postgres.h:315
#define VARDATA_ANY(PTR)
Definition: postgres.h:361
#define SET_VARSIZE(PTR, len)
Definition: postgres.h:342
#define PointerGetDatum(X)
Definition: postgres.h:600
#define VARSIZE_ANY_EXHDR(PTR)
Definition: postgres.h:354
unsigned int Oid
Definition: postgres_ext.h:31
static size_t qunique(void *array, size_t elements, size_t width, int(*compare)(const void *, const void *))
Definition: qunique.h:21
static int cmp(const chr *x, const chr *y, size_t len)
Definition: regc_locale.c:747
uint16 StrategyNumber
Definition: stratnum.h:22
Definition: trgm.h:67
uint8 flag
Definition: trgm.h:69
Definition: type.h:90
int index
Definition: trgm_op.c:46
trgm trg
Definition: trgm_op.c:45
Definition: c.h:622
#define CALCSML(count, len1, len2)
Definition: trgm.h:116
#define ISWORDCHR(c)
Definition: trgm.h:55
#define WordSimilarityStrategyNumber
Definition: trgm.h:36
#define StrictWordSimilarityStrategyNumber
Definition: trgm.h:38
#define ISESCAPECHAR(x)
Definition: trgm.h:63
#define ARRNELEM(x)
Definition: trgm.h:107
#define RPADDING
Definition: trgm.h:17
#define LPADDING
Definition: trgm.h:16
#define SimilarityStrategyNumber
Definition: trgm.h:30
#define ISWILDCARDCHAR(x)
Definition: trgm.h:64
#define CMPTRGM(a, b)
Definition: trgm.h:46
char trgm[3]
Definition: trgm.h:42
#define CPTRGM(a, b)
Definition: trgm.h:48
#define ISPRINTABLETRGM(t)
Definition: trgm.h:61
#define GETARR(x)
Definition: trgm.h:106
#define ARRKEY
Definition: trgm.h:96
#define TRGMHDRSIZE
Definition: trgm.h:73
static float4 iterate_word_similarity(int *trg2indexes, bool *found, int ulen1, int len2, int len, uint8 flags, TrgmBound *bounds)
Definition: trgm_op.c:457
uint8 TrgmBound
Definition: trgm_op.c:50
bool * trgm_presence_map(TRGM *query, TRGM *key)
Definition: trgm_op.c:1078
Datum strict_word_similarity_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1271
TRGM * generate_trgm(char *str, int slen)
Definition: trgm_op.c:356
void _PG_init(void)
Definition: trgm_op.c:62
Datum set_limit(PG_FUNCTION_ARGS)
Definition: trgm_op.c:110
static const char * get_wildcard_part(const char *str, int lenstr, char *buf, int *bytelen, int *charlen)
Definition: trgm_op.c:721
static int comp_trgm(const void *a, const void *b)
Definition: trgm_op.c:161
double strict_word_similarity_threshold
Definition: trgm_op.c:21
TRGM * generate_wildcard_trgm(const char *str, int slen)
Definition: trgm_op.c:866
PG_MODULE_MAGIC
Definition: trgm_op.c:16
uint32 trgm2int(trgm *ptr)
Definition: trgm_op.c:935
#define WORD_SIMILARITY_CHECK_ONLY
Definition: trgm_op.c:55
static void protect_out_of_mem(int slen)
Definition: trgm_op.c:339
Datum word_similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1139
Datum strict_word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1303
void compact_trigram(trgm *tptr, char *str, int bytelen)
Definition: trgm_op.c:198
PG_FUNCTION_INFO_V1(set_limit)
static float4 calc_word_similarity(char *str1, int slen1, char *str2, int slen2, uint8 flags)
Definition: trgm_op.c:621
double word_similarity_threshold
Definition: trgm_op.c:20
Datum word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1239
double index_strategy_get_limit(StrategyNumber strategy)
Definition: trgm_op.c:132
static pos_trgm * make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
Definition: trgm_op.c:397
Datum similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1117
double similarity_threshold
Definition: trgm_op.c:19
static int comp_ptrgm(const void *v1, const void *v2)
Definition: trgm_op.c:424
Datum show_trgm(PG_FUNCTION_ARGS)
Definition: trgm_op.c:949
static trgm * make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
Definition: trgm_op.c:223
bool trgm_contained_by(TRGM *trg1, TRGM *trg2)
Definition: trgm_op.c:1039
static char * find_word(char *str, int lenstr, char **endword, int *charlen)
Definition: trgm_op.c:171
#define TRGM_BOUND_RIGHT
Definition: trgm_op.c:52
Datum show_limit(PG_FUNCTION_ARGS)
Definition: trgm_op.c:155
Datum strict_word_similarity_dist_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1287
Datum word_similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1191
Datum word_similarity_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1207
#define WORD_SIMILARITY_STRICT
Definition: trgm_op.c:56
#define TRGM_BOUND_LEFT
Definition: trgm_op.c:51
float4 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
Definition: trgm_op.c:991
static int generate_trgm_only(trgm *trg, char *str, int slen, TrgmBound *bounds)
Definition: trgm_op.c:275
Datum similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1181
Datum word_similarity_dist_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1223
Datum strict_word_similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1155
Datum similarity_dist(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1171
Datum strict_word_similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1255
char * lowerstr_with_len(const char *str, int len)
Definition: ts_locale.c:257