PostgreSQL Source Code  git master
trgm_op.c
Go to the documentation of this file.
1 /*
2  * contrib/pg_trgm/trgm_op.c
3  */
4 #include "postgres.h"
5 
6 #include <ctype.h>
7 
8 #include "catalog/pg_type.h"
9 #include "common/int.h"
10 #include "lib/qunique.h"
11 #include "miscadmin.h"
12 #include "trgm.h"
13 #include "tsearch/ts_locale.h"
14 #include "utils/guc.h"
15 #include "utils/lsyscache.h"
16 #include "utils/memutils.h"
17 #include "utils/pg_crc.h"
18 
20 
21 /* GUC variables */
22 double similarity_threshold = 0.3f;
25 
42 
43 /* Trigram with position */
44 typedef struct
45 {
47  int index;
48 } pos_trgm;
49 
50 /* Trigram bound type */
51 typedef uint8 TrgmBound;
52 #define TRGM_BOUND_LEFT 0x01 /* trigram is left bound of word */
53 #define TRGM_BOUND_RIGHT 0x02 /* trigram is right bound of word */
54 
55 /* Word similarity flags */
56 #define WORD_SIMILARITY_CHECK_ONLY 0x01 /* only check existence of similar
57  * search pattern in text */
58 #define WORD_SIMILARITY_STRICT 0x02 /* force bounds of extent to match
59  * word bounds */
60 
61 /*
62  * Module load callback
63  */
64 void
65 _PG_init(void)
66 {
67  /* Define custom GUC variables. */
68  DefineCustomRealVariable("pg_trgm.similarity_threshold",
69  "Sets the threshold used by the % operator.",
70  "Valid range is 0.0 .. 1.0.",
72  0.3f,
73  0.0,
74  1.0,
76  0,
77  NULL,
78  NULL,
79  NULL);
80  DefineCustomRealVariable("pg_trgm.word_similarity_threshold",
81  "Sets the threshold used by the <% operator.",
82  "Valid range is 0.0 .. 1.0.",
84  0.6f,
85  0.0,
86  1.0,
88  0,
89  NULL,
90  NULL,
91  NULL);
92  DefineCustomRealVariable("pg_trgm.strict_word_similarity_threshold",
93  "Sets the threshold used by the <<% operator.",
94  "Valid range is 0.0 .. 1.0.",
96  0.5f,
97  0.0,
98  1.0,
100  0,
101  NULL,
102  NULL,
103  NULL);
104 
105  MarkGUCPrefixReserved("pg_trgm");
106 }
107 
108 /*
109  * Deprecated function.
110  * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
111  */
112 Datum
114 {
115  float4 nlimit = PG_GETARG_FLOAT4(0);
116  char *nlimit_str;
117  Oid func_out_oid;
118  bool is_varlena;
119 
120  getTypeOutputInfo(FLOAT4OID, &func_out_oid, &is_varlena);
121 
122  nlimit_str = OidOutputFunctionCall(func_out_oid, Float4GetDatum(nlimit));
123 
124  SetConfigOption("pg_trgm.similarity_threshold", nlimit_str,
126 
128 }
129 
130 
131 /*
132  * Get similarity threshold for given index scan strategy number.
133  */
134 double
136 {
137  switch (strategy)
138  {
140  return similarity_threshold;
145  default:
146  elog(ERROR, "unrecognized strategy number: %d", strategy);
147  break;
148  }
149 
150  return 0.0; /* keep compiler quiet */
151 }
152 
153 /*
154  * Deprecated function.
155  * Use "pg_trgm.similarity_threshold" GUC variable instead of this function.
156  */
157 Datum
159 {
161 }
162 
163 static int
164 comp_trgm(const void *a, const void *b)
165 {
166  return CMPTRGM(a, b);
167 }
168 
169 /*
170  * Finds first word in string, returns pointer to the word,
171  * endword points to the character after word
172  */
173 static char *
174 find_word(char *str, int lenstr, char **endword, int *charlen)
175 {
176  char *beginword = str;
177 
178  while (beginword - str < lenstr && !ISWORDCHR(beginword))
179  beginword += pg_mblen(beginword);
180 
181  if (beginword - str >= lenstr)
182  return NULL;
183 
184  *endword = beginword;
185  *charlen = 0;
186  while (*endword - str < lenstr && ISWORDCHR(*endword))
187  {
188  *endword += pg_mblen(*endword);
189  (*charlen)++;
190  }
191 
192  return beginword;
193 }
194 
195 /*
196  * Reduce a trigram (three possibly multi-byte characters) to a trgm,
197  * which is always exactly three bytes. If we have three single-byte
198  * characters, we just use them as-is; otherwise we form a hash value.
199  */
200 void
201 compact_trigram(trgm *tptr, char *str, int bytelen)
202 {
203  if (bytelen == 3)
204  {
205  CPTRGM(tptr, str);
206  }
207  else
208  {
209  pg_crc32 crc;
210 
212  COMP_LEGACY_CRC32(crc, str, bytelen);
214 
215  /*
216  * use only 3 upper bytes from crc, hope, it's good enough hashing
217  */
218  CPTRGM(tptr, &crc);
219  }
220 }
221 
222 /*
223  * Adds trigrams from words (already padded).
224  */
225 static trgm *
226 make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
227 {
228  char *ptr = str;
229 
230  if (charlen < 3)
231  return tptr;
232 
233  if (bytelen > charlen)
234  {
235  /* Find multibyte character boundaries and apply compact_trigram */
236  int lenfirst = pg_mblen(str),
237  lenmiddle = pg_mblen(str + lenfirst),
238  lenlast = pg_mblen(str + lenfirst + lenmiddle);
239 
240  while ((ptr - str) + lenfirst + lenmiddle + lenlast <= bytelen)
241  {
242  compact_trigram(tptr, ptr, lenfirst + lenmiddle + lenlast);
243 
244  ptr += lenfirst;
245  tptr++;
246 
247  lenfirst = lenmiddle;
248  lenmiddle = lenlast;
249  lenlast = pg_mblen(ptr + lenfirst + lenmiddle);
250  }
251  }
252  else
253  {
254  /* Fast path when there are no multibyte characters */
255  Assert(bytelen == charlen);
256 
257  while (ptr - str < bytelen - 2 /* number of trigrams = strlen - 2 */ )
258  {
259  CPTRGM(tptr, ptr);
260  ptr++;
261  tptr++;
262  }
263  }
264 
265  return tptr;
266 }
267 
268 /*
269  * Make array of trigrams without sorting and removing duplicate items.
270  *
271  * trg: where to return the array of trigrams.
272  * str: source string, of length slen bytes.
273  * bounds: where to return bounds of trigrams (if needed).
274  *
275  * Returns length of the generated array.
276  */
277 static int
278 generate_trgm_only(trgm *trg, char *str, int slen, TrgmBound *bounds)
279 {
280  trgm *tptr;
281  char *buf;
282  int charlen,
283  bytelen;
284  char *bword,
285  *eword;
286 
287  if (slen + LPADDING + RPADDING < 3 || slen == 0)
288  return 0;
289 
290  tptr = trg;
291 
292  /* Allocate a buffer for case-folded, blank-padded words */
293  buf = (char *) palloc(slen * pg_database_encoding_max_length() + 4);
294 
295  if (LPADDING > 0)
296  {
297  *buf = ' ';
298  if (LPADDING > 1)
299  *(buf + 1) = ' ';
300  }
301 
302  eword = str;
303  while ((bword = find_word(eword, slen - (eword - str), &eword, &charlen)) != NULL)
304  {
305 #ifdef IGNORECASE
306  bword = lowerstr_with_len(bword, eword - bword);
307  bytelen = strlen(bword);
308 #else
309  bytelen = eword - bword;
310 #endif
311 
312  memcpy(buf + LPADDING, bword, bytelen);
313 
314 #ifdef IGNORECASE
315  pfree(bword);
316 #endif
317 
318  buf[LPADDING + bytelen] = ' ';
319  buf[LPADDING + bytelen + 1] = ' ';
320 
321  /* Calculate trigrams marking their bounds if needed */
322  if (bounds)
323  bounds[tptr - trg] |= TRGM_BOUND_LEFT;
324  tptr = make_trigrams(tptr, buf, bytelen + LPADDING + RPADDING,
325  charlen + LPADDING + RPADDING);
326  if (bounds)
327  bounds[tptr - trg - 1] |= TRGM_BOUND_RIGHT;
328  }
329 
330  pfree(buf);
331 
332  return tptr - trg;
333 }
334 
335 /*
336  * Guard against possible overflow in the palloc requests below. (We
337  * don't worry about the additive constants, since palloc can detect
338  * requests that are a little above MaxAllocSize --- we just need to
339  * prevent integer overflow in the multiplications.)
340  */
341 static void
342 protect_out_of_mem(int slen)
343 {
344  if ((Size) (slen / 2) >= (MaxAllocSize / (sizeof(trgm) * 3)) ||
346  ereport(ERROR,
347  (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED),
348  errmsg("out of memory")));
349 }
350 
351 /*
352  * Make array of trigrams with sorting and removing duplicate items.
353  *
354  * str: source string, of length slen bytes.
355  *
356  * Returns the sorted array of unique trigrams.
357  */
358 TRGM *
359 generate_trgm(char *str, int slen)
360 {
361  TRGM *trg;
362  int len;
363 
364  protect_out_of_mem(slen);
365 
366  trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
367  trg->flag = ARRKEY;
368 
369  len = generate_trgm_only(GETARR(trg), str, slen, NULL);
371 
372  if (len == 0)
373  return trg;
374 
375  /*
376  * Make trigrams unique.
377  */
378  if (len > 1)
379  {
380  qsort(GETARR(trg), len, sizeof(trgm), comp_trgm);
381  len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
382  }
383 
385 
386  return trg;
387 }
388 
389 /*
390  * Make array of positional trigrams from two trigram arrays trg1 and trg2.
391  *
392  * trg1: trigram array of search pattern, of length len1. trg1 is required
393  * word which positions don't matter and replaced with -1.
394  * trg2: trigram array of text, of length len2. trg2 is haystack where we
395  * search and have to store its positions.
396  *
397  * Returns concatenated trigram array.
398  */
399 static pos_trgm *
400 make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
401 {
402  pos_trgm *result;
403  int i,
404  len = len1 + len2;
405 
406  result = (pos_trgm *) palloc(sizeof(pos_trgm) * len);
407 
408  for (i = 0; i < len1; i++)
409  {
410  memcpy(&result[i].trg, &trg1[i], sizeof(trgm));
411  result[i].index = -1;
412  }
413 
414  for (i = 0; i < len2; i++)
415  {
416  memcpy(&result[i + len1].trg, &trg2[i], sizeof(trgm));
417  result[i + len1].index = i;
418  }
419 
420  return result;
421 }
422 
423 /*
424  * Compare position trigrams: compare trigrams first and position second.
425  */
426 static int
427 comp_ptrgm(const void *v1, const void *v2)
428 {
429  const pos_trgm *p1 = (const pos_trgm *) v1;
430  const pos_trgm *p2 = (const pos_trgm *) v2;
431  int cmp;
432 
433  cmp = CMPTRGM(p1->trg, p2->trg);
434  if (cmp != 0)
435  return cmp;
436 
437  return pg_cmp_s32(p1->index, p2->index);
438 }
439 
440 /*
441  * Iterative search function which calculates maximum similarity with word in
442  * the string. Maximum similarity is only calculated only if the flag
443  * WORD_SIMILARITY_CHECK_ONLY isn't set.
444  *
445  * trg2indexes: array which stores indexes of the array "found".
446  * found: array which stores true of false values.
447  * ulen1: count of unique trigrams of array "trg1".
448  * len2: length of array "trg2" and array "trg2indexes".
449  * len: length of the array "found".
450  * flags: set of boolean flags parameterizing similarity calculation.
451  * bounds: whether each trigram is left/right bound of word.
452  *
453  * Returns word similarity.
454  */
455 static float4
456 iterate_word_similarity(int *trg2indexes,
457  bool *found,
458  int ulen1,
459  int len2,
460  int len,
461  uint8 flags,
462  TrgmBound *bounds)
463 {
464  int *lastpos,
465  i,
466  ulen2 = 0,
467  count = 0,
468  upper = -1,
469  lower;
470  float4 smlr_cur,
471  smlr_max = 0.0f;
472  double threshold;
473 
474  Assert(bounds || !(flags & WORD_SIMILARITY_STRICT));
475 
476  /* Select appropriate threshold */
477  threshold = (flags & WORD_SIMILARITY_STRICT) ?
480 
481  /*
482  * Consider first trigram as initial lower bound for strict word
483  * similarity, or initialize it later with first trigram present for plain
484  * word similarity.
485  */
486  lower = (flags & WORD_SIMILARITY_STRICT) ? 0 : -1;
487 
488  /* Memorise last position of each trigram */
489  lastpos = (int *) palloc(sizeof(int) * len);
490  memset(lastpos, -1, sizeof(int) * len);
491 
492  for (i = 0; i < len2; i++)
493  {
494  int trgindex;
495 
497 
498  /* Get index of next trigram */
499  trgindex = trg2indexes[i];
500 
501  /* Update last position of this trigram */
502  if (lower >= 0 || found[trgindex])
503  {
504  if (lastpos[trgindex] < 0)
505  {
506  ulen2++;
507  if (found[trgindex])
508  count++;
509  }
510  lastpos[trgindex] = i;
511  }
512 
513  /*
514  * Adjust upper bound if trigram is upper bound of word for strict
515  * word similarity, or if trigram is present in required substring for
516  * plain word similarity
517  */
518  if ((flags & WORD_SIMILARITY_STRICT) ? (bounds[i] & TRGM_BOUND_RIGHT)
519  : found[trgindex])
520  {
521  int prev_lower,
522  tmp_ulen2,
523  tmp_lower,
524  tmp_count;
525 
526  upper = i;
527  if (lower == -1)
528  {
529  lower = i;
530  ulen2 = 1;
531  }
532 
533  smlr_cur = CALCSML(count, ulen1, ulen2);
534 
535  /* Also try to adjust lower bound for greater similarity */
536  tmp_count = count;
537  tmp_ulen2 = ulen2;
538  prev_lower = lower;
539  for (tmp_lower = lower; tmp_lower <= upper; tmp_lower++)
540  {
541  float smlr_tmp;
542  int tmp_trgindex;
543 
544  /*
545  * Adjust lower bound only if trigram is lower bound of word
546  * for strict word similarity, or consider every trigram as
547  * lower bound for plain word similarity.
548  */
549  if (!(flags & WORD_SIMILARITY_STRICT)
550  || (bounds[tmp_lower] & TRGM_BOUND_LEFT))
551  {
552  smlr_tmp = CALCSML(tmp_count, ulen1, tmp_ulen2);
553  if (smlr_tmp > smlr_cur)
554  {
555  smlr_cur = smlr_tmp;
556  ulen2 = tmp_ulen2;
557  lower = tmp_lower;
558  count = tmp_count;
559  }
560 
561  /*
562  * If we only check that word similarity is greater than
563  * threshold we do not need to calculate a maximum
564  * similarity.
565  */
566  if ((flags & WORD_SIMILARITY_CHECK_ONLY)
567  && smlr_cur >= threshold)
568  break;
569  }
570 
571  tmp_trgindex = trg2indexes[tmp_lower];
572  if (lastpos[tmp_trgindex] == tmp_lower)
573  {
574  tmp_ulen2--;
575  if (found[tmp_trgindex])
576  tmp_count--;
577  }
578  }
579 
580  smlr_max = Max(smlr_max, smlr_cur);
581 
582  /*
583  * if we only check that word similarity is greater than threshold
584  * we do not need to calculate a maximum similarity.
585  */
586  if ((flags & WORD_SIMILARITY_CHECK_ONLY) && smlr_max >= threshold)
587  break;
588 
589  for (tmp_lower = prev_lower; tmp_lower < lower; tmp_lower++)
590  {
591  int tmp_trgindex;
592 
593  tmp_trgindex = trg2indexes[tmp_lower];
594  if (lastpos[tmp_trgindex] == tmp_lower)
595  lastpos[tmp_trgindex] = -1;
596  }
597  }
598  }
599 
600  pfree(lastpos);
601 
602  return smlr_max;
603 }
604 
605 /*
606  * Calculate word similarity.
607  * This function prepare two arrays: "trg2indexes" and "found". Then this arrays
608  * are used to calculate word similarity using iterate_word_similarity().
609  *
610  * "trg2indexes" is array which stores indexes of the array "found".
611  * In other words:
612  * trg2indexes[j] = i;
613  * found[i] = true (or false);
614  * If found[i] == true then there is trigram trg2[j] in array "trg1".
615  * If found[i] == false then there is not trigram trg2[j] in array "trg1".
616  *
617  * str1: search pattern string, of length slen1 bytes.
618  * str2: text in which we are looking for a word, of length slen2 bytes.
619  * flags: set of boolean flags parameterizing similarity calculation.
620  *
621  * Returns word similarity.
622  */
623 static float4
624 calc_word_similarity(char *str1, int slen1, char *str2, int slen2,
625  uint8 flags)
626 {
627  bool *found;
628  pos_trgm *ptrg;
629  trgm *trg1;
630  trgm *trg2;
631  int len1,
632  len2,
633  len,
634  i,
635  j,
636  ulen1;
637  int *trg2indexes;
638  float4 result;
639  TrgmBound *bounds;
640 
641  protect_out_of_mem(slen1 + slen2);
642 
643  /* Make positional trigrams */
644  trg1 = (trgm *) palloc(sizeof(trgm) * (slen1 / 2 + 1) * 3);
645  trg2 = (trgm *) palloc(sizeof(trgm) * (slen2 / 2 + 1) * 3);
646  if (flags & WORD_SIMILARITY_STRICT)
647  bounds = (TrgmBound *) palloc0(sizeof(TrgmBound) * (slen2 / 2 + 1) * 3);
648  else
649  bounds = NULL;
650 
651  len1 = generate_trgm_only(trg1, str1, slen1, NULL);
652  len2 = generate_trgm_only(trg2, str2, slen2, bounds);
653 
654  ptrg = make_positional_trgm(trg1, len1, trg2, len2);
655  len = len1 + len2;
656  qsort(ptrg, len, sizeof(pos_trgm), comp_ptrgm);
657 
658  pfree(trg1);
659  pfree(trg2);
660 
661  /*
662  * Merge positional trigrams array: enumerate each trigram and find its
663  * presence in required word.
664  */
665  trg2indexes = (int *) palloc(sizeof(int) * len2);
666  found = (bool *) palloc0(sizeof(bool) * len);
667 
668  ulen1 = 0;
669  j = 0;
670  for (i = 0; i < len; i++)
671  {
672  if (i > 0)
673  {
674  int cmp = CMPTRGM(ptrg[i - 1].trg, ptrg[i].trg);
675 
676  if (cmp != 0)
677  {
678  if (found[j])
679  ulen1++;
680  j++;
681  }
682  }
683 
684  if (ptrg[i].index >= 0)
685  {
686  trg2indexes[ptrg[i].index] = j;
687  }
688  else
689  {
690  found[j] = true;
691  }
692  }
693  if (found[j])
694  ulen1++;
695 
696  /* Run iterative procedure to find maximum similarity with word */
697  result = iterate_word_similarity(trg2indexes, found, ulen1, len2, len,
698  flags, bounds);
699 
700  pfree(trg2indexes);
701  pfree(found);
702  pfree(ptrg);
703 
704  return result;
705 }
706 
707 
708 /*
709  * Extract the next non-wildcard part of a search string, i.e. a word bounded
710  * by '_' or '%' meta-characters, non-word characters or string end.
711  *
712  * str: source string, of length lenstr bytes (need not be null-terminated)
713  * buf: where to return the substring (must be long enough)
714  * *bytelen: receives byte length of the found substring
715  * *charlen: receives character length of the found substring
716  *
717  * Returns pointer to end+1 of the found substring in the source string.
718  * Returns NULL if no word found (in which case buf, bytelen, charlen not set)
719  *
720  * If the found word is bounded by non-word characters or string boundaries
721  * then this function will include corresponding padding spaces into buf.
722  */
723 static const char *
724 get_wildcard_part(const char *str, int lenstr,
725  char *buf, int *bytelen, int *charlen)
726 {
727  const char *beginword = str;
728  const char *endword;
729  char *s = buf;
730  bool in_leading_wildcard_meta = false;
731  bool in_trailing_wildcard_meta = false;
732  bool in_escape = false;
733  int clen;
734 
735  /*
736  * Find the first word character, remembering whether preceding character
737  * was wildcard meta-character. Note that the in_escape state persists
738  * from this loop to the next one, since we may exit at a word character
739  * that is in_escape.
740  */
741  while (beginword - str < lenstr)
742  {
743  if (in_escape)
744  {
745  if (ISWORDCHR(beginword))
746  break;
747  in_escape = false;
748  in_leading_wildcard_meta = false;
749  }
750  else
751  {
752  if (ISESCAPECHAR(beginword))
753  in_escape = true;
754  else if (ISWILDCARDCHAR(beginword))
755  in_leading_wildcard_meta = true;
756  else if (ISWORDCHR(beginword))
757  break;
758  else
759  in_leading_wildcard_meta = false;
760  }
761  beginword += pg_mblen(beginword);
762  }
763 
764  /*
765  * Handle string end.
766  */
767  if (beginword - str >= lenstr)
768  return NULL;
769 
770  /*
771  * Add left padding spaces if preceding character wasn't wildcard
772  * meta-character.
773  */
774  *charlen = 0;
775  if (!in_leading_wildcard_meta)
776  {
777  if (LPADDING > 0)
778  {
779  *s++ = ' ';
780  (*charlen)++;
781  if (LPADDING > 1)
782  {
783  *s++ = ' ';
784  (*charlen)++;
785  }
786  }
787  }
788 
789  /*
790  * Copy data into buf until wildcard meta-character, non-word character or
791  * string boundary. Strip escapes during copy.
792  */
793  endword = beginword;
794  while (endword - str < lenstr)
795  {
796  clen = pg_mblen(endword);
797  if (in_escape)
798  {
799  if (ISWORDCHR(endword))
800  {
801  memcpy(s, endword, clen);
802  (*charlen)++;
803  s += clen;
804  }
805  else
806  {
807  /*
808  * Back up endword to the escape character when stopping at an
809  * escaped char, so that subsequent get_wildcard_part will
810  * restart from the escape character. We assume here that
811  * escape chars are single-byte.
812  */
813  endword--;
814  break;
815  }
816  in_escape = false;
817  }
818  else
819  {
820  if (ISESCAPECHAR(endword))
821  in_escape = true;
822  else if (ISWILDCARDCHAR(endword))
823  {
824  in_trailing_wildcard_meta = true;
825  break;
826  }
827  else if (ISWORDCHR(endword))
828  {
829  memcpy(s, endword, clen);
830  (*charlen)++;
831  s += clen;
832  }
833  else
834  break;
835  }
836  endword += clen;
837  }
838 
839  /*
840  * Add right padding spaces if next character isn't wildcard
841  * meta-character.
842  */
843  if (!in_trailing_wildcard_meta)
844  {
845  if (RPADDING > 0)
846  {
847  *s++ = ' ';
848  (*charlen)++;
849  if (RPADDING > 1)
850  {
851  *s++ = ' ';
852  (*charlen)++;
853  }
854  }
855  }
856 
857  *bytelen = s - buf;
858  return endword;
859 }
860 
861 /*
862  * Generates trigrams for wildcard search string.
863  *
864  * Returns array of trigrams that must occur in any string that matches the
865  * wildcard string. For example, given pattern "a%bcd%" the trigrams
866  * " a", "bcd" would be extracted.
867  */
868 TRGM *
869 generate_wildcard_trgm(const char *str, int slen)
870 {
871  TRGM *trg;
872  char *buf,
873  *buf2;
874  trgm *tptr;
875  int len,
876  charlen,
877  bytelen;
878  const char *eword;
879 
880  protect_out_of_mem(slen);
881 
882  trg = (TRGM *) palloc(TRGMHDRSIZE + sizeof(trgm) * (slen / 2 + 1) * 3);
883  trg->flag = ARRKEY;
884  SET_VARSIZE(trg, TRGMHDRSIZE);
885 
886  if (slen + LPADDING + RPADDING < 3 || slen == 0)
887  return trg;
888 
889  tptr = GETARR(trg);
890 
891  /* Allocate a buffer for blank-padded, but not yet case-folded, words */
892  buf = palloc(sizeof(char) * (slen + 4));
893 
894  /*
895  * Extract trigrams from each substring extracted by get_wildcard_part.
896  */
897  eword = str;
898  while ((eword = get_wildcard_part(eword, slen - (eword - str),
899  buf, &bytelen, &charlen)) != NULL)
900  {
901 #ifdef IGNORECASE
902  buf2 = lowerstr_with_len(buf, bytelen);
903  bytelen = strlen(buf2);
904 #else
905  buf2 = buf;
906 #endif
907 
908  /*
909  * count trigrams
910  */
911  tptr = make_trigrams(tptr, buf2, bytelen, charlen);
912 
913 #ifdef IGNORECASE
914  pfree(buf2);
915 #endif
916  }
917 
918  pfree(buf);
919 
920  if ((len = tptr - GETARR(trg)) == 0)
921  return trg;
922 
923  /*
924  * Make trigrams unique.
925  */
926  if (len > 1)
927  {
928  qsort(GETARR(trg), len, sizeof(trgm), comp_trgm);
929  len = qunique(GETARR(trg), len, sizeof(trgm), comp_trgm);
930  }
931 
933 
934  return trg;
935 }
936 
937 uint32
938 trgm2int(trgm *ptr)
939 {
940  uint32 val = 0;
941 
942  val |= *(((unsigned char *) ptr));
943  val <<= 8;
944  val |= *(((unsigned char *) ptr) + 1);
945  val <<= 8;
946  val |= *(((unsigned char *) ptr) + 2);
947 
948  return val;
949 }
950 
951 Datum
953 {
954  text *in = PG_GETARG_TEXT_PP(0);
955  TRGM *trg;
956  Datum *d;
957  ArrayType *a;
958  trgm *ptr;
959  int i;
960 
962  d = (Datum *) palloc(sizeof(Datum) * (1 + ARRNELEM(trg)));
963 
964  for (i = 0, ptr = GETARR(trg); i < ARRNELEM(trg); i++, ptr++)
965  {
966  text *item = (text *) palloc(VARHDRSZ + Max(12, pg_database_encoding_max_length() * 3));
967 
969  {
970  snprintf(VARDATA(item), 12, "0x%06x", trgm2int(ptr));
971  SET_VARSIZE(item, VARHDRSZ + strlen(VARDATA(item)));
972  }
973  else
974  {
975  SET_VARSIZE(item, VARHDRSZ + 3);
976  CPTRGM(VARDATA(item), ptr);
977  }
978  d[i] = PointerGetDatum(item);
979  }
980 
981  a = construct_array_builtin(d, ARRNELEM(trg), TEXTOID);
982 
983  for (i = 0; i < ARRNELEM(trg); i++)
984  pfree(DatumGetPointer(d[i]));
985 
986  pfree(d);
987  pfree(trg);
988  PG_FREE_IF_COPY(in, 0);
989 
991 }
992 
993 float4
994 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
995 {
996  trgm *ptr1,
997  *ptr2;
998  int count = 0;
999  int len1,
1000  len2;
1001 
1002  ptr1 = GETARR(trg1);
1003  ptr2 = GETARR(trg2);
1004 
1005  len1 = ARRNELEM(trg1);
1006  len2 = ARRNELEM(trg2);
1007 
1008  /* explicit test is needed to avoid 0/0 division when both lengths are 0 */
1009  if (len1 <= 0 || len2 <= 0)
1010  return (float4) 0.0;
1011 
1012  while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
1013  {
1014  int res = CMPTRGM(ptr1, ptr2);
1015 
1016  if (res < 0)
1017  ptr1++;
1018  else if (res > 0)
1019  ptr2++;
1020  else
1021  {
1022  ptr1++;
1023  ptr2++;
1024  count++;
1025  }
1026  }
1027 
1028  /*
1029  * If inexact then len2 is equal to count, because we don't know actual
1030  * length of second string in inexact search and we can assume that count
1031  * is a lower bound of len2.
1032  */
1033  return CALCSML(count, len1, inexact ? count : len2);
1034 }
1035 
1036 
1037 /*
1038  * Returns whether trg2 contains all trigrams in trg1.
1039  * This relies on the trigram arrays being sorted.
1040  */
1041 bool
1042 trgm_contained_by(TRGM *trg1, TRGM *trg2)
1043 {
1044  trgm *ptr1,
1045  *ptr2;
1046  int len1,
1047  len2;
1048 
1049  ptr1 = GETARR(trg1);
1050  ptr2 = GETARR(trg2);
1051 
1052  len1 = ARRNELEM(trg1);
1053  len2 = ARRNELEM(trg2);
1054 
1055  while (ptr1 - GETARR(trg1) < len1 && ptr2 - GETARR(trg2) < len2)
1056  {
1057  int res = CMPTRGM(ptr1, ptr2);
1058 
1059  if (res < 0)
1060  return false;
1061  else if (res > 0)
1062  ptr2++;
1063  else
1064  {
1065  ptr1++;
1066  ptr2++;
1067  }
1068  }
1069  if (ptr1 - GETARR(trg1) < len1)
1070  return false;
1071  else
1072  return true;
1073 }
1074 
1075 /*
1076  * Return a palloc'd boolean array showing, for each trigram in "query",
1077  * whether it is present in the trigram array "key".
1078  * This relies on the "key" array being sorted, but "query" need not be.
1079  */
1080 bool *
1081 trgm_presence_map(TRGM *query, TRGM *key)
1082 {
1083  bool *result;
1084  trgm *ptrq = GETARR(query),
1085  *ptrk = GETARR(key);
1086  int lenq = ARRNELEM(query),
1087  lenk = ARRNELEM(key),
1088  i;
1089 
1090  result = (bool *) palloc0(lenq * sizeof(bool));
1091 
1092  /* for each query trigram, do a binary search in the key array */
1093  for (i = 0; i < lenq; i++)
1094  {
1095  int lo = 0;
1096  int hi = lenk;
1097 
1098  while (lo < hi)
1099  {
1100  int mid = (lo + hi) / 2;
1101  int res = CMPTRGM(ptrq, ptrk + mid);
1102 
1103  if (res < 0)
1104  hi = mid;
1105  else if (res > 0)
1106  lo = mid + 1;
1107  else
1108  {
1109  result[i] = true;
1110  break;
1111  }
1112  }
1113  ptrq++;
1114  }
1115 
1116  return result;
1117 }
1119 Datum
1121 {
1122  text *in1 = PG_GETARG_TEXT_PP(0);
1123  text *in2 = PG_GETARG_TEXT_PP(1);
1124  TRGM *trg1,
1125  *trg2;
1126  float4 res;
1127 
1128  trg1 = generate_trgm(VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1));
1129  trg2 = generate_trgm(VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2));
1130 
1131  res = cnt_sml(trg1, trg2, false);
1132 
1133  pfree(trg1);
1134  pfree(trg2);
1135  PG_FREE_IF_COPY(in1, 0);
1136  PG_FREE_IF_COPY(in2, 1);
1137 
1139 }
1141 Datum
1143 {
1144  text *in1 = PG_GETARG_TEXT_PP(0);
1145  text *in2 = PG_GETARG_TEXT_PP(1);
1146  float4 res;
1147 
1149  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1150  0);
1151 
1152  PG_FREE_IF_COPY(in1, 0);
1153  PG_FREE_IF_COPY(in2, 1);
1155 }
1157 Datum
1159 {
1160  text *in1 = PG_GETARG_TEXT_PP(0);
1161  text *in2 = PG_GETARG_TEXT_PP(1);
1162  float4 res;
1163 
1165  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1167 
1168  PG_FREE_IF_COPY(in1, 0);
1169  PG_FREE_IF_COPY(in2, 1);
1171 }
1173 Datum
1175 {
1177  PG_GETARG_DATUM(0),
1178  PG_GETARG_DATUM(1)));
1179 
1180  PG_RETURN_FLOAT4(1.0 - res);
1181 }
1183 Datum
1185 {
1187  PG_GETARG_DATUM(0),
1188  PG_GETARG_DATUM(1)));
1189 
1191 }
1193 Datum
1195 {
1196  text *in1 = PG_GETARG_TEXT_PP(0);
1197  text *in2 = PG_GETARG_TEXT_PP(1);
1198  float4 res;
1199 
1201  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1203 
1204  PG_FREE_IF_COPY(in1, 0);
1205  PG_FREE_IF_COPY(in2, 1);
1207 }
1209 Datum
1211 {
1212  text *in1 = PG_GETARG_TEXT_PP(0);
1213  text *in2 = PG_GETARG_TEXT_PP(1);
1214  float4 res;
1215 
1217  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1219 
1220  PG_FREE_IF_COPY(in1, 0);
1221  PG_FREE_IF_COPY(in2, 1);
1223 }
1225 Datum
1227 {
1228  text *in1 = PG_GETARG_TEXT_PP(0);
1229  text *in2 = PG_GETARG_TEXT_PP(1);
1230  float4 res;
1231 
1233  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1234  0);
1235 
1236  PG_FREE_IF_COPY(in1, 0);
1237  PG_FREE_IF_COPY(in2, 1);
1238  PG_RETURN_FLOAT4(1.0 - res);
1239 }
1241 Datum
1243 {
1244  text *in1 = PG_GETARG_TEXT_PP(0);
1245  text *in2 = PG_GETARG_TEXT_PP(1);
1246  float4 res;
1247 
1249  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1250  0);
1251 
1252  PG_FREE_IF_COPY(in1, 0);
1253  PG_FREE_IF_COPY(in2, 1);
1254  PG_RETURN_FLOAT4(1.0 - res);
1255 }
1257 Datum
1259 {
1260  text *in1 = PG_GETARG_TEXT_PP(0);
1261  text *in2 = PG_GETARG_TEXT_PP(1);
1262  float4 res;
1263 
1265  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1267 
1268  PG_FREE_IF_COPY(in1, 0);
1269  PG_FREE_IF_COPY(in2, 1);
1271 }
1273 Datum
1275 {
1276  text *in1 = PG_GETARG_TEXT_PP(0);
1277  text *in2 = PG_GETARG_TEXT_PP(1);
1278  float4 res;
1279 
1281  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1283 
1284  PG_FREE_IF_COPY(in1, 0);
1285  PG_FREE_IF_COPY(in2, 1);
1287 }
1289 Datum
1291 {
1292  text *in1 = PG_GETARG_TEXT_PP(0);
1293  text *in2 = PG_GETARG_TEXT_PP(1);
1294  float4 res;
1295 
1297  VARDATA_ANY(in2), VARSIZE_ANY_EXHDR(in2),
1299 
1300  PG_FREE_IF_COPY(in1, 0);
1301  PG_FREE_IF_COPY(in2, 1);
1302  PG_RETURN_FLOAT4(1.0 - res);
1303 }
1305 Datum
1307 {
1308  text *in1 = PG_GETARG_TEXT_PP(0);
1309  text *in2 = PG_GETARG_TEXT_PP(1);
1310  float4 res;
1311 
1313  VARDATA_ANY(in1), VARSIZE_ANY_EXHDR(in1),
1315 
1316  PG_FREE_IF_COPY(in1, 0);
1317  PG_FREE_IF_COPY(in2, 1);
1318  PG_RETURN_FLOAT4(1.0 - res);
1319 }
ArrayType * construct_array_builtin(Datum *elems, int nelems, Oid elmtype)
Definition: arrayfuncs.c:3374
unsigned int uint32
Definition: c.h:493
#define Max(x, y)
Definition: c.h:985
#define VARHDRSZ
Definition: c.h:679
float float4
Definition: c.h:616
unsigned char uint8
Definition: c.h:491
size_t Size
Definition: c.h:592
int errcode(int sqlerrcode)
Definition: elog.c:859
int errmsg(const char *fmt,...)
Definition: elog.c:1072
#define ERROR
Definition: elog.h:39
#define elog(elevel,...)
Definition: elog.h:224
#define ereport(elevel,...)
Definition: elog.h:149
char * OidOutputFunctionCall(Oid functionId, Datum val)
Definition: fmgr.c:1763
#define PG_FREE_IF_COPY(ptr, n)
Definition: fmgr.h:260
#define PG_GETARG_TEXT_PP(n)
Definition: fmgr.h:309
#define DirectFunctionCall2(func, arg1, arg2)
Definition: fmgr.h:644
#define PG_GETARG_DATUM(n)
Definition: fmgr.h:268
#define PG_GETARG_FLOAT4(n)
Definition: fmgr.h:281
#define PG_RETURN_POINTER(x)
Definition: fmgr.h:361
#define PG_RETURN_FLOAT4(x)
Definition: fmgr.h:366
#define PG_FUNCTION_ARGS
Definition: fmgr.h:193
#define PG_RETURN_BOOL(x)
Definition: fmgr.h:359
void DefineCustomRealVariable(const char *name, const char *short_desc, const char *long_desc, double *valueAddr, double bootValue, double minValue, double maxValue, GucContext context, int flags, GucRealCheckHook check_hook, GucRealAssignHook assign_hook, GucShowHook show_hook)
Definition: guc.c:5126
void SetConfigOption(const char *name, const char *value, GucContext context, GucSource source)
Definition: guc.c:4275
void MarkGUCPrefixReserved(const char *className)
Definition: guc.c:5217
@ PGC_S_SESSION
Definition: guc.h:122
@ PGC_USERSET
Definition: guc.h:75
#define CALCGTSIZE(flag, siglen)
Definition: hstore_gist.c:60
long val
Definition: informix.c:664
static int pg_cmp_s32(int32 a, int32 b)
Definition: int.h:483
int b
Definition: isn.c:70
int a
Definition: isn.c:69
int j
Definition: isn.c:74
int i
Definition: isn.c:73
Assert(fmt[strlen(fmt) - 1] !='\n')
void getTypeOutputInfo(Oid type, Oid *typOutput, bool *typIsVarlena)
Definition: lsyscache.c:2863
int pg_database_encoding_max_length(void)
Definition: mbutils.c:1546
int pg_mblen(const char *mbstr)
Definition: mbutils.c:1023
void pfree(void *pointer)
Definition: mcxt.c:1508
void * palloc0(Size size)
Definition: mcxt.c:1334
void * palloc(Size size)
Definition: mcxt.c:1304
#define MaxAllocSize
Definition: memutils.h:40
#define CHECK_FOR_INTERRUPTS()
Definition: miscadmin.h:122
Datum lower(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:49
Datum upper(PG_FUNCTION_ARGS)
Definition: oracle_compat.c:80
const void size_t len
return crc
uint32 pg_crc32
Definition: pg_crc.h:37
#define INIT_LEGACY_CRC32(crc)
Definition: pg_crc.h:79
#define COMP_LEGACY_CRC32(crc, data, len)
Definition: pg_crc.h:81
#define FIN_LEGACY_CRC32(crc)
Definition: pg_crc.h:80
static char * buf
Definition: pg_test_fsync.c:73
#define snprintf
Definition: port.h:238
#define qsort(a, b, c, d)
Definition: port.h:449
static Datum PointerGetDatum(const void *X)
Definition: postgres.h:322
static Datum Float4GetDatum(float4 X)
Definition: postgres.h:475
uintptr_t Datum
Definition: postgres.h:64
static float4 DatumGetFloat4(Datum X)
Definition: postgres.h:458
static Pointer DatumGetPointer(Datum X)
Definition: postgres.h:312
unsigned int Oid
Definition: postgres_ext.h:31
static size_t qunique(void *array, size_t elements, size_t width, int(*compare)(const void *, const void *))
Definition: qunique.h:21
static int cmp(const chr *x, const chr *y, size_t len)
Definition: regc_locale.c:743
uint16 StrategyNumber
Definition: stratnum.h:22
Definition: trgm.h:67
uint8 flag
Definition: trgm.h:69
Definition: type.h:95
int index
Definition: trgm_op.c:47
trgm trg
Definition: trgm_op.c:46
Definition: c.h:674
#define CALCSML(count, len1, len2)
Definition: trgm.h:116
#define ISWORDCHR(c)
Definition: trgm.h:55
#define WordSimilarityStrategyNumber
Definition: trgm.h:36
#define StrictWordSimilarityStrategyNumber
Definition: trgm.h:38
#define ISESCAPECHAR(x)
Definition: trgm.h:63
#define ARRNELEM(x)
Definition: trgm.h:107
#define RPADDING
Definition: trgm.h:17
#define LPADDING
Definition: trgm.h:16
#define SimilarityStrategyNumber
Definition: trgm.h:30
#define ISWILDCARDCHAR(x)
Definition: trgm.h:64
#define CMPTRGM(a, b)
Definition: trgm.h:46
char trgm[3]
Definition: trgm.h:42
#define CPTRGM(a, b)
Definition: trgm.h:48
#define ISPRINTABLETRGM(t)
Definition: trgm.h:61
#define GETARR(x)
Definition: trgm.h:106
#define ARRKEY
Definition: trgm.h:96
#define TRGMHDRSIZE
Definition: trgm.h:73
static float4 iterate_word_similarity(int *trg2indexes, bool *found, int ulen1, int len2, int len, uint8 flags, TrgmBound *bounds)
Definition: trgm_op.c:454
uint8 TrgmBound
Definition: trgm_op.c:51
bool * trgm_presence_map(TRGM *query, TRGM *key)
Definition: trgm_op.c:1079
Datum strict_word_similarity_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1272
TRGM * generate_trgm(char *str, int slen)
Definition: trgm_op.c:357
void _PG_init(void)
Definition: trgm_op.c:63
Datum set_limit(PG_FUNCTION_ARGS)
Definition: trgm_op.c:111
static const char * get_wildcard_part(const char *str, int lenstr, char *buf, int *bytelen, int *charlen)
Definition: trgm_op.c:722
static int comp_trgm(const void *a, const void *b)
Definition: trgm_op.c:162
double strict_word_similarity_threshold
Definition: trgm_op.c:24
TRGM * generate_wildcard_trgm(const char *str, int slen)
Definition: trgm_op.c:867
PG_MODULE_MAGIC
Definition: trgm_op.c:19
uint32 trgm2int(trgm *ptr)
Definition: trgm_op.c:936
#define WORD_SIMILARITY_CHECK_ONLY
Definition: trgm_op.c:56
static void protect_out_of_mem(int slen)
Definition: trgm_op.c:340
Datum word_similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1140
Datum strict_word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1304
void compact_trigram(trgm *tptr, char *str, int bytelen)
Definition: trgm_op.c:199
PG_FUNCTION_INFO_V1(set_limit)
static float4 calc_word_similarity(char *str1, int slen1, char *str2, int slen2, uint8 flags)
Definition: trgm_op.c:622
double word_similarity_threshold
Definition: trgm_op.c:23
Datum word_similarity_dist_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1240
double index_strategy_get_limit(StrategyNumber strategy)
Definition: trgm_op.c:133
static pos_trgm * make_positional_trgm(trgm *trg1, int len1, trgm *trg2, int len2)
Definition: trgm_op.c:398
Datum similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1118
double similarity_threshold
Definition: trgm_op.c:22
static int comp_ptrgm(const void *v1, const void *v2)
Definition: trgm_op.c:425
Datum show_trgm(PG_FUNCTION_ARGS)
Definition: trgm_op.c:950
static trgm * make_trigrams(trgm *tptr, char *str, int bytelen, int charlen)
Definition: trgm_op.c:224
bool trgm_contained_by(TRGM *trg1, TRGM *trg2)
Definition: trgm_op.c:1040
static char * find_word(char *str, int lenstr, char **endword, int *charlen)
Definition: trgm_op.c:172
#define TRGM_BOUND_RIGHT
Definition: trgm_op.c:53
Datum show_limit(PG_FUNCTION_ARGS)
Definition: trgm_op.c:156
Datum strict_word_similarity_dist_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1288
Datum word_similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1192
Datum word_similarity_commutator_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1208
#define WORD_SIMILARITY_STRICT
Definition: trgm_op.c:57
#define TRGM_BOUND_LEFT
Definition: trgm_op.c:52
float4 cnt_sml(TRGM *trg1, TRGM *trg2, bool inexact)
Definition: trgm_op.c:992
static int generate_trgm_only(trgm *trg, char *str, int slen, TrgmBound *bounds)
Definition: trgm_op.c:276
Datum similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1182
Datum word_similarity_dist_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1224
Datum strict_word_similarity(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1156
Datum similarity_dist(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1172
Datum strict_word_similarity_op(PG_FUNCTION_ARGS)
Definition: trgm_op.c:1256
char * lowerstr_with_len(const char *str, int len)
Definition: ts_locale.c:266
#define VARDATA(PTR)
Definition: varatt.h:278
#define VARDATA_ANY(PTR)
Definition: varatt.h:324
#define SET_VARSIZE(PTR, len)
Definition: varatt.h:305
#define VARSIZE_ANY_EXHDR(PTR)
Definition: varatt.h:317