PostgreSQL Source Code  git master
ts_typanalyze.c File Reference
#include "postgres.h"
#include "catalog/pg_collation.h"
#include "catalog/pg_operator.h"
#include "commands/vacuum.h"
#include "tsearch/ts_type.h"
#include "utils/builtins.h"
#include "utils/hashutils.h"
Include dependency graph for ts_typanalyze.c:

Go to the source code of this file.

Data Structures

struct  LexemeHashKey
 
struct  TrackItem
 

Functions

static void compute_tsvector_stats (VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows)
 
static void prune_lexemes_hashtable (HTAB *lexemes_tab, int b_current)
 
static uint32 lexeme_hash (const void *key, Size keysize)
 
static int lexeme_match (const void *key1, const void *key2, Size keysize)
 
static int lexeme_compare (const void *key1, const void *key2)
 
static int trackitem_compare_frequencies_desc (const void *e1, const void *e2)
 
static int trackitem_compare_lexemes (const void *e1, const void *e2)
 
Datum ts_typanalyze (PG_FUNCTION_ARGS)
 

Function Documentation

◆ compute_tsvector_stats()

static void compute_tsvector_stats ( VacAttrStats stats,
AnalyzeAttrFetchFunc  fetchfunc,
int  samplerows,
double  totalrows 
)
static

Definition at line 140 of file ts_typanalyze.c.

References VacAttrStats::anl_context, ARRPTR, Assert, VacAttrStats::attr, cstring_to_text_with_len(), CurrentMemoryContext, DatumGetPointer, DatumGetTSVector, DEBUG3, TrackItem::delta, elog, HASHCTL::entrysize, TrackItem::frequency, HASHCTL::hash, HASH_COMPARE, HASH_CONTEXT, hash_create(), HASH_ELEM, HASH_ENTER, HASH_FUNCTION, hash_get_num_entries(), hash_key(), hash_search(), hash_seq_init(), hash_seq_search(), HASHCTL::hcxt, i, TrackItem::key, HASHCTL::keysize, WordEntry::len, LexemeHashKey::length, LexemeHashKey::lexeme, lexeme_hash(), lexeme_match(), HASHCTL::match, Max, MemoryContextSwitchTo(), MemSet, Min, VacAttrStats::numnumbers, VacAttrStats::numvalues, palloc(), pfree(), PointerGetDatum, WordEntry::pos, prune_lexemes_hashtable(), qsort, TSVectorData::size, VacAttrStats::stacoll, VacAttrStats::stadistinct, VacAttrStats::stakind, VacAttrStats::stanullfrac, VacAttrStats::stanumbers, VacAttrStats::staop, VacAttrStats::stats_valid, VacAttrStats::statypalign, VacAttrStats::statypbyval, VacAttrStats::statypid, VacAttrStats::statyplen, VacAttrStats::stavalues, VacAttrStats::stawidth, STRPTR, trackitem_compare_frequencies_desc(), trackitem_compare_lexemes(), TSVectorGetDatum, vacuum_delay_point(), value, and VARSIZE_ANY.

Referenced by ts_typanalyze().

144 {
145  int num_mcelem;
146  int null_cnt = 0;
147  double total_width = 0;
148 
149  /* This is D from the LC algorithm. */
150  HTAB *lexemes_tab;
151  HASHCTL hash_ctl;
152  HASH_SEQ_STATUS scan_status;
153 
154  /* This is the current bucket number from the LC algorithm */
155  int b_current;
156 
157  /* This is 'w' from the LC algorithm */
158  int bucket_width;
159  int vector_no,
160  lexeme_no;
162  TrackItem *item;
163 
164  /*
165  * We want statistics_target * 10 lexemes in the MCELEM array. This
166  * multiplier is pretty arbitrary, but is meant to reflect the fact that
167  * the number of individual lexeme values tracked in pg_statistic ought to
168  * be more than the number of values for a simple scalar column.
169  */
170  num_mcelem = stats->attr->attstattarget * 10;
171 
172  /*
173  * We set bucket width equal to (num_mcelem + 10) / 0.007 as per the
174  * comment above.
175  */
176  bucket_width = (num_mcelem + 10) * 1000 / 7;
177 
178  /*
179  * Create the hashtable. It will be in local memory, so we don't need to
180  * worry about overflowing the initial size. Also we don't need to pay any
181  * attention to locking and memory management.
182  */
183  MemSet(&hash_ctl, 0, sizeof(hash_ctl));
184  hash_ctl.keysize = sizeof(LexemeHashKey);
185  hash_ctl.entrysize = sizeof(TrackItem);
186  hash_ctl.hash = lexeme_hash;
187  hash_ctl.match = lexeme_match;
188  hash_ctl.hcxt = CurrentMemoryContext;
189  lexemes_tab = hash_create("Analyzed lexemes table",
190  num_mcelem,
191  &hash_ctl,
193 
194  /* Initialize counters. */
195  b_current = 1;
196  lexeme_no = 0;
197 
198  /* Loop over the tsvectors. */
199  for (vector_no = 0; vector_no < samplerows; vector_no++)
200  {
201  Datum value;
202  bool isnull;
203  TSVector vector;
204  WordEntry *curentryptr;
205  char *lexemesptr;
206  int j;
207 
209 
210  value = fetchfunc(stats, vector_no, &isnull);
211 
212  /*
213  * Check for null/nonnull.
214  */
215  if (isnull)
216  {
217  null_cnt++;
218  continue;
219  }
220 
221  /*
222  * Add up widths for average-width calculation. Since it's a
223  * tsvector, we know it's varlena. As in the regular
224  * compute_minimal_stats function, we use the toasted width for this
225  * calculation.
226  */
227  total_width += VARSIZE_ANY(DatumGetPointer(value));
228 
229  /*
230  * Now detoast the tsvector if needed.
231  */
232  vector = DatumGetTSVector(value);
233 
234  /*
235  * We loop through the lexemes in the tsvector and add them to our
236  * tracking hashtable.
237  */
238  lexemesptr = STRPTR(vector);
239  curentryptr = ARRPTR(vector);
240  for (j = 0; j < vector->size; j++)
241  {
242  bool found;
243 
244  /*
245  * Construct a hash key. The key points into the (detoasted)
246  * tsvector value at this point, but if a new entry is created, we
247  * make a copy of it. This way we can free the tsvector value
248  * once we've processed all its lexemes.
249  */
250  hash_key.lexeme = lexemesptr + curentryptr->pos;
251  hash_key.length = curentryptr->len;
252 
253  /* Lookup current lexeme in hashtable, adding it if new */
254  item = (TrackItem *) hash_search(lexemes_tab,
255  (const void *) &hash_key,
256  HASH_ENTER, &found);
257 
258  if (found)
259  {
260  /* The lexeme is already on the tracking list */
261  item->frequency++;
262  }
263  else
264  {
265  /* Initialize new tracking list element */
266  item->frequency = 1;
267  item->delta = b_current - 1;
268 
269  item->key.lexeme = palloc(hash_key.length);
270  memcpy(item->key.lexeme, hash_key.lexeme, hash_key.length);
271  }
272 
273  /* lexeme_no is the number of elements processed (ie N) */
274  lexeme_no++;
275 
276  /* We prune the D structure after processing each bucket */
277  if (lexeme_no % bucket_width == 0)
278  {
279  prune_lexemes_hashtable(lexemes_tab, b_current);
280  b_current++;
281  }
282 
283  /* Advance to the next WordEntry in the tsvector */
284  curentryptr++;
285  }
286 
287  /* If the vector was toasted, free the detoasted copy. */
288  if (TSVectorGetDatum(vector) != value)
289  pfree(vector);
290  }
291 
292  /* We can only compute real stats if we found some non-null values. */
293  if (null_cnt < samplerows)
294  {
295  int nonnull_cnt = samplerows - null_cnt;
296  int i;
297  TrackItem **sort_table;
298  int track_len;
299  int cutoff_freq;
300  int minfreq,
301  maxfreq;
302 
303  stats->stats_valid = true;
304  /* Do the simple null-frac and average width stats */
305  stats->stanullfrac = (double) null_cnt / (double) samplerows;
306  stats->stawidth = total_width / (double) nonnull_cnt;
307 
308  /* Assume it's a unique column (see notes above) */
309  stats->stadistinct = -1.0 * (1.0 - stats->stanullfrac);
310 
311  /*
312  * Construct an array of the interesting hashtable items, that is,
313  * those meeting the cutoff frequency (s - epsilon)*N. Also identify
314  * the minimum and maximum frequencies among these items.
315  *
316  * Since epsilon = s/10 and bucket_width = 1/epsilon, the cutoff
317  * frequency is 9*N / bucket_width.
318  */
319  cutoff_freq = 9 * lexeme_no / bucket_width;
320 
321  i = hash_get_num_entries(lexemes_tab); /* surely enough space */
322  sort_table = (TrackItem **) palloc(sizeof(TrackItem *) * i);
323 
324  hash_seq_init(&scan_status, lexemes_tab);
325  track_len = 0;
326  minfreq = lexeme_no;
327  maxfreq = 0;
328  while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
329  {
330  if (item->frequency > cutoff_freq)
331  {
332  sort_table[track_len++] = item;
333  minfreq = Min(minfreq, item->frequency);
334  maxfreq = Max(maxfreq, item->frequency);
335  }
336  }
337  Assert(track_len <= i);
338 
339  /* emit some statistics for debug purposes */
340  elog(DEBUG3, "tsvector_stats: target # mces = %d, bucket width = %d, "
341  "# lexemes = %d, hashtable size = %d, usable entries = %d",
342  num_mcelem, bucket_width, lexeme_no, i, track_len);
343 
344  /*
345  * If we obtained more lexemes than we really want, get rid of those
346  * with least frequencies. The easiest way is to qsort the array into
347  * descending frequency order and truncate the array.
348  */
349  if (num_mcelem < track_len)
350  {
351  qsort(sort_table, track_len, sizeof(TrackItem *),
353  /* reset minfreq to the smallest frequency we're keeping */
354  minfreq = sort_table[num_mcelem - 1]->frequency;
355  }
356  else
357  num_mcelem = track_len;
358 
359  /* Generate MCELEM slot entry */
360  if (num_mcelem > 0)
361  {
362  MemoryContext old_context;
363  Datum *mcelem_values;
364  float4 *mcelem_freqs;
365 
366  /*
367  * We want to store statistics sorted on the lexeme value using
368  * first length, then byte-for-byte comparison. The reason for
369  * doing length comparison first is that we don't care about the
370  * ordering so long as it's consistent, and comparing lengths
371  * first gives us a chance to avoid a strncmp() call.
372  *
373  * This is different from what we do with scalar statistics --
374  * they get sorted on frequencies. The rationale is that we
375  * usually search through most common elements looking for a
376  * specific value, so we can grab its frequency. When values are
377  * presorted we can employ binary search for that. See
378  * ts_selfuncs.c for a real usage scenario.
379  */
380  qsort(sort_table, num_mcelem, sizeof(TrackItem *),
382 
383  /* Must copy the target values into anl_context */
384  old_context = MemoryContextSwitchTo(stats->anl_context);
385 
386  /*
387  * We sorted statistics on the lexeme value, but we want to be
388  * able to find out the minimal and maximal frequency without
389  * going through all the values. We keep those two extra
390  * frequencies in two extra cells in mcelem_freqs.
391  *
392  * (Note: the MCELEM statistics slot definition allows for a third
393  * extra number containing the frequency of nulls, but we don't
394  * create that for a tsvector column, since null elements aren't
395  * possible.)
396  */
397  mcelem_values = (Datum *) palloc(num_mcelem * sizeof(Datum));
398  mcelem_freqs = (float4 *) palloc((num_mcelem + 2) * sizeof(float4));
399 
400  /*
401  * See comments above about use of nonnull_cnt as the divisor for
402  * the final frequency estimates.
403  */
404  for (i = 0; i < num_mcelem; i++)
405  {
406  TrackItem *item = sort_table[i];
407 
408  mcelem_values[i] =
410  item->key.length));
411  mcelem_freqs[i] = (double) item->frequency / (double) nonnull_cnt;
412  }
413  mcelem_freqs[i++] = (double) minfreq / (double) nonnull_cnt;
414  mcelem_freqs[i] = (double) maxfreq / (double) nonnull_cnt;
415  MemoryContextSwitchTo(old_context);
416 
417  stats->stakind[0] = STATISTIC_KIND_MCELEM;
418  stats->staop[0] = TextEqualOperator;
419  stats->stacoll[0] = DEFAULT_COLLATION_OID;
420  stats->stanumbers[0] = mcelem_freqs;
421  /* See above comment about two extra frequency fields */
422  stats->numnumbers[0] = num_mcelem + 2;
423  stats->stavalues[0] = mcelem_values;
424  stats->numvalues[0] = num_mcelem;
425  /* We are storing text values */
426  stats->statypid[0] = TEXTOID;
427  stats->statyplen[0] = -1; /* typlen, -1 for varlena */
428  stats->statypbyval[0] = false;
429  stats->statypalign[0] = 'i';
430  }
431  }
432  else
433  {
434  /* We found only nulls; assume the column is entirely null */
435  stats->stats_valid = true;
436  stats->stanullfrac = 1.0;
437  stats->stawidth = 0; /* "unknown" */
438  stats->stadistinct = 0.0; /* "unknown" */
439  }
440 
441  /*
442  * We don't need to bother cleaning up any of our temporary palloc's. The
443  * hashtable should also go away, as it used a child memory context.
444  */
445 }
#define HASH_CONTEXT
Definition: hsearch.h:93
#define HASH_ELEM
Definition: hsearch.h:87
MemoryContext hcxt
Definition: hsearch.h:78
#define DEBUG3
Definition: elog.h:23
#define PointerGetDatum(X)
Definition: postgres.h:556
#define Min(x, y)
Definition: c.h:911
Datum * stavalues[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:114
static MemoryContext MemoryContextSwitchTo(MemoryContext context)
Definition: palloc.h:109
Size entrysize
Definition: hsearch.h:73
static struct @145 value
uint32 len
Definition: ts_type.h:44
#define MemSet(start, val, len)
Definition: c.h:962
long hash_get_num_entries(HTAB *hashp)
Definition: dynahash.c:1335
bool statypbyval[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:124
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:906
char statypalign[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:125
Form_pg_attribute attr
Definition: vacuum.h:85
Definition: dynahash.c:208
void pfree(void *pointer)
Definition: mcxt.c:1056
int32 stawidth
Definition: vacuum.h:106
Oid stacoll[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:110
text * cstring_to_text_with_len(const char *s, int len)
Definition: varlena.c:183
int numnumbers[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:111
LexemeHashKey key
Definition: ts_typanalyze.c:34
int32 size
Definition: ts_type.h:93
static int lexeme_match(const void *key1, const void *key2, Size keysize)
static void prune_lexemes_hashtable(HTAB *lexemes_tab, int b_current)
#define TSVectorGetDatum(X)
Definition: ts_type.h:119
MemoryContext CurrentMemoryContext
Definition: mcxt.c:38
#define DatumGetTSVector(X)
Definition: ts_type.h:117
float4 stanullfrac
Definition: vacuum.h:105
Oid staop[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:109
bool stats_valid
Definition: vacuum.h:104
float float4
Definition: c.h:491
uintptr_t Datum
Definition: postgres.h:367
HTAB * hash_create(const char *tabname, long nelem, HASHCTL *info, int flags)
Definition: dynahash.c:316
int16 stakind[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:108
Size keysize
Definition: hsearch.h:72
HashCompareFunc match
Definition: hsearch.h:75
#define VARSIZE_ANY(PTR)
Definition: postgres.h:335
Oid statypid[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:122
static uint32 lexeme_hash(const void *key, Size keysize)
#define Max(x, y)
Definition: c.h:905
#define Assert(condition)
Definition: c.h:739
static int trackitem_compare_frequencies_desc(const void *e1, const void *e2)
float4 * stanumbers[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:112
#define HASH_COMPARE
Definition: hsearch.h:90
void * hash_seq_search(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1389
uint32 pos
Definition: ts_type.h:44
void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp)
Definition: dynahash.c:1379
MemoryContext anl_context
Definition: vacuum.h:90
static int trackitem_compare_lexemes(const void *e1, const void *e2)
#define DatumGetPointer(X)
Definition: postgres.h:549
static dshash_hash hash_key(dshash_table *hash_table, const void *key)
Definition: dshash.c:883
int numvalues[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:113
void * palloc(Size size)
Definition: mcxt.c:949
#define STRPTR(x)
Definition: hstore.h:76
int16 statyplen[STATISTIC_NUM_SLOTS]
Definition: vacuum.h:123
#define elog(elevel,...)
Definition: elog.h:228
int i
#define ARRPTR(x)
Definition: cube.c:24
#define qsort(a, b, c, d)
Definition: port.h:488
void vacuum_delay_point(void)
Definition: vacuum.c:1942
HashValueFunc hash
Definition: hsearch.h:74
#define HASH_FUNCTION
Definition: hsearch.h:89
float4 stadistinct
Definition: vacuum.h:107

◆ lexeme_compare()

static int lexeme_compare ( const void *  key1,
const void *  key2 
)
static

Definition at line 499 of file ts_typanalyze.c.

References LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by lexeme_match(), and trackitem_compare_lexemes().

500 {
501  const LexemeHashKey *d1 = (const LexemeHashKey *) key1;
502  const LexemeHashKey *d2 = (const LexemeHashKey *) key2;
503 
504  /* First, compare by length */
505  if (d1->length > d2->length)
506  return 1;
507  else if (d1->length < d2->length)
508  return -1;
509  /* Lengths are equal, do a byte-by-byte comparison */
510  return strncmp(d1->lexeme, d2->lexeme, d1->length);
511 }

◆ lexeme_hash()

static uint32 lexeme_hash ( const void *  key,
Size  keysize 
)
static

Definition at line 477 of file ts_typanalyze.c.

References DatumGetUInt32, hash_any(), LexemeHashKey::length, and LexemeHashKey::lexeme.

Referenced by compute_tsvector_stats().

478 {
479  const LexemeHashKey *l = (const LexemeHashKey *) key;
480 
481  return DatumGetUInt32(hash_any((const unsigned char *) l->lexeme,
482  l->length));
483 }
#define DatumGetUInt32(X)
Definition: postgres.h:486
Datum hash_any(const unsigned char *k, int keylen)
Definition: hashfn.c:148

◆ lexeme_match()

static int lexeme_match ( const void *  key1,
const void *  key2,
Size  keysize 
)
static

Definition at line 489 of file ts_typanalyze.c.

References lexeme_compare().

Referenced by compute_tsvector_stats().

490 {
491  /* The keysize parameter is superfluous, the keys store their lengths */
492  return lexeme_compare(key1, key2);
493 }
static int lexeme_compare(const void *key1, const void *key2)

◆ prune_lexemes_hashtable()

static void prune_lexemes_hashtable ( HTAB lexemes_tab,
int  b_current 
)
static

Definition at line 452 of file ts_typanalyze.c.

References TrackItem::delta, elog, ERROR, TrackItem::frequency, HASH_REMOVE, hash_search(), hash_seq_init(), hash_seq_search(), TrackItem::key, LexemeHashKey::lexeme, and pfree().

Referenced by compute_tsvector_stats().

453 {
454  HASH_SEQ_STATUS scan_status;
455  TrackItem *item;
456 
457  hash_seq_init(&scan_status, lexemes_tab);
458  while ((item = (TrackItem *) hash_seq_search(&scan_status)) != NULL)
459  {
460  if (item->frequency + item->delta <= b_current)
461  {
462  char *lexeme = item->key.lexeme;
463 
464  if (hash_search(lexemes_tab, (const void *) &item->key,
465  HASH_REMOVE, NULL) == NULL)
466  elog(ERROR, "hash table corrupted");
467  pfree(lexeme);
468  }
469  }
470 }
void * hash_search(HTAB *hashp, const void *keyPtr, HASHACTION action, bool *foundPtr)
Definition: dynahash.c:906
void pfree(void *pointer)
Definition: mcxt.c:1056
#define ERROR
Definition: elog.h:43
LexemeHashKey key
Definition: ts_typanalyze.c:34
void * hash_seq_search(HASH_SEQ_STATUS *status)
Definition: dynahash.c:1389
void hash_seq_init(HASH_SEQ_STATUS *status, HTAB *hashp)
Definition: dynahash.c:1379
#define elog(elevel,...)
Definition: elog.h:228

◆ trackitem_compare_frequencies_desc()

static int trackitem_compare_frequencies_desc ( const void *  e1,
const void *  e2 
)
static

Definition at line 517 of file ts_typanalyze.c.

References TrackItem::frequency.

Referenced by compute_tsvector_stats().

518 {
519  const TrackItem *const *t1 = (const TrackItem *const *) e1;
520  const TrackItem *const *t2 = (const TrackItem *const *) e2;
521 
522  return (*t2)->frequency - (*t1)->frequency;
523 }

◆ trackitem_compare_lexemes()

static int trackitem_compare_lexemes ( const void *  e1,
const void *  e2 
)
static

Definition at line 529 of file ts_typanalyze.c.

References lexeme_compare().

Referenced by compute_tsvector_stats().

530 {
531  const TrackItem *const *t1 = (const TrackItem *const *) e1;
532  const TrackItem *const *t2 = (const TrackItem *const *) e2;
533 
534  return lexeme_compare(&(*t1)->key, &(*t2)->key);
535 }
static int lexeme_compare(const void *key1, const void *key2)

◆ ts_typanalyze()

Datum ts_typanalyze ( PG_FUNCTION_ARGS  )

Definition at line 55 of file ts_typanalyze.c.

References VacAttrStats::attr, VacAttrStats::compute_stats, compute_tsvector_stats(), default_statistics_target, VacAttrStats::minrows, PG_GETARG_POINTER, and PG_RETURN_BOOL.

56 {
58  Form_pg_attribute attr = stats->attr;
59 
60  /* If the attstattarget column is negative, use the default value */
61  /* NB: it is okay to scribble on stats->attr since it's a copy */
62  if (attr->attstattarget < 0)
63  attr->attstattarget = default_statistics_target;
64 
66  /* see comment about the choice of minrows in commands/analyze.c */
67  stats->minrows = 300 * attr->attstattarget;
68 
69  PG_RETURN_BOOL(true);
70 }
int minrows
Definition: vacuum.h:97
#define PG_GETARG_POINTER(n)
Definition: fmgr.h:271
Form_pg_attribute attr
Definition: vacuum.h:85
static void compute_tsvector_stats(VacAttrStats *stats, AnalyzeAttrFetchFunc fetchfunc, int samplerows, double totalrows)
FormData_pg_attribute * Form_pg_attribute
Definition: pg_attribute.h:200
#define PG_RETURN_BOOL(x)
Definition: fmgr.h:349
AnalyzeAttrComputeStatsFunc compute_stats
Definition: vacuum.h:96
int default_statistics_target
Definition: analyze.c:80